diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..4cc5222786 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,34 @@ +--- +name: Bug report +about: Create a report to help us improve +title: 'Potential Bug: [Insert title here]' +labels: bug +assignees: WiseQA + +--- + +**Where was the issue found:** +- Committee ID: +- Environment: +- Browser: + +**Please describe the issue clearly and concisely** +A clear and concise description of what the bug is. + +Approximate time the issue was found: + +**To Reproduce** +How to replicate the issue: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..15e6b4c5b5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Ticket for feature requirements +title: '' +labels: '' +assignees: '' + +--- + +### Business Reason +As a [role], I will be able to [blank] so that I can [business reason] + +### Acceptance Criteria +**Given** [precedent] +**When** [action] +**Then** [result] + +### QA Notes + +### DEV Notes + +### Design diff --git a/.github/ISSUE_TEMPLATE/issue-template.md b/.github/ISSUE_TEMPLATE/issue-template.md deleted file mode 100644 index c1e3d33d8c..0000000000 --- a/.github/ISSUE_TEMPLATE/issue-template.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: Issue template -about: Issue template for fecfile-web-api -title: '' -labels: '' -assignees: '' - ---- - -### Business Reason ### - -As a [role], I will be able to [blank] so that I can [business reason] - -### Acceptance Criteria ### - -**If** [precedent] -**When** [action] -**Then** [result] diff --git a/.gitignore b/.gitignore index b733090575..0d2bea196d 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,7 @@ django-backend/.local/ *.bower-tmp # Locust Testing Data -locust-testing/locust-data/*.locust.json +performance-testing/locust-data/*.locust.json # System Files diff --git a/bin/bulk-testing-data-fixture-generator.py b/bin/bulk-testing-data-fixture-generator.py new file mode 100644 index 0000000000..5a92a5d3ad --- /dev/null +++ b/bin/bulk-testing-data-fixture-generator.py @@ -0,0 +1,429 @@ +import json +from random import choice, choices, randrange +import string +import os +from uuid import UUID, uuid4 + + +PRIMARY_COMMITTEE_FEC_ID = os.environ.get("LOCAL_TEST_USER", "C00000000") +PRIMARY_COMMITTEE_UUID = os.environ.get("LOCAL_TEST_COMMITTEE_UUID", uuid4()) +CONTACT_TYPES = ["IND", "ORG", "COM", "CAN"] +SCHEDULE_FORMATS = { + "A": { + "schedule_name": "schedulea", + "amount_prefix": "contribution", + "date_prefix": "contribution", + "schedule_id_field": "schedule_a_id", + }, + "B": { + "schedule_name": "scheduleb", + "amount_prefix": "expenditure", + "date_prefix": "expenditure", + "schedule_id_field": "schedule_b_id", + }, + "C": { + "schedule_name": "schedulec", + "amount_prefix": "loan", + "date_prefix": "loan_incurred", + "schedule_id_field": "schedule_c_id", + }, + # Including Schedule D transactions was blocking Django from + # importing the resulting fixture cleanly. + # "D": { + # "schedule_name": "scheduled", + # "amount_prefix": "incurred", + # "date_prefix": None, + # "schedule_id_field": "schedule_d_id", + # }, + "E": { + "schedule_name": "schedulee", + "amount_prefix": "expenditure", + "date_prefix": "disbursement", + "schedule_id_field": "schedule_e_id", + }, +} +F3X_DATE_DATA = [ + { + "report_code": "M2", + "year": 2000, + "from_date": "01-01", + "through_date": "01-31" + }, + { + "report_code": "M3", + "year": 2000, + "from_date": "02-01", + "through_date": "02-28" + }, + { + "report_code": "M4", + "year": 2000, + "from_date": "03-01", + "through_date": "03-31" + }, + { + "report_code": "M5", + "year": 2000, + "from_date": "04-01", + "through_date": "04-30" + }, + { + "report_code": "M6", + "year": 2000, + "from_date": "05-01", + "through_date": "05-31" + }, + { + "report_code": "M7", + "year": 2000, + "from_date": "06-01", + "through_date": "06-30" + }, + { + "report_code": "M8", + "year": 2000, + "from_date": "07-01", + "through_date": "07-31" + }, + { + "report_code": "M9", + "year": 2000, + "from_date": "08-01", + "through_date": "08-31" + }, + { + "report_code": "M10", + "year": 2000, + "from_date": "09-01", + "through_date": "09-30" + }, + { + "report_code": "M11", + "year": 2000, + "from_date": "10-01", + "through_date": "10-31" + }, + { + "report_code": "M12", + "year": 2000, + "from_date": "11-01", + "through_date": "11-30" + }, + { + "report_code": "YE", + "year": 2000, + "from_date": "12-01", + "through_date": "12-31" + }, +] + + +def random_string(length, use_letters=True, use_digits=False, use_punctuation=False): + character_pool = "" + if use_letters: + character_pool += string.ascii_letters + if use_digits: + character_pool += string.digits + if use_punctuation: + character_pool += string.punctuation + + if len(character_pool) == 0: + return "" + + return "".join(choices(character_pool, k=length)) + + +def get_record_uuid(record): + return record["fields"]["id"] + + +def get_committee(committee_fec_id, committee_uuid): + return { + "model": "committee_accounts.CommitteeAccount", + "fields": { + "id": committee_uuid, + "committee_id": committee_fec_id, + "created": "2024-01-01T00:00:00.000Z", + "updated": "2024-01-01T00:00:00.000Z" + } + } + + +def create_committee(committee_fec_id=None, committee_uuid=None): + if committee_fec_id is None: + committee_fec_id = f"C{random_string(8, use_letters=False, use_digits=True)}" + if committee_uuid is None: + committee_uuid = uuid4() + + return get_committee(committee_fec_id, committee_uuid) + + +def get_form_3x_date_data(): + data = F3X_DATE_DATA.pop(0) + out_data = { + "report_code": data["report_code"], + "coverage_from_date": f"{data['year']}-{data['from_date']}", + "coverage_through_date": f"{data['year']}-{data['through_date']}", + "date_signed": f"{data['year']}-{data['through_date']}" + } + data["year"] += 1 + F3X_DATE_DATA.append(data) + return out_data + + +def get_form_3x(form_3x_id): + return { + "model": "reports.Form3X", + "fields": { + "id": form_3x_id, + "change_of_address": False, + "qualified_committee": False, + } + } + + +def get_report(report_id, form_3x_id, committee_uuid): + return { + "model": "reports.report", + "fields": { + "id": report_id, + "form_3x_id": form_3x_id, + "committee_account_id": committee_uuid, + "form_type": "F3XN", + "calculation_status": None, + "treasurer_last_name": random_string(8), + "treasurer_first_name": random_string(8), + "treasurer_middle_name": random_string(5), + "treasurer_prefix": random_string(3, use_punctuation=True), + "treasurer_suffix": random_string(3, use_punctuation=True), + "created": "2024-01-01T00:00:00.000Z", + "updated": "2024-01-01T00:00:00.000Z" + } | get_form_3x_date_data() + } + + +def create_report(committee_uuid, report_id=None, form_3x_id=None): + if report_id is None: + report_id = uuid4() + if form_3x_id is None: + form_3x_id = uuid4() + + return [ + get_form_3x(form_3x_id), + get_report(report_id, form_3x_id, committee_uuid) + ] + + +def get_contact(contact_id, committee_uuid, contact_type): + contact = { + "model": "contacts.contact", + "fields": { + "id": contact_id, + "type": contact_type, + "committee_account_id": committee_uuid, + "street_1": random_string(16, use_digits=True), + "street_2": random_string(6, use_digits=True), + "city": random_string(10), + "state": random_string(2), + "zip": random_string(2, use_letters=False, use_digits=True), + "country": "USA", + "created": "2024-01-01T00:00:00.000Z", + "updated": "2024-01-01T00:00:00.000Z" + } + } + + if contact_type in ["COM", "ORG"]: + contact["fields"] |= { + "name": random_string(16, use_digits=True, use_punctuation=True) + } + if contact_type == "COM": + contact["fields"] |= { + "committee_id": f"C{random_string(8, use_letters=False, use_digits=True)}" + } + if contact_type in ["IND", "CAN"]: + contact["fields"] |= { + "telephone": random_string(10, use_letters=False, use_digits=True), + "last_name": random_string(8), + "first_name": random_string(8), + "middle_name": random_string(5), + "prefix": random_string(3, use_punctuation=True), + "suffix": random_string(3, use_punctuation=True), + "employer": random_string(16, use_digits=True, use_punctuation=True), + "occupation": random_string(12, use_digits=True, use_punctuation=True), + } + if contact_type == "CAN": + candidate_office = choice(["H", "S", "P"]) + candidate_district = random_string(1, use_letters=False, use_digits=True) + candidate_state = random_string(2) + candidate_id = f"""{candidate_office}{candidate_district}{candidate_state}{ + random_string(5, use_letters=False, use_digits=True) + }""" + contact["fields"] |= { + "candidate_state": candidate_state, + "candidate_office": candidate_office, + "candidate_district": candidate_district, + "candidate_id": candidate_id + } + + return contact + + +def create_contact(committee_uuid, contact_id=None): + if contact_id is None: + contact_id = uuid4() + + contact_format = choice(CONTACT_TYPES) + return get_contact(contact_id, committee_uuid, contact_format) + + +def get_sched_transaction(sched_transaction_id, amount, date, schedule_format): + amount_prefix = schedule_format["amount_prefix"] + date_prefix = schedule_format["date_prefix"] + + return { + "model": f"transactions.{schedule_format['schedule_name']}", + "fields": { + "id": f"{sched_transaction_id}", + f"{amount_prefix}_amount": amount, + } | {f"{date_prefix}_date": f"{date}"} if date_prefix else {} + } + + +def get_transaction( + transaction_id, sched_transaction_id, committee_uuid, contact_id, schedule +): + return { + "model": "transactions.transaction", + "fields": { + "id": f"{transaction_id}", + f"{schedule['schedule_id_field']}": f"{sched_transaction_id}", + "committee_account_id": f"{committee_uuid}", + "_form_type": "SA11AI", + "memo_code": choices([True, False], [0.8, 0.2], k=1)[0], + "contact_1_id": f"{contact_id}", + "aggregation_group": "GENERAL", + "transaction_type_identifier": "INDIVIDUAL_RECEIPT", + "created": "2024-01-01T00:00:00.000Z", + "updated": "2024-01-01T00:00:00.000Z" + } + } + + +def get_transaction_report(transaction_id, report_id): + return { + "model": "reports.reporttransaction", + "fields": { + "transaction_id": f"{transaction_id}", + "report_id": f"{report_id}", + "created": "2024-01-01T00:00:00.000Z", + "updated": "2024-01-01T00:00:00.000Z" + } + } + + +def create_transaction(committee_uuid, contact_id, report): + schedule = choice(list(SCHEDULE_FORMATS.values())) + report_id = get_record_uuid(report) + report_date = report["fields"]["coverage_from_date"] + + sched_transaction_id = uuid4() + transaction_id = uuid4() + return [ + get_sched_transaction( + sched_transaction_id, randrange(100, 500), report_date, schedule + ), + get_transaction( + transaction_id, sched_transaction_id, committee_uuid, contact_id, schedule + ), + get_transaction_report(transaction_id, report_id) + ] + + +def get_records(): + committees = [ + create_committee() for _ in range(4) + ] + committees.append(create_committee(PRIMARY_COMMITTEE_FEC_ID, PRIMARY_COMMITTEE_UUID)) + committee_records = {} + for comm in committees: + committee_records[get_record_uuid(comm)] = { + "committee_record": comm, + "form_3xs": [], + "reports": [], + "contacts": [], + "sched_transactions": [], + "transactions": [], + "transaction_reports": [] + } + + for _ in range(50): + committee = choice(committees) + committee_uuid = get_record_uuid(committee) + form_3x, report = create_report(committee_uuid) + committee_records[committee_uuid]["form_3xs"].append(form_3x) + committee_records[committee_uuid]["reports"].append(report) + + for _ in range(2395): + committee = choice(committees) + committee_uuid = get_record_uuid(committee) + contact = create_contact(committee_uuid) + committee_records[committee_uuid]["contacts"].append(contact) + + for _ in range(32500): + committee = choice(committees) + committee_uuid = get_record_uuid(committee) + report = choice(committee_records[committee_uuid]["reports"]) + contact = choice(committee_records[committee_uuid]["contacts"]) + contact_id = get_record_uuid(contact) + sched, trans, trans_rep = create_transaction( + committee_uuid, contact_id, report + ) + committee_records[committee_uuid]["sched_transactions"].append(sched) + committee_records[committee_uuid]["transactions"].append(trans) + committee_records[committee_uuid]["transaction_reports"].append(trans_rep) + + return committee_records + + +def prepare_records(records): + out_records = [] + for c in records.values(): + if c["committee_record"]["fields"]["committee_id"] != PRIMARY_COMMITTEE_FEC_ID: + out_records.append(c["committee_record"]) + out_records += ( + [c["form_3xs"]] + + c["reports"] + + c["contacts"] + + c["sched_transactions"] + + c["transactions"] + + c["transaction_reports"] + ) + return out_records + + +class UUIDEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, UUID): + return obj.hex + return json.JSONEncoder.default(self, obj) + + +def save_records_to_fixture(records): + directory = os.getcwd() + filename = "bulk-testing-data.locust.json" + full_filename = os.path.join(directory, filename) + file = open(full_filename, "w") + file.write(json.dumps(records, cls=UUIDEncoder)) + file.close() + + +records = get_records() +sorted_records = prepare_records(records) +save_records_to_fixture(sorted_records) +print(PRIMARY_COMMITTEE_FEC_ID, PRIMARY_COMMITTEE_UUID) +for c in records.values(): + print( + c["committee_record"]["fields"]["id"], + c["committee_record"]["fields"]["committee_id"] + ) +print(f"Generated fixture with {'{:,}'.format(len(sorted_records))} records") diff --git a/django-backend/fecfiler/committee_accounts/test_views.py b/django-backend/fecfiler/committee_accounts/test_views.py index dbe02ae428..2ce13981d3 100644 --- a/django-backend/fecfiler/committee_accounts/test_views.py +++ b/django-backend/fecfiler/committee_accounts/test_views.py @@ -125,6 +125,29 @@ def test_add_membership_for_preexisting_user(self): self.assertEqual(response.data["email"], "test_user_0001@fec.gov") self.assertEqual(response.data["is_active"], True) + def test_add_membership_for_preexisting_user_email_case_test(self): + # This test covers a bug found where the email entered is treated + # as case when determining membership when it should not be. + + view = CommitteeMembershipViewSet() + request = self.factory.get("/api/v1/committee-members/add-member") + request.user = self.user + request.session = { + "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), + "committee_id": "C01234567", + } + request.method = "POST" + request.data = { + "role": Membership.CommitteeRole.COMMITTEE_ADMINISTRATOR, + "email": "TEST_USER_0001@fec.gov", + } + view.request = request + response = view.add_member(request) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.data["email"], "test_user_0001@fec.gov") + self.assertEqual(response.data["is_active"], True) + def test_add_membership_requires_correct_parameters(self): view = CommitteeMembershipViewSet() request = self.factory.get("/api/v1/committee-members/add-member") diff --git a/django-backend/fecfiler/committee_accounts/views.py b/django-backend/fecfiler/committee_accounts/views.py index 96ebe6836e..bb14bba0a8 100644 --- a/django-backend/fecfiler/committee_accounts/views.py +++ b/django-backend/fecfiler/committee_accounts/views.py @@ -176,7 +176,7 @@ def add_member(self, request): ) # Create new membership - user = User.objects.filter(email=email).first() + user = User.objects.filter(email__iexact=email).first() membership_args = { "committee_account": committee, diff --git a/django-backend/fecfiler/contacts/tests/utils.py b/django-backend/fecfiler/contacts/tests/utils.py index 669775584f..8f4cba7bf2 100644 --- a/django-backend/fecfiler/contacts/tests/utils.py +++ b/django-backend/fecfiler/contacts/tests/utils.py @@ -9,3 +9,25 @@ def create_test_individual_contact(last_name, first_name, committee_account_id): committee_account_id=committee_account_id, ) return contact + + +def create_test_candidate_contact( + last_name, + first_name, + committee_account_id, + candidate_id, + candidate_office, + candidate_state, + candidate_district, +): + contact = Contact.objects.create( + type=Contact.ContactType.CANDIDATE, + last_name=last_name, + first_name=first_name, + committee_account_id=committee_account_id, + candidate_id=candidate_id, + candidate_office=candidate_office, + candidate_state=candidate_state, + candidate_district=candidate_district, + ) + return contact diff --git a/django-backend/fecfiler/contacts/views.py b/django-backend/fecfiler/contacts/views.py index f6a0e3b37d..8dd93342da 100644 --- a/django-backend/fecfiler/contacts/views.py +++ b/django-backend/fecfiler/contacts/views.py @@ -349,7 +349,5 @@ def restore(self, request): "Contact Ids are invalid", status=status.HTTP_400_BAD_REQUEST, ) - for contact in contacts: - contact.deleted = None - contact.save() + contacts.update(deleted=None) return Response(ids_to_restore) diff --git a/django-backend/fecfiler/feedback/test_views.py b/django-backend/fecfiler/feedback/test_views.py index 23257814fe..12fdeaae5e 100644 --- a/django-backend/fecfiler/feedback/test_views.py +++ b/django-backend/fecfiler/feedback/test_views.py @@ -32,7 +32,7 @@ def test_submit_feedback_happy_path( } response = client.post( - "/api/v1/feedback/submit/", data=test_post_data, **test_headers + "/api/v1/feedback/submit/", data=test_post_data, secure=True, **test_headers ) mock_github3_login.assert_called_once() diff --git a/django-backend/fecfiler/memo_text/test_models.py b/django-backend/fecfiler/memo_text/test_models.py index 20e277d347..509ff90871 100644 --- a/django-backend/fecfiler/memo_text/test_models.py +++ b/django-backend/fecfiler/memo_text/test_models.py @@ -1,23 +1,30 @@ import uuid from django.test import TestCase from .models import MemoText +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x class MemoTextTestCase(TestCase): - fixtures = ["test_f3x_reports", "test_memo_text", "C01234567_user_and_committee"] def setUp(self): + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + self.valid_memo_text = MemoText( id="94777fb3-6d3a-4e2c-87dc-5e6ed326e65b", rec_type="TEXT", text4000="tessst4", - committee_account_id="11111111-2222-3333-4444-555555555555", - report_id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + committee_account_id=self.committee.id, + report_id=q1_report.id, ) def test_get_memo_text(self): - memo_text = MemoText.objects.get(id="1dee28f8-4cfa-4f70-8658-7a9e7f02ab1d") - self.assertEquals(memo_text.text4000, "dahtest2") + self.valid_memo_text.save() + memo_text = MemoText.objects.get(id="94777fb3-6d3a-4e2c-87dc-5e6ed326e65b") + self.assertEquals(memo_text.text4000, "tessst4") def test_save_and_delete(self): self.valid_memo_text.save() @@ -28,5 +35,5 @@ def test_save_and_delete(self): self.assertEquals( memo_text_from_db.id, uuid.UUID("94777fb3-6d3a-4e2c-87dc-5e6ed326e65b") ) - self.assertEquals(memo_text_from_db.transaction_id_number, "REPORT_MEMO_TEXT2") + self.assertEquals(memo_text_from_db.transaction_id_number, "REPORT_MEMO_TEXT1") self.assertEquals(memo_text_from_db.text4000, "tessst4") diff --git a/django-backend/fecfiler/memo_text/test_serializers.py b/django-backend/fecfiler/memo_text/test_serializers.py index 857c825925..eeffd73ae1 100644 --- a/django-backend/fecfiler/memo_text/test_serializers.py +++ b/django-backend/fecfiler/memo_text/test_serializers.py @@ -2,35 +2,40 @@ from .serializers import MemoTextSerializer from fecfiler.user.models import User from rest_framework.request import Request, HttpRequest +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x class MemoTextSerializerTestCase(TestCase): - fixtures = ["test_memo_text", "C01234567_user_and_committee", "test_f3x_reports"] def setUp(self): + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) + q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + self.valid_memo_text = { "rec_type": "TEXT", - "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "report_id": q1_report.id, "text4000": "tessst4", "transaction_id_number": "REPORT_MEMO_TEXT_3", - "committee_account": "11111111-2222-3333-4444-555555555555", + "committee_account": self.committee.id, } self.invalid_memo_text = { "rec_type": "Invalid_rec_type", - "report_id": "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + "report_id": q1_report.id, "text4000": "tessst4", "transaction_id_number": "REPORT_MEMO_TEXT_3", - "committee_account": "11111111-2222-3333-4444-555555555555", + "committee_account": self.committee.id, } self.mock_request = Request(HttpRequest()) - self.mock_request.user = User.objects.get( - id="12345678-aaaa-bbbb-cccc-111122223333" - ) + self.mock_request.user = self.user self.mock_request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } def test_serializer_validate(self): diff --git a/django-backend/fecfiler/memo_text/test_views.py b/django-backend/fecfiler/memo_text/test_views.py index 1eae3fcf4f..649e163434 100644 --- a/django-backend/fecfiler/memo_text/test_views.py +++ b/django-backend/fecfiler/memo_text/test_views.py @@ -2,23 +2,26 @@ from fecfiler.user.models import User from rest_framework.test import APIClient from .views import MemoTextViewSet +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x class MemoTextViewSetTest(TestCase): - fixtures = ["test_f3x_reports", "test_memo_text", "C01234567_user_and_committee"] def setUp(self): - self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) + self.q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) self.factory = RequestFactory() def test_get_memo_texts_for_report_id(self): - request = self.factory.get( - "/api/v1/memo-text/?report_id=b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + request = self.factory.get(f"/api/v1/memo-text/?report_id={self.q1_report.id}") request.user = self.user request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } response = MemoTextViewSet.as_view({"get": "list"})(request) self.assertEqual(response.status_code, 200) @@ -28,14 +31,14 @@ def test_create_new_report_memo_text(self): client.force_authenticate(user=self.user) session = client.session._get_session_from_db() session.session_data = client.session.encode( - {"committee_uuid": "11111111-2222-3333-4444-555555555555"} + {"committee_uuid": str(self.committee.id)} ) session.save() data = { - "report_id": "1406535e-f99f-42c4-97a8-247904b7d297", + "report_id": self.q1_report.id, "rec_type": "TEXT", "text4000": "test_new_text", - "committee_account": "11111111-2222-3333-4444-555555555555", + "committee_account": str(self.committee.id), "transaction_id_number": "id_number", "transaction_uuid": None, "fields_to_validate": [ @@ -47,7 +50,7 @@ def test_create_new_report_memo_text(self): "transaction_uuid", ], } - response = client.post("/api/v1/memo-text/", data, format="json") + response = client.post("/api/v1/memo-text/", data, format="json", secure=True) self.assertEqual(response.status_code, 201) self.assertEqual(response.data["text4000"], "test_new_text") @@ -56,17 +59,17 @@ def test_create_existing_report_memo_text(self): client.force_authenticate(user=self.user) session = client.session._get_session_from_db() session.session_data = client.session.encode( - {"committee_uuid": "11111111-2222-3333-4444-555555555555"} + {"committee_uuid": str(self.committee.id)} ) session.save() data = { - "report_id": "a07c8c65-1b2d-4e6e-bcaa-fa8d39e50965", + "report_id": str(self.q1_report.id), "rec_type": "TEXT", "text4000": "test_existing_text", - "committee_account": "11111111-2222-3333-4444-555555555555", + "committee_account": str(self.committee.id), "transaction_id_number": "id_number", "transaction_uuid": None, } - response = client.post("/api/v1/memo-text/", data, format="json") + response = client.post("/api/v1/memo-text/", data, format="json", secure=True) self.assertEqual(response.status_code, 201) self.assertEqual(response.data["text4000"], "test_existing_text") diff --git a/django-backend/fecfiler/reports/form_3x/tests/test_models.py b/django-backend/fecfiler/reports/form_3x/tests/test_models.py index 9bb232a741..e49a11a248 100644 --- a/django-backend/fecfiler/reports/form_3x/tests/test_models.py +++ b/django-backend/fecfiler/reports/form_3x/tests/test_models.py @@ -1,11 +1,21 @@ from django.test import TestCase from ..models import Form3X +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x class F3XTestCase(TestCase): - fixtures = ["C01234567_user_and_committee", "test_f3x_reports"] def setUp(self): + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.q1_report = create_form3x( + self.committee, + "2024-01-01", + "2024-02-01", + {"election_code": "test_string_value"}, + ) self.valid_f3x_summary = Form3X( change_of_address=False, election_code="P2020", @@ -16,8 +26,7 @@ def setUp(self): ) def test_get_f3x_summary(self): - f3x_summary = Form3X.objects.get(id="a6d60d2d-d926-4e89-ad4b-c47d152a66ae") - self.assertEquals(f3x_summary.election_code, "test_string_value") + self.assertEquals(self.q1_report.form_3x.election_code, "test_string_value") def test_save_and_delete(self): self.valid_f3x_summary.save() diff --git a/django-backend/fecfiler/reports/form_3x/tests/test_views.py b/django-backend/fecfiler/reports/form_3x/tests/test_views.py index d6d0411cf7..0e1b0d2060 100644 --- a/django-backend/fecfiler/reports/form_3x/tests/test_views.py +++ b/django-backend/fecfiler/reports/form_3x/tests/test_views.py @@ -3,15 +3,49 @@ from fecfiler.reports.models import Report from ..views import Form3XViewSet from fecfiler.user.models import User - +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x from rest_framework.test import force_authenticate class Form3XViewSetTest(TestCase): - fixtures = ["test_f3x_reports", "C01234567_user_and_committee"] def setUp(self): - self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) + self.q1_report = create_form3x( + self.committee, + "2004-01-01", + "2004-02-28", + {}, + "Q1", + ) + + self.m4_report = create_form3x( + self.committee, + "2005-03-01", + "2005-03-31", + {}, + "M4", + ) + + self.my_report = create_form3x( + self.committee, + "2006-01-30", + "2006-02-28", + {}, + "MY", + ) + + self.twelve_c_report = create_form3x( + self.committee, + "2007-01-30", + "2007-02-28", + {}, + "12C", + ) self.factory = RequestFactory() def test_coverage_dates_happy_path(self): @@ -19,16 +53,16 @@ def test_coverage_dates_happy_path(self): request = self.factory.get("/api/v1/reports/form-f3x/coverage_dates") request.user = self.user request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } response = Form3XViewSet.as_view({"get": "coverage_dates"})(request) expected_json = [ { - "coverage_from_date": "2005-01-30", - "coverage_through_date": "2005-02-28", + "coverage_from_date": "2004-01-01", + "coverage_through_date": "2004-02-28", "report_code": "Q1", }, { @@ -52,21 +86,17 @@ def test_coverage_dates_happy_path(self): self.assertJSONEqual(str(response.content, encoding="utf8"), expected_json) def test_amend(self): - request = self.factory.post( - "/api/v1/reports/1406535e-f99f-42c4-97a8-247904b7d297/amend/" - ) + request = self.factory.post(f"/api/v1/reports/{self.q1_report.id}/amend/") request.user = self.user request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } force_authenticate(request, self.user) view = Form3XViewSet.as_view({"post": "amend"}) - response = view(request, pk="1406535e-f99f-42c4-97a8-247904b7d297") + response = view(request, pk=self.q1_report.id) self.assertEqual(response.status_code, 200) self.assertEqual( - Report.objects.filter(id="1406535e-f99f-42c4-97a8-247904b7d297") - .first() - .form_type, + Report.objects.filter(id=self.q1_report.id).first().form_type, "F3XA", ) diff --git a/django-backend/fecfiler/reports/models.py b/django-backend/fecfiler/reports/models.py index ad2835a6f2..672dd13687 100644 --- a/django-backend/fecfiler/reports/models.py +++ b/django-backend/fecfiler/reports/models.py @@ -1,6 +1,8 @@ import uuid +from rest_framework.exceptions import ValidationError +from rest_framework import status from django.db import models, transaction as db_transaction -from django.db.models import Q +from django.db.models import OuterRef, Subquery, Exists, Q from fecfiler.committee_accounts.models import CommitteeOwnedModel from .managers import ReportManager from .form_3x.models import Form3X @@ -106,7 +108,7 @@ def save(self, *args, **kwargs): if create_action and self.coverage_through_date: carry_forward_loans(self) carry_forward_debts(self) - update_recalculation(self) + flag_reports_for_recalculation(self) def get_future_in_progress_reports( self, @@ -135,6 +137,8 @@ def amend(self): self.save() def delete(self): + if not self.can_delete(): + raise ValidationError("Cannot delete report", status.HTTP_400_BAD_REQUEST) if not self.form_24: """only delete transactions if the report is the source of the tranaction""" @@ -152,6 +156,61 @@ def delete(self): super(CommitteeOwnedModel, self).delete() + def can_delete(self): + """ + can't delete if submitted + can't delete if amended + can't delete form3x if there exists any transactions in this report or + where any transactions in a different report back reference to them + """ + return ( + not self.upload_submission + and ( + self.report_version is None + or self.report_version == "0" + or self.report_version == 0 + ) + and ( + not self.form_3x + or ( + not bool(self.form_24) + and not ReportTransaction.objects.filter( + Exists( + Subquery( + ReportTransaction.objects.filter( + ~Q(report_id=self.id), + Q( + Q(transaction__id=OuterRef("transaction_id")) + | Q( + transaction__reatt_redes_id=OuterRef( + "transaction_id" + ) + ) + | Q( + transaction__parent_transaction_id=OuterRef( + "transaction_id" + ) + ) + | Q( + transaction__debt_id=OuterRef( + "transaction_id" + ) + ) + | Q( + transaction__loan_id=OuterRef( + "transaction_id" + ) + ) + ), + ) + ) + ), + report_id=self.id, + ).exists() + ) + ) + ) + TABLE_TO_FORM = { "form_3x": "F3X", @@ -165,22 +224,29 @@ def delete(self): ] -def update_recalculation(report: Report): - if report: +def flag_reports_for_recalculation(report: Report): + if report and report.get_form_name() in FORMS_TO_CALCULATE: committee = report.committee_account report_date = report.coverage_from_date - if report_date is not None: - reports_to_flag_for_recalculation = Report.objects.filter( + reports_to_flag = [] + if report_date is None: + reports_to_flag = Report.objects.get(id=report.id) + else: + reports_to_flag = Report.objects.filter( committee_account=committee, - coverage_from_date__gte=report_date, + coverage_from_date__gte=report_date ) - else: - reports_to_flag_for_recalculation = [report] - for report_to_recalc in reports_to_flag_for_recalculation: - report_to_recalc.calculation_status = None - report_to_recalc.save() - logger.info(f"Report: {report_to_recalc.id} marked for recalcuation") + flagged_count = reports_to_flag.update( + calculation_status=None + ) + logger.info( + f"""Report { + report.id + } marked for recalculation along with { + flagged_count-1 + } subsequent reports""" + ) class ReportMixin(models.Model): @@ -202,9 +268,7 @@ class ReportTransaction(models.Model): serialize=False, unique=True, ) - transaction = models.ForeignKey( - "transactions.Transaction", on_delete=models.CASCADE - ) + transaction = models.ForeignKey("transactions.Transaction", on_delete=models.CASCADE) report = models.ForeignKey(Report, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True) updated = models.DateTimeField(auto_now=True) diff --git a/django-backend/fecfiler/reports/serializers.py b/django-backend/fecfiler/reports/serializers.py index 73033cd9bd..a43c0f4111 100644 --- a/django-backend/fecfiler/reports/serializers.py +++ b/django-backend/fecfiler/reports/serializers.py @@ -1,4 +1,4 @@ -from .models import Report, ReportTransaction +from .models import Report from rest_framework.serializers import ( ModelSerializer, CharField, @@ -17,7 +17,6 @@ from fecfiler.reports.form_99.models import Form99 from fecfiler.reports.form_1m.models import Form1M from fecfiler.reports.form_1m.utils import add_form_1m_contact_fields -from django.db.models import OuterRef, Subquery, Exists, Q import structlog logger = structlog.get_logger(__name__) @@ -137,45 +136,10 @@ def to_representation(self, instance: Report, depth=0): this_report = Report.objects.get(id=representation["id"]) representation["is_first"] = this_report.is_first if this_report else True - representation["can_delete"] = self.can_delete(representation) + representation["can_delete"] = instance.can_delete() return representation - def can_delete(self, representation): - """can delete if there exist no transactions in this report - where any transactions in a different report back reference to them""" - no_check = ["F24", "F1M", "F99"] - return representation.get("report_status") == "In progress" and ( - representation["report_type"] in no_check - or not ( - ReportTransaction.objects.filter( - Exists( - Subquery( - ReportTransaction.objects.filter( - ~Q(report_id=representation["id"]), - Q( - Q(transaction__id=OuterRef("transaction_id")) - | Q( - transaction__reatt_redes_id=OuterRef( - "transaction_id" - ) - ) - | Q( - transaction__parent_transaction_id=OuterRef( - "transaction_id" - ) - ) - | Q(transaction__debt_id=OuterRef("transaction_id")) - | Q(transaction__loan_id=OuterRef("transaction_id")) - ), - ) - ) - ), - report_id=representation["id"], - ).exists() - ) - ) - def validate(self, data): self._context = self.context.copy() self._context["fields_to_ignore"] = self._context.get( diff --git a/django-backend/fecfiler/reports/test_views.py b/django-backend/fecfiler/reports/test_views.py index 6845a4fd0f..977ff0eb4d 100644 --- a/django-backend/fecfiler/reports/test_views.py +++ b/django-backend/fecfiler/reports/test_views.py @@ -1,32 +1,33 @@ from uuid import UUID +from django.http import QueryDict from django.test import RequestFactory, TestCase from fecfiler.reports.views import ReportViewSet from fecfiler.user.models import User +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x import structlog logger = structlog.get_logger(__name__) class CommitteeMemberViewSetTest(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_f24_reports", - "test_f99_reports", - ] - def setUp(self): - self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) self.factory = RequestFactory() def test_list_paginated(self): + for _ in range(10): + create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) view = ReportViewSet() view.format_kwarg = "format" request = self.factory.get("/api/v1/reports") request.user = self.user request.session = { - "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } request.method = "GET" request.query_params = {"page": 1} @@ -40,17 +41,52 @@ def test_list_no_pagination(self): request = self.factory.get("/api/v1/reports") request.user = self.user request.session = { - "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } request.method = "GET" request.query_params = {} view.request = request response = view.list(request) try: - response.data[ - "results" - ] # A non-paginated response will throw an error here + response.data["results"] # A non-paginated response will throw an error here self.assertTrue(response is None) except TypeError: self.assertTrue(response is not None) + + def test_ordering(self): + view = ReportViewSet() + view.format_kwarg = "format" + request = self.factory.get("/api/v1/reports") + request.user = self.user + request.session = { + "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), + "committee_id": "C01234567", + } + request.method = "GET" + q = QueryDict(mutable=True) + q["ordering"] = "form_type" + q["page"] = 1 + request.query_params = q + view.request = request + response = view.list(request) + + self.assertEqual(response.status_code, 200) + results = response.data["results"] + # Check that the results are ordered correctly + form_type_ordering = { + "F1MN": 10, + "F1MA": 10, + "F3XN": 20, + "F3XA": 20, + "F3XT": 20, + "F24N": 30, + "F24A": 30, + "F99": 40, + } + last_ordering = -1 + for result in results: + form_type = result["form_type"] + ordering = form_type_ordering.get(form_type, 0) + self.assertGreaterEqual(ordering, last_ordering) + last_ordering = ordering diff --git a/django-backend/fecfiler/reports/tests/test_models.py b/django-backend/fecfiler/reports/tests/test_models.py index d3bdacd77c..5976862b1e 100644 --- a/django-backend/fecfiler/reports/tests/test_models.py +++ b/django-backend/fecfiler/reports/tests/test_models.py @@ -1,9 +1,11 @@ +from decimal import Decimal from django.test import TestCase from fecfiler.web_services.models import UploadSubmission from fecfiler.reports.models import Report, Form24, Form3X -from fecfiler.reports.tests.utils import create_form3x, create_form24 +from fecfiler.reports.tests.utils import create_form, create_form3x, create_form24 +from fecfiler.committee_accounts.views import create_committee_view from fecfiler.committee_accounts.models import CommitteeAccount -from fecfiler.transactions.tests.utils import create_ie +from fecfiler.transactions.tests.utils import create_ie, create_debt from fecfiler.contacts.models import Contact from fecfiler.transactions.models import Transaction import structlog @@ -12,36 +14,36 @@ class ReportModelTestCase(TestCase): - fixtures = ["C01234567_user_and_committee", "test_f3x_reports", "test_f24_reports"] def setUp(self): self.missing_type_transaction = {} self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.f24_report = create_form24(self.committee) + self.f3x_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + self.contact_1 = Contact.objects.create(committee_account_id=self.committee.id) def test_amending(self): - f3x_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") + self.f3x_report.amend() + self.assertEqual(self.f3x_report.form_type, "F3XA") + self.assertEqual(self.f3x_report.report_version, 1) - f3x_report.amend() - self.assertEqual(f3x_report.form_type, "F3XA") - self.assertEqual(f3x_report.report_version, 1) - - f3x_report.amend() - self.assertEqual(f3x_report.report_version, 2) + self.f3x_report.amend() + self.assertEqual(self.f3x_report.report_version, 2) def test_amending_f24(self): - f24_report = Report.objects.get(id="10000f24-d926-4e89-ad4b-000000000001") new_upload_submission = UploadSubmission() - f24_report.upload_submission = new_upload_submission + self.f24_report.upload_submission = new_upload_submission - f24_report.amend() + self.f24_report.amend() self.assertEquals( - f24_report.form_24.original_amendment_date, new_upload_submission.created + self.f24_report.form_24.original_amendment_date, new_upload_submission.created ) - self.assertEquals(f24_report.form_type, "F24A") + self.assertEquals(self.f24_report.form_type, "F24A") def test_delete(self): - f24_report = create_form24(self.committee, {}) + f24_report = create_form24(self.committee) f24_report_id = f24_report.id f24_id = f24_report.form_24.id f3x_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) @@ -53,7 +55,15 @@ def test_delete(self): candidate_state="MD", candidate_district="99", ) - ie = create_ie(self.committee, candidate_a, "2023-01-01", "123.45", "H2024") + ie = create_ie( + self.committee, + candidate_a, + "2023-01-01", + "2023-02-01", + "123.45", + "H2024", + candidate_a, + ) ie.reports.set([f24_report_id, f3x_report_id]) ie.save() ie_id = ie.id @@ -70,3 +80,33 @@ def test_delete(self): ie = Transaction.all_objects.filter(id=ie_id).first() self.assertIsNotNone(ie.deleted) + + def test_can_delete(self): + q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + # Test when conditions are met for deletion + self.assertTrue(q1_report.can_delete()) + + # Test when upload_submission exists + upload_submission = UploadSubmission() + q1_report.upload_submission = upload_submission + self.assertFalse(q1_report.can_delete()) + q1_report.upload_submission = None + + # Test when report_version is not None, "0", or 0 + q1_report.report_version = "1" + self.assertFalse(q1_report.can_delete()) + q1_report.report_version = "0" + + # Test when form_24, form_1m, or form_99 exists with Form 3x + f24_form = create_form(Form24, {}) + q1_report.form_24 = f24_form + self.assertFalse(q1_report.can_delete()) + q1_report.form_24 = None + + # Test when there exists a transaction in this report + # where any transactions in a different report back reference to them + debt = create_debt(self.committee, self.contact_1, Decimal("123.00")) + q2_report = create_form3x(self.committee, "2024-03-01", "2024-05-01", {}) + debt.reports.add(q1_report) + debt.reports.add(q2_report) + self.assertFalse(q2_report.can_delete()) diff --git a/django-backend/fecfiler/reports/tests/utils.py b/django-backend/fecfiler/reports/tests/utils.py index 26dfd6ec47..5bb4ea7919 100644 --- a/django-backend/fecfiler/reports/tests/utils.py +++ b/django-backend/fecfiler/reports/tests/utils.py @@ -6,30 +6,46 @@ from fecfiler.reports.form_99.models import Form99 -def create_form3x(committee, coverage_from, coverage_through, data={}): - return create_test_report(Form3X, committee, coverage_from, coverage_through, data) +def create_form3x( + committee, + coverage_from, + coverage_through, + data, + report_code="Q1", +): + return create_test_report( + Form3X, "F3XN", committee, report_code, coverage_from, coverage_through, data + ) def create_form24(committee, data={}): - return create_test_report(Form24, committee, data=data) + return create_test_report(Form24, "F24N", committee, data=data) def create_form99(committee, data={}): - return create_test_report(Form99, committee, data=data) + return create_test_report(Form99, "F99", committee, data=data) def create_form1m(committee, data={}): - return create_test_report(Form1M, committee, data=data) + return create_test_report(Form1M, "F1MN", committee, data=data) def create_test_report( - form, committee, coverage_from=None, coverage_through=None, data={} + form, + form_type, + committee, + report_code=None, + coverage_from=None, + coverage_through=None, + data=None, ): form_object = create_form(form, data) report = Report.objects.create( + form_type=form_type, committee_account=committee, coverage_from_date=coverage_from, coverage_through_date=coverage_through, + report_code=report_code, **{FORM_CLASS_TO_FIELD[form]: form_object}, ) return report diff --git a/django-backend/fecfiler/reports/views.py b/django-backend/fecfiler/reports/views.py index f0968f4509..2f45c8992f 100644 --- a/django-backend/fecfiler/reports/views.py +++ b/django-backend/fecfiler/reports/views.py @@ -9,7 +9,7 @@ from fecfiler.memo_text.models import MemoText from fecfiler.web_services.models import DotFEC, UploadSubmission, WebPrintSubmission from .serializers import ReportSerializer -from django.db.models import Case, Value, When, CharField, F +from django.db.models import Case, Value, When, CharField, IntegerField, F from django.db.models.functions import Concat, Trim import structlog @@ -27,6 +27,17 @@ "F99": "Original", } +form_type_ordering = { + "F1MN": 10, + "F1MA": 10, + "F3XN": 20, + "F3XA": 20, + "F3XT": 20, + "F24N": 30, + "F24A": 30, + "F99": 40, +} + class ReportListPagination(pagination.PageNumberPagination): page_size = 10 @@ -79,17 +90,30 @@ class ReportViewSet(CommitteeOwnedViewMixin, ModelViewSet): ordering_fields = [ "report_code_label", "coverage_through_date", - "form_type", "upload_submission__created", "report_status", "version_label", + "form_type_ordering", ] - ordering = ["form_type"] + ordering = ["form_type_ordering"] # Allow requests to filter reports output based on report type by # passing a query parameter def get_queryset(self): - queryset = super().get_queryset() + ordering_whens = [ + When(form_type=k, then=Value(v)) for k, v in form_type_ordering.items() + ] + queryset = ( + super() + .get_queryset() + .annotate( + form_type_ordering=Case( + *ordering_whens, + default=Value(0), + output_field=IntegerField(), + ), + ) + ) report_type_filters = self.request.query_params.get("report_type") if report_type_filters is not None: report_type_list = report_type_filters.split(",") @@ -103,6 +127,7 @@ def get_queryset(self): queryset = queryset.filter(form_99__isnull=True) if "f1m" not in report_type_list: queryset = queryset.filter(form_1m__isnull=True) + return queryset @action(detail=True, methods=["post"], url_name="amend") @@ -164,6 +189,19 @@ def partial_update(self, request, pk=None): response = {"message": "Update function is not offered in this path."} return Response(response, status=status.HTTP_405_METHOD_NOT_ALLOWED) + def list(self, request, *args, **kwargs): + ordering = request.query_params.get("ordering") + if ordering in ["form_type", "-form_type"]: + new_ordering = ( + "-form_type_ordering" + if ordering.startswith("-") + else "form_type_ordering" + ) + request.query_params._mutable = True + request.query_params["ordering"] = new_ordering + request.query_params._mutable = False + return super().list(request, args, kwargs) + class ReportViewMixin(GenericViewSet): def get_queryset(self): diff --git a/django-backend/fecfiler/settings/base.py b/django-backend/fecfiler/settings/base.py index 5a3a322108..494b16beaf 100644 --- a/django-backend/fecfiler/settings/base.py +++ b/django-backend/fecfiler/settings/base.py @@ -64,7 +64,6 @@ "django.contrib.messages", "django.contrib.staticfiles", "rest_framework", - "drf_spectacular", "corsheaders", "storages", "django_structlog", @@ -188,6 +187,8 @@ OIDC_USERNAME_ALGO = "fecfiler.authentication.views.generate_username" # OpenID Connect settings end +USE_X_FORWARDED_HOST = True + LANGUAGE_CODE = "en-us" TIME_ZONE = "America/New_York" @@ -213,7 +214,7 @@ "rest_framework.authentication.SessionAuthentication", "rest_framework.authentication.BasicAuthentication", ), - "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", + "DEFAULT_RENDERER_CLASSES": ("rest_framework.renderers.JSONRenderer",), "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination", "PAGE_SIZE": 10, "EXCEPTION_HANDLER": "fecfiler.utils.custom_exception_handler", @@ -324,17 +325,13 @@ def get_logging_processors(): CELERY_LOCAL_STORAGE_DIRECTORY = os.path.join(BASE_DIR, "web_services/dot_fec/output") -CELERY_WORKER_STORAGE = env.get_credential( - "CELERY_WORKER_STORAGE", CeleryStorageType.AWS -) +CELERY_WORKER_STORAGE = env.get_credential("CELERY_WORKER_STORAGE", CeleryStorageType.AWS) """FEC Webload settings """ FEC_FILING_API = env.get_credential("FEC_FILING_API") FEC_FILING_API_KEY = env.get_credential("FEC_FILING_API_KEY") FEC_AGENCY_ID = env.get_credential("FEC_AGENCY_ID") -FILE_AS_TEST_COMMITTEE = env.get_credential("FILE_AS_TEST_COMMITTEE") -TEST_COMMITTEE_PASSWORD = env.get_credential("TEST_COMMITTEE_PASSWORD") WEBPRINT_EMAIL = env.get_credential("WEBPRINT_EMAIL") """OUTPUT_TEST_INFO_IN_DOT_FEC will configure the .fec writer to output extra info for testing purposes diff --git a/django-backend/fecfiler/transactions/managers.py b/django-backend/fecfiler/transactions/managers.py index 3ed4e95b22..74c1f5f07e 100644 --- a/django-backend/fecfiler/transactions/managers.py +++ b/django-backend/fecfiler/transactions/managers.py @@ -26,8 +26,6 @@ BooleanField, TextField, DecimalField, - RowRange, - Window, Manager, ) from decimal import Decimal @@ -41,47 +39,13 @@ class TransactionManager(SoftDeleteManager): - entity_aggregate_window = { - "partition_by": [ - F("contact_1_id"), - F("date__year"), - F("aggregation_group"), - ], - "order_by": ["date", "created"], - "frame": RowRange(None, 0), - } - election_aggregate_window = { - "partition_by": [ - F("schedule_e__election_code"), - F("contact_2__candidate_office"), - F("contact_2__candidate_state"), - F("contact_2__candidate_district"), - F("date__year"), - F("aggregation_group"), - ], - "order_by": ["date", "created"], - "frame": RowRange(None, 0), - } - loan_payment_window = { - "partition_by": [ - Case( - When(loan_id__isnull=False, then=F("loan_id")), - When(schedule_c__isnull=False, then=F("id")), - default=None, - ) - ], - "order_by": ["loan_key"], - "frame": RowRange(None, 0), - } def get_queryset(self): return super().get_queryset() def ITEMIZATION_CLAUSE(self): # noqa: N802 over_two_hundred_types = ( - schedule_a_over_two_hundred_types - + schedule_b_over_two_hundred_types - + schedule_e_over_two_hundred_types + schedule_a_over_two_hundred_types + schedule_b_over_two_hundred_types ) return Case( When(force_itemized__isnull=False, then=F("force_itemized")), @@ -90,6 +54,29 @@ def ITEMIZATION_CLAUSE(self): # noqa: N802 transaction_type_identifier__in=over_two_hundred_types, then=Q(aggregate__gt=Value(Decimal(200))), ), + When( + transaction_type_identifier__in=schedule_e_over_two_hundred_types, + then=Case( + When( + parent_transaction__parent_transaction___calendar_ytd_per_election_office__gt=Value( # noqa + Decimal(200) + ), + then=True, + ), + When( + parent_transaction___calendar_ytd_per_election_office__gt=Value( + Decimal(200) + ), + then=True, + ), + When( + _calendar_ytd_per_election_office__gt=Value(Decimal(200)), + then=True, + ), + default=False, + output_field=BooleanField(), + ), + ), default=Value(True), output_field=BooleanField(), ) @@ -133,16 +120,6 @@ def SCHEDULE_CLAUSE(self): # noqa: N802 "schedule_d__incurred_amount", ) - AGGREGATE = Case( - When(force_unaggregated=True, then=Decimal(0)), default=F("effective_amount") - ) - - def ENTITY_AGGREGGATE_CLAUSE(self): # noqa: N802 - return Window(expression=Sum(self.AGGREGATE), **self.entity_aggregate_window) - - def ELECTION_AGGREGATE_CLAUSE(self): # noqa: N802 - return Window(expression=Sum(self.AGGREGATE), **self.election_aggregate_window) - BACK_REFERENCE_CLAUSE = Coalesce( F("reatt_redes__transaction_id"), F("parent_transaction__transaction_id"), @@ -217,19 +194,6 @@ def LOAN_KEY_CLAUSE(self): # noqa: N802 output_field=TextField(), ) - def LOAN_PAYMENT_CLAUSE(self): # noqa: N802 - return Case( - When( - schedule_c__isnull=False, - then=Coalesce( - Window( - expression=Sum("effective_amount"), **self.loan_payment_window - ), - Value(Decimal(0)), - ), - ) - ) - def INCURRED_PRIOR_CLAUSE(self): # noqa: N802 return Case( When( @@ -313,13 +277,10 @@ def transaction_view(self): date=self.DATE_CLAUSE, amount=self.AMOUNT_CLAUSE, effective_amount=self.EFFECTIVE_AMOUNT_CLAUSE, - aggregate=self.ENTITY_AGGREGGATE_CLAUSE(), - _calendar_ytd_per_election_office=self.ELECTION_AGGREGATE_CLAUSE(), incurred_prior=self.INCURRED_PRIOR_CLAUSE(), payment_prior=self.PAYMENT_PRIOR_CLAUSE(), payment_amount=self.PAYMENT_AMOUNT_CLAUSE(), loan_key=self.LOAN_KEY_CLAUSE(), - loan_payment_to_date=self.LOAN_PAYMENT_CLAUSE(), _itemized=self.ITEMIZATION_CLAUSE(), form_type=self.FORM_TYPE_CLAUSE, name=self.DISPLAY_NAME_CLAUSE, diff --git a/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py b/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py new file mode 100644 index 0000000000..119c96d10f --- /dev/null +++ b/django-backend/fecfiler/transactions/migrations/0008_transaction__calendar_ytd_per_election_office_and_more.py @@ -0,0 +1,261 @@ +# Generated by Django 4.2.11 on 2024-05-21 20:49 + +from django.db import migrations, models + + +def populate_existing_rows(apps, schema_editor): + transaction = apps.get_model("transactions", "Transaction") + for row in transaction.objects.all(): + row.aggregate = 0.0 + row.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ("transactions", "0007_schedulee_so_candidate_state"), + ] + + operations = [ + migrations.AddField( + model_name="transaction", + name="_calendar_ytd_per_election_office", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + migrations.AddField( + model_name="transaction", + name="aggregate", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + migrations.AddField( + model_name="transaction", + name="loan_payment_to_date", + field=models.DecimalField( + blank=True, decimal_places=2, max_digits=11, null=True + ), + ), + migrations.RunSQL( + """ + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_entity_aggregates( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + BEGIN + IF txn.schedule_a_id IS NOT NULL THEN + SELECT contribution_date + INTO schedule_date + FROM transactions_schedulea + WHERE id = txn.schedule_a_id; + ELSIF txn.schedule_b_id IS NOT NULL THEN + SELECT expenditure_date + INTO schedule_date + FROM transactions_scheduleb + WHERE id = txn.schedule_b_id; + END IF; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + SUM(effective_amount) OVER (ORDER BY date, created) + AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE + contact_1_id = $1 + AND EXTRACT(YEAR FROM date) = $2 + AND aggregation_group = $3 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET aggregate = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + txn.contact_1_id, + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_calendar_ytd_per_election_office( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + DECLARE + schedule_date DATE; + v_election_code TEXT; + v_candidate_office TEXT; + v_candidate_state TEXT; + v_candidate_district TEXT; + BEGIN + SELECT COALESCE(disbursement_date, dissemination_date) + INTO schedule_date FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT election_code + INTO v_election_code + FROM transactions_schedulee + WHERE id = txn.schedule_e_id; + SELECT candidate_office, candidate_state, candidate_district + INTO v_candidate_office, v_candidate_state, v_candidate_district + FROM contacts WHERE id = txn.contact_2_id; + + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + t.id, + SUM(t.effective_amount) OVER + (ORDER BY t.date, t.created) AS new_sum + FROM transactions_schedulee e + JOIN transaction_view__' || sql_committee_id || ' t + ON e.id = t.schedule_e_id + JOIN contacts c + ON t.contact_2_id = c.id + WHERE + e.election_code = $1 + AND c.candidate_office = $2 + AND ( + c.candidate_state = $3 + OR ( + c.candidate_state IS NULL + AND $3 = '''' + ) + ) + AND ( + c.candidate_district = $4 + OR ( + c.candidate_district IS NULL + AND $4 = '''' + ) + ) + AND EXTRACT(YEAR FROM t.date) = $5 + AND aggregation_group = $6 + AND force_unaggregated IS NOT TRUE; + + UPDATE transactions_transaction AS t + SET _calendar_ytd_per_election_office = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id; + ' + USING + v_election_code, + v_candidate_office, + COALESCE(v_candidate_state, ''), + COALESCE(v_candidate_district, ''), + EXTRACT(YEAR FROM schedule_date), + txn.aggregation_group; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_loan_payment_to_date( + txn RECORD, + sql_committee_id TEXT, + temp_table_name TEXT + ) + RETURNS VOID AS $$ + BEGIN + EXECUTE ' + CREATE TEMPORARY TABLE ' || temp_table_name || ' + ON COMMIT DROP AS + SELECT + id, + loan_key, + SUM(effective_amount) OVER (ORDER BY loan_key) AS new_sum + FROM transaction_view__' || sql_committee_id || ' + WHERE loan_key LIKE ( + SELECT transaction_id FROM transactions_transaction + WHERE id = $1 + ) || ''%%''; -- Match the loan_key with a transaction_id prefix + + UPDATE transactions_transaction AS t + SET loan_payment_to_date = tt.new_sum + FROM ' || temp_table_name || ' AS tt + WHERE t.id = tt.id + AND tt.loan_key LIKE ''%%LOAN''; + ' + USING txn.id; + END; + $$ LANGUAGE plpgsql; + """ + ), + migrations.RunSQL( + """ + CREATE OR REPLACE FUNCTION calculate_aggregates() + RETURNS TRIGGER AS $$ + DECLARE + sql_committee_id TEXT; + temp_table_name TEXT; + BEGIN + sql_committee_id := REPLACE(NEW.committee_account_id::TEXT, '-', '_'); + temp_table_name := 'temp_' || REPLACE(uuid_generate_v4()::TEXT, '-', '_'); + RAISE NOTICE 'TESTING TRIGGER'; + + -- If schedule_c2_id or schedule_d_id is not null, stop processing + IF NEW.schedule_c2_id IS NOT NULL OR NEW.schedule_d_id IS NOT NULL + THEN + RETURN NEW; + END IF; + + IF NEW.schedule_a_id IS NOT NULL OR NEW.schedule_b_id IS NOT NULL + THEN + PERFORM calculate_entity_aggregates( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' + AND NEW.contact_1_id <> OLD.contact_1_id + THEN + PERFORM calculate_entity_aggregates( + OLD, sql_committee_id, temp_table_name || 'OLD'); + END IF; + + ELSIF NEW.schedule_c_id IS NOT NULL OR NEW.schedule_c1_id IS NOT NULL + THEN + PERFORM calculate_loan_payment_to_date( + NEW, sql_committee_id, temp_table_name || 'NEW'); + + ELSIF NEW.schedule_e_id IS NOT NULL + THEN + PERFORM calculate_calendar_ytd_per_election_office( + NEW, sql_committee_id, temp_table_name || 'NEW'); + IF TG_OP = 'UPDATE' + THEN + PERFORM calculate_calendar_ytd_per_election_office( + OLD, sql_committee_id, temp_table_name || 'OLD'); + END IF; + END IF; + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER calculate_aggregates_trigger + AFTER INSERT OR UPDATE ON transactions_transaction + FOR EACH ROW + WHEN (pg_trigger_depth() = 0) -- Prevent infinite trigger loop + EXECUTE FUNCTION calculate_aggregates(); + """ + ), + migrations.RunPython(populate_existing_rows), + ] diff --git a/django-backend/fecfiler/transactions/models.py b/django-backend/fecfiler/transactions/models.py index 4c754e6c3e..3f34d8f8f3 100644 --- a/django-backend/fecfiler/transactions/models.py +++ b/django-backend/fecfiler/transactions/models.py @@ -1,7 +1,7 @@ from django.db import models from fecfiler.soft_delete.models import SoftDeleteModel from fecfiler.committee_accounts.models import CommitteeOwnedModel -from fecfiler.reports.models import update_recalculation +from fecfiler.reports.models import flag_reports_for_recalculation from fecfiler.shared.utilities import generate_fec_uid from fecfiler.transactions.managers import ( TransactionManager, @@ -15,6 +15,7 @@ from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.schedule_d.models import ScheduleD from fecfiler.transactions.schedule_e.models import ScheduleE + import uuid import structlog @@ -136,6 +137,17 @@ def form_type(self, value): ScheduleE, on_delete=models.CASCADE, null=True, blank=True ) + # Calculated fields + aggregate = models.DecimalField( + null=True, blank=True, max_digits=11, decimal_places=2 + ) + _calendar_ytd_per_election_office = models.DecimalField( + null=True, blank=True, max_digits=11, decimal_places=2 + ) + loan_payment_to_date = models.DecimalField( + null=True, blank=True, max_digits=11, decimal_places=2 + ) + objects = TransactionManager() def get_schedule_name(self): @@ -169,7 +181,7 @@ def save(self, *args, **kwargs): super(Transaction, self).save(*args, **kwargs) for report in self.reports.all(): - update_recalculation(report) + flag_reports_for_recalculation(report) class Meta: indexes = [models.Index(fields=["_form_type"])] @@ -193,11 +205,8 @@ class T(Transaction): date = models.DateField() form_type = models.TextField() effective_amount = models.DecimalField() - aggregate = models.DecimalField() - _calendar_ytd_per_election_office = models.DecimalField() back_reference_tran_id_number = models.TextField() loan_key = models.TextField() - loan_payment_to_date = models.DecimalField() incurred_prior = models.DecimalField() payment_prior = models.DecimalField() payment_amount = models.DecimalField() diff --git a/django-backend/fecfiler/transactions/tests/test_manager.py b/django-backend/fecfiler/transactions/tests/test_manager.py index 2d5e2dc0df..7e0d9358fa 100644 --- a/django-backend/fecfiler/transactions/tests/test_manager.py +++ b/django-backend/fecfiler/transactions/tests/test_manager.py @@ -14,6 +14,10 @@ create_debt, ) from decimal import Decimal +from django.db import transaction +import structlog + +logger = structlog.get_logger(__name__) class TransactionViewTestCase(TestCase): @@ -43,14 +47,15 @@ def test_transaction_view(self): transactions = ( view.filter(committee_account_id=self.committee.id).all().order_by("date") ) - + for t in transactions: + t.refresh_from_db() self.assertEqual(transactions[0].aggregate, Decimal("123.45")) self.assertEqual(transactions[2].aggregate, Decimal("300")) self.assertEqual(transactions[3].aggregate, Decimal("100")) def test_force_unaggregated(self): - unaggregated = create_test_transaction( # noqa: F841 + create_test_transaction( # noqa: F841 "INDIVIDUAL_RECEIPT", ScheduleA, self.committee, @@ -80,7 +85,9 @@ def test_force_unaggregated(self): transactions = view.filter(committee_account_id=self.committee.id).order_by( "date" ) - self.assertEqual(transactions[0].aggregate, Decimal("0")) + for t in transactions: + t.refresh_from_db() + self.assertEqual(transactions[0].aggregate, None) self.assertEqual(transactions[0].force_unaggregated, True) self.assertEqual(transactions[1].aggregate, Decimal("200")) @@ -113,44 +120,49 @@ def test_transaction_view_parent(self): ) def test_transaction_view_list_with_grandparent(self): - jf_transfer = create_schedule_a( - "JOINT_FUNDRAISING_TRANSFER", - self.committee, - self.contact_1, - "2024-01-01", - amount="500.00", - ) - jf_transfer.save() + with transaction.atomic(): + jf_transfer = create_schedule_a( + "JOINT_FUNDRAISING_TRANSFER", + self.committee, + self.contact_1, + "2024-01-01", + amount="500.00", + itemized=True, + ) + jf_transfer.save() - parnership_jf_transfer_memo = create_schedule_a( - "PARTNERSHIP_JF_TRANSFER_MEMO", - self.committee, - self.contact_1, - "2024-01-01", - amount="50.00", - ) - parnership_jf_transfer_memo.parent_transaction = jf_transfer - parnership_jf_transfer_memo.save() + parnership_jf_transfer_memo = create_schedule_a( + "PARTNERSHIP_JF_TRANSFER_MEMO", + self.committee, + self.contact_1, + "2024-01-02", + amount="50.00", + itemized=True, + ) + parnership_jf_transfer_memo.parent_transaction = jf_transfer + parnership_jf_transfer_memo.save() - parnership_attribution_jf_transfer_memo = create_schedule_a( - "PARTNERSHIP_ATTRIBUTION_JF_TRANSFER_MEMO", - self.committee, - self.contact_1, - "2024-01-01", - amount="5.00", - ) - parnership_attribution_jf_transfer_memo.parent_transaction = ( - parnership_jf_transfer_memo - ) - parnership_attribution_jf_transfer_memo.save() + parnership_attribution_jf_transfer_memo = create_schedule_a( + "PARTNERSHIP_ATTRIBUTION_JF_TRANSFER_MEMO", + self.committee, + self.contact_1, + "2024-01-03", + amount="5.00", + itemized=True, + ) + parnership_attribution_jf_transfer_memo.parent_transaction = ( + parnership_jf_transfer_memo + ) + parnership_attribution_jf_transfer_memo.save() + + view = get_read_model(self.committee.id).objects.all().order_by("date") - view = get_read_model(self.committee.id).objects.all() self.assertEqual(view[0].itemized, True) - self.assertEqual(view[0].calendar_ytd_per_election_office, 500) + self.assertEqual(view[0].aggregate, 500) self.assertEqual(view[1].itemized, True) - self.assertEqual(view[1].calendar_ytd_per_election_office, 500) + self.assertEqual(view[1].aggregate, 550) self.assertEqual(view[2].itemized, True) - self.assertEqual(view[2].calendar_ytd_per_election_office, 500) + self.assertEqual(view[2].aggregate, 555) def test_refund_aggregate(self): create_schedule_a( @@ -195,37 +207,43 @@ def test_election_aggregate(self): ) ies = [ { # same election previous year - "date": "2023-01-01", + "disbursement_date": "2023-01-01", + "dissemination_date": "2023-01-01", "amount": "123.45", "contact": candidate_a, "code": "H2024", }, { # same election same year - "date": "2024-01-01", + "disbursement_date": "2024-01-01", + "dissemination_date": "2024-01-01", "amount": "1.00", "contact": candidate_a, "code": "H2024", }, { # same election same year - "date": "2024-01-02", + "disbursement_date": "2024-01-02", + "dissemination_date": "2024-01-02", "amount": "1.00", "contact": candidate_a, "code": "H2024", }, { # same election same year - "date": "2024-01-03", + "disbursement_date": "2024-01-03", + "dissemination_date": "2024-01-03", "amount": "2.00", "contact": candidate_b, "code": "H2024", }, { # different election same year - "date": "2024-01-04", + "disbursement_date": "2024-01-04", + "dissemination_date": "2024-01-04", "amount": "3.00", "contact": candidate_c, "code": "H2024", }, { # different election same year - "date": "2024-01-05", + "disbursement_date": "2024-01-05", + "dissemination_date": "2024-01-05", "amount": "4.00", "contact": candidate_a, "code": "Z2024", @@ -234,31 +252,23 @@ def test_election_aggregate(self): for ie in ies: create_ie( - self.committee, ie["contact"], ie["date"], ie["amount"], ie["code"] + self.committee, + ie["contact"], + ie["disbursement_date"], + ie["dissemination_date"], + ie["amount"], + ie["code"], + ie["contact"], ) - view: QuerySet = Transaction.objects.transaction_view() - transactions = view.filter(committee_account_id=self.committee.id).order_by( - "date" - ) - self.assertEqual( - transactions[0]._calendar_ytd_per_election_office, Decimal("123.45") - ) - self.assertEqual( - transactions[1]._calendar_ytd_per_election_office, Decimal("1.00") - ) - self.assertEqual( - transactions[2]._calendar_ytd_per_election_office, Decimal("2.00") - ) - self.assertEqual( - transactions[3]._calendar_ytd_per_election_office, Decimal("4.00") - ) - self.assertEqual( - transactions[4]._calendar_ytd_per_election_office, Decimal("3.00") - ) - self.assertEqual( - transactions[5]._calendar_ytd_per_election_office, Decimal("4.00") - ) + view = get_read_model(self.committee.id).objects.all() + + self.assertEqual(view[0]._calendar_ytd_per_election_office, Decimal("123.45")) + self.assertEqual(view[1]._calendar_ytd_per_election_office, Decimal("1.00")) + self.assertEqual(view[2]._calendar_ytd_per_election_office, Decimal("2.00")) + self.assertEqual(view[3]._calendar_ytd_per_election_office, Decimal("4.00")) + self.assertEqual(view[4]._calendar_ytd_per_election_office, Decimal("3.00")) + self.assertEqual(view[5]._calendar_ytd_per_election_office, Decimal("4.00")) def test_debts(self): q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) @@ -294,17 +304,17 @@ def test_debts(self): print("DEBT SUCCESS") def test_line_label(self): - first_schedule_a = create_schedule_a( + create_schedule_a( "INDIVIDUAL_RECEIPT", self.committee, self.contact_1, "2024-01-01", "1.00", "GENERAL", - "SA11AII", + "SA11AI", + False, + True, ) - first_schedule_a.force_itemized = True - first_schedule_a.save() create_schedule_a( "INDIVIDUAL_RECEIPT", self.committee, @@ -312,7 +322,7 @@ def test_line_label(self): "2024-01-02", "2.00", "GENERAl", - "SA11AII", + "SA11AI", ) create_schedule_a( "INDIVIDUAL_RECEIPT", @@ -321,20 +331,103 @@ def test_line_label(self): "2024-01-03", "1000.00", "GENERAL", - "SA11AI", + "SA11AII", ) create_schedule_b( "OPERATING_EXPENDITURE", self.committee, self.contact_1, - "2024-01-01", + "2024-01-04", "100.00", "GENERAL_DISBURSEMENT", "SB21B", ) - view = get_read_model(self.committee.id).objects.all() + view = get_read_model(self.committee.id).objects.all().order_by("date") self.assertEqual(view[0].line_label, "11(a)(i)") - self.assertEqual(view[1].line_label, "11(a)(i)") + self.assertEqual(view[1].line_label, "11(a)(ii)") self.assertEqual(view[2].line_label, "11(a)(ii)") self.assertEqual(view[3].line_label, "21(b)") + + def test_itemization(self): + scha = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + "2024-01-01", + "20.00", + "GENERAL", + "SA11AI", + False, + None, + ) + obs = Transaction.objects.transaction_view().filter(id=scha.id) + self.assertFalse(obs[0]._itemized) + + schb = create_schedule_b( + "OPERATING_EXPENDITURE", + self.committee, + self.contact_1, + "2024-01-04", + "20.00", + "GENERAL_DISBURSEMENT", + "SB21B", + ) + obs = Transaction.objects.transaction_view().filter(id=schb.id) + self.assertFalse(obs[0]._itemized) + + scha = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + "2024-01-01", + "250.00", + "GENERAL", + "SA11AI", + False, + None, + ) + obs = Transaction.objects.transaction_view().filter(id=scha.id) + self.assertTrue(obs[0]._itemized) + + schb = create_schedule_b( + "OPERATING_EXPENDITURE", + self.committee, + self.contact_1, + "2024-01-04", + "250.00", + "GENERAL_DISBURSEMENT", + "SB21B", + ) + obs = Transaction.objects.transaction_view().filter(id=schb.id) + self.assertTrue(obs[0]._itemized) + + candidate_a = Contact.objects.create( + committee_account_id=self.committee.id, + candidate_office="H", + candidate_state="MD", + candidate_district="99", + ) + ie = create_ie( + self.committee, + candidate_a, + "2023-01-01", + "2023-01-01", + "123.45", + "H2024", + candidate_a, + ) + obs = Transaction.objects.transaction_view().filter(id=ie.id) + self.assertFalse(obs[0]._itemized) + + ie = create_ie( + self.committee, + candidate_a, + "2023-01-01", + "2023-01-01", + "250.45", + "H2024", + candidate_a, + ) + obs = Transaction.objects.transaction_view().filter(id=ie.id) + self.assertTrue(obs[0]._itemized) diff --git a/django-backend/fecfiler/transactions/tests/test_serializers.py b/django-backend/fecfiler/transactions/tests/test_serializers.py index 93e2810466..49b5705fbb 100644 --- a/django-backend/fecfiler/transactions/tests/test_serializers.py +++ b/django-backend/fecfiler/transactions/tests/test_serializers.py @@ -2,25 +2,22 @@ from fecfiler.user.models import User from rest_framework.request import HttpRequest, Request from fecfiler.transactions.serializers import TransactionSerializer +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view class TransactionSerializerBaseTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_transaction_serializer", - ] def setUp(self): self.missing_type_transaction = {} - + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) self.mock_request = Request(HttpRequest()) - self.mock_request.user = User.objects.get( - id="12345678-aaaa-bbbb-cccc-111122223333" - ) + self.mock_request.user = self.user self.mock_request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } def test_no_transaction_type_identifier(self): diff --git a/django-backend/fecfiler/transactions/tests/test_views.py b/django-backend/fecfiler/transactions/tests/test_views.py index fbcacbbbac..6f28969674 100644 --- a/django-backend/fecfiler/transactions/tests/test_views.py +++ b/django-backend/fecfiler/transactions/tests/test_views.py @@ -1,3 +1,4 @@ +from decimal import Decimal from django.test import TestCase from django.test.client import RequestFactory from rest_framework import status @@ -5,18 +6,27 @@ from fecfiler.reports.models import Report import json from copy import deepcopy -from fecfiler.transactions.views import TransactionViewSet +from fecfiler.transactions.views import TransactionViewSet, TransactionOrderingFilter from fecfiler.transactions.models import Transaction +from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x +from fecfiler.contacts.tests.utils import ( + create_test_individual_contact, + create_test_candidate_contact, +) +from fecfiler.transactions.tests.utils import ( + create_schedule_a, + create_schedule_b, + create_loan, + create_ie, +) +import structlog + +logger = structlog.get_logger(__name__) class TransactionViewsTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_transaction_views_transactions", - "test_election_aggregation_data", - ] json_content_type = "application/json" @@ -25,13 +35,66 @@ def setUpClass(cls): return super().setUpClass() def setUp(self): - print("SETUP TEST_VEW") - create_committee_view("11111111-2222-3333-4444-555555555555") self.factory = RequestFactory() - self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) + self.q1_report = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + self.contact_1 = create_test_individual_contact( + "last name", "First name", self.committee.id + ) + self.contact_2 = create_test_candidate_contact( + "last name", "First name", self.committee.id, "H8MA03131", "S", "AK", "01" + ) + self.transaction = create_ie( + self.committee, + self.contact_1, + "2023-01-12", + "2023-01-15", + "153.00", + "C2012", + self.contact_2, + ) self.payloads = json.load( open("fecfiler/transactions/fixtures/view_payloads.json") ) + create_schedule_b( + "GENERAL_DISBURSEMENT", + self.committee, + self.contact_1, + "2023-09-02", + "3.00", + "GENERAL_DISBURSEMENT", + ) + + self.ordering_filter = TransactionOrderingFilter() + self.view = TransactionViewSet() + + request = self.factory.get( + "/api/v1/transactions/", + content_type=self.json_content_type, + ) + request.query_params = { + "ordering": "memo_code", + "report_id": self.q1_report.id, + } + request.session = { + "committee_uuid": self.committee.id, + "committee_id": self.committee.committee_id, + } + self.view.request = request + + def create_trans_from_data(self, receipt_data): + create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + receipt_data["date"], + receipt_data["amount"], + group=receipt_data["group"], + report=self.q1_report, + memo_code=receipt_data["memo"], + ) def request(self, payload, params={}): request = self.factory.post( @@ -43,8 +106,8 @@ def request(self, payload, params={}): request.data = deepcopy(payload) request.query_params = params request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } return request @@ -71,6 +134,21 @@ def test_update(self): ) def test_get_queryset(self): + for i in range(8): + create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + "2023-01-01", + str((i + 1) * 10), + "GENERAL", + ) + + for i in range(2): + create_loan( + self.committee, self.contact_1, Decimal((i + 1) * 10), "2023-09-20", "2.0" + ) + view_set = TransactionViewSet() view_set.request = self.request({}, {"schedules": "A,B,C,C2,D,E"}) self.assertEqual(view_set.get_queryset().count(), 12) @@ -82,9 +160,7 @@ def test_get_queryset(self): def test_get_previous_entity(self): view_set = TransactionViewSet() view_set.format_kwarg = {} - view_set.request = self.request( - {}, {"contact_1_id": "00000000-6486-4062-944f-aa0c4cbe4073"} - ) + view_set.request = self.request({}, {"contact_1_id": str(self.contact_1.id)}) # leave out required params response = view_set.previous_transaction_by_entity(view_set.request) self.assertEqual(response.status_code, 400) @@ -93,7 +169,7 @@ def test_get_previous_entity(self): self.request( {}, { - "contact_1_id": "00000000-6486-4062-944f-aa0c4cbe4073", + "contact_1_id": str(self.contact_1.id), "date": "2023-09-20", "aggregation_group": "GENERAL_DISBURSEMENT", }, @@ -105,7 +181,7 @@ def test_get_previous_entity(self): self.request( {}, { - "contact_1_id": "00000000-6486-4062-944f-aa0c4cbe4073", + "contact_1_id": str(self.contact_1.id), "date": "2024-09-20", "aggregation_group": "GENERAL_DISBURSEMENT", }, @@ -150,30 +226,31 @@ def test_get_previous_election(self): "election_code": "C2012", "candidate_office": "S", "candidate_state": "AK", + "candidate_district": "01", }, ) response = view_set.previous_transaction_by_election(view_set.request) - self.assertEqual(response.data["date"], "2023-10-31") + transaction = response.data + + self.assertEqual(transaction.get("date"), "2023-01-12") def test_inherited_election_aggregate(self): - request = self.factory.get( - "/api/v1/transactions/aaaaaaaa-607f-4f5d-bfb4-0fa1776d4e35/" - ) + request = self.factory.get(f"/api/v1/transactions/{self.transaction.id}/") request.user = self.user request.query_params = {} request.data = {} request.session = { - "committee_uuid": "11111111-2222-3333-4444-555555555555", - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } view = TransactionViewSet view.request = request - response = view.as_view({"get": "retrieve"})( - request, pk="aaaaaaaa-607f-4f5d-bfb4-0fa1776d4e35" - ) + + response = view.as_view({"get": "retrieve"})(request, pk=str(self.transaction.id)) transaction = response.data - self.assertEqual(transaction.get("calendar_ytd_per_election_office"), "58.00") + logger.debug(transaction) + self.assertEqual(transaction.get("_calendar_ytd_per_election_office"), "153.00") def test_reatt_redes_multisave_transactions(self): txn1 = deepcopy(self.payloads["IN_KIND"]) @@ -191,8 +268,8 @@ def test_reatt_redes_multisave_transactions(self): self.assertEqual(len(transactions), 2) def test_add_transaction_to_report(self): - report_id = "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - transaction_id = "474a1a10-da68-4d71-9a11-cccccccccccc" + report_id = str(self.q1_report.id) + transaction_id = str(self.transaction.id) payload = {"transaction_id": transaction_id, "report_id": report_id} view_set = TransactionViewSet() @@ -218,15 +295,15 @@ def test_add_transaction_to_report(self): self.assertEqual(response.data, "No transaction matching id provided") # Verify response when non existing report id provided - payload["transaction_id"] = "474a1a10-da68-4d71-9a11-cccccccccccc" + payload["transaction_id"] = str(self.transaction.id) payload["report_id"] = "474a1a10-da68-4d71-9a11-cccccccccccc" response = view_set.add_transaction_to_report(self.request(payload)) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) self.assertEqual(response.data, "No report matching id provided") def test_remove_transaction_from_report(self): - report_id = "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - transaction_id = "0b0b9776-df8b-4f5f-b4c5-d751167417e7" + report_id = str(self.q1_report.id) + transaction_id = str(self.transaction.id) payload = {"transaction_id": transaction_id, "report_id": report_id} view_set = TransactionViewSet() @@ -251,7 +328,7 @@ def test_remove_transaction_from_report(self): self.assertEqual(response.data, "No transaction matching id provided") # Verify response when non existing report id provided - payload["transaction_id"] = "474a1a10-da68-4d71-9a11-cccccccccccc" + payload["transaction_id"] = str(self.transaction.id) payload["report_id"] = "474a1a10-da68-4d71-9a11-cccccccccccc" response = view_set.remove_transaction_from_report(self.request(payload)) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) @@ -259,14 +336,102 @@ def test_remove_transaction_from_report(self): def test_save_debt(self): payload = self.payloads["DEBT"] + payload["report_ids"] = [str(self.q1_report.id)] view_set = TransactionViewSet() response = view_set.create(self.request(payload)) - report_coverage_from_date = Report.objects.get( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).coverage_from_date + report_coverage_from_date = self.q1_report.coverage_from_date debt_id = response.data self.assertEqual(response.status_code, 200) debt = Transaction.objects.get(id=debt_id) self.assertEqual( - debt.schedule_d.report_coverage_from_date, report_coverage_from_date + debt.schedule_d.report_coverage_from_date.strftime("%Y-%m-%d"), + report_coverage_from_date, + ) + + def test_sorting_memo_code(self): + indiviual_receipt_data = [ + {"date": "2023-01-01", "amount": "123.45", "group": "GENERAL", "memo": False}, + {"date": "2024-01-01", "amount": "100.00", "group": "GENERAL", "memo": None}, + {"date": "2024-01-02", "amount": "200.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-03", "amount": "100.00", "group": "OTHER", "memo": True}, + ] + for receipt_data in indiviual_receipt_data: + self.create_trans_from_data(receipt_data) + + transactions = Transaction.objects.filter(committee_account_id=self.committee.id) + memos_sorted = transactions.order_by("memo_code") + + ordered_queryset = self.ordering_filter.filter_queryset( + self.view.request, transactions, self.view + ) + self.assertEqual(ordered_queryset.first().id, memos_sorted.first().id) + + def test_sorting_memo_code_inverted(self): + indiviual_receipt_data = [ + {"date": "2023-01-01", "amount": "123.45", "group": "GENERAL", "memo": False}, + {"date": "2024-01-01", "amount": "100.00", "group": "GENERAL", "memo": None}, + {"date": "2024-01-02", "amount": "200.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-03", "amount": "100.00", "group": "OTHER", "memo": True}, + ] + for receipt_data in indiviual_receipt_data: + self.create_trans_from_data(receipt_data) + self.view.request.query_params["ordering"] = "-memo_code" + + transactions = self.view.get_queryset().filter( + committee_account_id=self.committee.id + ) + memos_inverted = transactions.order_by("-memo_code") + + ordered_queryset = self.ordering_filter.filter_queryset( + self.view.request, transactions, self.view + ) + self.assertEqual(ordered_queryset.first().id, memos_inverted.first().id) + + def test_sorting_memos_only_true(self): + indiviual_receipt_data = [ + {"date": "2023-01-01", "amount": "123.45", "group": "GENERAL", "memo": True}, + {"date": "2024-01-01", "amount": "100.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-02", "amount": "200.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-03", "amount": "100.00", "group": "OTHER", "memo": True}, + ] + for receipt_data in indiviual_receipt_data: + self.create_trans_from_data(receipt_data) + self.view.request.query_params["ordering"] = "-memo_code" + + transactions = self.view.get_queryset().filter( + committee_account_id=self.committee.id + ) + memos_sorted = transactions.order_by("memo_code") + + parsed_ordering = self.ordering_filter.get_ordering( + self.view.request, transactions, self.view + ) + self.assertListEqual(parsed_ordering, ["memo_code"]) + + ordered_queryset = self.ordering_filter.filter_queryset( + self.view.request, transactions, self.view + ) + self.assertEqual(ordered_queryset.first().id, memos_sorted.first().id) + + def test_multi_sorting(self): + indiviual_receipt_data = [ + {"date": "2023-01-01", "amount": "200.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-01", "amount": "300.00", "group": "GENERAL", "memo": True}, + {"date": "2024-01-02", "amount": "100.00", "group": "GENERAL", "memo": False}, + {"date": "2024-01-03", "amount": "400.00", "group": "OTHER", "memo": False}, + ] + for receipt_data in indiviual_receipt_data: + self.create_trans_from_data(receipt_data) + + transactions = self.view.get_queryset().filter( + committee_account_id=self.committee.id + ) + self.view.request.query_params["ordering"] = "memo_code, amount" + memos_sorted = transactions.order_by("memo_code", "amount") + + ordered_queryset = self.ordering_filter.filter_queryset( + self.view.request, transactions, self.view ) + self.assertEqual(ordered_queryset.count(), len(indiviual_receipt_data)) + for i in range(ordered_queryset.count()): + self.assertEqual(ordered_queryset[i].id, memos_sorted[i].id) diff --git a/django-backend/fecfiler/transactions/tests/utils.py b/django-backend/fecfiler/transactions/tests/utils.py index 531c0e3e3d..96cf8251c6 100644 --- a/django-backend/fecfiler/transactions/tests/utils.py +++ b/django-backend/fecfiler/transactions/tests/utils.py @@ -7,10 +7,21 @@ from fecfiler.transactions.schedule_c2.models import ScheduleC2 from fecfiler.transactions.schedule_d.models import ScheduleD from fecfiler.transactions.schedule_e.models import ScheduleE +from fecfiler.contacts.models import Contact +from fecfiler.reports.models import ReportTransaction def create_schedule_a( - type, committee, contact, date, amount, group="GENERAL", form_type="SA11I" + type, + committee, + contact, + date, + amount, + group="GENERAL", + form_type="SA11I", + memo_code=False, + itemized=False, + report=None, ): return create_test_transaction( type, @@ -18,46 +29,112 @@ def create_schedule_a( committee, contact_1=contact, group=group, + report=report, schedule_data={"contribution_date": date, "contribution_amount": amount}, - transaction_data={"_form_type": form_type}, + transaction_data={ + "_form_type": form_type, + "memo_code": memo_code, + "force_itemized": itemized, + }, ) def create_schedule_b( - type, committee, contact, date, amount, group="GENERAL", form_type="SB" + type, + committee, + contact, + date, + amount, + group="GENERAL", + form_type="SB", + memo_code=False, + report=None, ): return create_test_transaction( type, - ScheduleA, + ScheduleB, committee, contact_1=contact, group=group, - schedule_data={"contribution_date": date, "contribution_amount": amount}, - transaction_data={"_form_type": form_type}, + report=report, + schedule_data={"expenditure_date": date, "expenditure_amount": amount}, + transaction_data={"_form_type": form_type, "memo_code": memo_code}, ) -def create_ie(committee, contact, date, amount, code): +def create_ie( + committee, + contact: Contact, + disbursement_date, + dissemination_date, + amount, + code, + candidate: Contact, + memo_code=False, + report=None, +): return create_test_transaction( "INDEPENDENT_EXPENDITURE", ScheduleE, committee, - contact_2=contact, + contact_1=contact, + contact_2=candidate, + report=report, + group="INDEPENDENT_EXPENDITURE", schedule_data={ - "disbursement_date": date, + "disbursement_date": disbursement_date, + "dissemination_date": dissemination_date, "expenditure_amount": amount, "election_code": code, }, + transaction_data={ + "_form_type": "SE", + "memo_code": memo_code, + }, ) -def create_debt(committee, contact, incurred_amount): +def create_debt( + committee, + contact, + incurred_amount, + form_type="SD9", + type="DEBT_OWED_BY_COMMITTEE", + report=None, +): return create_test_transaction( - "DEBT_OWED_BY_COMMITTEE", + type, ScheduleD, committee, contact_1=contact, + report=report, schedule_data={"incurred_amount": incurred_amount}, + transaction_data={"_form_type": form_type}, + ) + + +def create_loan( + committee, + contact, + loan_amount, + loan_due_date, + loan_interest_rate, + secured=False, + type="LOAN_RECEIVED_FROM_INDIVIDUAL", + form_type="SC/9", +): + return create_test_transaction( + type, + ScheduleC, + committee, + contact_1=contact, + schedule_data={ + "loan_amount": loan_amount, + "loan_due_date": loan_due_date, + "loan_interest_rate": loan_interest_rate, + "secured": secured, + }, + transaction_data={"_form_type": form_type}, ) @@ -70,6 +147,7 @@ def create_test_transaction( group=None, schedule_data=None, transaction_data=None, + report=None, ): schedule_object = create_schedule(schedule, schedule_data) transaction = Transaction.objects.create( @@ -81,6 +159,8 @@ def create_test_transaction( **{SCHEDULE_CLASS_TO_FIELD[schedule]: schedule_object}, **(transaction_data or {}) ) + if report: + create_report_transaction(report, transaction) return transaction @@ -88,6 +168,13 @@ def create_schedule(schedule: Model, data): return schedule.objects.create(**data) +def create_report_transaction(report, transaction): + if transaction and report: + return ReportTransaction.objects.create( + report_id=report.id, transaction_id=transaction.id + ) + + SCHEDULE_CLASS_TO_FIELD = { ScheduleA: "schedule_a", ScheduleB: "schedule_b", diff --git a/django-backend/fecfiler/transactions/views.py b/django-backend/fecfiler/transactions/views.py index 4fdd214ff9..1976fb5fd7 100644 --- a/django-backend/fecfiler/transactions/views.py +++ b/django-backend/fecfiler/transactions/views.py @@ -1,5 +1,6 @@ from django.db import transaction as db_transaction -from rest_framework import filters, pagination +from rest_framework import pagination +from rest_framework.filters import OrderingFilter from rest_framework.decorators import action from rest_framework.response import Response @@ -18,7 +19,7 @@ TransactionSerializer, SCHEDULE_SERIALIZERS, ) -from fecfiler.reports.models import Report, update_recalculation +from fecfiler.reports.models import Report, flag_reports_for_recalculation from fecfiler.contacts.models import Contact from fecfiler.contacts.serializers import create_or_update_contact from fecfiler.transactions.schedule_c.views import save_hook as schedule_c_save_hook @@ -37,10 +38,38 @@ class TransactionListPagination(pagination.PageNumberPagination): page_size_query_param = "page_size" +class TransactionOrderingFilter(OrderingFilter): + def get_ordering(self, request, queryset, view): + ordering_query_param = request.query_params.get(self.ordering_param) + ordering_fields = getattr(view, "ordering_fields", []) + + if ordering_query_param: + fields = [param.strip() for param in ordering_query_param.split(',')] + ordering = [] + for field in fields: + if field.strip('-') in ordering_fields: + if field == '-memo_code' and not ( + queryset.filter(memo_code=True).exists() + and queryset.exclude(memo_code=True).exists() + ): + field = 'memo_code' + ordering.append(field) + if ordering: + return ordering + + return self.get_default_ordering(view) + + def filter_queryset(self, request, queryset, view): + ordering = self.get_ordering(request, queryset, view) + if ordering: + return queryset.order_by(*ordering) + return queryset + + class TransactionViewSet(CommitteeOwnedViewMixin, ModelViewSet): serializer_class = TransactionSerializer pagination_class = TransactionListPagination - filter_backends = [filters.OrderingFilter] + filter_backends = [TransactionOrderingFilter] ordering_fields = [ "line_label", "created", @@ -263,7 +292,7 @@ def save_transaction(self, transaction_data, request): schedule_instance.report_coverage_from_date = report.coverage_from_date schedule_instance.save() - update_recalculation(report) + flag_reports_for_recalculation(report) logger.info( f"Transaction {transaction_instance.id} " f"linked to report(s): {', '.join(report_ids)}" @@ -327,7 +356,7 @@ def add_transaction_to_report(self, request): return Response("No transaction matching id provided", status=404) transaction.reports.add(report) - update_recalculation(report) + flag_reports_for_recalculation(report) return Response("Transaction added to report") @action(detail=False, methods=["post"], url_path=r"remove-from-report") @@ -343,7 +372,7 @@ def remove_transaction_from_report(self, request): return Response("No transaction matching id provided", status=404) transaction.reports.remove(report) - update_recalculation(report) + flag_reports_for_recalculation(report) return Response("Transaction removed from report") diff --git a/django-backend/fecfiler/urls.py b/django-backend/fecfiler/urls.py index dfaa6f92c3..f3668fd479 100644 --- a/django-backend/fecfiler/urls.py +++ b/django-backend/fecfiler/urls.py @@ -1,30 +1,28 @@ from django.conf.urls import include -from django.urls import re_path -from rest_framework.decorators import api_view +from django.urls import re_path, path +from rest_framework.decorators import api_view, permission_classes from rest_framework.response import Response -from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView +from django.views.generic.base import RedirectView +from fecfiler.settings import LOGIN_REDIRECT_CLIENT_URL BASE_V1_URL = r"^api/v1/" @api_view(["GET"]) -def test_celery(request): +def test_celery(_request): from fecfiler.celery import debug_task debug_task.delay() return Response(status=200) +@api_view(["GET", "HEAD"]) +@permission_classes([]) +def get_api_status(_request): + return Response(status=200) + + urlpatterns = [ - re_path( - r"^api/schema/", SpectacularAPIView.as_view(api_version="v1"), name="schema" - ), - re_path( - r"^api/docs/", - SpectacularSwaggerView.as_view( - template_name="swagger-ui.html", url_name="schema" - ), - ), re_path(BASE_V1_URL, include("fecfiler.committee_accounts.urls")), re_path(BASE_V1_URL, include("fecfiler.contacts.urls")), re_path(BASE_V1_URL, include("fecfiler.reports.urls")), @@ -37,4 +35,6 @@ def test_celery(request): re_path(BASE_V1_URL, include("fecfiler.feedback.urls")), re_path(r"^oidc/", include("mozilla_django_oidc.urls")), re_path(r"^celery-test/", test_celery), + path("", RedirectView.as_view(url=LOGIN_REDIRECT_CLIENT_URL)), + re_path(BASE_V1_URL + "status/", get_api_status) ] diff --git a/django-backend/fecfiler/user/managers.py b/django-backend/fecfiler/user/managers.py index 16e4278301..ec2260420f 100644 --- a/django-backend/fecfiler/user/managers.py +++ b/django-backend/fecfiler/user/managers.py @@ -10,8 +10,7 @@ def create_user(self, user_id, **obj_data): new_user = super().create_user(user_id, **obj_data) pending_memberships = Membership.objects.filter( - user=None, - pending_email=obj_data['email'] + user=None, pending_email__iexact=obj_data["email"] ) logger.info( diff --git a/django-backend/fecfiler/user/test_views.py b/django-backend/fecfiler/user/test_views.py index 2b66b12aaf..3e570467bd 100644 --- a/django-backend/fecfiler/user/test_views.py +++ b/django-backend/fecfiler/user/test_views.py @@ -15,7 +15,7 @@ def test_get_current_user_happy_path(self): client = APIClient() client.force_authenticate(user=self.user) - response = client.get("/api/v1/users/current/") + response = client.get("/api/v1/users/current/", secure=True) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["first_name"], "First") @@ -33,7 +33,7 @@ def test_update_current_user_happy_path(self): "security_consent_exp_date": "2025-03-12", } - response = client.put("/api/v1/users/current/", test_put_data) + response = client.put("/api/v1/users/current/", test_put_data, secure=True) self.assertEqual(response.status_code, 200) self.assertEqual(response.data["first_name"], "test_first_name_updated") diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py index bfd1301d82..a67a069173 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py @@ -5,7 +5,7 @@ from fecfiler.transactions.managers import Schedule from django.core.exceptions import ObjectDoesNotExist from .dot_fec_serializer import serialize_instance, CRLF_STR -from fecfiler.settings import FILE_AS_TEST_COMMITTEE, OUTPUT_TEST_INFO_IN_DOT_FEC +from fecfiler.settings import OUTPUT_TEST_INFO_IN_DOT_FEC from fecfiler.transactions.schedule_a.utils import add_schedule_a_contact_fields from fecfiler.transactions.schedule_b.utils import add_schedule_b_contact_fields from fecfiler.transactions.schedule_c.utils import add_schedule_c_contact_fields @@ -27,9 +27,7 @@ def compose_report(report_id, upload_submission_record_id): logger.info(f"composing report: {report_id}") report = report_result.first() """Compose derived fields""" - report.filer_committee_id_number = ( - FILE_AS_TEST_COMMITTEE or report.committee_account.committee_id - ) + report.filer_committee_id_number = report.committee_account.committee_id if upload_submission_result.exists(): report.date_signed = upload_submission_result.first().created return report @@ -46,7 +44,7 @@ def compose_transactions(report_id): """Compose derived fields""" for transaction in transactions: transaction.filer_committee_id_number = ( - FILE_AS_TEST_COMMITTEE or transaction.committee_account.committee_id + transaction.committee_account.committee_id ) if transaction.schedule_a: @@ -78,9 +76,7 @@ def compose_report_level_memos(report_id): if report_level_memos.exists(): logger.info(f"composing report level memos: {report_id}") for memo in report_level_memos: - memo.filer_committee_id_number = ( - FILE_AS_TEST_COMMITTEE or memo.committee_account.committee_id - ) + memo.filer_committee_id_number = memo.committee_account.committee_id return report_level_memos else: logger.info(f"no report level memos found for report: {report_id}") @@ -199,9 +195,7 @@ def compose_dot_fec(report_id, upload_submission_record_id): ) if transaction.memo_text: memo = transaction.memo_text - memo.filer_committee_id_number = ( - FILE_AS_TEST_COMMITTEE or memo.committee_account.committee_id - ) + memo.filer_committee_id_number = memo.committee_account.committee_id memo.back_reference_tran_id_number = transaction.transaction_id memo.back_reference_sched_form_name = transaction.form_type serialized_memo = serialize_instance("Text", memo) diff --git a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py index 5dcc0c23d7..760c31b8f4 100644 --- a/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/dot_fec_submitter.py @@ -5,8 +5,6 @@ from fecfiler.web_services.models import FECStatus from fecfiler.settings import ( FEC_FILING_API_KEY, - FILE_AS_TEST_COMMITTEE, - TEST_COMMITTEE_PASSWORD, FEC_AGENCY_ID, ) import structlog @@ -23,11 +21,8 @@ def __init__(self, api): def get_submission_json(self, dot_fec_record, e_filing_password, backdoor_code=None): json_obj = { - "committee_id": FILE_AS_TEST_COMMITTEE - or dot_fec_record.report.committee_account.committee_id, - "password": ( - TEST_COMMITTEE_PASSWORD if FILE_AS_TEST_COMMITTEE else e_filing_password - ), + "committee_id": dot_fec_record.report.committee_account.committee_id, + "password": e_filing_password, "api_key": FEC_FILING_API_KEY, "email_1": dot_fec_record.report.confirmation_email_1, "email_2": dot_fec_record.report.confirmation_email_2, diff --git a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py index 82ad8ba937..5f2e519ad6 100644 --- a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py +++ b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py @@ -1,39 +1,64 @@ from django.test import TestCase -from fecfiler.reports.models import Report from fecfiler.memo_text.models import MemoText -from fecfiler.transactions.models import Transaction from .dot_fec_composer import compose_dot_fec, add_row_to_content from fecfiler.committee_accounts.views import create_committee_view from .dot_fec_serializer import serialize_instance, CRLF_STR, FS_STR +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.reports.tests.utils import create_form3x, create_form99 +from fecfiler.transactions.tests.utils import create_schedule_a +from fecfiler.contacts.tests.utils import create_test_individual_contact +from datetime import datetime class DotFECSerializerTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_f99", - "test_individual_receipt", - "test_memo_text", - ] def setUp(self): - create_committee_view("11111111-2222-3333-4444-555555555555") - self.f3x = Report.objects.filter( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).first() - self.transaction = Transaction.objects.filter( - id="e7880981-9ee7-486f-b288-7a607e4cd0dd" - ).first() - self.report_level_memo_text = MemoText.objects.filter( - id="1dee28f8-4cfa-4f70-8658-7a9e7f02ab1d" - ).first() + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + coverage_from = datetime.strptime("2024-01-01", "%Y-%m-%d") + coverage_through = datetime.strptime("2024-02-01", "%Y-%m-%d") + self.f3x = create_form3x( + self.committee, + coverage_from, + coverage_through, + {"L38_net_operating_expenditures_ytd": format(381.00, ".2f")}, + ) + self.f99 = create_form99( + self.committee, + { + "text_code": "ABC", + "message_text": "\nBEHOLD! A large text string\nwith new lines", + }, + ) + self.contact_1 = create_test_individual_contact( + "last name", "First name", self.committee.id + ) + + self.transaction = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + datetime.strptime("2024-01-03", "%Y-%m-%d"), + "1.00", + "GENERAL", + "SA11AI", + ) + self.transaction.reports.add(self.f3x) + self.transaction.save() + self.report_level_memo_text = MemoText( + id="94777fb3-6d3a-4e2c-87dc-5e6ed326e65b", + rec_type="TEXT", + text4000="dahtest2", + committee_account_id=self.committee.id, + report_id=self.f3x.id, + ) def test_compose_dot_fec(self): with self.assertRaisesMessage(Exception, "header: 100000000 not found"): compose_dot_fec(100000000, None) - file_content = compose_dot_fec("b6d60d2d-d926-4e89-ad4b-c47d152a66ae", None) - self.assertEqual(file_content.count(CRLF_STR), 5) + file_content = compose_dot_fec(self.f3x.id, None) + self.assertEqual(file_content.count(CRLF_STR), 2) def test_add_row_to_content(self): summary_row = serialize_instance("F3X", self.f3x) @@ -51,7 +76,7 @@ def test_add_row_to_content(self): self.assertEqual(split_dot_fec_str[2].split(FS_STR)[-1], "dahtest2") def test_f99(self): - content = compose_dot_fec("11111111-1111-1111-1111-111111111111", None) + content = compose_dot_fec(self.f99.id, None) split_content = content.split("\n") split_report_row = split_content[1].split(FS_STR) self.assertEqual(split_report_row[14], "ABC\r") diff --git a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_serializer.py b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_serializer.py index 507f788200..96d9a85371 100644 --- a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_serializer.py +++ b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_serializer.py @@ -1,5 +1,4 @@ from django.test import TestCase -import datetime from decimal import Decimal from .dot_fec_serializer import ( serialize_field, @@ -13,34 +12,78 @@ from fecfiler.transactions.models import Transaction from fecfiler.transactions.schedule_a.models import ScheduleA from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x +from fecfiler.transactions.tests.utils import create_schedule_a, create_loan +from fecfiler.contacts.tests.utils import create_test_individual_contact +from datetime import datetime, date class DotFECSerializerTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_individual_receipt", - "test_memo_text", - "test_fake_schedule_c", - ] - def setUp(self): - self.f3x = Report.objects.filter( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).first() - self.transaction = Transaction.objects.filter( - id="e7880981-9ee7-486f-b288-7a607e4cd0dd" - ).first() - self.schc_transaction1 = Transaction.objects.filter( - id="57b95c3a-f7bc-4c36-a3fd-66bb52976eec" - ).first() - self.schc_transaction2 = Transaction.objects.filter( - id="3fce473a-7939-4733-a0a8-298d16a058cd" - ).first() - self.report_level_memo_text = MemoText.objects.filter( - id="1dee28f8-4cfa-4f70-8658-7a9e7f02ab1d" - ).first() - self.report_level_memo_text.filer_committee_id_number = "C00601211" + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + coverage_from = datetime.strptime("2024-01-01", "%Y-%m-%d") + coverage_through = datetime.strptime("2024-02-01", "%Y-%m-%d") + self.f3x = create_form3x( + self.committee, + coverage_from, + coverage_through, + { + "L38_net_operating_expenditures_ytd": format(381.00, ".2f"), + "L6b_cash_on_hand_beginning_period": format(6.00, ".2f"), + "change_of_address": True, + }, + ) + self.f3x.treasurer_last_name = "Lastname" + self.f3x.date_signed = datetime.strptime("2004-07-29", "%Y-%m-%d") + + self.f3x.save() + + self.contact_1 = create_test_individual_contact( + "last name", "First name", self.committee.id + ) + + self.transaction = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + datetime.strptime("2020-04-19", "%Y-%m-%d"), + "1234.56", + "GENERAL", + "SA11AI", + ) + self.transaction.reports.add(self.f3x) + self.transaction.save() + + self.schc_transaction1 = create_loan( + self.committee, + self.contact_1, + 10, + datetime.strptime("2023-09-20", "%Y-%m-%d"), + "2.0", + True, + ) + + self.schc_transaction2 = create_loan( + self.committee, + self.contact_1, + 10, + datetime.strptime("2023-09-20", "%Y-%m-%d"), + "2.0", + ) + + self.report_level_memo_text = MemoText( + id="94777fb3-6d3a-4e2c-87dc-5e6ed326e65b", + rec_type="TEXT", + text4000="dahtest2", + committee_account_id=self.committee.id, + report_id=self.f3x.id, + transaction_id_number="REPORT_MEMO_TEXT_1", + ) + + self.report_level_memo_text.filer_committee_id_number = "C00000000" self.report_level_memo_text.back_reference_sched_form_name = "F3XN" self.header = Header("HDR", "FEC", "8.4", "FECFile Online", "0.0.1") @@ -125,6 +168,7 @@ def test_serialize_field(self): self.assertEqual(serialized_boolean_yn_undefined, "N") def test_serialize_f3x_summary_instance(self): + self.assertEqual(self.f3x.form_type, "F3XN") summary_row = serialize_instance("F3X", self.f3x) split_row = summary_row.split(FS_STR) self.assertEqual(split_row[0], "F3XN") @@ -141,7 +185,7 @@ def test_serialize_report_level_memo_instance(self): report_level_memo_row = serialize_instance("Text", self.report_level_memo_text) split_row = report_level_memo_row.split(FS_STR) self.assertEqual(split_row[0], "TEXT") - self.assertEqual(split_row[1], "C00601211") + self.assertEqual(split_row[1], "C00000000") self.assertEqual(split_row[2], "REPORT_MEMO_TEXT_1") self.assertEqual(split_row[4], "F3XN") self.assertEqual(split_row[5], "dahtest2") @@ -161,6 +205,6 @@ def test_get_value_from_path(self): contribution_date = get_value_from_path( self.transaction, "schedule_a.contribution_date" ) - self.assertEqual(contribution_date, datetime.date(2020, 4, 19)) + self.assertEqual(contribution_date.date(), date(2020, 4, 19)) bogus_value = get_value_from_path(self.transaction, "not.real.path") self.assertIsNone(bogus_value) diff --git a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py index 3d9b2d8a2e..9d09e52231 100644 --- a/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py +++ b/django-backend/fecfiler/web_services/dot_fec/test_dot_fec_submitter.py @@ -4,23 +4,19 @@ from .dot_fec_submitter import DotFECSubmitter from fecfiler.web_services.models import DotFEC from fecfiler.web_services.tasks import create_dot_fec -from fecfiler.reports.models import Report from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.reports.tests.utils import create_form3x class DotFECSubmitterTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - ] def setUp(self): - create_committee_view("11111111-2222-3333-4444-555555555555") - self.f3x = Report.objects.filter( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).first() + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.f3x = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) self.dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + str(self.f3x.id), force_write_to_disk=True, ) self.dot_fec_record = DotFEC.objects.get(id=self.dot_fec_id) diff --git a/django-backend/fecfiler/web_services/summary/test_summary.py b/django-backend/fecfiler/web_services/summary/test_summary.py index 4ef7c0889f..2d269229b1 100644 --- a/django-backend/fecfiler/web_services/summary/test_summary.py +++ b/django-backend/fecfiler/web_services/summary/test_summary.py @@ -1,29 +1,34 @@ from decimal import Decimal from django.test import TestCase -from fecfiler.reports.models import Report -from fecfiler.transactions.models import get_read_model +from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.committee_accounts.views import create_committee_view from .summary import SummaryService +from fecfiler.reports.tests.utils import create_form3x +from datetime import datetime +from fecfiler.contacts.tests.utils import create_test_individual_contact +from .tests.utils import generate_data class F3XReportTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_schedulea_summary_transactions", - "test_scheduleb_summary_transactions", - "test_schedulec_summary_transactions", - "test_scheduled_summary_transactions", - "test_schedulee_summary_transactions", - "test_contacts", - ] def setUp(self): - create_committee_view("11111111-2222-3333-4444-555555555555") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.contact_1 = create_test_individual_contact( + "last name", "First name", self.committee.id + ) def test_calculate_summary_column_a(self): - f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") - + f3x = create_form3x( + self.committee, + datetime.strptime("2005-01-30", "%Y-%m-%d").date(), + datetime.strptime("2005-02-28", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": 61}, + "12C", + ) + self.debt = generate_data( + self.committee, self.contact_1, f3x, ["a", "b", "c", "d", "e"] + ) summary_service = SummaryService(f3x) summary_a, _ = summary_service.calculate_summary() @@ -33,7 +38,7 @@ def test_calculate_summary_column_a(self): Decimal("0") + +Decimal("18146.17"), # line_6b # line_6c ) self.assertEqual(summary_a["line_8"], summary_a["line_6d"] - summary_a["line_7"]) - self.assertEqual(summary_a["line_9"], Decimal("215.00")) + self.assertEqual(summary_a["line_9"], Decimal("250.00")) self.assertEqual(summary_a["line_10"], Decimal("250.00")) self.assertEqual(summary_a["line_11ai"], Decimal("10000.23")) self.assertEqual(summary_a["line_11aii"], Decimal("3.77")) @@ -111,14 +116,22 @@ def test_calculate_summary_column_a(self): ) def test_calculate_summary_column_b(self): - f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") + f3x = create_form3x( + self.committee, + datetime.strptime("2005-01-30", "%Y-%m-%d").date(), + datetime.strptime("2005-02-28", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": 61}, + "12C", + ) + self.debt = generate_data( + self.committee, self.contact_1, f3x, ["a", "b", "c", "d", "e"] + ) summary_service = SummaryService(f3x) _, summary_b = summary_service.calculate_summary() - debt = get_read_model(f3x.committee_account.id).objects.get( - id="aaaaaaaa-4d75-46f0-bce2-111000000001" - ) - self.assertEqual(debt.itemized, False) + self.assertIsNotNone(self.debt) + if self.debt is not None: + self.assertEqual(self.debt.force_itemized, False) self.assertEqual(summary_b["line_6c"], Decimal("18985.17")) self.assertEqual( @@ -199,24 +212,35 @@ def test_calculate_summary_column_b(self): ) def test_report_with_no_transactions(self): - f3x = Report.objects.get(id="a07c8c65-1b2d-4e6e-bcaa-fa8d39e50965") + f3x = create_form3x( + self.committee, + datetime.strptime("2024-01-01", "%Y-%m-%d").date(), + datetime.strptime("2024-02-01", "%Y-%m-%d").date(), + {}, + ) summary_service = SummaryService(f3x) summary_a, _ = summary_service.calculate_summary() self.assertEqual(summary_a["line_15"], Decimal("0")) self.assertEqual(summary_a["line_17"], Decimal("0")) def test_report_with_zero_cash_on_hand(self): - f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") - f3x.form_3x.L6a_cash_on_hand_jan_1_ytd = 0 - f3x.form_3x.save() + f3x = create_form3x( + self.committee, + datetime.strptime("2024-01-01", "%Y-%m-%d").date(), + datetime.strptime("2024-02-01", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": 0}, + ) summary_service = SummaryService(f3x) summary_a, _ = summary_service.calculate_summary() self.assertTrue("line_8" in summary_a) def test_report_with_none_cash_on_hand(self): - f3x = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") - f3x.form_3x.L6a_cash_on_hand_jan_1_ytd = None - f3x.form_3x.save() + f3x = create_form3x( + self.committee, + datetime.strptime("2024-01-01", "%Y-%m-%d").date(), + datetime.strptime("2024-02-01", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": None}, + ) summary_service = SummaryService(f3x) summary_a, _ = summary_service.calculate_summary() self.assertFalse("line_8" in summary_a) diff --git a/django-backend/fecfiler/web_services/summary/test_tasks.py b/django-backend/fecfiler/web_services/summary/test_tasks.py index 2828f68c92..aca797464f 100644 --- a/django-backend/fecfiler/web_services/summary/test_tasks.py +++ b/django-backend/fecfiler/web_services/summary/test_tasks.py @@ -1,26 +1,50 @@ +from datetime import datetime from decimal import Decimal from django.test import TestCase from .tasks import CalculationState, calculate_summary from fecfiler.committee_accounts.views import create_committee_view from fecfiler.reports.models import Report +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.reports.tests.utils import create_form3x +from fecfiler.contacts.tests.utils import create_test_individual_contact +from .tests.utils import generate_data class F3XSerializerTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_contacts", - "test_f3x_reports", - "test_schedulea_summary_transactions", - ] def setUp(self): - create_committee_view("11111111-2222-3333-4444-555555555555") + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.contact_1 = create_test_individual_contact( + "last name", "First name", self.committee.id + ) + self.q1_report = create_form3x( + self.committee, + datetime.strptime("2005-01-30", "%Y-%m-%d").date(), + datetime.strptime("2005-02-28", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": 61}, + ) + self.q2_report = create_form3x( + self.committee, + datetime.strptime("2005-03-01", "%Y-%m-%d").date(), + datetime.strptime("2005-05-01", "%Y-%m-%d").date(), + {}, + "Q2", + ) + self.q3_report = create_form3x( + self.committee, + datetime.strptime("2005-09-01", "%Y-%m-%d").date(), + datetime.strptime("2005-10-01", "%Y-%m-%d").date(), + {}, + "Q3", + ) def test_summary_task(self): + generate_data(self.committee, self.contact_1, self.q1_report, ["a"]) report_id = calculate_summary("not_an_existing_report") self.assertIsNone(report_id) - report_id = calculate_summary("b6d60d2d-d926-4e89-ad4b-c47d152a66ae") + report_id = calculate_summary(self.q1_report.id) report = Report.objects.get(id=report_id) self.assertEqual( report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, @@ -33,7 +57,7 @@ def test_summary_task(self): self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value) def test_report_with_no_transactions(self): - report_id = calculate_summary("a07c8c65-1b2d-4e6e-bcaa-fa8d39e50965") + report_id = calculate_summary(self.q2_report.id) report = Report.objects.get(id=report_id) self.assertEqual( report.form_3x.L15_offsets_to_operating_expenditures_refunds_period, @@ -45,22 +69,17 @@ def test_report_with_no_transactions(self): self.assertEqual(report.calculation_status, CalculationState.SUCCEEDED.value) def test_report_group_recalculation(self): - report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-000000000002") - previous_report = Report.objects.get(id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae") - + generate_data(self.committee, self.contact_1, self.q1_report, ["a"]) + previous_report = self.q1_report previous_report.calculation_status = None previous_report.save() - report.calculation_status = None - report.save() + self.q2_report.calculation_status = None + self.q2_report.save() - calculate_summary("b6d60d2d-d926-4e89-ad4b-000000000002") - calculated_report = Report.objects.get( - id="b6d60d2d-d926-4e89-ad4b-000000000002" - ) - calculated_prev_report = Report.objects.get( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) # noqa: E501 + calculate_summary(self.q2_report.id) + calculated_report = Report.objects.get(id=self.q2_report.id) + calculated_prev_report = Report.objects.get(id=self.q1_report.id) self.assertEqual( calculated_report.form_3x.L6b_cash_on_hand_beginning_period, diff --git a/locust-testing/__init__.py b/django-backend/fecfiler/web_services/summary/tests/__init__.py similarity index 100% rename from locust-testing/__init__.py rename to django-backend/fecfiler/web_services/summary/tests/__init__.py diff --git a/django-backend/fecfiler/web_services/summary/tests/utils.py b/django-backend/fecfiler/web_services/summary/tests/utils.py new file mode 100644 index 0000000000..ea499ef064 --- /dev/null +++ b/django-backend/fecfiler/web_services/summary/tests/utils.py @@ -0,0 +1,758 @@ +from fecfiler.transactions.tests.utils import ( + create_ie, + create_schedule_b, + create_schedule_a, + create_loan, + create_debt, +) +from fecfiler.reports.tests.utils import create_form3x +from datetime import datetime +from fecfiler.contacts.models import Contact + +sc10 = "SC/10" + + +def generate_data(committee, contact, f3x, schedules): + debt = None + other_f3x = create_form3x( + committee, + datetime.strptime("2007-01-30", "%Y-%m-%d").date(), + datetime.strptime("2007-02-28", "%Y-%m-%d").date(), + {"L6a_cash_on_hand_jan_1_ytd": 61}, + ) + if "a" in schedules: + sch_a_transactions = [ + { + "date": "2005-02-01", + "amount": "10000.23", + "group": "GENERAL", + "form_type": "SA11AI", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-02-08", + "amount": "3.77", + "group": "GENERAL", + "form_type": "SA11AII", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "444.44", + "group": "GENERAL", + "form_type": "SA11B", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "555.55", + "group": "OTHER", + "form_type": "SA11C", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "1212.12", + "group": "GENERAL", + "form_type": "SA12", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "1313.13", + "group": "GENERAL", + "form_type": "SA13", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "1414.14", + "group": "GENERAL", + "form_type": "SA14", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "1234.56", + "group": "GENERAL", + "form_type": "SA15", + "tti": "OFFSET_TO_OPEX", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "891.23", + "group": "GENERAL", + "form_type": "SA15", + "tti": "OFFSET_TO_OPEX", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "10000.23", + "group": "GENERAL", + "form_type": "SA15", + "tti": "OFFSET_TO_OPEX", + "memo": True, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "16", + "group": "GENERAL", + "form_type": "SA16", + "tti": "REFUND_TO_FEDERAL_CANDIDATE", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "200.50", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "-1", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2005-02-01", + "amount": "800.50", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + ] + debt_a = gen_schedule_a(sch_a_transactions, f3x, committee, contact) + + sch_a_transactions = [ + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA11AI", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": False, + }, + { + "date": "2022-03-01", + "amount": "8.23", + "group": "GENERAL", + "form_type": "SA11AI", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA11B", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA11B", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA11C", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA11C", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA12", + "tti": "PARTY_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA12", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA13", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA13", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA14", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA14", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA15", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA15", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA16", + "tti": "REFUND_TO_FEDERAL_CANDIDATE", + "memo": False, + "itemized": True, + }, + { + "date": "2005-05-01", + "amount": "1000.00", + "group": "GENERAL", + "form_type": "SA16", + "tti": "REFUND_TO_FEDERAL_CANDIDATE", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "300.00", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-01-01", + "amount": "100", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": False, + "itemized": True, + }, + { + "date": "2005-03-01", + "amount": "500.00", + "group": "GENERAL", + "form_type": "SA17", + "tti": "INDIVIDUAL_RECEIPT", + "memo": True, + "itemized": True, + }, + ] + debt_b = gen_schedule_a(sch_a_transactions, other_f3x, committee, contact) + debt = debt_b or debt_a + + if "b" in schedules: + sch_b_transactions = [ + { + "amount": 150, + "date": "2005-02-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB21B", + }, + { + "amount": 22, + "date": "2005-02-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB22", + }, + { + "amount": 14, + "date": "2005-02-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB23", + }, + { + "amount": 44, + "date": "2005-02-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB26", + }, + { + "amount": 31, + "date": "2005-02-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB27", + }, + { + "amount": 101.50, + "date": "2005-02-01", + "type": "REFUND_INDIVIDUAL_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28A", + }, + { + "amount": 201.50, + "date": "2005-02-01", + "type": "REFUND_PARTY_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28B", + }, + { + "amount": 301.50, + "date": "2005-02-01", + "type": "REFUND_PAC_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28C", + }, + { + "amount": 201.50, + "date": "2005-02-01", + "type": "REFUND_PAC_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB29", + }, + { + "amount": 102.25, + "date": "2005-02-01", + "type": "FEDERAL_ELECTION_ACTIVITY_100PCT_PAYMENT", + "group": "GENERAL", + "form_type": "SB30B", + }, + ] + gen_schedule_b(sch_b_transactions, f3x, committee, contact) + sch_b_transactions = [ + { + "amount": 100, + "date": "2005-01-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB21B", + }, + { + "amount": 100, + "date": "2004-12-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB21B", + }, + { + "amount": 100, + "date": "2005-01-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB22", + }, + { + "amount": 1000, + "date": "2005-05-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB22", + }, + { + "amount": 50, + "date": "2005-01-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB23", + }, + { + "amount": 1000, + "date": "2005-05-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB23", + }, + { + "amount": 17, + "date": "2005-01-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB26", + }, + { + "amount": 1000, + "date": "2005-05-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB26", + }, + { + "amount": 10, + "date": "2005-01-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB27", + }, + { + "amount": 100, + "date": "2005-05-01", + "type": "TRANSFER_TO_AFFILIATES", + "group": "GENERAL", + "form_type": "SB27", + }, + { + "amount": 1000.00, + "date": "2005-01-01", + "type": "REFUND_INDIVIDUAL_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28A", + }, + { + "amount": 500, + "date": "2005-03-01", + "type": "REFUND_INDIVIDUAL_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28A", + }, + { + "amount": 2000.00, + "date": "2005-01-01", + "type": "REFUND_PARTY_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28B", + }, + { + "amount": 500, + "date": "2005-03-01", + "type": "REFUND_PARTY_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28B", + }, + { + "amount": 3000.00, + "date": "2005-01-01", + "type": "REFUND_PAC_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28C", + }, + { + "amount": 500, + "date": "2005-03-01", + "type": "REFUND_PAC_CONTRIBUTION", + "group": "GENERAL", + "form_type": "SB28C", + }, + { + "amount": 1000.00, + "date": "2005-01-01", + "type": "OTHER_DISBURSEMENT", + "group": "GENERAL", + "form_type": "SB29", + }, + { + "amount": 500, + "date": "2005-03-01", + "type": "OTHER_DISBURSEMENT", + "group": "GENERAL", + "form_type": "SB29", + }, + { + "amount": 600.00, + "date": "2005-03-01", + "type": "FEDERAL_ELECTION_ACTIVITY_100PCT_PAYMENT", + "group": "GENERAL", + "form_type": "SB30B", + }, + { + "amount": 1000.00, + "date": "2005-01-01", + "type": "FEDERAL_ELECTION_ACTIVITY_100PCT_PAYMENT", + "group": "GENERAL", + "form_type": "SB30B", + }, + ] + gen_schedule_b(sch_b_transactions, other_f3x, committee, contact) + + if "c" in schedules: + sch_c_transactions = [ + { + "amount": 150, + "date": "2005-02-01", + "percent": "2.0", + "form_type": "SC/9", + }, + { + "amount": 30, + "date": "2005-02-01", + "percent": "2.0", + "form_type": sc10, + }, + ] + gen_schedule_c(sch_c_transactions, f3x, committee, contact) + sch_c_transactions = [ + { + "amount": 100, + "date": "2005-01-01", + "percent": "2.0", + "form_type": "SC/9", + }, + { + "amount": 100, + "date": "2004-12-01", + "percent": "2.0", + "form_type": "SC/9", + }, + { + "amount": 100, + "date": "2005-01-01", + "percent": "2.0", + "form_type": sc10, + }, + { + "amount": 100, + "date": "2004-12-01", + "percent": "2.0", + "form_type": sc10, + }, + ] + gen_schedule_c(sch_c_transactions, other_f3x, committee, contact) + + if "d" in schedules: + sch_d_transactions = [ + { + "amount": 100, + "date": "2005-02-01", + "form_type": "SD9", + }, + { + "amount": 220, + "date": "2005-02-01", + "form_type": "SD10", + }, + ] + gen_schedule_d(sch_d_transactions, f3x, committee, contact) + sch_d_transactions = [ + { + "amount": 100, + "date": "2005-01-01", + "form_type": "SD9", + }, + { + "amount": 100, + "date": "2004-01-01", + "form_type": "SD9", + }, + { + "amount": 220, + "date": "2005-02-01", + "form_type": "SD10", + }, + { + "amount": 220, + "date": "2009-02-01", + "form_type": "SD10", + }, + ] + gen_schedule_d(sch_d_transactions, other_f3x, committee, contact) + + if "e" in schedules: + candidate = Contact.objects.create( + committee_account_id=committee.id, + candidate_office="H", + candidate_state="MD", + candidate_district="99", + ) + sch_e_transactions = [ + { + "amount": 65, + "disbursement_date": "2005-01-30", + "dissemination_date": "2005-01-30", + "memo_code": False, + }, + { + "amount": 76, + "disbursement_date": "2005-01-30", + "dissemination_date": "2005-01-30", + "memo_code": False, + }, + { + "amount": 10, + "disbursement_date": "2005-01-30", + "dissemination_date": "2005-01-30", + "memo_code": False, + }, + { + "amount": 57, + "disbursement_date": "2005-01-30", + "dissemination_date": "2005-01-30", + "memo_code": True, + }, + ] + gen_schedule_e(sch_e_transactions, f3x, committee, contact, candidate) + sch_e_transactions = [ + { + "amount": 145, + "disbursement_date": "2005-01-01", + "dissemination_date": "2005-01-01", + "memo_code": False, + }, + { + "amount": 77, + "disbursement_date": "1969-08-01", + "dissemination_date": "1969-08-01", + "memo_code": False, + }, + ] + + gen_schedule_e(sch_e_transactions, other_f3x, committee, contact, candidate) + return debt + + +def gen_schedule_a(transaction_data, f3x, committee, contact): + debt = None + for data in transaction_data: + scha = create_schedule_a( + data["tti"], + committee, + contact, + data["date"], + data["amount"], + data["group"], + data["form_type"], + data["memo"], + data["itemized"], + ) + scha.reports.add(f3x) + scha.save() + if data["form_type"] == "SA11AII": + debt = scha + return debt + + +def gen_schedule_b(transaction_data, f3x, committee, contact): + for data in transaction_data: + schb = create_schedule_b( + data["type"], + committee, + contact, + data["date"], + data["amount"], + data["group"], + data["form_type"], + ) + + schb.reports.add(f3x) + schb.save() + + +def gen_schedule_c(transaction_data, f3x, committee, contact): + for data in transaction_data: + schc = create_loan( + committee, + contact, + data["amount"], + data["date"], + data["percent"], + False, + "LOAN_RECEIVED_FROM_INDIVIDUAL", + data["form_type"], + ) + schc.reports.add(f3x) + + +def gen_schedule_d(transaction_data, f3x, committee, contact): + for data in transaction_data: + schd = create_debt( + committee, + contact, + data["amount"], + data["form_type"], + ) + schd.reports.add(f3x) + + +def gen_schedule_e(transaction_data, f3x, committee, contact, candidate): + for data in transaction_data: + sche = create_ie( + committee, + contact, + data["disbursement_date"], + data["dissemination_date"], + data["amount"], + None, + candidate, + data["memo_code"], + ) + sche.reports.add(f3x) diff --git a/django-backend/fecfiler/web_services/test_models.py b/django-backend/fecfiler/web_services/test_models.py index ba61bf52dd..05008bbee1 100644 --- a/django-backend/fecfiler/web_services/test_models.py +++ b/django-backend/fecfiler/web_services/test_models.py @@ -1,24 +1,24 @@ import json from django.test import TestCase -from fecfiler.reports.models import Report from fecfiler.web_services.models import ( FECStatus, UploadSubmission, FECSubmissionState, WebPrintSubmission, ) +from fecfiler.committee_accounts.models import CommitteeAccount +from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x +from fecfiler.user.models import User class UploadSubmissionTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - ] def setUp(self): - self.f3x = Report.objects.filter( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).first() + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") + create_committee_view(self.committee.id) + self.f3x = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) self.upload_submission = UploadSubmission() self.webprint_submission = WebPrintSubmission() @@ -28,9 +28,7 @@ def setUp(self): def test_initiate_submission(self): self.assertIsNone(self.f3x.upload_submission_id) - submission = UploadSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + submission = UploadSubmission.objects.initiate_submission(str(self.f3x.id)) self.assertEqual( submission.fecfile_task_state, FECSubmissionState.INITIALIZING.value, @@ -69,9 +67,7 @@ def test_save_error(self): def test_webprint_initiate_submission(self): self.assertIsNone(self.f3x.webprint_submission_id) - submission = WebPrintSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + submission = WebPrintSubmission.objects.initiate_submission(str(self.f3x.id)) self.assertEqual( submission.fecfile_task_state, FECSubmissionState.INITIALIZING.value, diff --git a/django-backend/fecfiler/web_services/test_tasks.py b/django-backend/fecfiler/web_services/test_tasks.py index b081d47e90..d87903e8ca 100644 --- a/django-backend/fecfiler/web_services/test_tasks.py +++ b/django-backend/fecfiler/web_services/test_tasks.py @@ -1,7 +1,5 @@ from django.test import TestCase from .tasks import create_dot_fec, submit_to_fec, submit_to_webprint -from fecfiler.reports.models import Report -from fecfiler.transactions.models import Transaction from .models import ( DotFEC, FECStatus, @@ -12,34 +10,42 @@ from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR from pathlib import Path from fecfiler.settings import CELERY_LOCAL_STORAGE_DIRECTORY +from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.committee_accounts.views import create_committee_view +from fecfiler.reports.tests.utils import create_form3x +from fecfiler.contacts.tests.utils import create_test_individual_contact +from fecfiler.transactions.tests.utils import create_schedule_a class TasksTestCase(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - "test_individual_receipt", - "test_memo_text", - ] def setUp(self): - create_committee_view("11111111-2222-3333-4444-555555555555") - self.f3x = Report.objects.filter( - id="b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ).first() - self.transaction = Transaction.objects.filter( - id="e7880981-9ee7-486f-b288-7a607e4cd0dd" - ).first() + self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + create_committee_view(self.committee.id) + self.f3x = create_form3x(self.committee, "2024-01-01", "2024-02-01", {}) + self.contact_1 = create_test_individual_contact( + "Smith", "John", self.committee.id + ) + + self.transaction = create_schedule_a( + "INDIVIDUAL_RECEIPT", + self.committee, + self.contact_1, + "2023-01-05", + "123.45", + "GENERAL", + "SA11AI", + ) + self.transaction.reports.add(self.f3x) + self.transaction.force_itemized = True + self.transaction.save() """ CREATE DOT FEC TESTS """ def test_create_dot_fec(self): - dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", None, None, True - ) + dot_fec_id = create_dot_fec(str(self.f3x.id), None, None, True) dot_fec_record = DotFEC.objects.get(id=dot_fec_id) result_dot_fec = Path(CELERY_LOCAL_STORAGE_DIRECTORY).joinpath( dot_fec_record.file_name @@ -59,11 +65,9 @@ def test_create_dot_fec(self): """ def test_submit_to_fec(self): - upload_submission = UploadSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + upload_submission = UploadSubmission.objects.initiate_submission(str(self.f3x.id)) dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + str(self.f3x.id), upload_submission_id=upload_submission.id, force_write_to_disk=True, ) @@ -89,11 +93,9 @@ def test_submit_to_fec(self): self.assertEqual(len(upload_submission.fec_report_id), 36) def test_submit_no_password(self): - upload_submission = UploadSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + upload_submission = UploadSubmission.objects.initiate_submission(str(self.f3x.id)) dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + str(self.f3x.id), upload_submission_id=upload_submission.id, force_write_to_disk=True, ) @@ -102,16 +104,12 @@ def test_submit_no_password(self): self.assertEqual( upload_submission.fecfile_task_state, FECSubmissionState.FAILED.value ) - self.assertEqual( - upload_submission.fecfile_error, "No E-Filing Password provided" - ) + self.assertEqual(upload_submission.fecfile_error, "No E-Filing Password provided") def test_submit_missing_file(self): - upload_submission = UploadSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - ) + upload_submission = UploadSubmission.objects.initiate_submission(str(self.f3x.id)) dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + str(self.f3x.id), upload_submission_id=upload_submission.id, force_write_to_disk=True, ) @@ -123,9 +121,7 @@ def test_submit_missing_file(self): self.assertEqual( upload_submission.fecfile_task_state, FECSubmissionState.FAILED.value ) - self.assertEqual( - upload_submission.fecfile_error, "Could not retrieve .FEC bytes" - ) + self.assertEqual(upload_submission.fecfile_error, "Could not retrieve .FEC bytes") """ SUBMIT TO WEBPRINT TESTS @@ -133,10 +129,10 @@ def test_submit_missing_file(self): def test_submit_to_webprint(self): webprint_submission = WebPrintSubmission.objects.initiate_submission( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" + str(self.f3x.id) ) dot_fec_id = create_dot_fec( - "b6d60d2d-d926-4e89-ad4b-c47d152a66ae", + str(self.f3x.id), webprint_submission_id=webprint_submission.id, force_write_to_disk=True, ) diff --git a/django-backend/fecfiler/web_services/test_views.py b/django-backend/fecfiler/web_services/test_views.py index 54765d6cb7..0bac0e48b8 100644 --- a/django-backend/fecfiler/web_services/test_views.py +++ b/django-backend/fecfiler/web_services/test_views.py @@ -6,24 +6,22 @@ from fecfiler.committee_accounts.models import CommitteeAccount from fecfiler.committee_accounts.views import create_committee_view from fecfiler.reports.tests.utils import ( - create_form3x, create_form24, create_form99, create_form1m + create_form3x, + create_form24, + create_form99, + create_form1m, ) from fecfiler.web_services.summary.tasks import CalculationState from unittest.mock import patch -from uuid import UUID @override_settings(CELERY_TASK_ALWAYS_EAGER=True, CELERY_TASK_EAGER_PROPOGATES=True) class WebServicesViewSetTest(TestCase): - fixtures = [ - "C01234567_user_and_committee", - "test_f3x_reports", - ] def setUp(self): - self.user = User.objects.get(id="12345678-aaaa-bbbb-cccc-111122223333") self.committee = CommitteeAccount.objects.create(committee_id="C00000000") + self.user = User.objects.create(email="test@fec.gov", username="gov") create_committee_view(self.committee.id) self.committee.members.add(self.user) self.factory = RequestFactory() @@ -170,15 +168,17 @@ def test_check_dot_fec_not_ready(self, mock_async_result): self.assertEqual(response.data, {"done": False}) def test_get_dot_fec_not_exists(self): - id = "b6d60d2d-d926-4e89-ad4b-c47d152a66ae" - - request = self.factory.get(f"api/v1/web-services/dot-fec/{id}") + report = create_form3x( + self.committee, "2024-01-01", "2024-02-01", {"L6a_cash_on_hand_jan_1_ytd": 1} + ) + test_id = report.id + request = self.factory.get(f"api/v1/web-services/dot-fec/{test_id}") request.session = { - "committee_uuid": UUID("11111111-2222-3333-4444-555555555555"), - "committee_id": "C01234567", + "committee_uuid": str(self.committee.id), + "committee_id": str(self.committee.committee_id), } force_authenticate(request, user=self.user) - response = self.view.get_dot_fec(request, id) + response = self.view.get_dot_fec(request, test_id) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, f"No .FEC was found for id: {id}") + self.assertEqual(response.data, f"No .FEC was found for id: {test_id}") diff --git a/django-backend/fixtures/bulk-testing-data.locust.json.gz b/django-backend/fixtures/bulk-testing-data.locust.json.gz new file mode 100644 index 0000000000..3ad717d3c0 Binary files /dev/null and b/django-backend/fixtures/bulk-testing-data.locust.json.gz differ diff --git a/docker-compose.yml b/docker-compose.yml index 52afbc4ea4..bc4e889261 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,8 +55,6 @@ services: FEC_FILING_API: FEC_FILING_API_KEY: FEC_AGENCY_ID: - FILE_AS_TEST_COMMITTEE: - TEST_COMMITTEE_PASSWORD: WEBPRINT_EMAIL: OUTPUT_TEST_INFO_IN_DOT_FEC: LOG_FORMAT: @@ -69,8 +67,6 @@ services: container_name: fecfile-api volumes: - ./django-backend:/opt/nxg_fec - ports: - - 8080:8080 extra_hosts: - "host.docker.internal:host-gateway" depends_on: @@ -99,8 +95,6 @@ services: FEC_FILING_API: FEC_FILING_API_KEY: FEC_AGENCY_ID: - FILE_AS_TEST_COMMITTEE: - TEST_COMMITTEE_PASSWORD: WEBPRINT_EMAIL: OUTPUT_TEST_INFO_IN_DOT_FEC: FEC_API: @@ -109,13 +103,21 @@ services: MOCK_OPENFEC: REDIS FECFILE_GITHUB_TOKEN: + nginx: + build: https://github.com/fecgov/fecfile-api-proxy.git#develop + container_name: fecfile-api-proxy + ports: + - 8080:8080 + depends_on: + - api + locust-leader: image: locustio/locust ports: - "8089:8089" volumes: - ./:/mnt/locust - command: -f /mnt/locust/locust-testing/locust_run.py --master -H http://fecfile-api:8080 + command: -f /mnt/locust/performance-testing/locust_run.py --master -H http://fecfile-api:8080 profiles: [locust] environment: LOCAL_TEST_USER: @@ -126,7 +128,7 @@ services: image: locustio/locust volumes: - ./:/mnt/locust - command: -f /mnt/locust/locust-testing/locust_run.py --worker --master-host locust-leader -L DEBUG + command: -f /mnt/locust/performance-testing/locust_run.py --worker --master-host locust-leader -L DEBUG profiles: [locust] environment: LOCAL_TEST_USER: diff --git a/manifests/manifest-dev-api.yml b/manifests/manifest-dev-api.yml index 1cc686c7de..1acf3edb41 100644 --- a/manifests/manifest-dev-api.yml +++ b/manifests/manifest-dev-api.yml @@ -3,7 +3,7 @@ applications: - name: fecfile-web-api instances: 2 routes: - - route: fecfile-web-api-dev.app.cloud.gov + - route: fecfile-web-api-dev.apps.internal stack: cflinuxfs4 buildpacks: - python_buildpack diff --git a/manifests/manifest-prod-api.yml b/manifests/manifest-prod-api.yml index eee8a04fa6..22ef8528f2 100644 --- a/manifests/manifest-prod-api.yml +++ b/manifests/manifest-prod-api.yml @@ -3,7 +3,7 @@ applications: - name: fecfile-web-api instances: 2 routes: - - route: fecfile-web-api-prod.app.cloud.gov + - route: fecfile-web-api-prod.apps.internal stack: cflinuxfs4 buildpacks: - python_buildpack diff --git a/manifests/manifest-stage-api.yml b/manifests/manifest-stage-api.yml index 7e23e555b1..960dd8e32b 100644 --- a/manifests/manifest-stage-api.yml +++ b/manifests/manifest-stage-api.yml @@ -3,7 +3,7 @@ applications: - name: fecfile-web-api instances: 2 routes: - - route: fecfile-web-api-stage.app.cloud.gov + - route: fecfile-web-api-stage.apps.internal stack: cflinuxfs4 buildpacks: - python_buildpack diff --git a/locust-testing/README.md b/performance-testing/README.md similarity index 86% rename from locust-testing/README.md rename to performance-testing/README.md index 9d342c1888..a7b18ef902 100644 --- a/locust-testing/README.md +++ b/performance-testing/README.md @@ -16,7 +16,7 @@ before creating additional resources randomly as needed. Inter-resource links ( `contact_id` and `report_id` fields on a transaction) are not pre-generated and are instead determined randomly at run-time. -You can generate these .json files by running `python locust-testing/locust_data_generator.py` +You can generate these .json files by running `python performance-testing/locust_data_generator.py` Run the script with the `-h` flag for additional information. ## Setup - Environment variables @@ -108,4 +108,12 @@ the duration of the testing session. There are (as of writing) four tasks: In addition to load testing, Silk query profiling can be installed to inspect queries and response times. -Installation instructions for local development can be found [here](https://github.com/jazzband/django-silk?tab=readme-ov-file#installation). +For a jump-start in setting up for Silk testing, consider merging in the `silk-profiling-base` branch. +The branch contains the necessary configuration changes and marks a selection of functions for profiling. +Silk requires changes to the database, so after merging, be sure to run `python manage.py migrate` +or spin up a fresh container. + +Once set up, silk profiling will run automatically as the API receives and processes requests. +To view the results, visit the API's `/silk` endpoint (for local development: `localhost:8080/silk/`) + +If setting up from scratch or looking for usage instructions, you can find documentation [here](https://github.com/jazzband/django-silk?tab=readme-ov-file#installation). diff --git a/locust-testing/locust-data/__init__.py b/performance-testing/__init__.py similarity index 100% rename from locust-testing/locust-data/__init__.py rename to performance-testing/__init__.py diff --git a/performance-testing/locust-data/__init__.py b/performance-testing/locust-data/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/locust-testing/locust_data_generator.py b/performance-testing/locust_data_generator.py similarity index 100% rename from locust-testing/locust_data_generator.py rename to performance-testing/locust_data_generator.py diff --git a/locust-testing/locust_run.py b/performance-testing/locust_run.py similarity index 100% rename from locust-testing/locust_run.py rename to performance-testing/locust_run.py diff --git a/requirements.txt b/requirements.txt index ccbb22ee7a..9c2ad92bde 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ Django==4.2.11 # when updating this, also update requirements/tox versions in f django-cors-headers==3.13.0 django-storages==1.13.1 djangorestframework==3.14.0 -drf-spectacular==0.24.2 git+https://github.com/fecgov/fecfile-validate@0f8b966b623fbca644aebb2054fe4829eb0e0a93#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.42 github3.py==4.0.1 diff --git a/tasks.py b/tasks.py index 25b4b40994..08aa6bc97e 100644 --- a/tasks.py +++ b/tasks.py @@ -81,9 +81,7 @@ def _login_to_cf(ctx, space): " cf create-service cloud-gov-service-account space-deployer" " [my-service-account-name]" ) - print( - " cf create-service-key [my-server-account-name] [my-service-key-name]" - ) + print(" cf create-service-key [my-server-account-name] [my-service-key-name]") print(" cf service-key [my-server-account-name] [my-service-key-name]") exit(1) @@ -97,7 +95,9 @@ def _do_deploy(ctx, space, app): cmd = "push --strategy rolling" if existing_deploy.ok else "push" new_deploy = ctx.run( - f"cf {cmd} {app} -f {manifest_filename}", echo=True, warn=True, + f"cf {cmd} {app} -f {manifest_filename}", + echo=True, + warn=True, ) return new_deploy @@ -137,17 +137,13 @@ def _rollback(ctx, app): hide=True, warn=True, ) - active_deployments = ( - json.loads(status.stdout).get("pagination").get("total_results") - ) + active_deployments = json.loads(status.stdout).get("pagination").get("total_results") # Try to roll back if active_deployments > 0: print("Attempting to roll back any deployment in progress...") # Show the in-between state ctx.run(f"cf app {app}", echo=True, warn=True) - cancel_deploy = ctx.run( - f"cf cancel-deployment {app}", echo=True, warn=True - ) + cancel_deploy = ctx.run(f"cf cancel-deployment {app}", echo=True, warn=True) if cancel_deploy.ok: print("Successfully cancelled deploy. Check logs.") else: @@ -197,5 +193,20 @@ def deploy(ctx, space=None, branch=None, login=False, help=False): _rollback(ctx, app) return sys.exit(1) + # Allow proxy to connect to api via internal route + add_network_policy = ctx.run( + "cf add-network-policy fecfile-api-proxy fecfile-web-api", + echo=True, + warn=True, + ) + if not add_network_policy.ok: + print( + "Unable to add network policy. Make sure the proxy app is deployed.\n" + "For more information, check logs." + ) + + # Fail the build because the api will be down until the proxy can connect + return sys.exit(1) + # Needed for CircleCI return sys.exit(0)