Merge pull request #613 from fecgov/feature/558

Feature/558 Generic dot_fec_composer

django-backend/fecfiler/reports/form_99/models.py

@@ -24,5 +24,7 @@ class Form99(models.Model):
     zip = models.TextField(null=True, blank=True)
     text_code = models.TextField(null=True, blank=True)
 
+    text = models.TextField(null=True, blank=True)
+
     class Meta:
         app_label = "reports"

django-backend/fecfiler/reports/form_99/serializers.py

@@ -19,6 +19,7 @@ class Form99Serializer(ReportSerializer):
     state = CharField(required=False, allow_null=True)
     zip = CharField(required=False, allow_null=True)
     text_code = CharField(required=False, allow_null=True)
+    text = CharField(required=False, allow_null=True)
 
     def to_internal_value(self, data):
         internal = super().to_internal_value(data)

django-backend/fecfiler/reports/migrations/0005_form99_text.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.3 on 2023-11-22 13:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reports', '0004_form1m_report_form_1m'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='form99',
+            name='text',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

django-backend/fecfiler/reports/models.py

@@ -179,6 +179,19 @@ def get_future_in_progress_reports(
             coverage_through_date__gte=self.coverage_through_date,
         )
 
+    def get_form_name(self):
+        for form_key in TABLE_TO_FORM:
+            if getattr(self, form_key, None):
+                return TABLE_TO_FORM.get(form_key)
+
+
+TABLE_TO_FORM = {
+    "form_3x": "F3X",
+    "form_24": "F24",
+    "form_99": "F99",
+    "form_1m": "F1M",
+}
+
 
 class ReportMixin(models.Model):
     """Abstract model for tracking reports"""

django-backend/fecfiler/web_services/dot_fec/dot_fec_composer.py

@@ -12,21 +12,21 @@
 logger = logging.getLogger(__name__)
 
 
-def compose_f3x_report(report_id, upload_submission_record_id):
+def compose_report(report_id, upload_submission_record_id):
     report_result = Report.objects.filter(id=report_id)
     upload_submission_result = UploadSubmission.objects.filter(
         id=upload_submission_record_id
     )
     if report_result.exists():
-        logger.info(f"composing f3x report: {report_id}")
-        f3x_report = report_result.first()
+        logger.info(f"composing report: {report_id}")
+        report = report_result.first()
         """Compose derived fields"""
-        f3x_report.filer_committee_id_number = (
-            FILE_AS_TEST_COMMITTEE or f3x_report.committee_account.committee_id
+        report.filer_committee_id_number = (
+            FILE_AS_TEST_COMMITTEE or report.committee_account.committee_id
         )
         if upload_submission_result.exists():
-            f3x_report.date_signed = upload_submission_result.first().created
-        return f3x_report
+            report.date_signed = upload_submission_result.first().created
+        return report
     else:
         raise ObjectDoesNotExist(f"report: {report_id} not found")
 
@@ -129,6 +129,19 @@ def add_row_to_content(content, row):
     return (content or "") + str(row) + CRLF_STR
 
 
+def add_free_text(content, text):
+    """Returns new string with free text wrapped with begin/end markers"""
+    return (
+        (content or "")
+        + "[BEGINTEXT]"
+        + CRLF_STR
+        + text
+        + CRLF_STR
+        + "[ENDTEXT]"
+        + CRLF_STR
+    )
+
+
 def get_schema_name(schedule):
     return {
         Schedule.A.value.value: "SchA",
@@ -159,11 +172,11 @@ def compose_dot_fec(report_id, upload_submission_record_id):
         logger.debug(header_row)
         file_content = add_row_to_content(None, header_row)
 
-        f3x_report = compose_f3x_report(report_id, upload_submission_record_id)
-        f3x_report_row = serialize_instance("F3X", f3x_report)
-        logger.debug("Serialized F3X Report:")
-        logger.debug(f3x_report_row)
-        file_content = add_row_to_content(file_content, f3x_report_row)
+        report = compose_report(report_id, upload_submission_record_id)
+        report_row = serialize_instance(report.get_form_name(), report)
+        logger.debug("Serialized Report:")
+        logger.debug(report_row)
+        file_content = add_row_to_content(file_content, report_row)
 
         transactions = compose_transactions(report_id)
         for transaction in transactions:
@@ -190,12 +203,16 @@ def compose_dot_fec(report_id, upload_submission_record_id):
 
         report_level_memos = compose_report_level_memos(report_id)
         for memo in report_level_memos:
-            memo.back_reference_sched_form_name = f3x_report.form_type
+            memo.back_reference_sched_form_name = report.form_type
             serialized_memo = serialize_instance("Text", memo)
             logger.debug("Serialized Report Level Memo:")
             logger.debug(memo)
             file_content = add_row_to_content(file_content, serialized_memo)
 
+        """Free text"""
+        if report.get_form_name() == "F99":
+            file_content = add_free_text(file_content, report.form_99.text)
+
         return file_content
     except Exception as error:
         logger.error(f"failed to compose .FEC for report {report_id}: {str(error)}")

django-backend/fecfiler/web_services/dot_fec/dot_fec_serializer.py

@@ -9,6 +9,7 @@
 logger = logging.getLogger(__name__)
 
 CRLF_STR = str(chr(ascii.CR) + chr(ascii.LF))
+FS_STR = chr(ascii.FS)
 
 
 def get_value_from_path(object, path):
@@ -93,7 +94,7 @@ def serialize_instance(schema_name, instance):
         else ""
         for column_index in range(0, row_length)
     ]
-    return chr(ascii.FS).join(row)
+    return FS_STR.join(row)
 
 
 def get_field_mappings(schema_name):

django-backend/fecfiler/web_services/dot_fec/test_dot_fec_composer.py

@@ -1,16 +1,16 @@
 from django.test import TestCase
-from curses import ascii
 from fecfiler.reports.models import Report
 from fecfiler.memo_text.models import MemoText
 from fecfiler.transactions.models import Transaction
 from .dot_fec_composer import compose_dot_fec, add_row_to_content
-from .dot_fec_serializer import serialize_instance, CRLF_STR
+from .dot_fec_serializer import serialize_instance, CRLF_STR, FS_STR
 
 
 class DotFECSerializerTestCase(TestCase):
     fixtures = [
         "test_committee_accounts",
         "test_f3x_reports",
+        "test_f99",
         "test_individual_receipt",
         "test_memo_text",
     ]
@@ -44,6 +44,18 @@ def test_add_row_to_content(self):
         dot_fec_str = add_row_to_content(dot_fec_str, report_level_memo_row)
         self.assertEqual(dot_fec_str[-2:], CRLF_STR)
         split_dot_fec_str = dot_fec_str.split(CRLF_STR)
-        self.assertEqual(split_dot_fec_str[0].split(chr(ascii.FS))[-1], "381.00")
-        self.assertEqual(split_dot_fec_str[1].split(chr(ascii.FS))[0], "SA11AI")
-        self.assertEqual(split_dot_fec_str[2].split(chr(ascii.FS))[-1], "dahtest2")
+        self.assertEqual(split_dot_fec_str[0].split(FS_STR)[-1], "381.00")
+        self.assertEqual(split_dot_fec_str[1].split(FS_STR)[0], "SA11AI")
+        self.assertEqual(split_dot_fec_str[2].split(FS_STR)[-1], "dahtest2")
+
+    def test_f99(self):
+        content = compose_dot_fec("11111111-1111-1111-1111-111111111111", None)
+        split_content = content.split("\n")
+        split_report_row = split_content[1].split(FS_STR)
+        self.assertEqual(split_report_row[15], "ABC\r")
+        free_text = content[content.find("[BEGINTEXT]"):]
+        self.assertEqual(
+            free_text,
+            "[BEGINTEXT]\r\n\nBEHOLD! A large text string"
+            + "\nwith new lines\r\n[ENDTEXT]\r\n",
+        )

django-backend/fecfiler/web_services/dot_fec/test_dot_fec_serializer.py

@@ -11,7 +11,7 @@
 from fecfiler.memo_text.models import MemoText
 from fecfiler.transactions.models import Transaction
 from fecfiler.transactions.schedule_a.models import ScheduleA
-from curses import ascii
+from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR
 
 
 class DotFECSerializerTestCase(TestCase):
@@ -118,20 +118,20 @@ def test_serialize_field(self):
 
     def test_serialize_f3x_summary_instance(self):
         summary_row = serialize_instance("F3X", self.f3x)
-        split_row = summary_row.split(chr(ascii.FS))
+        split_row = summary_row.split(FS_STR)
         self.assertEqual(split_row[0], "F3XN")
         self.assertEqual(split_row[21], "20040729")
         self.assertEqual(split_row[3], "X")
         self.assertEqual(split_row[122], "381.00")
 
     def test_serialize_schedule_a_transaction_instance(self):
         transaction_row = serialize_instance("SchA", self.transaction)
-        split_row = transaction_row.split(chr(ascii.FS))
+        split_row = transaction_row.split(FS_STR)
         self.assertEqual(split_row[0], "SA11AI")
 
     def test_serialize_report_level_memo_instance(self):
         report_level_memo_row = serialize_instance("Text", self.report_level_memo_text)
-        split_row = report_level_memo_row.split(chr(ascii.FS))
+        split_row = report_level_memo_row.split(FS_STR)
         self.assertEqual(split_row[0], "TEXT")
         self.assertEqual(split_row[1], "C00601211")
         self.assertEqual(split_row[2], "REPORT_MEMO_TEXT_1")
@@ -140,7 +140,7 @@ def test_serialize_report_level_memo_instance(self):
 
     def test_serialize_header_instance(self):
         report_level_memo_row = serialize_instance("HDR", self.header)
-        split_row = report_level_memo_row.split(chr(ascii.FS))
+        split_row = report_level_memo_row.split(FS_STR)
         self.assertEqual(split_row[0], "HDR")
         self.assertEqual(split_row[1], "FEC")
         self.assertEqual(split_row[2], "8.4")

django-backend/fecfiler/web_services/fixtures/test_f99.json

@@ -0,0 +1,36 @@
+[
+    {
+        "model": "reports.Form99",
+        "fields": {
+            "id": "11111111-1111-1111-1111-111111111111",
+            "street_1": "test_string_value",
+            "street_2": "test_string_value",
+            "city": "test_string_value",
+            "state": "test_string_value",
+            "zip": "test_string_value",
+            "text_code": "ABC",
+            "text": "\nBEHOLD! A large text string\nwith new lines"
+        }
+    },
+    {
+        "model": "reports.report",
+        "fields": {
+            "id": "11111111-1111-1111-1111-111111111111",
+            "form_99_id": "11111111-1111-1111-1111-111111111111",
+            "committee_account_id": "735db943-9446-462a-9be0-c820baadb622",
+            "form_type": "F99",
+            "report_code": "Q1",
+            "calculation_status": null,
+            "coverage_from_date": "2005-01-30",
+            "coverage_through_date": "2005-02-28",
+            "treasurer_last_name": "Lastname",
+            "treasurer_first_name": "Firstname",
+            "treasurer_middle_name": "test_string_value",
+            "treasurer_prefix": "test_string_value",
+            "treasurer_suffix": "test_string_value",
+            "date_signed": "2004-07-29",
+            "created": "2022-02-09T00:00:00.000Z",
+            "updated": "2022-02-09T00:00:00.000Z"
+        }
+    }
+]

django-backend/fecfiler/web_services/test_tasks.py

@@ -9,7 +9,7 @@
     UploadSubmission,
     WebPrintSubmission,
 )
-from curses import ascii
+from fecfiler.web_services.dot_fec.dot_fec_serializer import FS_STR
 from pathlib import Path
 from fecfiler.settings import CELERY_LOCAL_STORAGE_DIRECTORY
 
@@ -45,9 +45,9 @@ def test_create_dot_fec(self):
         try:
             with open(result_dot_fec, encoding="utf-8") as f:
                 lines = f.readlines()
-                self.assertEqual(lines[0][:4], "HDR" + chr(ascii.FS))
-                self.assertEqual(lines[1][:5], "F3XN" + chr(ascii.FS))
-                self.assertEqual(lines[2][:7], "SA11AI" + chr(ascii.FS))
+                self.assertEqual(lines[0][:4], "HDR" + FS_STR)
+                self.assertEqual(lines[1][:5], "F3XN" + FS_STR)
+                self.assertEqual(lines[2][:7], "SA11AI" + FS_STR)
         finally:
             if result_dot_fec.exists():
                 result_dot_fec.unlink()
@@ -66,7 +66,11 @@ def test_submit_to_fec(self):
             force_write_to_disk=True,
         )
         upload_id = submit_to_fec(
-            dot_fec_id, upload_submission.id, "test_password", None, True,
+            dot_fec_id,
+            upload_submission.id,
+            "test_password",
+            None,
+            True,
         )
         upload_submission.refresh_from_db()
         self.assertEqual(upload_id, upload_submission.id)