From 085b6ee6244748d8423d688cd76fef91f26833cc Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 18 May 2022 14:14:59 -0400 Subject: [PATCH 01/23] Corrected typo in a comment: "limmited" to "limited" --- django-backend/fecfiler/f3x_summaries/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index a9381d9299..802222d618 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -11,7 +11,7 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): `update` and `destroy` actions. Note that this ViewSet inherits from CommitteeOwnedViewSet - The queryset will be further limmited by the user's committee + The queryset will be further limited by the user's committee in CommitteeOwnedViewSet's implementation of get_queryset() """ From f653586d6bd2ce703582b1d550ae5fbdd9b839c9 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 May 2022 08:37:46 -0400 Subject: [PATCH 02/23] Very WIP --- .../fecfiler/f3x_summaries/models.py | 35 +++++++++++++++++++ .../fecfiler/f3x_summaries/views.py | 10 +++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 0b2f8b44c9..135497d548 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -1,3 +1,5 @@ +from locale import MON_3, MON_8 +from tkinter.messagebox import YES from django.db import models from fecfiler.soft_delete.models import SoftDeleteModel from fecfiler.committee_accounts.models import CommitteeOwnedModel @@ -209,3 +211,36 @@ class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): class Meta: db_table = "f3x_summaries" + + +class ReportCodeLabels(SoftDeleteModel): + labels = { + "Q1":"APRIL 15 (Q1)", + "Q2": "JULY 15 (Q2)", + "Q3":"OCTOBER 15 (Q3)", + "YE":"JANUARY 31 (YE)", + "TER":"TERMINATION (TER)", + "MY":"JANUARY 31 (31)", + "12G":"GENERAL (12G)", + "12P":"PRIMARY (12P)", + "12R":"RUNOFF (12R)", + "12S":"SPECIAL (12S)", + "12C":"CONVENTION (12C)", + "30G":"GENERAL (30G)", + "30R":"RUNOFF (30R)", + "30S":"SPECIAL (30S)", + "M2":"FEBRUARY 20 (M2)", + "M3":"MARCH 30 (M3)", + "M4":"APRIL 20 (M4)", + "M5":"MAY 20 (M5)", + "M6":"JUNE 20 (M6)", + "M7":"JULY 20 (M7)", + "M8":"AUGUST 20 (M8)", + "M9":"SEPTEMBER 20 (M9)", + "M10":"OCTOBER 20 (M10)", + "M11":"NOVEMBER 20 (M11)", + "M12":"DECEMBER 20 (M12)" + } + + class Meta: + db_table = "report_code_labels" \ No newline at end of file diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 802222d618..57093e48f3 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -1,7 +1,7 @@ from rest_framework import filters from django.db.models import Case, Value, When from fecfiler.committee_accounts.views import CommitteeOwnedViewSet -from .models import F3XSummary +from .models import F3XSummary, ReportCodeLabels from .serializers import F3XSummarySerializer @@ -15,6 +15,12 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): in CommitteeOwnedViewSet's implementation of get_queryset() """ + report_code_labels = ReportCodeLabels() + for label in report_code_labels.labels: + "" + + + """ queryset = F3XSummary.objects.alias(report_code_label=Case( When(report_code='Q1', then=Value('APRIL 15 (Q1)')), When(report_code='Q2', then=Value('JULY 15 (Q2)')), @@ -43,6 +49,8 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): When(report_code='M12', then=Value('DECEMBER 20 (M12)')), default=Value('_NOT_FOUND_'), )).all() + """ + serializer_class = F3XSummarySerializer permission_classes = [] filter_backends = [filters.OrderingFilter] From 0def1f57d4744a951607cb1b4e14adb292b27510 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 May 2022 14:41:00 -0400 Subject: [PATCH 03/23] Creates a new model, ReportCodeLabel, with fields "report_code" and "label", an index on "report_code", and the necessary Serializer, ViewSet, and routes --- .../migrations/0010_auto_20220524_1436.py | 29 +++++++++++++++++++ .../fecfiler/f3x_summaries/models.py | 13 +++++++-- .../fecfiler/f3x_summaries/serializers.py | 11 ++++++- django-backend/fecfiler/f3x_summaries/urls.py | 3 +- .../fecfiler/f3x_summaries/views.py | 26 ++++++++++++----- 5 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py new file mode 100644 index 0000000000..3a10da95c6 --- /dev/null +++ b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.12 on 2022-05-24 18:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('f3x_summaries', '0009_auto_20220515_2116'), + ] + + operations = [ + migrations.CreateModel( + name='ReportCodeLabel', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('deleted', models.DateTimeField(blank=True, null=True)), + ('label', models.TextField(blank=True, null=True)), + ('report_code', models.TextField(blank=True, null=True)), + ], + options={ + 'db_table': 'report_code_labels', + }, + ), + migrations.AddIndex( + model_name='reportcodelabel', + index=models.Index(fields=['report_code'], name='report_code_report__9cd13b_idx'), + ), + ] diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 135497d548..095e4c840b 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -1,4 +1,5 @@ from locale import MON_3, MON_8 +from logging import raiseExceptions from tkinter.messagebox import YES from django.db import models from fecfiler.soft_delete.models import SoftDeleteModel @@ -213,7 +214,8 @@ class Meta: db_table = "f3x_summaries" -class ReportCodeLabels(SoftDeleteModel): +class ReportCodeLabel(SoftDeleteModel): + """ labels = { "Q1":"APRIL 15 (Q1)", "Q2": "JULY 15 (Q2)", @@ -241,6 +243,13 @@ class ReportCodeLabels(SoftDeleteModel): "M11":"NOVEMBER 20 (M11)", "M12":"DECEMBER 20 (M12)" } + """ + label = models.TextField(null=True, blank=True); + report_code = models.TextField(null=True, blank=True); + class Meta: - db_table = "report_code_labels" \ No newline at end of file + db_table = "report_code_labels" + indexes = [ + models.Index(fields=["report_code"]) + ] \ No newline at end of file diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index faf4f3c25e..5de95374be 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -1,4 +1,5 @@ -from .models import F3XSummary +from .models import F3XSummary, ReportCodeLabel +from rest_framework.serializers import ModelSerializer from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.validation import serializers import logging @@ -20,3 +21,11 @@ class Meta: "created", "updated", ] + +class ReportCodeLabelSerializer(ModelSerializer): + class Meta: + model = ReportCodeLabel + fields = ( + 'label', + 'report_code' + ) \ No newline at end of file diff --git a/django-backend/fecfiler/f3x_summaries/urls.py b/django-backend/fecfiler/f3x_summaries/urls.py index 26f721684d..b3365812cb 100644 --- a/django-backend/fecfiler/f3x_summaries/urls.py +++ b/django-backend/fecfiler/f3x_summaries/urls.py @@ -1,10 +1,11 @@ from django.urls import path, include from rest_framework.routers import DefaultRouter -from .views import F3XSummaryViewSet +from .views import F3XSummaryViewSet, ReportCodeLabelViewSet # Create a router and register our viewsets with it. router = DefaultRouter() router.register(r"f3x-summaries", F3XSummaryViewSet, basename="f3x-summaries") +router.register(r"report-code-labels",ReportCodeLabelViewSet, basename="report-code-labels") # The API URLs are now determined automatically by the router. urlpatterns = [ diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 57093e48f3..350a5587f6 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -1,8 +1,10 @@ from rest_framework import filters +from rest_framework.viewsets import GenericViewSet +from rest_framework.mixins import ListModelMixin from django.db.models import Case, Value, When from fecfiler.committee_accounts.views import CommitteeOwnedViewSet -from .models import F3XSummary, ReportCodeLabels -from .serializers import F3XSummarySerializer +from .models import F3XSummary, ReportCodeLabel +from .serializers import F3XSummarySerializer, ReportCodeLabelSerializer class F3XSummaryViewSet(CommitteeOwnedViewSet): @@ -15,12 +17,16 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): in CommitteeOwnedViewSet's implementation of get_queryset() """ - report_code_labels = ReportCodeLabels() - for label in report_code_labels.labels: - "" - - """ + report_code_labels = ReportCodeLabel.labels.keys() + case_objects = [] + for label in report_code_labels: + case_objects.append(When(report_code=label, then=Value(ReportCodeLabels.labels[label]))) + + ## The "*" dereferences the case_objects list, making the When() objects inside it the arguments for the Case() constructor + queryset = F3XSummary.objects.alias(report_code_label=Case(*case_objects)); + """ + queryset = F3XSummary.objects.alias(report_code_label=Case( When(report_code='Q1', then=Value('APRIL 15 (Q1)')), When(report_code='Q2', then=Value('JULY 15 (Q2)')), @@ -49,10 +55,14 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): When(report_code='M12', then=Value('DECEMBER 20 (M12)')), default=Value('_NOT_FOUND_'), )).all() - """ serializer_class = F3XSummarySerializer permission_classes = [] filter_backends = [filters.OrderingFilter] ordering_fields = ['form_type', 'report_code_label', 'coverage_through_date'] ordering = ['form_type'] + +class ReportCodeLabelViewSet(GenericViewSet, ListModelMixin): + queryset = ReportCodeLabel.objects.all() + serializer_class = ReportCodeLabelSerializer + pagination_class = None \ No newline at end of file From a6a3407fbcb0a0dbad9f44ba5297eb494233d8dc Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 May 2022 14:42:18 -0400 Subject: [PATCH 04/23] Creates a fixture for quickly importing the 25 current Report Codes and their labels into the database --- .../fixtures/report_code_labels.json | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 django-backend/fixtures/report_code_labels.json diff --git a/django-backend/fixtures/report_code_labels.json b/django-backend/fixtures/report_code_labels.json new file mode 100644 index 0000000000..05e62d60e7 --- /dev/null +++ b/django-backend/fixtures/report_code_labels.json @@ -0,0 +1,227 @@ +[ +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 1, + "fields": { + "deleted": null, + "label": "APRIL 15 (Q1)", + "report_code": "Q1" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 2, + "fields": { + "deleted": null, + "label": "JULY 15 (Q2)", + "report_code": "Q2" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 3, + "fields": { + "deleted": null, + "label": "OCTOBER 15 (Q3)", + "report_code": "Q3" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 4, + "fields": { + "deleted": null, + "label": "JANUARY 31 (YE)", + "report_code": "YE" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 5, + "fields": { + "deleted": null, + "label": "TERMINATION (TER)", + "report_code": "TER" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 6, + "fields": { + "deleted": null, + "label": "JANUARY 31 (31)", + "report_code": "MY" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 7, + "fields": { + "deleted": null, + "label": "GENERAL (12G)", + "report_code": "12G" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 8, + "fields": { + "deleted": null, + "label": "PRIMARY (12P)", + "report_code": "12P" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 9, + "fields": { + "deleted": null, + "label": "RUNOFF (12R)", + "report_code": "12R" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 10, + "fields": { + "deleted": null, + "label": "SPECIAL (12S)", + "report_code": "12S" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 11, + "fields": { + "deleted": null, + "label": "CONVENTION (12C)", + "report_code": "12C" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 12, + "fields": { + "deleted": null, + "label": "GENERAL (30G)", + "report_code": "30G" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 13, + "fields": { + "deleted": null, + "label": "RUNOFF (30R)", + "report_code": "30R" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 14, + "fields": { + "deleted": null, + "label": "SPECIAL (30S)", + "report_code": "30S" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 15, + "fields": { + "deleted": null, + "label": "FEBRUARY 20 (M2)", + "report_code": "M2" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 16, + "fields": { + "deleted": null, + "label": "MARCH 30 (M3)", + "report_code": "M3" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 17, + "fields": { + "deleted": null, + "label": "APRIL 20 (M4)", + "report_code": "M4" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 18, + "fields": { + "deleted": null, + "label": "MAY 20 (M5)", + "report_code": "M5" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 19, + "fields": { + "deleted": null, + "label": "JUNE 20 (M6)", + "report_code": "M6" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 20, + "fields": { + "deleted": null, + "label": "JULY 20 (M7)", + "report_code": "M7" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 21, + "fields": { + "deleted": null, + "label": "AUGUST 20 (M8)", + "report_code": "M8" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 22, + "fields": { + "deleted": null, + "label": "SEPTEMBER 20 (M9)", + "report_code": "M9" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 23, + "fields": { + "deleted": null, + "label": "OCTOBER 20 (M10)", + "report_code": "M10" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 24, + "fields": { + "deleted": null, + "label": "NOVEMBER 20 (M11)", + "report_code": "M11" + } +}, +{ + "model": "f3x_summaries.reportcodelabel", + "pk": 25, + "fields": { + "deleted": null, + "label": "DECEMBER 20 (M12)", + "report_code": "M12" + } +} +] From df2d65a18410749d6e3dc904d56cad7a70f87ef5 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Tue, 24 May 2022 16:44:37 -0400 Subject: [PATCH 05/23] Made the field "report_code" unique --- django-backend/fecfiler/f3x_summaries/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 095e4c840b..8177fd8db3 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -245,7 +245,7 @@ class ReportCodeLabel(SoftDeleteModel): } """ - label = models.TextField(null=True, blank=True); + label = models.TextField(null=True, blank=True, unique=True); report_code = models.TextField(null=True, blank=True); class Meta: From 0da7324c3e8e6e29e33212b7ce4002a2e9d67945 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Wed, 25 May 2022 11:27:29 -0400 Subject: [PATCH 06/23] Currently working INNER JOIN, but sorting isn't behaving --- django-backend/fecfiler/f3x_summaries/views.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 350a5587f6..eea1a45b0c 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -2,6 +2,7 @@ from rest_framework.viewsets import GenericViewSet from rest_framework.mixins import ListModelMixin from django.db.models import Case, Value, When +from django.db.models.expressions import RawSQL from fecfiler.committee_accounts.views import CommitteeOwnedViewSet from .models import F3XSummary, ReportCodeLabel from .serializers import F3XSummarySerializer, ReportCodeLabelSerializer @@ -27,6 +28,7 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): queryset = F3XSummary.objects.alias(report_code_label=Case(*case_objects)); """ + """ queryset = F3XSummary.objects.alias(report_code_label=Case( When(report_code='Q1', then=Value('APRIL 15 (Q1)')), When(report_code='Q2', then=Value('JULY 15 (Q2)')), @@ -55,6 +57,19 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): When(report_code='M12', then=Value('DECEMBER 20 (M12)')), default=Value('_NOT_FOUND_'), )).all() + """ + + def get_queryset(self): + SQLCommand = """ + SELECT * FROM f3x_summaries + INNER JOIN (SELECT label AS report_code_label, report_code from report_code_labels) report_code_labels + ON f3x_summaries.report_code = report_code_labels.report_code + """ + + rawqueryset = F3XSummary.objects.raw(SQLCommand) + rawqueryset.order_by = F3XSummary.objects.order_by + ##print(rawqueryset[0].report_code_label) + return rawqueryset serializer_class = F3XSummarySerializer permission_classes = [] From 71940e7fac3e9755bb3bea6360d58fd198881d46 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 25 May 2022 12:11:26 -0400 Subject: [PATCH 07/23] refactor report label relation --- .../fixtures/report_code_labels.json | 25 ------ .../migrations/0010_auto_20220524_1436.py | 29 ------ .../migrations/0010_auto_20220525_1157.py | 60 +++++++++++++ .../fecfiler/f3x_summaries/models.py | 89 ++++++++++--------- 4 files changed, 107 insertions(+), 96 deletions(-) rename django-backend/{ => fecfiler/f3x_summaries}/fixtures/report_code_labels.json (81%) delete mode 100644 django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py create mode 100644 django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py diff --git a/django-backend/fixtures/report_code_labels.json b/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json similarity index 81% rename from django-backend/fixtures/report_code_labels.json rename to django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json index 05e62d60e7..c5cb2735e9 100644 --- a/django-backend/fixtures/report_code_labels.json +++ b/django-backend/fecfiler/f3x_summaries/fixtures/report_code_labels.json @@ -3,7 +3,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 1, "fields": { - "deleted": null, "label": "APRIL 15 (Q1)", "report_code": "Q1" } @@ -12,7 +11,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 2, "fields": { - "deleted": null, "label": "JULY 15 (Q2)", "report_code": "Q2" } @@ -21,7 +19,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 3, "fields": { - "deleted": null, "label": "OCTOBER 15 (Q3)", "report_code": "Q3" } @@ -30,7 +27,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 4, "fields": { - "deleted": null, "label": "JANUARY 31 (YE)", "report_code": "YE" } @@ -39,7 +35,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 5, "fields": { - "deleted": null, "label": "TERMINATION (TER)", "report_code": "TER" } @@ -48,7 +43,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 6, "fields": { - "deleted": null, "label": "JANUARY 31 (31)", "report_code": "MY" } @@ -57,7 +51,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 7, "fields": { - "deleted": null, "label": "GENERAL (12G)", "report_code": "12G" } @@ -66,7 +59,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 8, "fields": { - "deleted": null, "label": "PRIMARY (12P)", "report_code": "12P" } @@ -75,7 +67,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 9, "fields": { - "deleted": null, "label": "RUNOFF (12R)", "report_code": "12R" } @@ -84,7 +75,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 10, "fields": { - "deleted": null, "label": "SPECIAL (12S)", "report_code": "12S" } @@ -93,7 +83,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 11, "fields": { - "deleted": null, "label": "CONVENTION (12C)", "report_code": "12C" } @@ -102,7 +91,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 12, "fields": { - "deleted": null, "label": "GENERAL (30G)", "report_code": "30G" } @@ -111,7 +99,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 13, "fields": { - "deleted": null, "label": "RUNOFF (30R)", "report_code": "30R" } @@ -120,7 +107,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 14, "fields": { - "deleted": null, "label": "SPECIAL (30S)", "report_code": "30S" } @@ -129,7 +115,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 15, "fields": { - "deleted": null, "label": "FEBRUARY 20 (M2)", "report_code": "M2" } @@ -138,7 +123,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 16, "fields": { - "deleted": null, "label": "MARCH 30 (M3)", "report_code": "M3" } @@ -147,7 +131,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 17, "fields": { - "deleted": null, "label": "APRIL 20 (M4)", "report_code": "M4" } @@ -156,7 +139,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 18, "fields": { - "deleted": null, "label": "MAY 20 (M5)", "report_code": "M5" } @@ -165,7 +147,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 19, "fields": { - "deleted": null, "label": "JUNE 20 (M6)", "report_code": "M6" } @@ -174,7 +155,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 20, "fields": { - "deleted": null, "label": "JULY 20 (M7)", "report_code": "M7" } @@ -183,7 +163,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 21, "fields": { - "deleted": null, "label": "AUGUST 20 (M8)", "report_code": "M8" } @@ -192,7 +171,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 22, "fields": { - "deleted": null, "label": "SEPTEMBER 20 (M9)", "report_code": "M9" } @@ -201,7 +179,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 23, "fields": { - "deleted": null, "label": "OCTOBER 20 (M10)", "report_code": "M10" } @@ -210,7 +187,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 24, "fields": { - "deleted": null, "label": "NOVEMBER 20 (M11)", "report_code": "M11" } @@ -219,7 +195,6 @@ "model": "f3x_summaries.reportcodelabel", "pk": 25, "fields": { - "deleted": null, "label": "DECEMBER 20 (M12)", "report_code": "M12" } diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py deleted file mode 100644 index 3a10da95c6..0000000000 --- a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220524_1436.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 3.2.12 on 2022-05-24 18:36 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('f3x_summaries', '0009_auto_20220515_2116'), - ] - - operations = [ - migrations.CreateModel( - name='ReportCodeLabel', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('deleted', models.DateTimeField(blank=True, null=True)), - ('label', models.TextField(blank=True, null=True)), - ('report_code', models.TextField(blank=True, null=True)), - ], - options={ - 'db_table': 'report_code_labels', - }, - ), - migrations.AddIndex( - model_name='reportcodelabel', - index=models.Index(fields=['report_code'], name='report_code_report__9cd13b_idx'), - ), - ] diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py new file mode 100644 index 0000000000..edf037ded6 --- /dev/null +++ b/django-backend/fecfiler/f3x_summaries/migrations/0010_auto_20220525_1157.py @@ -0,0 +1,60 @@ +# Generated by Django 3.2.12 on 2022-05-25 15:57 + +from django.db import migrations, models +import django.db.models.deletion +from django.core.management import call_command + + +def forwards_func(apps, schema_editor): + call_command( + "loaddata", + "fecfiler/f3x_summaries/fixtures/report_code_labels.json", + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("f3x_summaries", "0009_auto_20220515_2116"), + ] + + operations = [ + migrations.CreateModel( + name="ReportCodeLabel", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("label", models.TextField(blank=True, null=True)), + ("report_code", models.TextField(blank=True, null=True, unique=True)), + ], + options={ + "db_table": "report_code_labels", + }, + ), + migrations.AddIndex( + model_name="reportcodelabel", + index=models.Index( + fields=["report_code"], name="report_code_report__9cd13b_idx" + ), + ), + migrations.RunPython(forwards_func), + migrations.AlterField( + model_name="f3xsummary", + name="report_code", + field=models.ForeignKey( + blank=True, + db_column="report_code", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="f3x_summaries.reportcodelabel", + to_field="report_code", + ), + ), + ] diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 8177fd8db3..1e499ed02b 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -6,6 +6,45 @@ from fecfiler.committee_accounts.models import CommitteeOwnedModel +class ReportCodeLabel(models.Model): + """ + labels = { + "Q1":"APRIL 15 (Q1)", + "Q2": "JULY 15 (Q2)", + "Q3":"OCTOBER 15 (Q3)", + "YE":"JANUARY 31 (YE)", + "TER":"TERMINATION (TER)", + "MY":"JANUARY 31 (31)", + "12G":"GENERAL (12G)", + "12P":"PRIMARY (12P)", + "12R":"RUNOFF (12R)", + "12S":"SPECIAL (12S)", + "12C":"CONVENTION (12C)", + "30G":"GENERAL (30G)", + "30R":"RUNOFF (30R)", + "30S":"SPECIAL (30S)", + "M2":"FEBRUARY 20 (M2)", + "M3":"MARCH 30 (M3)", + "M4":"APRIL 20 (M4)", + "M5":"MAY 20 (M5)", + "M6":"JUNE 20 (M6)", + "M7":"JULY 20 (M7)", + "M8":"AUGUST 20 (M8)", + "M9":"SEPTEMBER 20 (M9)", + "M10":"OCTOBER 20 (M10)", + "M11":"NOVEMBER 20 (M11)", + "M12":"DECEMBER 20 (M12)" + } + """ + + label = models.TextField(null=True, blank=True) + report_code = models.TextField(null=True, blank=True, unique=True) + + class Meta: + db_table = "report_code_labels" + indexes = [models.Index(fields=["report_code"])] + + class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): """Generated model from json schema""" @@ -19,7 +58,14 @@ class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): state = models.TextField(null=True, blank=True) zip = models.TextField(null=True, blank=True) - report_code = models.TextField(null=True, blank=True) + report_code = models.ForeignKey( + ReportCodeLabel, + models.SET_NULL, + null=True, + blank=True, + to_field="report_code", + db_column="report_code", + ) election_code = models.TextField(null=True, blank=True) date_of_election = models.TextField(null=True, blank=True) state_of_election = models.TextField(null=True, blank=True) @@ -212,44 +258,3 @@ class F3XSummary(SoftDeleteModel, CommitteeOwnedModel): class Meta: db_table = "f3x_summaries" - - -class ReportCodeLabel(SoftDeleteModel): - """ - labels = { - "Q1":"APRIL 15 (Q1)", - "Q2": "JULY 15 (Q2)", - "Q3":"OCTOBER 15 (Q3)", - "YE":"JANUARY 31 (YE)", - "TER":"TERMINATION (TER)", - "MY":"JANUARY 31 (31)", - "12G":"GENERAL (12G)", - "12P":"PRIMARY (12P)", - "12R":"RUNOFF (12R)", - "12S":"SPECIAL (12S)", - "12C":"CONVENTION (12C)", - "30G":"GENERAL (30G)", - "30R":"RUNOFF (30R)", - "30S":"SPECIAL (30S)", - "M2":"FEBRUARY 20 (M2)", - "M3":"MARCH 30 (M3)", - "M4":"APRIL 20 (M4)", - "M5":"MAY 20 (M5)", - "M6":"JUNE 20 (M6)", - "M7":"JULY 20 (M7)", - "M8":"AUGUST 20 (M8)", - "M9":"SEPTEMBER 20 (M9)", - "M10":"OCTOBER 20 (M10)", - "M11":"NOVEMBER 20 (M11)", - "M12":"DECEMBER 20 (M12)" - } - """ - - label = models.TextField(null=True, blank=True, unique=True); - report_code = models.TextField(null=True, blank=True); - - class Meta: - db_table = "report_code_labels" - indexes = [ - models.Index(fields=["report_code"]) - ] \ No newline at end of file From 0a9528a22b9333c8f6b6598ab396fdfc864d7bfd Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 25 May 2022 18:49:49 -0400 Subject: [PATCH 08/23] introduces foreign_key_fields to validate mixin --- .../fecfiler/f3x_summaries/serializers.py | 16 +++-- .../fecfiler/f3x_summaries/views.py | 61 ++----------------- .../fecfiler/validation/serializers.py | 22 ++++++- 3 files changed, 38 insertions(+), 61 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index 5de95374be..a60a7e9457 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -1,5 +1,6 @@ +from django.forms import SlugField from .models import F3XSummary, ReportCodeLabel -from rest_framework.serializers import ModelSerializer +from rest_framework.serializers import ModelSerializer, SlugRelatedField from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer from fecfiler.validation import serializers import logging @@ -11,6 +12,12 @@ class F3XSummarySerializer( CommitteeOwnedSerializer, serializers.FecSchemaValidatorSerializerMixin ): schema_name = "F3X" + report_code = SlugRelatedField( + many=False, + read_only=False, + slug_field="report_code", + queryset=ReportCodeLabel.objects.all(), + ) class Meta: model = F3XSummary @@ -21,11 +28,10 @@ class Meta: "created", "updated", ] + foreign_key_fields = {"report_code": "report_code"} + class ReportCodeLabelSerializer(ModelSerializer): class Meta: model = ReportCodeLabel - fields = ( - 'label', - 'report_code' - ) \ No newline at end of file + fields = ("label", "report_code") diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index eea1a45b0c..27d93e0bd2 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -18,66 +18,17 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): in CommitteeOwnedViewSet's implementation of get_queryset() """ - """ - report_code_labels = ReportCodeLabel.labels.keys() - case_objects = [] - for label in report_code_labels: - case_objects.append(When(report_code=label, then=Value(ReportCodeLabels.labels[label]))) - - ## The "*" dereferences the case_objects list, making the When() objects inside it the arguments for the Case() constructor - queryset = F3XSummary.objects.alias(report_code_label=Case(*case_objects)); - """ - - """ - queryset = F3XSummary.objects.alias(report_code_label=Case( - When(report_code='Q1', then=Value('APRIL 15 (Q1)')), - When(report_code='Q2', then=Value('JULY 15 (Q2)')), - When(report_code='Q3', then=Value('OCTOBER 15 (Q3)')), - When(report_code='YE', then=Value('JANUARY 31 (YE)')), - When(report_code='TER', then=Value('TERMINATION (TER)')), - When(report_code='MY', then=Value('JANUARY 31 (MY)')), - When(report_code='12G', then=Value('GENERAL (12G)')), - When(report_code='12P', then=Value('PRIMARY (12P)')), - When(report_code='12R', then=Value('RUNOFF (12R)')), - When(report_code='12S', then=Value('SPECIAL (12S)')), - When(report_code='12C', then=Value('CONVENTION (12C)')), - When(report_code='30G', then=Value('GENERAL (30G)')), - When(report_code='30R', then=Value('RUNOFF (30R)')), - When(report_code='30S', then=Value('SPECIAL (30S)')), - When(report_code='M2', then=Value('FEBRUARY 20 (M2)')), - When(report_code='M3', then=Value('MARCH 20 (M3)')), - When(report_code='M4', then=Value('APRIL 20 (M4)')), - When(report_code='M5', then=Value('MAY 20 (M5)')), - When(report_code='M6', then=Value('JUNE 20 (M6)')), - When(report_code='M7', then=Value('JULY 20 (M7)')), - When(report_code='M8', then=Value('AUGUST 20 (M8)')), - When(report_code='M9', then=Value('SEPTEMBER 20 (M9)')), - When(report_code='M10', then=Value('OCTOBER 20 (M10)')), - When(report_code='M11', then=Value('NOVEMBER 20 (M11)')), - When(report_code='M12', then=Value('DECEMBER 20 (M12)')), - default=Value('_NOT_FOUND_'), - )).all() - """ - - def get_queryset(self): - SQLCommand = """ - SELECT * FROM f3x_summaries - INNER JOIN (SELECT label AS report_code_label, report_code from report_code_labels) report_code_labels - ON f3x_summaries.report_code = report_code_labels.report_code - """ - - rawqueryset = F3XSummary.objects.raw(SQLCommand) - rawqueryset.order_by = F3XSummary.objects.order_by - ##print(rawqueryset[0].report_code_label) - return rawqueryset + queryset = F3XSummary.objects.select_related("report_code").all() + """Join on report code labels""" serializer_class = F3XSummarySerializer permission_classes = [] filter_backends = [filters.OrderingFilter] - ordering_fields = ['form_type', 'report_code_label', 'coverage_through_date'] - ordering = ['form_type'] + ordering_fields = ["form_type", "report_code__label", "coverage_through_date"] + ordering = ["form_type"] + class ReportCodeLabelViewSet(GenericViewSet, ListModelMixin): queryset = ReportCodeLabel.objects.all() serializer_class = ReportCodeLabelSerializer - pagination_class = None \ No newline at end of file + pagination_class = None diff --git a/django-backend/fecfiler/validation/serializers.py b/django-backend/fecfiler/validation/serializers.py index 8af5f7e69b..960cc77668 100644 --- a/django-backend/fecfiler/validation/serializers.py +++ b/django-backend/fecfiler/validation/serializers.py @@ -44,6 +44,23 @@ def get_schema_name(self, data): ) return self.schema_name + def get_validation_candidate(self, data): + validation_candidate = data.copy() + for (foreign_key_field, actual_key) in self.get_foreign_key_fields().items(): + validation_candidate[foreign_key_field] = getattr( + validation_candidate.get(foreign_key_field, {}), actual_key + ) + + return validation_candidate + + def get_foreign_key_fields(self): + """ + Returns a dictionary of foreign key fields + """ + meta = getattr(self, "Meta", None) + foreign_key_fields = getattr(meta, "foreign_key_fields", None) + return dict(foreign_key_fields) if foreign_key_fields else {} + def validate(self, data): """Overrides Django Rest Framework's Serializer validate to validate with fecfile_validate rules. @@ -59,8 +76,11 @@ def validate(self, data): fields_to_validate = ( fields_to_validate_str.split(",") if fields_to_validate_str else [] ) + validation_result = validate.validate( - self.get_schema_name(data), data, fields_to_validate + self.get_schema_name(data), + self.get_validation_candidate(data), + fields_to_validate, ) if validation_result.errors: From 47ef55f9f8c51cd35990f54390836907a35bc31a Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 25 May 2022 19:00:49 -0400 Subject: [PATCH 09/23] adds some documentation --- django-backend/fecfiler/validation/serializers.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/validation/serializers.py b/django-backend/fecfiler/validation/serializers.py index 960cc77668..158a88925f 100644 --- a/django-backend/fecfiler/validation/serializers.py +++ b/django-backend/fecfiler/validation/serializers.py @@ -45,6 +45,16 @@ def get_schema_name(self, data): return self.schema_name def get_validation_candidate(self, data): + """Returns a copy of data where foreign key fields are replaced with the + underlying foreign key field. + + Example: the f3x table is related to the report code label table by the + report code field. DRF gives us the whole + :py:class:`fecfiler.f3x_summaries.models.ReportCodeLabel` object. + for validating purposes, we just want the report code. With the + foreign_key_fields mapping defined in Meta, we replace the foreign-key- + related object with the key only. + """ validation_candidate = data.copy() for (foreign_key_field, actual_key) in self.get_foreign_key_fields().items(): validation_candidate[foreign_key_field] = getattr( @@ -54,9 +64,7 @@ def get_validation_candidate(self, data): return validation_candidate def get_foreign_key_fields(self): - """ - Returns a dictionary of foreign key fields - """ + """Returns a dictionary of foreign key fields""" meta = getattr(self, "Meta", None) foreign_key_fields = getattr(meta, "foreign_key_fields", None) return dict(foreign_key_fields) if foreign_key_fields else {} From 9320ae7e22a6db2e43a5530f0107bb6f6b898566 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 25 May 2022 19:06:23 -0400 Subject: [PATCH 10/23] report_code is not required --- django-backend/fecfiler/f3x_summaries/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index a60a7e9457..ca96f74b65 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -14,6 +14,7 @@ class F3XSummarySerializer( schema_name = "F3X" report_code = SlugRelatedField( many=False, + required=False, read_only=False, slug_field="report_code", queryset=ReportCodeLabel.objects.all(), From 71bdf6836a31bbe07b2e4a35d722d0fd3877ee09 Mon Sep 17 00:00:00 2001 From: toddlees Date: Wed, 25 May 2022 19:09:51 -0400 Subject: [PATCH 11/23] error proffing code --- django-backend/fecfiler/validation/serializers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/validation/serializers.py b/django-backend/fecfiler/validation/serializers.py index 158a88925f..4ff837e7c8 100644 --- a/django-backend/fecfiler/validation/serializers.py +++ b/django-backend/fecfiler/validation/serializers.py @@ -57,9 +57,10 @@ def get_validation_candidate(self, data): """ validation_candidate = data.copy() for (foreign_key_field, actual_key) in self.get_foreign_key_fields().items(): - validation_candidate[foreign_key_field] = getattr( - validation_candidate.get(foreign_key_field, {}), actual_key - ) + if hasattr(validation_candidate.get(foreign_key_field, {}), actual_key): + validation_candidate[foreign_key_field] = getattr( + validation_candidate.get(foreign_key_field, {}), actual_key + ) return validation_candidate From 3f9bf1026318808cbb83cd172739c5a4d91633d3 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Thu, 26 May 2022 08:58:28 -0400 Subject: [PATCH 12/23] Merged in Todd's work --- .../fecfiler/f3x_summaries/models.py | 33 ++----------------- .../fecfiler/f3x_summaries/views.py | 1 - 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 1e499ed02b..2d39cbfa79 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -7,39 +7,12 @@ class ReportCodeLabel(models.Model): - """ - labels = { - "Q1":"APRIL 15 (Q1)", - "Q2": "JULY 15 (Q2)", - "Q3":"OCTOBER 15 (Q3)", - "YE":"JANUARY 31 (YE)", - "TER":"TERMINATION (TER)", - "MY":"JANUARY 31 (31)", - "12G":"GENERAL (12G)", - "12P":"PRIMARY (12P)", - "12R":"RUNOFF (12R)", - "12S":"SPECIAL (12S)", - "12C":"CONVENTION (12C)", - "30G":"GENERAL (30G)", - "30R":"RUNOFF (30R)", - "30S":"SPECIAL (30S)", - "M2":"FEBRUARY 20 (M2)", - "M3":"MARCH 30 (M3)", - "M4":"APRIL 20 (M4)", - "M5":"MAY 20 (M5)", - "M6":"JUNE 20 (M6)", - "M7":"JULY 20 (M7)", - "M8":"AUGUST 20 (M8)", - "M9":"SEPTEMBER 20 (M9)", - "M10":"OCTOBER 20 (M10)", - "M11":"NOVEMBER 20 (M11)", - "M12":"DECEMBER 20 (M12)" - } - """ - label = models.TextField(null=True, blank=True) report_code = models.TextField(null=True, blank=True, unique=True) + def __str__(self): + return self.label + class Meta: db_table = "report_code_labels" indexes = [models.Index(fields=["report_code"])] diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 27d93e0bd2..ec20eb522b 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -27,7 +27,6 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): ordering_fields = ["form_type", "report_code__label", "coverage_through_date"] ordering = ["form_type"] - class ReportCodeLabelViewSet(GenericViewSet, ListModelMixin): queryset = ReportCodeLabel.objects.all() serializer_class = ReportCodeLabelSerializer From f4b30e8525e632fa2adeac7a1e12e251e10f4773 Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 27 May 2022 09:49:15 -0400 Subject: [PATCH 13/23] Incorporates CircleCI linting recommendations --- django-backend/fecfiler/f3x_summaries/models.py | 3 --- django-backend/fecfiler/f3x_summaries/serializers.py | 1 - django-backend/fecfiler/f3x_summaries/urls.py | 5 ++++- django-backend/fecfiler/f3x_summaries/views.py | 2 -- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/models.py b/django-backend/fecfiler/f3x_summaries/models.py index 2d39cbfa79..e9012d3409 100644 --- a/django-backend/fecfiler/f3x_summaries/models.py +++ b/django-backend/fecfiler/f3x_summaries/models.py @@ -1,6 +1,3 @@ -from locale import MON_3, MON_8 -from logging import raiseExceptions -from tkinter.messagebox import YES from django.db import models from fecfiler.soft_delete.models import SoftDeleteModel from fecfiler.committee_accounts.models import CommitteeOwnedModel diff --git a/django-backend/fecfiler/f3x_summaries/serializers.py b/django-backend/fecfiler/f3x_summaries/serializers.py index ca96f74b65..7fc286ec29 100644 --- a/django-backend/fecfiler/f3x_summaries/serializers.py +++ b/django-backend/fecfiler/f3x_summaries/serializers.py @@ -1,4 +1,3 @@ -from django.forms import SlugField from .models import F3XSummary, ReportCodeLabel from rest_framework.serializers import ModelSerializer, SlugRelatedField from fecfiler.committee_accounts.serializers import CommitteeOwnedSerializer diff --git a/django-backend/fecfiler/f3x_summaries/urls.py b/django-backend/fecfiler/f3x_summaries/urls.py index b3365812cb..cdfe849676 100644 --- a/django-backend/fecfiler/f3x_summaries/urls.py +++ b/django-backend/fecfiler/f3x_summaries/urls.py @@ -5,7 +5,10 @@ # Create a router and register our viewsets with it. router = DefaultRouter() router.register(r"f3x-summaries", F3XSummaryViewSet, basename="f3x-summaries") -router.register(r"report-code-labels",ReportCodeLabelViewSet, basename="report-code-labels") +router.register( + r"report-code-labels", + ReportCodeLabelViewSet, + basename="report-code-labels") # The API URLs are now determined automatically by the router. urlpatterns = [ diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index ec20eb522b..4a2359d983 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -1,8 +1,6 @@ from rest_framework import filters from rest_framework.viewsets import GenericViewSet from rest_framework.mixins import ListModelMixin -from django.db.models import Case, Value, When -from django.db.models.expressions import RawSQL from fecfiler.committee_accounts.views import CommitteeOwnedViewSet from .models import F3XSummary, ReportCodeLabel from .serializers import F3XSummarySerializer, ReportCodeLabelSerializer From fbcd29c96e5a8304b1fb81ae8c1bd5741b7de13c Mon Sep 17 00:00:00 2001 From: Elaine Krauss Date: Fri, 27 May 2022 10:07:01 -0400 Subject: [PATCH 14/23] More linting updates --- django-backend/fecfiler/f3x_summaries/urls.py | 4 ++-- django-backend/fecfiler/f3x_summaries/views.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/urls.py b/django-backend/fecfiler/f3x_summaries/urls.py index cdfe849676..9d4cf69267 100644 --- a/django-backend/fecfiler/f3x_summaries/urls.py +++ b/django-backend/fecfiler/f3x_summaries/urls.py @@ -6,8 +6,8 @@ router = DefaultRouter() router.register(r"f3x-summaries", F3XSummaryViewSet, basename="f3x-summaries") router.register( - r"report-code-labels", - ReportCodeLabelViewSet, + r"report-code-labels", + ReportCodeLabelViewSet, basename="report-code-labels") # The API URLs are now determined automatically by the router. diff --git a/django-backend/fecfiler/f3x_summaries/views.py b/django-backend/fecfiler/f3x_summaries/views.py index 4a2359d983..8ffadef517 100644 --- a/django-backend/fecfiler/f3x_summaries/views.py +++ b/django-backend/fecfiler/f3x_summaries/views.py @@ -25,6 +25,7 @@ class F3XSummaryViewSet(CommitteeOwnedViewSet): ordering_fields = ["form_type", "report_code__label", "coverage_through_date"] ordering = ["form_type"] + class ReportCodeLabelViewSet(GenericViewSet, ListModelMixin): queryset = ReportCodeLabel.objects.all() serializer_class = ReportCodeLabelSerializer From 70eaba7b50cd42cc186fa9defb8c4f9149246d2a Mon Sep 17 00:00:00 2001 From: Matt Travers Date: Mon, 6 Jun 2022 08:26:50 -0400 Subject: [PATCH 15/23] Updated requirements.txt to point to latest validate commit --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a83532f02b..29a795e058 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ django-cors-headers==3.11.0 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@64a35b25fc1b843866a2f612c08510064d4d162f#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@6e6ea5905f823b125ea548bebb0d05c92642d849#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==2.11.3 From d17bd62848ec9d86831e20f3a7ba34fa53da02c9 Mon Sep 17 00:00:00 2001 From: David Heitzer Date: Mon, 13 Jun 2022 15:44:17 -0400 Subject: [PATCH 16/23] update validation version hash for release --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 29a795e058..c0ff5acc13 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ django-cors-headers==3.11.0 djangorestframework==3.13.1 djangorestframework-jwt==1.11.0 drf-spectacular==0.21.2 -git+https://github.com/fecgov/fecfile-validate@6e6ea5905f823b125ea548bebb0d05c92642d849#egg=fecfile_validate&subdirectory=fecfile_validate_python +git+https://github.com/fecgov/fecfile-validate@be23cab922b2c263f2071be3d7d827d195c01397#egg=fecfile_validate&subdirectory=fecfile_validate_python GitPython==3.1.27 gunicorn==19.10.0 Jinja2==2.11.3 From 3be2411aa2341d57be39c33b2bdfe8a890f974c8 Mon Sep 17 00:00:00 2001 From: toddlees <97105825+toddlees@users.noreply.github.com> Date: Fri, 17 Jun 2022 09:46:13 -0400 Subject: [PATCH 17/23] output migration results to file --- bin/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run.sh b/bin/run.sh index ccdf306207..38cbb0e9e9 100755 --- a/bin/run.sh +++ b/bin/run.sh @@ -1,7 +1,7 @@ cd django-backend # Run migrations -./manage.py migrate --noinput +./manage.py migrate --noinput > migrate.out # Run application python wait_for_db.py && gunicorn --bind 0.0.0.0:8080 fecfiler.wsgi -w 9 -t 200 From 8fe912685b9059364d040340dc613f5d943ae2df Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 10:29:24 -0400 Subject: [PATCH 18/23] handle 'X' and '' in migrations --- .../migrations/0009_auto_20220515_2116.py | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py index 93eb6c83cd..0273ec045f 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py @@ -1,23 +1,36 @@ # Generated by Django 3.2.12 on 2022-05-16 01:16 +from fecfiler.f3x_summaries.models import F3XSummary from django.db import migrations, models +def convert_fecfile_booleans(): + for f3x_summary in F3XSummary.objects.all(): + f3x_summary.change_of_address = ( + "true" if f3x_summary.change_of_address in ["true", "X"] else "false" + ) + f3x_summary.qualified_committee = ( + "true" if f3x_summary.qualified_committee in ["true", "X"] else "false" + ) + f3x_summary.save() + + class Migration(migrations.Migration): dependencies = [ - ('f3x_summaries', '0008_auto_20220503_1411'), + ("f3x_summaries", "0008_auto_20220503_1411"), ] operations = [ + migrations.RunPython(convert_fecfile_booleans), migrations.AlterField( - model_name='f3xsummary', - name='change_of_address', + model_name="f3xsummary", + name="change_of_address", field=models.BooleanField(blank=True, default=False, null=True), ), migrations.AlterField( - model_name='f3xsummary', - name='qualified_committee', + model_name="f3xsummary", + name="qualified_committee", field=models.BooleanField(blank=True, default=False, null=True), ), ] From 414160603233d1a7fabdf125bef9f388829f979f Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 10:33:28 -0400 Subject: [PATCH 19/23] add args for runpython hook --- .../f3x_summaries/migrations/0009_auto_20220515_2116.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py index 0273ec045f..fa19c701e7 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py @@ -4,7 +4,7 @@ from django.db import migrations, models -def convert_fecfile_booleans(): +def convert_fecfile_booleans(apps, schema_editor): for f3x_summary in F3XSummary.objects.all(): f3x_summary.change_of_address = ( "true" if f3x_summary.change_of_address in ["true", "X"] else "false" From d72aff04b36ff26d93ed5f7a47ccec92b1ffa515 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 10:40:01 -0400 Subject: [PATCH 20/23] return actual booleans in transform --- .../f3x_summaries/migrations/0009_auto_20220515_2116.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py index fa19c701e7..40a02171a2 100644 --- a/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py +++ b/django-backend/fecfiler/f3x_summaries/migrations/0009_auto_20220515_2116.py @@ -7,10 +7,10 @@ def convert_fecfile_booleans(apps, schema_editor): for f3x_summary in F3XSummary.objects.all(): f3x_summary.change_of_address = ( - "true" if f3x_summary.change_of_address in ["true", "X"] else "false" + True if f3x_summary.change_of_address in ["true", "X"] else False ) f3x_summary.qualified_committee = ( - "true" if f3x_summary.qualified_committee in ["true", "X"] else "false" + True if f3x_summary.qualified_committee in ["true", "X"] else False ) f3x_summary.save() From b256e696d3257f31d78109089b066b9266e6a005 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 11:00:30 -0400 Subject: [PATCH 21/23] Handle schedule a boolean conversion as well --- .../0008_alter_schatransaction_memo_code.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py b/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py index 04ffdfbef7..9cc7e3083c 100644 --- a/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py +++ b/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py @@ -1,18 +1,27 @@ # Generated by Django 3.2.12 on 2022-05-16 11:44 from django.db import migrations, models +from fecfiler.scha_transactions.models import SchATransaction + + +def convert_fecfile_booleans(apps, schema_editor): + for transaction in SchATransaction.objects.all(): + transaction.memo_code = ( + True if transaction.memo_code in ["true", "X"] else False + ) + transaction.save() class Migration(migrations.Migration): dependencies = [ - ('scha_transactions', '0007_alter_schatransaction_committee_account'), + ("scha_transactions", "0007_alter_schatransaction_committee_account"), ] operations = [ migrations.AlterField( - model_name='schatransaction', - name='memo_code', + model_name="schatransaction", + name="memo_code", field=models.BooleanField(blank=True, default=False, null=True), ), ] From 8460a5de9056724bba6227b2016b48c17c5b9773 Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 11:06:10 -0400 Subject: [PATCH 22/23] actually call the convert function --- .../migrations/0008_alter_schatransaction_memo_code.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py b/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py index 9cc7e3083c..2ff1c6c697 100644 --- a/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py +++ b/django-backend/fecfiler/scha_transactions/migrations/0008_alter_schatransaction_memo_code.py @@ -19,6 +19,7 @@ class Migration(migrations.Migration): ] operations = [ + migrations.RunPython(convert_fecfile_booleans), migrations.AlterField( model_name="schatransaction", name="memo_code", From 025f911d4ae6cf4b6249fa44f5f42eee9a131d7f Mon Sep 17 00:00:00 2001 From: toddlees Date: Fri, 17 Jun 2022 11:27:13 -0400 Subject: [PATCH 23/23] ignore migrations for code coverage --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index feb9dc71cc..d3727a7b55 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -12,7 +12,7 @@ sonar.sources=django-backend #sonar.sourceEncoding=UTF-8 # Exclude utility script from coverage -sonar.coverage.exclusions=**/json_schema_to_django_model.py +sonar.coverage.exclusions=**/json_schema_to_django_model.py,**/migrations/** sonar.python.coverage.reportPaths=coverage-reports/coverage.xml sonar.python.bandit.reportPaths=bandit.out sonar.python.flake8.reportPaths=flake8.out