move clean content row to the model

praekeltfoundation · Feb 6, 2025 · 0c1ec40 · 0c1ec40
1 parent ca8b739
commit 0c1ec40
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 57 deletions.
diff --git a/home/import_content_pages.py b/home/import_content_pages.py
@@ -8,10 +8,7 @@
 from typing import Any
 from uuid import uuid4
 
-from django.core.exceptions import (
-    ObjectDoesNotExist,  # type: ignore
-    ValidationError,
-)
+from django.core.exceptions import ObjectDoesNotExist, ValidationError  # type: ignore
 from taggit.models import Tag  # type: ignore
 from treebeard.exceptions import NodeAlreadySaved  # type: ignore
 from wagtail.blocks import StructValue  # type: ignore

diff --git a/home/models.py b/home/models.py
@@ -14,28 +14,45 @@
 from modelcluster.models import ClusterableModel
 from taggit.models import ItemBase, TagBase, TaggedItemBase
 from wagtail import blocks
-from wagtail.admin.panels import (FieldPanel, MultiFieldPanel, ObjectList,
-                                  TabbedInterface, TitleFieldPanel)
+from wagtail.admin.panels import (
+    FieldPanel,
+    MultiFieldPanel,
+    ObjectList,
+    TabbedInterface,
+    TitleFieldPanel,
+)
 from wagtail.api import APIField
-from wagtail.blocks import (StreamBlockValidationError, StreamValue,
-                            StructBlockValidationError)
+from wagtail.blocks import (
+    StreamBlockValidationError,
+    StreamValue,
+    StructBlockValidationError,
+)
 from wagtail.contrib.settings.models import BaseSiteSetting, register_setting
 from wagtail.documents.blocks import DocumentChooserBlock
 from wagtail.fields import StreamField
 from wagtail.images.blocks import ImageChooserBlock
-from wagtail.models import (DraftStateMixin, Locale, LockableMixin, Page,
-                            ReferenceIndex, Revision, RevisionMixin,
-                            WorkflowMixin)
+from wagtail.models import (
+    DraftStateMixin,
+    Locale,
+    LockableMixin,
+    Page,
+    ReferenceIndex,
+    Revision,
+    RevisionMixin,
+    WorkflowMixin,
+)
 from wagtail.models.sites import Site
 from wagtail.search import index
 from wagtail.snippets.blocks import SnippetChooserBlock
 from wagtail_content_import.models import ContentImportMixin
 from wagtailmedia.blocks import AbstractMediaChooserBlock
 
 from .panels import PageRatingPanel
-from .whatsapp import (TemplateSubmissionException,
-                       create_standalone_whatsapp_template,
-                       create_whatsapp_template)
+from .whatsapp import (
+    TemplateSubmissionException,
+    create_standalone_whatsapp_template,
+    create_whatsapp_template,
+)
 
 from .constants import (  # isort:skip
     AGE_CHOICES,
@@ -962,7 +979,6 @@ def clean(self):
 
         # Clean the whatsapp body to remove hidden characters
         if self.whatsapp_body and isinstance(self.whatsapp_body, StreamValue):
-            whatsapp_block = []
             for block in self.whatsapp_body:
                 if block.block_type == "Whatsapp_Message":
                     message = block.value["message"]
@@ -973,8 +989,6 @@ def clean(self):
                     ).strip()
 
                     block.value["message"] = cleaned_message
-                whatsapp_block.append(block)
-            # self.whatsapp_body = StreamValue(self.whatsapp_body.stream_block, whatsapp_block)
 
         # The WA title is needed for all templates to generate a name for the template
         if self.is_whatsapp_template and not self.whatsapp_title:

diff --git a/home/tests/import-export-data/content2.csv b/home/tests/import-export-data/content2.csv
@@ -24,12 +24,12 @@ Reply with a number to take part in a *free* self-help program created by WHO.
 
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
+2. Manage your stress
 _Learn how to cope with stress and improve your wellbeing._",,UTILITY,[],,,,[],,,,,self-help,"*Self-help programs* 🌬️
 
 Reply with a number to take part in a *free* self-help program created by WHO.
 
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
+2. Manage your stress
 _Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,,[],,,,,,en
diff --git a/home/tests/import-export-data/content3.csv b/home/tests/import-export-data/content3.csv
diff --git a/home/tests/test_content_import_export.py b/home/tests/test_content_import_export.py
@@ -576,7 +576,7 @@ def test_simple(self, csv_impexp: ImportExport) -> None:
          * Should we set enable_web and friends based on body, title, or an
            enable field that we'll need to add to the export?
         """
-        csv_bytes = csv_impexp.import_file("content3.csv")
+        csv_bytes = csv_impexp.import_file("content2.csv")
         content = csv_impexp.export_content()
         src, dst = csv_impexp.csvs2dicts(csv_bytes, content)
         assert dst == src
@@ -744,7 +744,7 @@ def test_import_error(self, csv_impexp: ImportExport) -> None:
         (This uses content2.csv from test_api.py and broken.csv.)
         """
         # Start with some existing content.
-        csv_bytes = csv_impexp.import_file("content3.csv")
+        csv_bytes = csv_impexp.import_file("content2.csv")
 
         # This CSV doesn't have any of the fields we expect.
         with pytest.raises((KeyError, TypeError, ImportException)):
@@ -1670,7 +1670,8 @@ def test_hidden_characters(self, csv_impexp: ImportExport) -> None:
         """
         Import a page that has hidden characters in the whatsapp body
         """
-        csv_impexp.import_file("test_special_chars.csv")
+        csv_bytes = csv_impexp.import_file("test_special_chars.csv")
+        assert "\u2028\u2028" in csv_bytes.decode("utf-8")
         assert "\u2028" not in ContentPage.objects.all().values()[0]["whatsapp_body"]