Merge branch 'main' into add-slugs-locale-orderedset

CHANGELOG.md

@@ -7,11 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 <!--
 ## Unreleased
+### Fixed
+- ContentPage import: Accept xlsx where field formatting is numeric.
+-->
+
+## v1.3.0
 ### Added
 - Filter ordered content sets by profile field values.
 - Add list_button_title to Whatsapp messages.
 - Add ability to specify actions to Whatsapp list item messages.
 - Add slug and locale to OrderedContentSets.
+- Filter ordered content sets by slug and locale in import.
 
 ### Removed
 - Removed word embeddings search (`s` query parameter in API)
@@ -23,18 +29,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - API: Added page keyword to Assessments API
 - Ordered Content Sets: Import now allows for CSV values in time, unit, before or after, and contact field
 - Importing certain ContentPage fields with whitespace-only values will no longer throw unhandled exceptions
+- UI: ContentPage listed multiple times in sidebar
+- Ordered content set XLSX export
+
+### Changed
+- Submit WA templates with Buttons if they exist, only submit with Quick Replies if no Buttons
 
 ### Security
 - Updated sentry-sdk from 1.44.1 to 2.8.0
 - Updated wagtail from 5.2.4 to 5.2.6
 - Updated certifi from 2024.2.2 to 2024.7.4
-- Updated django from 4.2.11 to 4.2.15
+- Updated django from 4.2.11 to 4.2.16
 - Updated sqlparse from 0.4.4 to 0.5.0
 - Updated djangorestframework from 3.15.1 to 3.15.2
 - Updated urllib3 from 2.2.1 to 2.2.2
 - Updated requests from 2.31.0 to 2.32.2
 - Updated idna from 3.6 to 3.7
--->
 
 ## v1.2.1
 ### Added

home/import_content_pages.py

@@ -697,7 +697,6 @@ def wagtail_format(self) -> dict[str, str]:
 @dataclass(slots=True, frozen=True)
 class ContentRow:
     slug: str
-    page_id: int | None = None
     parent: str = ""
     web_title: str = ""
     web_subtitle: str = ""
@@ -763,7 +762,6 @@ def from_flat(cls, row: dict[str, str], row_num: int) -> "ContentRow":
                 for item in deserialise_list(row_list_items)
             ]
         return cls(
-            page_id=to_int_or_none(row.pop("page_id", None)),
             variation_title=deserialise_dict(row.pop("variation_title", "")),
             tags=deserialise_list(row.pop("tags", "")),
             quick_replies=deserialise_list(row.pop("quick_replies", "")),
@@ -859,4 +857,11 @@ def JSON_loader(row_num: int, value: str) -> list[dict[str, Any]]:
 
 
 def to_int_or_none(val: str | None) -> int | None:
-    return int(val) if val else None
+    if val is None:
+        return None
+    try:
+        return int(val)
+    except ValueError:
+        # If it's an excel document with number formatting, we get a float
+        # If it's not a valid number, then we let the exception bubble up
+        return int(float(val))

home/import_ordered_content_sets.py

@@ -22,6 +22,7 @@ class OrderedContentSetPage:
     before_or_after: str
     page_slug: str
     contact_field: str
+    locale: str
 
 
 class OrderedContentSetImporter:
@@ -43,7 +44,7 @@ def __init__(
         self.progress_queue = progress_queue
 
     def _get_or_init_ordered_content_set(
-        self, row: dict[str, str], set_slug: str, set_locale: str
+        self, index: int, row: dict[str, str], set_slug: str, set_locale: str
     ) -> OrderedContentSet:
         """
         Get or initialize an instance of OrderedContentSet from a row of a CSV file.
@@ -53,14 +54,19 @@ def _get_or_init_ordered_content_set(
         :param set_locale: The locale of the ordered content set.
         :return: An instance of OrderedContentSet.
         """
-        locale = Locale.objects.get(language_code=set_locale)
-        ordered_set = OrderedContentSet.objects.filter(
-            slug=set_slug, locale=locale
-        ).first()
-        if not ordered_set:
-            ordered_set = OrderedContentSet(slug=set_slug, locale=locale)
-
-        return ordered_set
+        try:
+            locale = Locale.objects.get(language_code=set_locale)
+            ordered_set = OrderedContentSet.objects.filter(
+                slug=set_slug, locale=locale
+            ).first()
+            if not ordered_set:
+                ordered_set = OrderedContentSet(slug=set_slug, locale=locale)
+
+            return ordered_set
+        except Locale.DoesNotExist:
+            if set_locale:
+                raise ImportException(f"Locale {set_locale} does not exist.", index)
+            raise ImportException("No locale specified.", index)
 
     def _add_profile_fields(
         self, ordered_set: OrderedContentSet, row: dict[str, str]
@@ -103,6 +109,7 @@ def _extract_ordered_content_set_pages(
                 before_or_after=before_or_after,
                 page_slug=page_slug,
                 contact_field=contact_field,
+                locale=row["Locale"],
             )
             for time, unit, before_or_after, page_slug, contact_field in zip(
                 times, units, before_or_afters, page_slugs, contact_fields, strict=False
@@ -132,10 +139,13 @@ def _add_pages(
         :param ordered_set: The ordered content set to add the pages to.
         :param pages: A list of OrderedContentSetPage objects.
         """
+        locale = ordered_set.locale
         ordered_set.pages = []
         for page in pages:
             if page.page_slug and page.page_slug != "-":
-                content_page = ContentPage.objects.filter(slug=page.page_slug).first()
+                content_page = ContentPage.objects.filter(
+                    slug=page.page_slug, locale=locale
+                ).first()
                 if content_page:
                     os_page = {
                         "contentpage": content_page,
@@ -147,7 +157,8 @@ def _add_pages(
                     ordered_set.pages.append(("pages", os_page))
                 else:
                     raise ImportException(
-                        f"Content page not found for slug '{page.page_slug}'", index
+                        f"Content page not found for slug '{page.page_slug}' in locale '{locale}'",
+                        index,
                     )
 
     def _create_ordered_set_from_row(
@@ -162,7 +173,7 @@ def _create_ordered_set_from_row(
         :raises ImportException: If time, units, before_or_afters, page_slugs and contact_fields are not all equal length.
         """
         ordered_set = self._get_or_init_ordered_content_set(
-            row, row["Slug"].lower(), row["Locale"]
+            index, row, row["Slug"].lower(), row["Locale"]
         )
         ordered_set.name = row["Name"]
         self._add_profile_fields(ordered_set, row)

home/models.py

@@ -806,14 +806,23 @@ def save_page_view(self, query_params, platform=None):
     def quick_reply_buttons(self):
         return self.quick_reply_items.all().values_list("tag__name", flat=True)
 
+    @property
+    def whatsapp_template_buttons(self):
+        # If Buttons and quick replies are present then Buttons are used
+        first_msg = self.whatsapp_body.raw_data[0]["value"]
+        if "buttons" in first_msg and first_msg["buttons"]:
+            buttons = [b["value"]["title"] for b in first_msg["buttons"]]
+            return buttons
+        return sorted(self.quick_reply_buttons)
+
     @property
     def whatsapp_template_fields(self):
         """
         Returns a tuple of fields that can be used to determine template equality
         """
         return (
             self.whatsapp_template_body,
-            sorted(self.quick_reply_buttons),
+            self.whatsapp_template_buttons,
             self.is_whatsapp_template,
             self.whatsapp_template_image,
             self.whatsapp_template_category,
@@ -850,7 +859,7 @@ def submit_whatsapp_template(self, previous_revision):
             self.whatsapp_template_body,
             str(self.whatsapp_template_category),
             self.locale,
-            sorted(self.quick_reply_buttons),
+            self.whatsapp_template_buttons,
             self.whatsapp_template_image,
             self.whatsapp_template_example_values,
         )

home/tests/import-export-data/contentpage_required_fields_multi_locale.csv

@@ -0,0 +1,7 @@
+message,slug,parent,web_title,locale,translation_tag
+0,main_menu,,Main Menu,English,9aab62ab-caf7-4606-b9bf-ac3148309319
+1,first_time_user,Main Menu,main menu first time user,English,38a22433-e474-4f5a-b06b-7367d1a7f664
+1,health_info,main menu first time user,health info,English,138da946-ee82-42ac-8e63-720771dfd674
+0,main_menu,,Main Menu,Portuguese,9aab62ab-caf7-4606-b9bf-ac3148309319
+1,first_time_user,Main Menu,main menu first time user,Portuguese,38a22433-e474-4f5a-b06b-7367d1a7f664
+1,health_info,main menu first time user,health info,Portuguese,138da946-ee82-42ac-8e63-720771dfd674

home/tests/import-export-data/ordered_content.csv

@@ -1,2 +1,3 @@
 Name,Profile Fields,Page Slugs,Time,Unit,Before Or After,Contact Field,Slug,Locale
 Test Set,"gender:male, relationship:in_a_relationship","first_time_user, first_time_user, first_time_user","2, 3, 4","days, months, years","before, before, after",edd,Test_Set,en
+Test Set PT,"gender:male, relationship:in_a_relationship",first_time_user,2,days,before,edd,test_set,pt

home/tests/import-export-data/ordered_content_incorrect_locale.csv

@@ -0,0 +1,3 @@
+Name,Profile Fields,Page Slugs,Time,Unit,Before Or After,Contact Field,Slug,Locale
+Test Set,"gender:male, relationship:in_a_relationship","first_time_user, first_time_user, first_time_user","2, 3, 4","days, months, years","before, before, after",edd,Test_Set,en
+Test Set PT,"gender:male, relationship:in_a_relationship",first_time_user,2,days,before,edd,test_set,pt

home/tests/import-export-data/ordered_content_no_locale.csv

@@ -0,0 +1,3 @@
+Name,Profile Fields,Page Slugs,Time,Unit,Before Or After,Contact Field,Slug,Locale
+Test Set,"gender:male, relationship:in_a_relationship","first_time_user, first_time_user, first_time_user","2, 3, 4","days, months, years","before, before, after",edd,Test_Set,en
+Test Set PT,"gender:male, relationship:in_a_relationship",first_time_user,2,days,before,edd,test_set,

home/tests/test_content_import_export.py

@@ -1230,19 +1230,53 @@ def test_import_ordered_content_sets_no_page_error(
             content = csv_impexp.read_bytes("ordered_content.csv")
             csv_impexp.import_ordered_sets(content)
 
-        assert e.value.message == ["Content page not found for slug 'first_time_user'"]
+        assert e.value.message == [
+            "Content page not found for slug 'first_time_user' in locale 'English'"
+        ]
+
+    def test_import_ordered_content_sets_no_locale_error(
+        self, csv_impexp: ImportExport
+    ) -> None:
+        """
+        Importing CSV for ordered content sets without a locale should throw an error.
+        """
+        csv_impexp.import_file("contentpage_required_fields.csv")
+
+        with pytest.raises(ImportException) as e:
+            content = csv_impexp.read_bytes("ordered_content_no_locale.csv")
+            csv_impexp.import_ordered_sets(content)
+
+        assert e.value.message == ["No locale specified."]
+
+    def test_import_ordered_content_sets_incorrect_locale_error(
+        self, csv_impexp: ImportExport
+    ) -> None:
+        """
+        Importing CSV for ordered content sets with an incorrect locale should throw an error.
+        """
+        csv_impexp.import_file("contentpage_required_fields.csv")
+
+        with pytest.raises(ImportException) as e:
+            content = csv_impexp.read_bytes("ordered_content_incorrect_locale.csv")
+            csv_impexp.import_ordered_sets(content)
+
+        assert e.value.message == ["Locale pt does not exist."]
 
     def test_import_ordered_sets_csv(self, csv_impexp: ImportExport) -> None:
         """
         Importing a CSV file with ordered content sets should not break
         """
-        csv_impexp.import_file("contentpage_required_fields.csv")
+        pt, _created = Locale.objects.get_or_create(language_code="pt")
+        HomePage.add_root(locale=pt, title="Home (pt)", slug="home-pt")
+
+        csv_impexp.import_file("contentpage_required_fields_multi_locale.csv")
         content = csv_impexp.read_bytes("ordered_content.csv")
         csv_impexp.import_ordered_sets(content)
 
-        locale = Locale.objects.get(language_code="en")
+        en = Locale.objects.get(language_code="en")
+
         ordered_set = OrderedContentSet.objects.filter(
-            name="Test Set", slug="test_set", locale=locale
+            slug="test_set", locale=en
         ).first()
 
         assert ordered_set.name == "Test Set"
@@ -1274,6 +1308,25 @@ def test_import_ordered_sets_csv(self, csv_impexp: ImportExport) -> None:
             ("relationship", "in_a_relationship"),
         ]
 
+        ordered_set_pt = OrderedContentSet.objects.filter(
+            slug="test_set", locale=pt
+        ).first()
+
+        pages = unwagtail(ordered_set_pt.pages)
+        assert len(pages) == 1
+
+        page = pages[0][1]
+        assert page["contentpage"].slug == "first_time_user"
+        assert page["time"] == "2"
+        assert page["unit"] == "days"
+        assert page["before_or_after"] == "before"
+        assert page["contact_field"] == "edd"
+
+        assert unwagtail(ordered_set_pt.profile_fields) == [
+            ("gender", "male"),
+            ("relationship", "in_a_relationship"),
+        ]
+
     def test_import_ordered_sets_no_profile_fields_csv(
         self, csv_impexp: ImportExport
     ) -> None:
@@ -1359,6 +1412,17 @@ def test_import_pages_xlsx(self, xlsx_impexp: ImportExport) -> None:
         content_pages = ContentPage.objects.all()
         assert len(content_pages) > 0
 
+    def test_import_pages_number_type(self, xlsx_impexp: ImportExport) -> None:
+        """
+        Importing an XLSX file where number fields have a number cell formatting
+        shouldn't break
+        """
+        home_page = HomePage.objects.first()
+        PageBuilder.build_cpi(home_page, "main-menu", "main menu first time user")
+        xlsx_impexp.import_file("contentpage_number_type.xlsx", purge=False)
+        content_pages = ContentPage.objects.all()
+        assert len(content_pages) > 0
+
     def test_invalid_page(self, csv_impexp: ImportExport) -> None:
         """
         Import an invalid page that matches a valid page already in the db