Add language column in import, remove locale in export

praekeltfoundation · Jan 30, 2025 · 58ff445 · 58ff445
1 parent 0025840
commit 58ff445
Show file tree

Hide file tree

Showing 18 changed files with 96 additions and 159 deletions.
diff --git a/home/export_content_pages.py b/home/export_content_pages.py
@@ -15,17 +15,9 @@
 from wagtail.models import Locale, Page  # type: ignore
 from wagtail.query import PageQuerySet  # type: ignore
 
-from .models import (
-    ContentPage,
-    ContentPageIndex,
-    HomePage,
-    MessengerBlock,
-    SMSBlock,
-    USSDBlock,
-    VariationBlock,
-    ViberBlock,
-    WhatsappBlock,
-)
+from .models import (ContentPage, ContentPageIndex, HomePage, MessengerBlock,
+                     SMSBlock, USSDBlock, VariationBlock, ViberBlock,
+                     WhatsappBlock)
 from .xlsx_helpers import get_active_sheet
 
 HP_CTYPE = HomePage._meta.verbose_name
@@ -73,7 +65,6 @@ class ExportRow:
     tags: str = ""
     quick_replies: str = ""
     triggers: str = ""
-    locale: str = ""
     next_prompt: str = ""
     buttons: str = ""
     image_link: str = ""
@@ -205,16 +196,9 @@ def _export_content_page(self, page: ContentPage, structure: str) -> None:
         Export a ContentPage.
 
         FIXME:
-         * The locale should be a language code rather than a language name to
-           make importing less messy.
          * We should use the parent slug (which is expected to be unique per
            locale (probably?)) instead of the parent title.
         """
-        export_lang_code = ""
-        for lang_code, lang_dn in get_content_languages().items():
-            if lang_dn == str(page.locale):
-                export_lang_code = lang_code
-
         row = ExportRow(
             structure=structure,
             message=1,
@@ -235,9 +219,8 @@ def _export_content_page(self, page: ContentPage, structure: str) -> None:
             tags=self._comma_sep_qs(page.tags.all()),
             quick_replies=self._comma_sep_qs(page.quick_replies.all()),
             triggers=self._comma_sep_qs(page.triggers.all()),
-            locale=str(page.locale),
             related_pages=self._comma_sep_qs(self._related_pages(page)),
-            language_code=export_lang_code,
+            language_code=page.locale.language_code,
         )
         self.rows.append(row)
         message_bodies = list(
@@ -268,8 +251,6 @@ def _export_cpi(self, page: ContentPageIndex, structure: str) -> None:
         Export a ContentPageIndex.
 
         FIXME:
-         * The locale should be a language code rather than a language name to
-           make importing less messy.
          * We should use the parent slug (which is expected to be unique per
            locale (probably?)) instead of the parent title.
         """
@@ -280,7 +261,7 @@ def _export_cpi(self, page: ContentPageIndex, structure: str) -> None:
             parent=self._parent_title(page),
             web_title=page.title,
             translation_tag=str(page.translation_key),
-            locale=str(page.locale),
+            language_code=page.locale.language_code,
         )
         self.rows.append(row)
 
@@ -361,7 +342,6 @@ def _set_xlsx_styles(wb: Workbook, sheet: Worksheet) -> None:
         "tags": 118,
         "quick_replies": 118,
         "triggers": 118,
-        "locale": 118,
         "next_prompt": 118,
         "buttons": 118,
         "image_link": 118,

diff --git a/home/import_content_pages.py b/home/import_content_pages.py
@@ -8,32 +8,23 @@
 from typing import Any
 from uuid import uuid4
 
-from django.core.exceptions import ObjectDoesNotExist, ValidationError  # type: ignore
+from django.core.exceptions import (ObjectDoesNotExist,  # type: ignore
+                                    ValidationError)
 from taggit.models import Tag  # type: ignore
 from treebeard.exceptions import NodeAlreadySaved  # type: ignore
-from wagtail.blocks import (  # type: ignore
-    StructValue,  # type: ignore
-)
+from wagtail.blocks import StructValue  # type: ignore; type: ignore
 from wagtail.coreutils import get_content_languages  # type: ignore
 from wagtail.models import Locale, Page  # type: ignore
 from wagtail.models.sites import Site  # type: ignore
 from wagtail.rich_text import RichText  # type: ignore
 
-from home.import_helpers import ImportException, parse_file, validate_using_form
-
-from .models import (
-    Assessment,
-    ContentPage,
-    ContentPageIndex,
-    ContentQuickReply,
-    ContentTrigger,
-    HomePage,
-    MessengerBlock,
-    SMSBlock,
-    USSDBlock,
-    ViberBlock,
-    WhatsappBlock,
-)
+from home.import_helpers import (ImportException, parse_file,
+                                 validate_using_form)
+
+from .models import (Assessment, ContentPage, ContentPageIndex,
+                     ContentQuickReply, ContentTrigger, HomePage,
+                     MessengerBlock, SMSBlock, USSDBlock, ViberBlock,
+                     WhatsappBlock)
 
 PageId = tuple[str, Locale]
 
@@ -99,14 +90,14 @@ def process_rows(self, rows: list["ContentRow"]) -> None:
         for i, row in enumerate(rows, start=2):
             try:
                 if row.is_page_index:
-                    if self.locale and row.locale != self.locale.get_display_name():
+                    prev_locale = self._get_locale_from_row(row)
+                    if self.locale and self.locale != prev_locale:
                         # This page index isn't for the locale we're importing, so skip it.
                         continue
                     self.create_content_page_index_from_row(row)
-                    prev_locale = self.locale_from_display_name(row.locale)
                 elif row.is_content_page:
                     self.create_shadow_content_page_from_row(row, i)
-                    prev_locale = self.locale_from_display_name(row.locale)
+                    prev_locale = self._get_locale_from_row(row)
                 elif row.is_variation_message:
                     self.add_variation_to_shadow_content_page_from_row(row, prev_locale)
                 else:
@@ -117,6 +108,15 @@ def process_rows(self, rows: list["ContentRow"]) -> None:
                 e.locale = row.locale
                 raise e
 
+    def _get_locale_from_row(self, row: "ContentRow") -> Locale:
+        if row.language_code:
+            try:
+                return Locale.objects.get(language_code=row.language_code)
+            except Locale.DoesNotExist:
+                raise ImportException(f"Language not found: {row.language_code}")
+        else:
+            return self.locale_from_display_name(row.locale)
+
     def save_pages(self) -> None:
         for i, page in enumerate(self.shadow_pages.values()):
             if self.locale and page.locale != self.locale:
@@ -242,7 +242,7 @@ def default_locale(self) -> Locale:
         return site.root_page.locale
 
     def create_content_page_index_from_row(self, row: "ContentRow") -> None:
-        locale = self.locale_from_display_name(row.locale)
+        locale = self._get_locale_from_row(row)
         try:
             index = ContentPageIndex.objects.get(slug=row.slug, locale=locale)
         except ContentPageIndex.DoesNotExist:
@@ -252,7 +252,7 @@ def create_content_page_index_from_row(self, row: "ContentRow") -> None:
         # but optional for the default locale.
         if row.translation_tag or locale != self.default_locale():
             index.translation_key = row.translation_tag
-        locale = self.locale_from_display_name(row.locale)
+        # locale = self.locale_from_display_name(row.locale)
         try:
             with contextlib.suppress(NodeAlreadySaved):
                 self.home_page(locale).add_child(instance=index)
@@ -268,10 +268,7 @@ def create_content_page_index_from_row(self, row: "ContentRow") -> None:
     def create_shadow_content_page_from_row(
         self, row: "ContentRow", row_num: int
     ) -> None:
-        if row.language_code:
-            locale = locale = Locale.objects.get(language_code=row.language_code)
-        else:
-            locale = self.locale_from_display_name(row.locale)
+        locale = self._get_locale_from_row(row)
         page = ShadowContentPage(
             row_num=row_num,
             slug=row.slug,

diff --git a/home/tests/import-export-data/content2.csv b/home/tests/import-export-data/content2.csv
@@ -1,5 +1,5 @@
-structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
-Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,English,,,,,,,,
+structure,message,page_id,Slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,whatsapp_template_category,example_values,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,footer,language_code
+Menu 1,0,4,main-menu,,Main Menu,,,,,,,,,,,,,,,,,,,,a0b85075-d01b-46bf-8997-8591e87ba171,,,,,,,,,,,en
 Sub 1.1,1,5,main-menu-first-time-user,Main Menu,main menu first time user,,,main menu first time user,"*Welcome to HealthAlert* 🌍
 
 This is a messaging service created by the _*World Health Organization*_ *(WHO)* that provides information on COVID-19 as well as emergency resources for disease outbreaks, natural, and man-made disasters.
@@ -12,12 +12,12 @@ This is a messaging service created by the World Health Organization (WHO) that
 
 You can return to this main menu at any time by replying 🏠
 
-Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,English,,[],,,,,,en
+Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,,[],,,,,,en
 Sub 1.1.1,1,6,health-info,main menu first time user,health info,,,health info,"*Health information* 🏥
 
 Get information and advice from WHO by tapping on a button below ⬇️",,UTILITY,[],,,,[],Health Info SMS Title,*Health Info SMS Body*,Health Info USSD Title,*Health Info USSD Body*,health info,"*Health information* 🏥
 
-Get information and advice from WHO by tapping on a button below ⬇️",,,c9d6309e-173f-4c1d-bbaf-440b1fd4415f,health_info,,,English,,[],,,,,,en
+Get information and advice from WHO by tapping on a button below ⬇️",,,c9d6309e-173f-4c1d-bbaf-440b1fd4415f,health_info,,,,[],,,,,,en
 Sub 1.1.2,1,7,self-help,main menu first time user,self-help,,,self-help,"*Self-help programs* 🌬️
 
 Reply with a number to take part in a *free* self-help program created by WHO.
@@ -32,4 +32,4 @@ Reply with a number to take part in a *free* self-help program created by WHO.
 1. Quit tobacco 🚭
 _Stop smoking with the help of a guided, 42-day program._
 2. Manage your stress 🧘🏽‍♀️
-_Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,English,,[],,,,,,en
+_Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,,[],,,,,,en
diff --git a/home/tests/import-export-data/contentpage_index_required_fields.csv b/home/tests/import-export-data/contentpage_index_required_fields.csv
@@ -1,2 +1,2 @@
-message,slug,parent,web_title,locale
-0,main-menu,,Main Menu,English
+message,slug,parent,web_title,language_code
+0,main-menu,,Main Menu,en
diff --git a/home/tests/import-export-data/contentpage_required_fields.csv b/home/tests/import-export-data/contentpage_required_fields.csv
@@ -1,4 +1,4 @@
-message,slug,parent,web_title,locale
-0,main_menu,,Main Menu,English
-1,first_time_user,Main Menu,main menu first time user,English
-1,health_info,main menu first time user,health info,English
+message,slug,parent,web_title,language_code
+0,main_menu,,Main Menu,en
+1,first_time_user,Main Menu,main menu first time user,en
+1,health_info,main menu first time user,health info,en
diff --git a/home/tests/import-export-data/exported_content_20230911-variations-linked-page.csv b/home/tests/import-export-data/exported_content_20230911-variations-linked-page.csv
@@ -1,12 +1,12 @@
-structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,locale,next_prompt,buttons,image_link,doc_link,media_link,related_pages,example_values,footer,language_code
-Menu 1,0,5,appointment-reminders,,Appointment reminders,,,,,,,,,,,,,,,,,,8cff04aa-cf08-4120-88b4-e2269b7d5d80,,,,English,,,,,,,,,
-Menu 2,0,6,stage-based-messages,,Stage-based messages,,,,,,,,,,,,,,,,,,5f2d9e63-f047-41ab-921a-c5ca7c04d643,,,,English,,,,,,,,,
-Menu 3,0,7,health-info-messages,,Health info messages,,,,,,,,,,,,,,,,,,3db069a4-7112-4e66-a9a2-f35c6c18055a,,,,English,,,,,,,,,
-Menu 4,0,17,whatsapp-template-testing,,whatsapp template testing,,,,,,,,,,,,,,,,,,5f7221f4-146a-48c2-b2e3-c8491aaead9d,,,,English,,,,,,,,,
-Menu 5,0,164,import-export,,Import Export,,,,,,,,,,,,,,,,,,497bdc1f-43fc-4925-80a1-e68cb942faa4,,,,English,,,,,,,,,
-Sub 5.1,1,165,cp-import-export,Import Export,CP-Import/export,,,WA import export data,Message 1 contains an image,,,,,[],,,,,,,,,8ac50daf-de21-4d05-b697-6d983b7ed3d5,"Tag2, Tag1",Quick reply1,"Trigger2, Trigger1",English,,[],/admin/images/usage/4/,,,ma_qa_temp,,,en
-,1,165,cp-import-export,,,,,,,,gender: male,Variation message one for Gender Male,,,,,,,,,,,,,,,,,,,,,,,,
-,2,165,cp-import-export,,,,,,Message2 has a document attached,,,,,[],,,,,,,,,,,,,,,[],,/admin/documents/usage/1/,,,,,
-,2,165,cp-import-export,,,,,,,,gender: empty,Variation message one for Gender Rather not say,,,,,,,,,,,,,,,,,,,,,,,,
-,3,165,cp-import-export,,,,,,Message 3 with no variation but has an audio clip,,,,,[],,,,,,,,,,,,,,,[],,,/admin/media/usage/1/,,,,
-Sub 5.2,1,166,ma_qa_temp,Import Export,MA QA Temp,,,,,,,,,,,,,,,,,,e9793b5f-f8c7-46c5-8a5e-bd9b8f00fee9,,,,English,,,,,,,,,en
+structure,message,page_id,slug,parent,web_title,web_subtitle,web_body,whatsapp_title,whatsapp_body,whatsapp_template_name,variation_title,variation_body,list_title,list_items,sms_title,sms_body,ussd_title,ussd_body,messenger_title,messenger_body,viber_title,viber_body,translation_tag,tags,quick_replies,triggers,next_prompt,buttons,image_link,doc_link,media_link,related_pages,example_values,footer,language_code
+Menu 1,0,5,appointment-reminders,,Appointment reminders,,,,,,,,,,,,,,,,,,8cff04aa-cf08-4120-88b4-e2269b7d5d80,,,,,,,,,,,,en
+Menu 2,0,6,stage-based-messages,,Stage-based messages,,,,,,,,,,,,,,,,,,5f2d9e63-f047-41ab-921a-c5ca7c04d643,,,,,,,,,,,,en
+Menu 3,0,7,health-info-messages,,Health info messages,,,,,,,,,,,,,,,,,,3db069a4-7112-4e66-a9a2-f35c6c18055a,,,,,,,,,,,,en
+Menu 4,0,17,whatsapp-template-testing,,whatsapp template testing,,,,,,,,,,,,,,,,,,5f7221f4-146a-48c2-b2e3-c8491aaead9d,,,,,,,,,,,,en
+Menu 5,0,164,import-export,,Import Export,,,,,,,,,,,,,,,,,,497bdc1f-43fc-4925-80a1-e68cb942faa4,,,,,,,,,,,,en
+Sub 5.1,1,165,cp-import-export,Import Export,CP-Import/export,,,WA import export data,Message 1 contains an image,,,,,[],,,,,,,,,8ac50daf-de21-4d05-b697-6d983b7ed3d5,"Tag2, Tag1",Quick reply1,"Trigger2, Trigger1",,[],/admin/images/usage/4/,,,ma_qa_temp,,,en
+,1,165,cp-import-export,,,,,,,,gender: male,Variation message one for Gender Male,,,,,,,,,,,,,,,,,,,,,,,
+,2,165,cp-import-export,,,,,,Message2 has a document attached,,,,,[],,,,,,,,,,,,,,[],,/admin/documents/usage/1/,,,,,
+,2,165,cp-import-export,,,,,,,,gender: empty,Variation message one for Gender Rather not say,,,,,,,,,,,,,,,,,,,,,,,
+,3,165,cp-import-export,,,,,,Message 3 with no variation but has an audio clip,,,,,[],,,,,,,,,,,,,,[],,,/admin/media/usage/1/,,,,
+Sub 5.2,1,166,ma_qa_temp,Import Export,MA QA Temp,,,,,,,,,,,,,,,,,,e9793b5f-f8c7-46c5-8a5e-bd9b8f00fee9,,,,,,,,,,,,en
diff --git a/home/tests/import-export-data/language_code_import.csv b/home/tests/import-export-data/language_code_import.csv
@@ -13,23 +13,3 @@ This is a messaging service created by the World Health Organization (WHO) that
 You can return to this main menu at any time by replying 🏠
 
 Choose what you'd like to know more about by tapping a button below ⬇️",,,5892bccd-8025-419d-9a8e-a6a37b755dbf,menu,"Self-help🌬️, Settings⚙️, Health Info🏥",Main menu🏠,English,,[],,,,,,
-Sub 1.1.1,1,6,health-info,main menu first time user,health info,,,health info,"*Health information* 🏥
-
-Get information and advice from WHO by tapping on a button below ⬇️",,UTILITY,[],,,,[],Health Info SMS Title,*Health Info SMS Body*,Health Info USSD Title,*Health Info USSD Body*,health info,"*Health information* 🏥
-
-Get information and advice from WHO by tapping on a button below ⬇️",,,c9d6309e-173f-4c1d-bbaf-440b1fd4415f,health_info,,,English,,[],,,,,,
-Sub 1.1.2,1,7,self-help,main menu first time user,self-help,,,self-help,"*Self-help programs* 🌬️
-
-Reply with a number to take part in a *free* self-help program created by WHO.
-
-1. Quit tobacco 🚭
-_Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
-_Learn how to cope with stress and improve your wellbeing._",,UTILITY,[],,,,[],,,,,self-help,"*Self-help programs* 🌬️
-
-Reply with a number to take part in a *free* self-help program created by WHO.
-
-1. Quit tobacco 🚭
-_Stop smoking with the help of a guided, 42-day program._
-2. Manage your stress 🧘🏽‍♀️
-_Learn how to cope with stress and improve your wellbeing._",,,3e5d77f7-4d34-430d-aad7-d9ca01f79732,self_help,,,English,,[],,,,,,