Mettre à jour et normaliser des codes de l'objet Source (#1187)

incubateur-ademe · Jan 13, 2025 · faa8e1a · faa8e1a
1 parent c688024
commit faa8e1a
Show file tree

Hide file tree

Showing 22 changed files with 319 additions and 60 deletions.
diff --git a/dags/sources/config/airflow_params.py b/dags/sources/config/airflow_params.py
@@ -14,6 +14,7 @@
     clean_souscategorie_codes_sinoe,
     clean_url,
     convert_opening_hours,
+    strip_lower_string,
     strip_string,
 )
 from sources.tasks.transform.transform_df import (
@@ -63,6 +64,7 @@
     "clean_url": clean_url,
     "clean_souscategorie_codes_sinoe": clean_souscategorie_codes_sinoe,
     "get_latlng_from_geopoint": get_latlng_from_geopoint,
+    "strip_lower_string": strip_lower_string,
 }
 
 

diff --git a/dags/sources/dags/source_aliapur.py b/dags/sources/dags/source_aliapur.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_citeo.py b/dags/sources/dags/source_citeo.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_cma.py b/dags/sources/dags/source_cma.py
@@ -91,7 +91,7 @@
             },
             {
                 "column": "source_code",
-                "value": "cma_reparacteur",
+                "value": "cmareparacteur",
             },
             # 4. Transformation du dataframe
             {

diff --git a/dags/sources/dags/source_corepile.py b/dags/sources/dags/source_corepile.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_ecodds.py b/dags/sources/dags/source_ecodds.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_ecologic.py b/dags/sources/dags/source_ecologic.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_ecomaison.py b/dags/sources/dags/source_ecomaison.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "site_web",
                 "transformation": "clean_url",

diff --git a/dags/sources/dags/source_ecosystem.py b/dags/sources/dags/source_ecosystem.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_ocab.py b/dags/sources/dags/source_ocab.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "type_de_point_de_collecte",
                 "transformation": "clean_acteur_type_code",

diff --git a/dags/sources/dags/source_ocad3e.py b/dags/sources/dags/source_ocad3e.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            # {
-            #     "origin": "ecoorganisme",
-            #     "destination": "source_code",
-            # },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",

diff --git a/dags/sources/dags/source_pyreo.py b/dags/sources/dags/source_pyreo.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             # {
             #     "origin": "site_web",
             #     "transformation": "clean_url",

diff --git a/dags/sources/dags/source_refashion.py b/dags/sources/dags/source_refashion.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -39,6 +35,11 @@
                 "destination": "horaires_description",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             {
                 "origin": "site_web",
                 "transformation": "clean_url",

diff --git a/dags/sources/dags/source_screlec.py b/dags/sources/dags/source_screlec.py
@@ -19,10 +19,6 @@
         ),
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -40,6 +36,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             # {
             #     "origin": "site_web",
             #     "transformation": "clean_url",

diff --git a/dags/sources/dags/source_sinoe.py b/dags/sources/dags/source_sinoe.py
@@ -76,7 +76,7 @@
             },
             {
                 "column": "source_code",
-                "value": "ADEME_SINOE_Decheteries",
+                "value": "ademesinoedecheteries",
             },
             # 4. Transformation du dataframe
             {

diff --git a/dags/sources/dags/source_soren.py b/dags/sources/dags/source_soren.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             # {
             #     "origin": "site_web",
             #     "transformation": "clean_url",

diff --git a/dags/sources/dags/source_valdelia.py b/dags/sources/dags/source_valdelia.py
@@ -14,10 +14,6 @@
     params={
         "normalization_rules": [
             # 1. Renommage des colonnes
-            {
-                "origin": "ecoorganisme",
-                "destination": "source_code",
-            },
             {
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
@@ -35,6 +31,11 @@
                 "destination": "latitude",
             },
             # 2. Transformation des colonnes
+            {
+                "origin": "ecoorganisme",
+                "transformation": "strip_lower_string",
+                "destination": "source_code",
+            },
             # {
             #     "origin": "site_web",
             #     "transformation": "clean_url",

diff --git a/dags/sources/tasks/transform/transform_column.py b/dags/sources/tasks/transform/transform_column.py
@@ -90,6 +90,10 @@ def strip_string(value: str | None, _) -> str:
     return str(value).strip() if not pd.isna(value) and value else ""
 
 
+def strip_lower_string(value: str | None, _) -> str:
+    return str(value).strip().lower() if not pd.isna(value) and value else ""
+
+
 def clean_acteur_type_code(value, _):
     mapping_dict = {
         # Here we store key without accents and special characters

diff --git a/dags_unit_tests/sources/tasks/transform/test_transform_column.py b/dags_unit_tests/sources/tasks/transform/test_transform_column.py
@@ -14,6 +14,7 @@
     clean_souscategorie_codes_sinoe,
     clean_url,
     convert_opening_hours,
+    strip_lower_string,
     strip_string,
 )
 
@@ -151,6 +152,25 @@ def test_strip_string(self, input, output):
         assert strip_string(input, None) == output
 
 
+class TestStriplowerString:
+
+    @pytest.mark.parametrize(
+        "input, output",
+        [
+            (None, ""),
+            (pd.NA, ""),
+            (np.nan, ""),
+            (" ", ""),
+            (75001, "75001"),
+            (" adresse postale ", "adresse postale"),
+            ("AdreSse posTale", "adresse postale"),
+            (" AdreSse posTale ", "adresse postale"),
+        ],
+    )
+    def test_strip_lower_string(self, input, output):
+        assert strip_lower_string(input, None) == output
+
+
 class TestCleanActeurTypeCode:
     @pytest.mark.parametrize(
         "value, expected_code",