Merge branch 'main' into contact-form-assistant

incubateur-ademe · Jan 21, 2025 · 46a2851 · 46a2851
2 parents 8684808 + b197b2f
commit 46a2851
Show file tree

Hide file tree

Showing 92 changed files with 1,971 additions and 2,998 deletions.
diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -124,7 +124,7 @@ jobs:
 
       - name: Populate database
         run: |
-          python manage.py loaddata actions acteur_services acteur_types categories objets
+          python manage.py loaddata actions acteur_services acteur_types categories objets produits synonymes
 
       - name: Run serveur
         run: |

diff --git a/Makefile b/Makefile
@@ -131,6 +131,10 @@ e2e-test:
 	npx playwright test
 	$(PYTEST) ./integration_tests
 
+.PHONY: a11y
+a11y:
+	npx playwright test --reporter=list ./e2e_tests/accessibility.spec.ts
+
 .PHONY: js-test
 js-test:
 	npm run test

diff --git a/core/models.py b/core/models.py
@@ -0,0 +1,10 @@
+from django.db import models
+from django.db.models.functions import Now
+
+
+class TimestampedModel(models.Model):
+    cree_le = models.DateTimeField(auto_now_add=True, db_default=Now())
+    modifie_le = models.DateTimeField(auto_now=True, db_default=Now())
+
+    class Meta:
+        abstract = True
diff --git a/core/settings.py b/core/settings.py
@@ -61,10 +61,10 @@
     "core",
     "qfdmd",
     "qfdmo",
+    "data",
     "corsheaders",
 ]
 
-# FIXME : check if we can manage django forms templating with jinja2
 FORM_RENDERER = "django.forms.renderers.TemplatesSetting"
 
 

diff --git a/core/urls.py b/core/urls.py
@@ -60,6 +60,7 @@ class PaginatedSitemap(GenericSitemap):
     path("dsfr/", include(("dsfr_hacks.urls", "dsfr_hacks"), namespace="dsfr_hacks")),
     path("", include(("qfdmo.urls", "qfdmo"), namespace="qfdmo")),
     path("", include(("qfdmd.urls", "qfdmd"), namespace="qfdmd")),
+    path("data/", include(("data.urls", "data"), namespace="data")),
     path("docs/", TemplateView.as_view(template_name="techdocs.html"), name="techdocs"),
 ]
 

diff --git a/core/views.py b/core/views.py
@@ -0,0 +1,8 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
+
+
+class IsStaffMixin(LoginRequiredMixin):
+    def dispatch(self, request, *args, **kwargs):
+        if not request.user.is_staff:
+            return self.handle_no_permission()
+        return super().dispatch(request, *args, **kwargs)
diff --git a/dags/ingest_validated_dataset_to_db.py b/dags/ingest_validated_dataset_to_db.py
@@ -1,3 +1,7 @@
+"""
+DEPRECATED : utiliser le dag apply_suggestions
+"""
+
 from datetime import timedelta
 
 import pandas as pd
@@ -20,9 +24,12 @@
 
 dag = DAG(
     dag_id="validate_and_process_dagruns",
-    dag_display_name="Traitement des cohortes de données validées",
+    dag_display_name="DEPRECATED : Traitement des cohortes de données validées",
     default_args=default_args,
-    description="Check for VALIDATE in qfdmo_dagrun and process qfdmo_dagrunchange",
+    description="""
+    DEPRECATED : Check for VALIDATE in qfdmo_dagrun and process qfdmo_dagrunchange
+    util uniquement pour les cohortes de siretisations
+    """,
     schedule="*/5 * * * *",
     catchup=False,
     max_active_runs=1,

diff --git a/dags/sources/config/airflow_params.py b/dags/sources/config/airflow_params.py
@@ -26,6 +26,7 @@
     clean_label_codes,
     clean_siret_and_siren,
     clean_telephone,
+    compute_location,
     get_latlng_from_geopoint,
     merge_and_clean_souscategorie_codes,
     merge_sous_categories_columns,
@@ -65,6 +66,7 @@
     "clean_souscategorie_codes_sinoe": clean_souscategorie_codes_sinoe,
     "get_latlng_from_geopoint": get_latlng_from_geopoint,
     "strip_lower_string": strip_lower_string,
+    "compute_location": compute_location,
 }
 
 

diff --git a/dags/sources/config/shared_constants.py b/dags/sources/config/shared_constants.py
@@ -1,9 +1,24 @@
-# DagRun statuts
+# DEPRECATED DagRun statuts
 DAGRUN_TOVALIDATE = "TO_VALIDATE"
 DAGRUN_TOINSERT = "TO_INSERT"
 DAGRUN_REJECTED = "REJECTED"
 DAGRUN_FINISHED = "FINISHED"
 
+# Suggestion statuts (pour cohorte et unitaire)
+SUGGESTION_AVALIDER = "AVALIDER"
+SUGGESTION_REJETEE = "REJETEE"
+SUGGESTION_PARTIEL = "PARTIEL"
+SUGGESTION_ATRAITER = "ATRAITER"
+SUGGESTION_ENCOURS = "ENCOURS"
+SUGGESTION_ERREUR = "ERREUR"
+SUGGESTION_SUCCES = "SUCCES"
+
+# SuggestionCohorte actions
+SUGGESTION_CLUSTERING = "CLUSTERING"
+SUGGESTION_SOURCE_AJOUT = "SOURCE_AJOUT"
+SUGGESTION_SOURCE_MODIFICATION = "SOURCE_MODIFICATION"
+SUGGESTION_SOURCE_SUPRESSION = "SOURCE_SUPRESSION"
+
 # Public accueilli
 PUBLIC_PAR = "Particuliers"
 PUBLIC_PRO = "Professionnels"

diff --git a/dags/sources/dags/source_aliapur.py b/dags/sources/dags/source_aliapur.py
@@ -46,21 +46,6 @@
                 "transformation": "clean_public_accueilli",
                 "destination": "public_accueilli",
             },
-            # {
-            #     "origin": "uniquement_sur_rdv",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "uniquement_sur_rdv",
-            # },
-            # {
-            #     "origin": "exclusivite_de_reprisereparation",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "exclusivite_de_reprisereparation",
-            # },
-            # {
-            #     "origin": "reprise",
-            #     "transformation": "clean_reprise",
-            #     "destination": "reprise",
-            # },
             {
                 "origin": "produitsdechets_acceptes",
                 "transformation": "clean_souscategorie_codes",
@@ -72,6 +57,11 @@
                 "value": constants.ACTEUR_ACTIF,
             },
             # 4. Transformation du dataframe
+            {
+                "origin": ["latitude", "longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["labels_etou_bonus", "acteur_type_code"],
                 "transformation": "clean_label_codes",

diff --git a/dags/sources/dags/source_citeo.py b/dags/sources/dags/source_citeo.py
@@ -18,10 +18,6 @@
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
             },
-            # {
-            #     "origin": "enseigne_commerciale",
-            #     "destination": "nom_commercial",
-            # },
             {
                 "origin": "longitudewgs84",
                 "destination": "longitude",
@@ -46,21 +42,6 @@
                 "transformation": "clean_public_accueilli",
                 "destination": "public_accueilli",
             },
-            # {
-            #     "origin": "uniquement_sur_rdv",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "uniquement_sur_rdv",
-            # },
-            # {
-            #     "origin": "exclusivite_de_reprisereparation",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "exclusivite_de_reprisereparation",
-            # },
-            # {
-            #     "origin": "reprise",
-            #     "transformation": "clean_reprise",
-            #     "destination": "reprise",
-            # },
             {
                 "origin": "produitsdechets_acceptes",
                 "transformation": "clean_souscategorie_codes",
@@ -76,6 +57,11 @@
                 "value": [],
             },
             # 4. Transformation du dataframe
+            {
+                "origin": ["latitude", "longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["id_point_apport_ou_reparation", "nom"],
                 "transformation": "clean_identifiant_externe",

diff --git a/dags/sources/dags/source_cma.py b/dags/sources/dags/source_cma.py
@@ -94,6 +94,11 @@
                 "value": "cmareparacteur",
             },
             # 4. Transformation du dataframe
+            {
+                "origin": ["final_latitude", "final_longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["telephone", "code_postal"],
                 "transformation": "clean_telephone",
@@ -160,7 +165,6 @@
         "ignore_duplicates": False,
         "validate_address_with_ban": False,
         "product_mapping": get_mapping_config(mapping_key="sous_categories_cma"),
-        "source_code": "cma_reparacteur",
     },
     schedule=None,
 ) as dag:

diff --git a/dags/sources/dags/source_corepile.py b/dags/sources/dags/source_corepile.py
@@ -46,21 +46,6 @@
                 "transformation": "clean_public_accueilli",
                 "destination": "public_accueilli",
             },
-            # {
-            #     "origin": "uniquement_sur_rdv",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "uniquement_sur_rdv",
-            # },
-            # {
-            #     "origin": "exclusivite_de_reprisereparation",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "exclusivite_de_reprisereparation",
-            # },
-            # {
-            #     "origin": "reprise",
-            #     "transformation": "clean_reprise",
-            #     "destination": "reprise",
-            # },
             {
                 "origin": "produitsdechets_acceptes",
                 "transformation": "clean_souscategorie_codes",
@@ -76,11 +61,11 @@
                 "value": [],
             },
             # 4. Transformation du dataframe
-            # {
-            #     "origin": ["labels_etou_bonus", "acteur_type_code"],
-            #     "transformation": "clean_label_codes",
-            #     "destination": ["label_codes"],
-            # },
+            {
+                "origin": ["latitude", "longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["id_point_apport_ou_reparation", "nom"],
                 "transformation": "clean_identifiant_externe",
@@ -140,15 +125,6 @@
             {"remove": "point_de_reparation"},
             # 6. Colonnes à garder (rien à faire, utilisé pour le controle)
         ],
-        "column_mapping": {
-            "id_point_apport_ou_reparation": "identifiant_externe",
-            "type_de_point_de_collecte": "acteur_type_id",
-            "ecoorganisme": "source_id",
-            "nom_de_lorganisme": "nom",
-            "enseigne_commerciale": "nom_commercial",
-            "longitudewgs84": "longitude",
-            "latitudewgs84": "latitude",
-        },
         "endpoint": (
             "https://data.pointsapport.ademe.fr/data-fair/api/v1/datasets/"
             "donnees-eo-corepile/lines?size=10000"

diff --git a/dags/sources/dags/source_ecodds.py b/dags/sources/dags/source_ecodds.py
@@ -46,11 +46,6 @@
                 "transformation": "clean_public_accueilli",
                 "destination": "public_accueilli",
             },
-            # {
-            #     "origin": "uniquement_sur_rdv",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "uniquement_sur_rdv",
-            # },
             {
                 "origin": "exclusivite_de_reprisereparation",
                 "transformation": "cast_eo_boolean_or_string_to_boolean",
@@ -72,6 +67,11 @@
                 "value": constants.ACTEUR_ACTIF,
             },
             # 4. Transformation du dataframe
+            {
+                "origin": ["latitude", "longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["labels_etou_bonus", "acteur_type_code"],
                 "transformation": "clean_label_codes",
@@ -90,11 +90,6 @@
                 "transformation": "clean_identifiant_unique",
                 "destination": ["identifiant_unique"],
             },
-            # {
-            #     "origin": ["siret"],
-            #     "transformation": "clean_siret_and_siren",
-            #     "destination": ["siret", "siren"],
-            # },
             {
                 "origin": ["adresse_format_ban"],
                 "transformation": "clean_adresse",

diff --git a/dags/sources/dags/source_ecologic.py b/dags/sources/dags/source_ecologic.py
@@ -18,10 +18,6 @@
                 "origin": "nom_de_lorganisme",
                 "destination": "nom",
             },
-            # {
-            #     "origin": "enseigne_commerciale",
-            #     "destination": "nom_commercial",
-            # },
             {
                 "origin": "longitudewgs84",
                 "destination": "longitude",
@@ -46,16 +42,6 @@
                 "transformation": "clean_public_accueilli",
                 "destination": "public_accueilli",
             },
-            # {
-            #     "origin": "uniquement_sur_rdv",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "uniquement_sur_rdv",
-            # },
-            # {
-            #     "origin": "exclusivite_de_reprisereparation",
-            #     "transformation": "cast_eo_boolean_or_string_to_boolean",
-            #     "destination": "exclusivite_de_reprisereparation",
-            # },
             {
                 "origin": "reprise",
                 "transformation": "clean_reprise",
@@ -76,11 +62,11 @@
                 "value": [],
             },
             # 4. Transformation du dataframe
-            # {
-            #     "origin": ["labels_etou_bonus", "acteur_type_code"],
-            #     "transformation": "clean_label_codes",
-            #     "destination": ["label_codes"],
-            # },
+            {
+                "origin": ["latitude", "longitude"],
+                "transformation": "compute_location",
+                "destination": ["location"],
+            },
             {
                 "origin": ["id_point_apport_ou_reparation", "nom"],
                 "transformation": "clean_identifiant_externe",
@@ -94,11 +80,6 @@
                 "transformation": "clean_identifiant_unique",
                 "destination": ["identifiant_unique"],
             },
-            # {
-            #     "origin": ["siret"],
-            #     "transformation": "clean_siret_and_siren",
-            #     "destination": ["siret", "siren"],
-            # },
             {
                 "origin": ["adresse_format_ban"],
                 "transformation": "clean_adresse",
@@ -135,7 +116,6 @@
             {"remove": "adresse_format_ban"},
             {"remove": "id_point_apport_ou_reparation"},
             {"remove": "point_de_collecte_ou_de_reprise_des_dechets"},
-            # {"remove": "labels_etou_bonus"},
             {"remove": "point_dapport_de_service_reparation"},
             {"remove": "point_dapport_pour_reemploi"},
             {"remove": "point_de_reparation"},
@@ -145,9 +125,6 @@
             "https://data.pointsapport.ademe.fr/data-fair/api/v1/datasets/"
             "donnees-eo-ecologic/lines?size=10000"
         ),
-        "columns_to_add_by_default": {
-            "statut": constants.ACTEUR_ACTIF,
-        },
         "ignore_duplicates": False,
         "validate_address_with_ban": False,
         "merge_duplicated_acteurs": True,  # In case of multi ecoorganisme or filiere