déplacement de fichiers et suppression de champs inutilisés

dags/sources/tasks/business_logic/db_write_suggestion.py

@@ -26,26 +26,26 @@ def db_write_suggestion(
         metadata=metadata,
         dag_name=f"{dag_name} - AJOUT",
         run_name=run_name,
-        action_type=constants.SUGGESTION_SOURCE_AJOUT,
+        type_action=constants.SUGGESTION_SOURCE_AJOUT,
     )
     insert_suggestion(
         df=df_acteur_to_delete,
         metadata=metadata,
         dag_name=f"{dag_name} - SUPRESSION",
         run_name=run_name,
-        action_type=constants.SUGGESTION_SOURCE_SUPRESSION,
+        type_action=constants.SUGGESTION_SOURCE_SUPRESSION,
     )
     insert_suggestion(
         df=df_acteur_to_update,
         metadata=metadata,
         dag_name=f"{dag_name} - MISES A JOUR",
         run_name=run_name,
-        action_type=constants.SUGGESTION_SOURCE_MISESAJOUR,
+        type_action=constants.SUGGESTION_SOURCE_MISESAJOUR,
     )
 
 
 def insert_suggestion(
-    df: pd.DataFrame, metadata: dict, dag_name: str, run_name: str, action_type: str
+    df: pd.DataFrame, metadata: dict, dag_name: str, run_name: str, type_action: str
 ):
     if df.empty:
         return
@@ -72,7 +72,7 @@ def insert_suggestion(
             (
                 dag_name,
                 run_name,
-                action_type,
+                type_action,
                 constants.SUGGESTION_AVALIDER,
                 json.dumps(metadata),
                 current_date,
@@ -82,10 +82,9 @@ def insert_suggestion(
         suggestion_cohorte_id = result.fetchone()[0]
 
     # Insert dag_run_change
-    df["type_action"] = action_type
     df["suggestion_cohorte_id"] = suggestion_cohorte_id
     df["statut"] = constants.SUGGESTION_AVALIDER
-    df[["suggestion", "suggestion_cohorte_id", "type_action", "statut"]].to_sql(
+    df[["suggestion", "suggestion_cohorte_id", "statut"]].to_sql(
         "data_suggestionunitaire",
         engine,
         if_exists="append",

dags/suggestions/dags/apply_suggestions.py

@@ -2,16 +2,10 @@
 
 from airflow.models import DAG
 from airflow.utils.dates import days_ago
-from suggestions.airflow_logic.db_normalize_suggestion_task import (
+from suggestions.tasks.airflow_logic import (
     db_normalize_suggestion_task,
-)
-from suggestions.airflow_logic.db_read_suggestiontoprocess_task import (
     db_read_suggestiontoprocess_task,
-)
-from suggestions.airflow_logic.db_write_validsuggestions_task import (
     db_write_validsuggestions_task,
-)
-from suggestions.airflow_logic.launch_compute_carte_acteur_task import (
     launch_compute_carte_acteur_task,
 )
 

dags/suggestions/tasks/airflow_logic/__init__.py

@@ -0,0 +1,4 @@
+from suggestions.tasks.airflow_logic.db_normalize_suggestion_task import *  # noqa
+from suggestions.tasks.airflow_logic.db_read_suggestiontoprocess_task import *  # noqa
+from suggestions.tasks.airflow_logic.db_write_validsuggestions_task import *  # noqa
+from suggestions.tasks.airflow_logic.launch_compute_carte_acteur_task import *  # noqa

dags/suggestions/airflow_logic/db_normalize_suggestion_task.py → dags/suggestions/tasks/airflow_logic/db_normalize_suggestion_task.py

@@ -1,6 +1,8 @@
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
-from suggestions.business_logic.db_normalize_suggestion import db_normalize_suggestion
+from suggestions.tasks.business_logic.db_normalize_suggestion import (
+    db_normalize_suggestion,
+)
 
 
 def db_normalize_suggestion_task(dag: DAG):

dags/suggestions/airflow_logic/db_read_suggestiontoprocess_task.py → dags/suggestions/tasks/airflow_logic/db_read_suggestiontoprocess_task.py

@@ -1,6 +1,6 @@
 from airflow.models import DAG
 from airflow.operators.python import ShortCircuitOperator
-from suggestions.business_logic.db_read_suggestiontoprocess import (
+from suggestions.tasks.business_logic.db_read_suggestiontoprocess import (
     db_read_suggestiontoprocess,
 )
 

dags/suggestions/airflow_logic/db_write_validsuggestions_task.py → dags/suggestions/tasks/airflow_logic/db_write_validsuggestions_task.py

@@ -1,6 +1,6 @@
 from airflow.models import DAG
 from airflow.operators.python import PythonOperator
-from suggestions.business_logic.db_write_validsuggestions import (
+from suggestions.tasks.business_logic.db_write_validsuggestions import (
     db_write_validsuggestions,
 )
 from utils import logging_utils as log

dags/suggestions/business_logic/db_normalize_suggestion.py → dags/suggestions/tasks/business_logic/db_normalize_suggestion.py

@@ -1,7 +1,7 @@
 import pandas as pd
 from shared.tasks.database_logic.db_manager import PostgresConnectionManager
 from sources.config import shared_constants as constants
-from suggestions.business_logic.db_read_suggestiontoprocess import (
+from suggestions.tasks.business_logic.db_read_suggestiontoprocess import (
     get_first_suggetsioncohorte_to_insert,
 )
 from utils import logging_utils as log

data/migrations/0001_bancache.py

@@ -0,0 +1,55 @@
+# Generated by Django 5.1.4 on 2025-01-09 14:04
+
+import django.contrib.gis.db.models.fields
+import django.core.validators
+import django.db.models.deletion
+import django.db.models.functions.datetime
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="BANCache",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("adresse", models.CharField(blank=True, max_length=255, null=True)),
+                (
+                    "code_postal",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ("ville", models.CharField(blank=True, max_length=255, null=True)),
+                (
+                    "location",
+                    django.contrib.gis.db.models.fields.PointField(
+                        blank=True, null=True, srid=4326
+                    ),
+                ),
+                ("ban_returned", models.JSONField(blank=True, null=True)),
+                (
+                    "modifie_le",
+                    models.DateTimeField(
+                        auto_now=True,
+                        db_default=django.db.models.functions.datetime.Now(),
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Cache BAN",
+                "verbose_name_plural": "Cache BAN",
+            },
+        ),
+    ]

data/migrations/0001_initial.py → data/migrations/0002_tables_suggestion.py

@@ -11,47 +11,11 @@ class Migration(migrations.Migration):
 
     initial = True
 
-    dependencies = []
+    dependencies = [
+        ("data", "0001_bancache"),
+    ]
 
     operations = [
-        migrations.CreateModel(
-            name="BANCache",
-            fields=[
-                (
-                    "id",
-                    models.BigAutoField(
-                        auto_created=True,
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="ID",
-                    ),
-                ),
-                ("adresse", models.CharField(blank=True, max_length=255, null=True)),
-                (
-                    "code_postal",
-                    models.CharField(blank=True, max_length=255, null=True),
-                ),
-                ("ville", models.CharField(blank=True, max_length=255, null=True)),
-                (
-                    "location",
-                    django.contrib.gis.db.models.fields.PointField(
-                        blank=True, null=True, srid=4326
-                    ),
-                ),
-                ("ban_returned", models.JSONField(blank=True, null=True)),
-                (
-                    "modifie_le",
-                    models.DateTimeField(
-                        auto_now=True,
-                        db_default=django.db.models.functions.datetime.Now(),
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "Cache BAN",
-                "verbose_name_plural": "Cache BAN",
-            },
-        ),
         migrations.CreateModel(
             name="SuggestionCohorte",
             fields=[
@@ -115,18 +79,6 @@ class Migration(migrations.Migration):
                         null=True,
                     ),
                 ),
-                (
-                    "pourcentage_erreurs_tolerees",
-                    models.IntegerField(
-                        db_default=0,
-                        default=0,
-                        help_text="Nombre d'erreurs tolérées en pourcentage",
-                        validators=[
-                            django.core.validators.MinValueValidator(0),
-                            django.core.validators.MaxValueValidator(100),
-                        ],
-                    ),
-                ),
                 (
                     "cree_le",
                     models.DateTimeField(

data/models.py

@@ -1,5 +1,4 @@
 from django.contrib.gis.db import models
-from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db.models.functions import Now
 
 from dags.sources.config.shared_constants import (
@@ -72,12 +71,6 @@ class SuggestionCohorte(models.Model):
     metadata = models.JSONField(
         null=True, blank=True, help_text="Metadata de la cohorte, données statistiques"
     )
-    pourcentage_erreurs_tolerees = models.IntegerField(
-        default=0,
-        db_default=0,
-        help_text="Nombre d'erreurs tolérées en pourcentage",
-        validators=[MinValueValidator(0), MaxValueValidator(100)],
-    )
     cree_le = models.DateTimeField(auto_now_add=True, db_default=Now())
     modifie_le = models.DateTimeField(auto_now=True, db_default=Now())
 
@@ -99,30 +92,12 @@ def is_clustering_type(self) -> bool:
     def __str__(self) -> str:
         return f"{self.identifiant_action} - {self.identifiant_execution}"
 
-    def display_meta_data(self) -> dict:
-        displayed_metadata = {}
-        displayed_metadata["Nombre d'acteur à créer ou mettre à jour"] = (
-            self.metadata.get("acteurs_to_add_or_update", 0)
-        )
-        displayed_metadata["Nombre de duplicats"] = self.metadata.get(
-            "number_of_duplicates", 0
-        )
-        displayed_metadata["Nombre d'acteur à supprimer"] = self.metadata.get(
-            "number_of_removed_actors", 0
-        )
-        return displayed_metadata
-
 
 class SuggestionUnitaire(models.Model):
     id = models.AutoField(primary_key=True)
     suggestion_cohorte = models.ForeignKey(
         SuggestionCohorte, on_delete=models.CASCADE, related_name="suggestion_unitaires"
     )
-    type_action = models.CharField(
-        choices=SuggestionAction.choices,
-        max_length=250,
-        blank=True,
-    )
     statut = models.CharField(
         max_length=50,
         choices=SuggestionStatut.choices,