Skip to content

Commit

Permalink
clean source dags files
Browse files Browse the repository at this point in the history
  • Loading branch information
@kolok
kolok committed Jan 16, 2025
1 parent b4cbe5f commit 5017db0
Show file tree
Hide file tree
Showing 15 changed files with 0 additions and 252 deletions.
15 changes: 0 additions & 15 deletions dags/sources/dags/source_aliapur.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand Down
19 changes: 0 additions & 19 deletions dags/sources/dags/source_citeo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,6 @@
"origin": "nom_de_lorganisme",
"destination": "nom",
},
# {
# "origin": "enseigne_commerciale",
# "destination": "nom_commercial",
# },
{
"origin": "longitudewgs84",
"destination": "longitude",
Expand All @@ -46,21 +42,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand Down
1 change: 0 additions & 1 deletion dags/sources/dags/source_cma.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,6 @@
"ignore_duplicates": False,
"validate_address_with_ban": False,
"product_mapping": get_mapping_config(mapping_key="sous_categories_cma"),
"source_code": "cma_reparacteur",
},
schedule=None,
) as dag:
Expand Down
29 changes: 0 additions & 29 deletions dags/sources/dags/source_corepile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand All @@ -81,11 +66,6 @@
"transformation": "compute_location",
"destination": ["location"],
},
# {
# "origin": ["labels_etou_bonus", "acteur_type_code"],
# "transformation": "clean_label_codes",
# "destination": ["label_codes"],
# },
{
"origin": ["id_point_apport_ou_reparation", "nom"],
"transformation": "clean_identifiant_externe",
Expand Down Expand Up @@ -145,15 +125,6 @@
{"remove": "point_de_reparation"},
# 6. Colonnes à garder (rien à faire, utilisé pour le controle)
],
"column_mapping": {
"id_point_apport_ou_reparation": "identifiant_externe",
"type_de_point_de_collecte": "acteur_type_id",
"ecoorganisme": "source_id",
"nom_de_lorganisme": "nom",
"enseigne_commerciale": "nom_commercial",
"longitudewgs84": "longitude",
"latitudewgs84": "latitude",
},
"endpoint": (
"https://data.pointsapport.ademe.fr/data-fair/api/v1/datasets/"
"donnees-eo-corepile/lines?size=10000"
Expand Down
10 changes: 0 additions & 10 deletions dags/sources/dags/source_ecodds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
{
"origin": "exclusivite_de_reprisereparation",
"transformation": "cast_eo_boolean_or_string_to_boolean",
Expand Down Expand Up @@ -95,11 +90,6 @@
"transformation": "clean_identifiant_unique",
"destination": ["identifiant_unique"],
},
# {
# "origin": ["siret"],
# "transformation": "clean_siret_and_siren",
# "destination": ["siret", "siren"],
# },
{
"origin": ["adresse_format_ban"],
"transformation": "clean_adresse",
Expand Down
25 changes: 0 additions & 25 deletions dags/sources/dags/source_ecologic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,6 @@
"origin": "nom_de_lorganisme",
"destination": "nom",
},
# {
# "origin": "enseigne_commerciale",
# "destination": "nom_commercial",
# },
{
"origin": "longitudewgs84",
"destination": "longitude",
Expand All @@ -46,16 +42,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
{
"origin": "reprise",
"transformation": "clean_reprise",
Expand All @@ -81,11 +67,6 @@
"transformation": "compute_location",
"destination": ["location"],
},
# {
# "origin": ["labels_etou_bonus", "acteur_type_code"],
# "transformation": "clean_label_codes",
# "destination": ["label_codes"],
# },
{
"origin": ["id_point_apport_ou_reparation", "nom"],
"transformation": "clean_identifiant_externe",
Expand All @@ -99,11 +80,6 @@
"transformation": "clean_identifiant_unique",
"destination": ["identifiant_unique"],
},
# {
# "origin": ["siret"],
# "transformation": "clean_siret_and_siren",
# "destination": ["siret", "siren"],
# },
{
"origin": ["adresse_format_ban"],
"transformation": "clean_adresse",
Expand Down Expand Up @@ -140,7 +116,6 @@
{"remove": "adresse_format_ban"},
{"remove": "id_point_apport_ou_reparation"},
{"remove": "point_de_collecte_ou_de_reprise_des_dechets"},
# {"remove": "labels_etou_bonus"},
{"remove": "point_dapport_de_service_reparation"},
{"remove": "point_dapport_pour_reemploi"},
{"remove": "point_de_reparation"},
Expand Down
15 changes: 0 additions & 15 deletions dags/sources/dags/source_ecomaison.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,16 +56,6 @@
"transformation": "cast_eo_boolean_or_string_to_boolean",
"destination": "uniquement_sur_rdv",
},
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand All @@ -86,11 +76,6 @@
"transformation": "compute_location",
"destination": ["location"],
},
# {
# "origin": ["labels_etou_bonus", "acteur_type_code"],
# "transformation": "clean_label_codes",
# "destination": ["label_codes"],
# },
{
"origin": ["id_point_apport_ou_reparation", "nom"],
"transformation": "clean_identifiant_externe",
Expand Down
27 changes: 0 additions & 27 deletions dags/sources/dags/source_ecosystem.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand All @@ -81,11 +66,6 @@
"transformation": "compute_location",
"destination": ["location"],
},
# {
# "origin": ["labels_etou_bonus", "acteur_type_code"],
# "transformation": "clean_label_codes",
# "destination": ["label_codes"],
# },
{
"origin": ["id_point_apport_ou_reparation", "nom"],
"transformation": "clean_identifiant_externe",
Expand All @@ -111,8 +91,6 @@
},
{
"origin": [
# "point_dapport_de_service_reparation",
# "point_de_reparation",
"point_dapport_pour_reemploi",
"point_de_collecte_ou_de_reprise_des_dechets",
],
Expand All @@ -121,8 +99,6 @@
},
{
"origin": [
# "point_dapport_de_service_reparation",
# "point_de_reparation",
"point_dapport_pour_reemploi",
"point_de_collecte_ou_de_reprise_des_dechets",
],
Expand All @@ -140,10 +116,7 @@
{"remove": "adresse_format_ban"},
{"remove": "id_point_apport_ou_reparation"},
{"remove": "point_de_collecte_ou_de_reprise_des_dechets"},
# {"remove": "labels_etou_bonus"},
# {"remove": "point_dapport_de_service_reparation"},
{"remove": "point_dapport_pour_reemploi"},
# {"remove": "point_de_reparation"},
{"remove": "siret"},
# 6. Colonnes à garder (rien à faire, utilisé pour le controle)
{"keep": "adresse_complement"},
Expand Down
23 changes: 0 additions & 23 deletions dags/sources/dags/source_ocab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,6 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
# {
# "origin": "exclusivite_de_reprisereparation",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "exclusivite_de_reprisereparation",
# },
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand All @@ -81,11 +66,6 @@
"transformation": "compute_location",
"destination": ["location"],
},
# {
# "origin": ["labels_etou_bonus", "acteur_type_code"],
# "transformation": "clean_label_codes",
# "destination": ["label_codes"],
# },
{
"origin": ["id_point_apport_ou_reparation", "nom"],
"transformation": "clean_identifiant_externe",
Expand Down Expand Up @@ -140,10 +120,7 @@
{"remove": "adresse_format_ban"},
{"remove": "id_point_apport_ou_reparation"},
{"remove": "point_de_collecte_ou_de_reprise_des_dechets"},
# {"remove": "labels_etou_bonus"},
# {"remove": "point_dapport_de_service_reparation"},
{"remove": "point_dapport_pour_reemploi"},
# {"remove": "point_de_reparation"},
# 6. Colonnes à garder (rien à faire, utilisé pour le controle)
],
"endpoint": (
Expand Down
20 changes: 0 additions & 20 deletions dags/sources/dags/source_ocad3e.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,21 +46,11 @@
"transformation": "clean_public_accueilli",
"destination": "public_accueilli",
},
# {
# "origin": "uniquement_sur_rdv",
# "transformation": "cast_eo_boolean_or_string_to_boolean",
# "destination": "uniquement_sur_rdv",
# },
{
"origin": "exclusivite_de_reprisereparation",
"transformation": "cast_eo_boolean_or_string_to_boolean",
"destination": "exclusivite_de_reprisereparation",
},
# {
# "origin": "reprise",
# "transformation": "clean_reprise",
# "destination": "reprise",
# },
{
"origin": "produitsdechets_acceptes",
"transformation": "clean_souscategorie_codes",
Expand All @@ -75,10 +65,6 @@
"column": "statut",
"value": constants.ACTEUR_ACTIF,
},
# {
# "column": "label_codes",
# "value": [],
# },
# 4. Transformation du dataframe
{
"origin": ["latitude", "longitude"],
Expand Down Expand Up @@ -122,8 +108,6 @@
"origin": [
"point_dapport_de_service_reparation",
"point_de_reparation",
# "point_dapport_pour_reemploi",
# "point_de_collecte_ou_de_reprise_des_dechets",
],
"transformation": "clean_acteurservice_codes",
"destination": ["acteurservice_codes"],
Expand All @@ -132,8 +116,6 @@
"origin": [
"point_dapport_de_service_reparation",
"point_de_reparation",
# "point_dapport_pour_reemploi",
# "point_de_collecte_ou_de_reprise_des_dechets",
],
"transformation": "clean_action_codes",
"destination": ["action_codes"],
Expand All @@ -148,10 +130,8 @@
{"remove": "_score"},
{"remove": "adresse_format_ban"},
{"remove": "id_point_apport_ou_reparation"},
# {"remove": "point_de_collecte_ou_de_reprise_des_dechets"},
{"remove": "labels_etou_bonus"},
{"remove": "point_dapport_de_service_reparation"},
# {"remove": "point_dapport_pour_reemploi"},
{"remove": "point_de_reparation"},
{"remove": "perimetre_dintervention"},
{"remove": "ecoorganisme"},
Expand Down
Loading

0 comments on commit 5017db0

Please sign in to comment.