Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: factorisation d'un copié-collé #1640

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat: factorisation d'un copié-collé #1640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9c1288b
Factorize copy paste
Guilouf Jan 20, 2025
95c12ab
Fix typo
Guilouf Feb 7, 2025
eadcc62
Fix
Guilouf Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions lemarche/siaes/management/base_update_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
import argparse
from datetime import timedelta

import requests
from django.conf import settings
from django.db.models import Q
from django.utils import timezone

from lemarche.siaes.models import Siae
from lemarche.utils.apis import api_slack
from lemarche.utils.commands import BaseCommand


class UpdateAPICommand(BaseCommand):
"""
Base class for QPV and ZRR API fetch commands
"""

API_NAME: str = None
FIELDS_TO_BULK_UPDATE = []
CLIENT = None

def __init__(self, stdout=None, stderr=None, no_color=False):
super().__init__(stdout, stderr, no_color)
self.success_count = {"etablissement": 0, "etablissement_target": 0}

def add_arguments(self, parser):
parser.add_argument(
"-L", "--limit", type=int, default=None, help="Limiter le nombre de structures à processer"
)
parser.add_argument(
"-F",
"--force",
action=argparse.BooleanOptionalAction,
default=None,
help="Forcer l'update sans la vérification de la dernière mise à jour",
)

@staticmethod
def get_filter_query(date_limit) -> Q:
"""Qobjects to filter siaes_queryset"""
raise NotImplementedError

@staticmethod
def is_in_target(latitude, longitude, client):
raise NotImplementedError

def get_query_set(self, **options):
siaes_queryset = Siae.objects.filter(Q(coords__isnull=False)).order_by("id")

if not options["force"]:
since_last_date_limit = timezone.now() - timedelta(days=settings.API_QPV_RELATIVE_DAYS_TO_UPDATE)
siaes_queryset = siaes_queryset.filter(self.get_filter_query(since_last_date_limit))

if options["limit"]:
siaes_queryset = siaes_queryset[: options["limit"]]
return siaes_queryset

def handle(self, *args, **options):
siae_list = self.get_query_set(**options)
self.stdout_messages_info([f"Populating API {self.API_NAME}...", f"Found {len(siae_list)} Siae"])

siaes_to_update = []
try:
for siae in siae_list:
# update siae from API
siae = self.update_siae(siae)
siaes_to_update.append(siae)
# log progress
count_siae_to_update = len(siaes_to_update)
if (count_siae_to_update % 50) == 0:
self.stdout_info(f"{count_siae_to_update}...")

except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
if status_code == 429:
# the real exceed request error have code=10005, with the api
# but requests send error code 429 "Unknown Status Code"
self.stdout_error("exceeded the requests limit for today (5000/per day)")
finally:
self.CLIENT.close()
# we still save siaes target status
Siae.objects.bulk_update(
siaes_to_update, self.FIELDS_TO_BULK_UPDATE, batch_size=settings.BATCH_SIZE_BULK_UPDATE
)

msg_success = [
f"----- Synchronisation API {self.API_NAME} -----",
f"Done! Processed {len(siaes_to_update)}/{len(siae_list)} siaes",
f"success count: {self.success_count['etablissement']}/{len(siaes_to_update)}",
f"True count: {self.success_count['etablissement_target']}/{len(siaes_to_update)}",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))
91 changes: 13 additions & 78 deletions lemarche/siaes/management/commands/update_api_qpv_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
import argparse
from datetime import timedelta

import requests
from django.conf import settings
from django.db.models import Q
from django.utils import timezone

from lemarche.siaes.models import Siae
from lemarche.utils.apis import api_slack
from lemarche.siaes.management.base_update_api import UpdateAPICommand
from lemarche.utils.apis.api_qpv import IS_QPV_KEY, QPV_CODE_KEY, QPV_NAME_KEY, get_default_client, is_in_qpv
from lemarche.utils.commands import BaseCommand


class Command(BaseCommand):
class Command(UpdateAPICommand):
"""
Populates API QPV

Expand All @@ -24,86 +17,28 @@ class Command(BaseCommand):
Usage: poetry run python manage.py update_api_qpv_fields --limit 100 --no-force
"""

def __init__(self, stdout=None, stderr=None, no_color=False):
super().__init__(stdout, stderr, no_color)
self.success_count = {"etablissement": 0, "etablissement_qpv": 0}

API_NAME = "QPV"
FIELDS_TO_BULK_UPDATE = ["is_qpv", "api_qpv_last_sync_date", "qpv_code", "qpv_name"]
CLIENT = get_default_client()

def add_arguments(self, parser):
parser.add_argument(
"-L", "--limit", type=int, default=None, help="Limiter le nombre de structures à processer"
)
parser.add_argument(
"-F",
"--force",
action=argparse.BooleanOptionalAction,
default=None,
help="Forcer l'update sans la vérification de la dernière mise à jour",
)

def get_query_set(self, **options):
siaes_queryset = Siae.objects.filter(Q(coords__isnull=False)).order_by("id")

if not options["force"]:
since_last_date_limit = timezone.now() - timedelta(days=settings.API_QPV_RELATIVE_DAYS_TO_UPDATE)
siaes_queryset = siaes_queryset.filter(
(Q(api_qpv_last_sync_date__lte=since_last_date_limit) | Q(api_qpv_last_sync_date__isnull=True))
)

if options["limit"]:
siaes_queryset = siaes_queryset[: options["limit"]]
return siaes_queryset

def handle(self, *args, **options):
siae_list = self.get_query_set(**options)
self.stdout_messages_info(["Populating API QPV...", f"Found {len(siae_list)} Siae"])

count_siae_to_update = 0
client = get_default_client()
try:
siaes_to_update = []
for siae in siae_list:
# update siae from API
siae = self.update_siae(client, siae)
siaes_to_update.append(siae)
# log progress
count_siae_to_update = len(siaes_to_update)
if (count_siae_to_update % 50) == 0:
self.stdout_info(f"{count_siae_to_update}...")

except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
if status_code == 429:
# the real exceed request error have code=10005, with the api
# but requests send error code 429 "Unknown Status Code"
self.stdout_error("exceeded the requests limit for today (5000/per day)")
finally:
client.close()
# we still save siaes qpv status
Siae.objects.bulk_update(
siaes_to_update, self.FIELDS_TO_BULK_UPDATE, batch_size=settings.BATCH_SIZE_BULK_UPDATE
)
@staticmethod
def get_filter_query(date_limit) -> Q:
return Q(api_qpv_last_sync_date__lte=date_limit) | Q(api_qpv_last_sync_date__isnull=True)

msg_success = [
"----- Synchronisation API QPV -----",
f"Done! Processed {len(siaes_to_update)}/{len(siae_list)} siaes",
f"success count: {self.success_count['etablissement']}/{len(siaes_to_update)}",
f"True count: {self.success_count['etablissement_qpv']}/{len(siaes_to_update)}",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))
@staticmethod
def is_in_target(latitude, longitude, client):
return is_in_qpv(latitude, longitude, client=client)

def update_siae(self, client, siae):
def update_siae(self, siae):
# call api is in qpv
result_is_in_qpv = is_in_qpv(siae.latitude, siae.longitude, client=client)
result_is_in_qpv = self.is_in_target(siae.latitude, siae.longitude, client=self.CLIENT)
self.success_count["etablissement"] += 1
siae.is_qpv = result_is_in_qpv[IS_QPV_KEY]
siae.api_qpv_last_sync_date = timezone.now()
if siae.is_qpv:
siae.qpv_code = result_is_in_qpv[QPV_CODE_KEY]
siae.qpv_name = result_is_in_qpv[QPV_NAME_KEY]
self.success_count["etablissement_qpv"] += 1
self.success_count["etablissement_target"] += 1
else:
siae.qpv_code = ""
siae.qpv_name = ""
Expand Down
91 changes: 13 additions & 78 deletions lemarche/siaes/management/commands/update_api_zrr_fields.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
import argparse
from datetime import timedelta

import requests
from django.conf import settings
from django.db.models import Q
from django.utils import timezone

from lemarche.siaes.models import Siae
from lemarche.utils.apis import api_slack
from lemarche.siaes.management.base_update_api import UpdateAPICommand
from lemarche.utils.apis.api_zrr import IS_ZRR_KEY, ZRR_CODE_KEY, ZRR_NAME_KEY, get_default_client, is_in_zrr
from lemarche.utils.commands import BaseCommand


class Command(BaseCommand):
class Command(UpdateAPICommand):
"""
Populates API ZRR

Expand All @@ -24,86 +17,28 @@ class Command(BaseCommand):
Usage: poetry run python manage.py update_api_zrr_fields --limit 100 --no-force
"""

def __init__(self, stdout=None, stderr=None, no_color=False):
super().__init__(stdout, stderr, no_color)
self.success_count = {"etablissement": 0, "etablissement_zrr": 0}

API_NAME = "ZRR"
FIELDS_TO_BULK_UPDATE = ["is_zrr", "api_zrr_last_sync_date", "zrr_code", "zrr_name"]
CLIENT = get_default_client()

def add_arguments(self, parser):
parser.add_argument(
"-L", "--limit", type=int, default=None, help="Limiter le nombre de structures à processer"
)
parser.add_argument(
"-F",
"--force",
action=argparse.BooleanOptionalAction,
default=None,
help="Forcer l'update sans la vérification de la dernière mise à jour",
)

def get_query_set(self, **options):
siaes_queryset = Siae.objects.filter(Q(coords__isnull=False)).order_by("id")

if not options["force"]:
since_last_date_limit = timezone.now() - timedelta(days=settings.API_QPV_RELATIVE_DAYS_TO_UPDATE)
siaes_queryset = siaes_queryset.filter(
(Q(api_zrr_last_sync_date__lte=since_last_date_limit) | Q(api_zrr_last_sync_date__isnull=True))
)

if options["limit"]:
siaes_queryset = siaes_queryset[: options["limit"]]
return siaes_queryset

def handle(self, *args, **options):
siae_list = self.get_query_set(**options)
self.stdout_messages_info(["Populating API ZRR...", f"Found {len(siae_list)} Siae"])

count_siae_to_update = 0
client = get_default_client()
try:
siaes_to_update = []
for siae in siae_list:
# update siae from API
siae = self.update_siae(client, siae)
siaes_to_update.append(siae)
# log progress
count_siae_to_update = len(siaes_to_update)
if (count_siae_to_update % 50) == 0:
self.stdout_info(f"{count_siae_to_update}...")

except requests.exceptions.HTTPError as e:
status_code = e.response.status_code
if status_code == 429:
# the real exceed request error have code=10005, with the api
# but requests send error code 429 "Unknown Status Code"
self.stdout_error("exceeded the requests limit for today (5000/per day)")
finally:
client.close()
# we still save siaes zrr status
Siae.objects.bulk_update(
siaes_to_update, self.FIELDS_TO_BULK_UPDATE, batch_size=settings.BATCH_SIZE_BULK_UPDATE
)
@staticmethod
def get_filter_query(date_limit) -> Q:
return Q(api_zrr_last_sync_date__lte=date_limit) | Q(api_zrr_last_sync_date__isnull=True)

msg_success = [
"----- Synchronisation API ZRR -----",
f"Done! Processed {len(siaes_to_update)}/{len(siae_list)} siaes",
f"success count: {self.success_count['etablissement']}/{len(siaes_to_update)}",
f"True count: {self.success_count['etablissement_zrr']}/{len(siaes_to_update)}",
]
self.stdout_messages_success(msg_success)
api_slack.send_message_to_channel("\n".join(msg_success))
@staticmethod
def is_in_target(latitude, longitude, client):
return is_in_zrr(latitude, longitude, client=client)

def update_siae(self, client, siae):
def update_siae(self, siae):
# call api is in zrr
result_is_in_zrr = is_in_zrr(siae.latitude, siae.longitude, client=client)
result_is_in_zrr = self.is_in_target(siae.latitude, siae.longitude, client=self.CLIENT)
self.success_count["etablissement"] += 1
siae.is_zrr = result_is_in_zrr[IS_ZRR_KEY]
siae.api_zrr_last_sync_date = timezone.now()
if siae.is_zrr:
siae.zrr_code = result_is_in_zrr[ZRR_CODE_KEY]
siae.zrr_name = result_is_in_zrr[ZRR_NAME_KEY]
self.success_count["etablissement_zrr"] += 1
self.success_count["etablissement_target"] += 1
else:
siae.zrr_code = ""
siae.zrr_name = ""
Expand Down