From 5794d9f8887c5aea141bccfb8d71ca3b3c4308d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Thu, 16 Jan 2025 17:38:36 +0100 Subject: [PATCH 1/6] Support new GTFS validation rule (#4416) --- .../resource/_gtfs_unusable_trip.html.heex | 16 ++++++++++++++++ .../lib/transport_web/views/resource_view.ex | 3 ++- .../en/LC_MESSAGES/validations-explanations.po | 4 ++++ .../fr/LC_MESSAGES/validations-explanations.po | 4 ++++ .../priv/gettext/validations-explanations.pot | 4 ++++ 5 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 apps/transport/lib/transport_web/templates/resource/_gtfs_unusable_trip.html.heex diff --git a/apps/transport/lib/transport_web/templates/resource/_gtfs_unusable_trip.html.heex b/apps/transport/lib/transport_web/templates/resource/_gtfs_unusable_trip.html.heex new file mode 100644 index 0000000000..08839e3ba1 --- /dev/null +++ b/apps/transport/lib/transport_web/templates/resource/_gtfs_unusable_trip.html.heex @@ -0,0 +1,16 @@ +

+ <%= dgettext("validations-explanations", "UnusableTrip") %> +

+ + + + + + + <%= for issue <- @issues do %> + + + + + <% end %> +
<%= dgettext("validations-explanations", "Object type") %><%= dgettext("validations-explanations", "Object ID") %>
<%= issue["object_type"] %><%= issue["object_id"] %>
diff --git a/apps/transport/lib/transport_web/views/resource_view.ex b/apps/transport/lib/transport_web/views/resource_view.ex index 25e4565a1a..cd1e0f4e25 100644 --- a/apps/transport/lib/transport_web/views/resource_view.ex +++ b/apps/transport/lib/transport_web/views/resource_view.ex @@ -40,7 +40,8 @@ defmodule TransportWeb.ResourceView do "MissingId" => "_missing_id_issue.html", "MissingName" => "_missing_name_issue.html", "SubFolder" => "_subfolder_issue.html", - "NegativeStopDuration" => "_negative_stop_duration_issue.html" + "NegativeStopDuration" => "_negative_stop_duration_issue.html", + "UnusableTrip" => "_unusable_trip.html" }, Transport.Validators.GTFSTransport.issue_type(issues.entries), "_generic_issue.html" diff --git a/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po b/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po index 094412efc1..1b167befb8 100644 --- a/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po +++ b/apps/transport/priv/gettext/en/LC_MESSAGES/validations-explanations.po @@ -146,3 +146,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Details for debugging purposes" msgstr "" + +#, elixir-autogen, elixir-format +msgid "UnusableTrip" +msgstr "A trip must visit more than one stop in stop_times.txt to be usable by passengers for boarding and alighting." diff --git a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po index 55adbefc56..e11d99bf4f 100644 --- a/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po +++ b/apps/transport/priv/gettext/fr/LC_MESSAGES/validations-explanations.po @@ -146,3 +146,7 @@ msgstr "Emplacement inconnu" #, elixir-autogen, elixir-format msgid "Details for debugging purposes" msgstr "Détails à fin de débogage" + +#, elixir-autogen, elixir-format +msgid "UnusableTrip" +msgstr "Un trajet doit passer par plus d’un arrêt dans stop_times.txt pour être utilisable par les voyageurs à la montée ou la descente." diff --git a/apps/transport/priv/gettext/validations-explanations.pot b/apps/transport/priv/gettext/validations-explanations.pot index e1e89c104f..ae81432ff4 100644 --- a/apps/transport/priv/gettext/validations-explanations.pot +++ b/apps/transport/priv/gettext/validations-explanations.pot @@ -145,3 +145,7 @@ msgstr "" #, elixir-autogen, elixir-format msgid "Details for debugging purposes" msgstr "" + +#, elixir-autogen, elixir-format +msgid "UnusableTrip" +msgstr "" From 4897113b913eb9f616e812030d5a33ef2ead24be Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Fri, 17 Jan 2025 11:28:25 +0100 Subject: [PATCH 2/6] Identifiants ZFE : ajout syndicats pour Tours et Lyon (#4415) --- apps/transport/priv/zfe_ids.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/transport/priv/zfe_ids.csv b/apps/transport/priv/zfe_ids.csv index 7e22b5c2e2..8d027ad45b 100644 --- a/apps/transport/priv/zfe_ids.csv +++ b/apps/transport/priv/zfe_ids.csv @@ -19,7 +19,7 @@ siren;code;epci_principal;autres_siren 247200132;LE MANS;Le Mans Métropole; 200093201;LILLE;Métropole Européenne de Lille; 248719312;LIMOGES;Limoges Métropole; -200046977;LYON;Métropole de Lyon; +200046977;LYON;Métropole de Lyon;256900994 200054807;MARSEILLE-AIX EN PROVENCE;Métropole Aix-Marseille-Provence; 200039865;METZ;Metz Métropole; 243400017;MONTPELLIER;Montpellier Méditerranée Métropole; @@ -40,5 +40,5 @@ siren;code;epci_principal;autres_siren 246700488;STRASBOURG;Eurométropole de Strasbourg; 248300543;TOULON;Métropole Toulon Provence Méditerranée; 253100986;TOULOUSE;Toulouse métropole;243100518 -243700754;TOURS;Tours Métropole Val de Loire; +243700754;TOURS;Tours Métropole Val de Loire;200085108 245901160;VALENCIENNES;Valenciennes métropole; From 4f8009934596a04b7ad4e4d0fb6ea8a1ea065a3f Mon Sep 17 00:00:00 2001 From: Antoine Augusti Date: Mon, 20 Jan 2025 13:02:21 +0100 Subject: [PATCH 3/6] =?UTF-8?q?Mise=20=C3=A0=20jour=20op=C3=A9rateurs=20GB?= =?UTF-8?q?FS=20(#4418)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/transport/priv/gbfs_operators.csv | 4 ++-- apps/transport/test/transport/csv_documents_test.exs | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/transport/priv/gbfs_operators.csv b/apps/transport/priv/gbfs_operators.csv index 53cb327bc8..aae44271f0 100644 --- a/apps/transport/priv/gbfs_operators.csv +++ b/apps/transport/priv/gbfs_operators.csv @@ -2,6 +2,7 @@ url;operator example.com;Example api.cyclocity.fr;JC Decaux api.saint-etienne-metropole.fr;Fifteen +backend.citiz.fr;Citiz bdx.mecatran.com;Cykleo bird.co;Bird clermontferrand.publicbikesystem.net;Citybike @@ -16,8 +17,6 @@ fifteen.site;Fifteen getapony.com;Pony lime.bike;Lime media.ilevia.fr/opendata;Cykleo -mobi-iti-nam.okina.fr/api-proxy/api/gbfs/1.0/gbfs/velibeo_ecovelo/gbfs;Ecovélo -mobi-iti-nam.okina.fr/api-proxy/api/gbfs/1.0/gbfs/perivelo_ecovelo/gbfs;Ecovélo nextbike.net;nextbike smovengo.cloud;Fifteen urbansharing.com/lovelolibreservice.fr;Citybike @@ -25,3 +24,4 @@ voiapp.io;Voi zoov.eu;Fifteen zoov.io;Fifteen zoov.site;Fifteen +_ecovelo/gbfs;Ecovélo diff --git a/apps/transport/test/transport/csv_documents_test.exs b/apps/transport/test/transport/csv_documents_test.exs index 24993ef940..b5c7267ae3 100644 --- a/apps/transport/test/transport/csv_documents_test.exs +++ b/apps/transport/test/transport/csv_documents_test.exs @@ -32,7 +32,8 @@ defmodule Transport.CSVDocumentsTest do # Check `operator` values. Prevent typos and ensure unique values. # Detect things like `Cykleo` VS `Cykléo`. for x <- operators, y <- operators, x != y do - assert String.jaro_distance(x, y) <= 0.75, "#{x} and #{y} look too similar. Is it the same operator?" + error_message = "#{x} and #{y} look too similar. Is it the same operator?" + assert String.jaro_distance(x, y) <= 0.75 || distinct_operators?(x, y), error_message end # Check `url` values. Make sure there is at most a single match per GBFS feed. @@ -43,4 +44,11 @@ defmodule Transport.CSVDocumentsTest do refute String.contains?(x, y), "#{x} is contained #{y}. A GBFS feed can only match for a single URL." end end + + def distinct_operators?(x, y) do + [ + ["Citiz", "Citybike"] + ] + |> Enum.member?(Enum.sort([x, y])) + end end From 410255dea98203a51681bf6d767b886bcda9f242 Mon Sep 17 00:00:00 2001 From: Vincent Degove Date: Mon, 20 Jan 2025 14:23:55 +0100 Subject: [PATCH 4/6] =?UTF-8?q?Notifications=20de=20p=C3=A9remption=20boug?= =?UTF-8?q?=C3=A9es=20vers=20un=20job=20Oban=20=E2=80=93=20et=20autres=20r?= =?UTF-8?q?efactos=20de=20DataChecker=20(#4390)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...iration_admin_producer_notification_job.ex | 79 ++++++ .../lib/jobs/expiration_notification_job.ex | 2 + .../lib/jobs/new_dataset_notifications_job.ex | 21 +- apps/transport/lib/transport/data_checker.ex | 100 +------- apps/transport/lib/transport/scheduler.ex | 2 - .../test/transport/data_checker_test.exs | 233 ------------------ ...n_admin_producer_notification_job_test.exs | 172 +++++++++++++ .../new_dataset_notifications_job_test.exs | 33 ++- config/runtime.exs | 3 +- 9 files changed, 303 insertions(+), 342 deletions(-) create mode 100644 apps/transport/lib/jobs/expiration_admin_producer_notification_job.ex create mode 100644 apps/transport/test/transport/jobs/expiration_admin_producer_notification_job_test.exs diff --git a/apps/transport/lib/jobs/expiration_admin_producer_notification_job.ex b/apps/transport/lib/jobs/expiration_admin_producer_notification_job.ex new file mode 100644 index 0000000000..ad493add7b --- /dev/null +++ b/apps/transport/lib/jobs/expiration_admin_producer_notification_job.ex @@ -0,0 +1,79 @@ +defmodule Transport.Jobs.ExpirationAdminProducerNotificationJob do + @moduledoc """ + This module is in charge of sending notifications to admins and producers when data is outdated. + It is similar to `Transport.Jobs.ExpirationNotificationJob`, dedicated to reusers. + Both could be merged in the future. + """ + + use Oban.Worker, max_attempts: 3, tags: ["notifications"] + import Ecto.Query + + @type delay_and_records :: {integer(), [{DB.Dataset.t(), [DB.Resource.t()]}]} + @expiration_reason Transport.NotificationReason.reason(:expiration) + # If delay < 0, the resource is already expired + @default_outdated_data_delays [-90, -60, -30, -45, -15, -7, -3, 0, 7, 14] + + @impl Oban.Worker + + def perform(%Oban.Job{id: job_id}) do + outdated_data(job_id) + :ok + end + + def outdated_data(job_id) do + for delay <- possible_delays(), + date = Date.add(Date.utc_today(), delay) do + {delay, gtfs_datasets_expiring_on(date)} + end + |> Enum.reject(fn {_, records} -> Enum.empty?(records) end) + |> send_outdated_data_admin_mail() + |> Enum.map(&send_outdated_data_producer_notifications(&1, job_id)) + end + + @spec gtfs_datasets_expiring_on(Date.t()) :: [{DB.Dataset.t(), [DB.Resource.t()]}] + def gtfs_datasets_expiring_on(%Date{} = date) do + DB.Dataset.base_query() + |> DB.Dataset.join_from_dataset_to_metadata(Transport.Validators.GTFSTransport.validator_name()) + |> where( + [metadata: m, resource: r], + fragment("TO_DATE(?->>'end_date', 'YYYY-MM-DD')", m.metadata) == ^date and r.format == "GTFS" + ) + |> select([dataset: d, resource: r], {d, r}) + |> distinct(true) + |> DB.Repo.all() + |> Enum.group_by(fn {%DB.Dataset{} = d, _} -> d end, fn {_, %DB.Resource{} = r} -> r end) + |> Enum.to_list() + end + + def possible_delays do + @default_outdated_data_delays + |> Enum.uniq() + |> Enum.sort() + end + + # A different email is sent to producers for every delay, containing all datasets expiring on this given delay + @spec send_outdated_data_producer_notifications(delay_and_records(), integer()) :: :ok + def send_outdated_data_producer_notifications({delay, records}, job_id) do + Enum.each(records, fn {%DB.Dataset{} = dataset, resources} -> + @expiration_reason + |> DB.NotificationSubscription.subscriptions_for_reason_dataset_and_role(dataset, :producer) + |> Enum.each(fn %DB.NotificationSubscription{contact: %DB.Contact{} = contact} = subscription -> + contact + |> Transport.UserNotifier.expiration_producer(dataset, resources, delay) + |> Transport.Mailer.deliver() + + DB.Notification.insert!(dataset, subscription, %{delay: delay, job_id: job_id}) + end) + end) + end + + @spec send_outdated_data_admin_mail([delay_and_records()]) :: [delay_and_records()] + defp send_outdated_data_admin_mail([] = _records), do: [] + + defp send_outdated_data_admin_mail(records) do + Transport.AdminNotifier.expiration(records) + |> Transport.Mailer.deliver() + + records + end +end diff --git a/apps/transport/lib/jobs/expiration_notification_job.ex b/apps/transport/lib/jobs/expiration_notification_job.ex index e8cba3cf6f..09f4f0dfce 100644 --- a/apps/transport/lib/jobs/expiration_notification_job.ex +++ b/apps/transport/lib/jobs/expiration_notification_job.ex @@ -6,6 +6,8 @@ defmodule Transport.Jobs.ExpirationNotificationJob do It has 2 `perform/1` methods: - a dispatcher one in charge of identifying contacts we should get in touch with today - another in charge of building the daily digest for a specific contact (with only their favorited datasets) + + It is similar to `Transport.Jobs.ExpirationAdminProducerNotificationJob`, dedicated to producers and admins. """ use Oban.Worker, max_attempts: 3, diff --git a/apps/transport/lib/jobs/new_dataset_notifications_job.ex b/apps/transport/lib/jobs/new_dataset_notifications_job.ex index f8be16d5b0..759f188371 100644 --- a/apps/transport/lib/jobs/new_dataset_notifications_job.ex +++ b/apps/transport/lib/jobs/new_dataset_notifications_job.ex @@ -4,10 +4,12 @@ defmodule Transport.Jobs.NewDatasetNotificationsJob do """ use Oban.Worker, max_attempts: 3, tags: ["notifications"] import Ecto.Query + @new_dataset_reason Transport.NotificationReason.reason(:new_dataset) @impl Oban.Worker - def perform(%Oban.Job{inserted_at: %DateTime{} = inserted_at}) do - inserted_at |> relevant_datasets() |> Transport.DataChecker.send_new_dataset_notifications() + + def perform(%Oban.Job{id: job_id, inserted_at: %DateTime{} = inserted_at}) do + inserted_at |> relevant_datasets() |> send_new_dataset_notifications(job_id) :ok end @@ -18,4 +20,19 @@ defmodule Transport.Jobs.NewDatasetNotificationsJob do |> where([dataset: d], d.inserted_at >= ^datetime_limit) |> DB.Repo.all() end + + @spec send_new_dataset_notifications([DB.Dataset.t()] | [], pos_integer()) :: no_return() | :ok + def send_new_dataset_notifications([], _job_id), do: :ok + + def send_new_dataset_notifications(datasets, job_id) do + @new_dataset_reason + |> DB.NotificationSubscription.subscriptions_for_reason_and_role(:reuser) + |> Enum.each(fn %DB.NotificationSubscription{contact: %DB.Contact{} = contact} = subscription -> + contact + |> Transport.UserNotifier.new_datasets(datasets) + |> Transport.Mailer.deliver() + + DB.Notification.insert!(subscription, %{dataset_ids: Enum.map(datasets, & &1.id), job_id: job_id}) + end) + end end diff --git a/apps/transport/lib/transport/data_checker.ex b/apps/transport/lib/transport/data_checker.ex index 60c6fe663b..6cdd2f3d1d 100644 --- a/apps/transport/lib/transport/data_checker.ex +++ b/apps/transport/lib/transport/data_checker.ex @@ -1,17 +1,12 @@ defmodule Transport.DataChecker do @moduledoc """ - Use to check data, and act about it, like send email + Use to check data for toggling on and off active status of datasets depending on status on data.gouv.fr """ alias DB.{Dataset, Repo} import Ecto.Query require Logger - @type delay_and_records :: {integer(), [{DB.Dataset.t(), [DB.Resource.t()]}]} @type dataset_status :: :active | :inactive | :ignore | :no_producer | {:archived, DateTime.t()} - @expiration_reason Transport.NotificationReason.reason(:expiration) - @new_dataset_reason Transport.NotificationReason.reason(:new_dataset) - # If delay < 0, the resource is already expired - @default_outdated_data_delays [-90, -60, -30, -45, -15, -7, -3, 0, 7, 14] @doc """ This method is a scheduled job which does two things: @@ -106,99 +101,6 @@ defmodule Transport.DataChecker do ) end - def outdated_data do - # Generated as an integer rather than a UUID because `payload.job_id` - # for other notifications are %Oban.Job.id (bigint). - job_id = Enum.random(1..Integer.pow(2, 63)) - - for delay <- possible_delays(), - date = Date.add(Date.utc_today(), delay) do - {delay, gtfs_datasets_expiring_on(date)} - end - |> Enum.reject(fn {_, records} -> Enum.empty?(records) end) - |> send_outdated_data_mail() - |> Enum.map(&send_outdated_data_notifications(&1, job_id)) - end - - @spec gtfs_datasets_expiring_on(Date.t()) :: [{DB.Dataset.t(), [DB.Resource.t()]}] - def gtfs_datasets_expiring_on(%Date{} = date) do - DB.Dataset.base_query() - |> DB.Dataset.join_from_dataset_to_metadata(Transport.Validators.GTFSTransport.validator_name()) - |> where( - [metadata: m, resource: r], - fragment("TO_DATE(?->>'end_date', 'YYYY-MM-DD')", m.metadata) == ^date and r.format == "GTFS" - ) - |> select([dataset: d, resource: r], {d, r}) - |> distinct(true) - |> DB.Repo.all() - |> Enum.group_by(fn {%DB.Dataset{} = d, _} -> d end, fn {_, %DB.Resource{} = r} -> r end) - |> Enum.to_list() - end - - def possible_delays do - @default_outdated_data_delays - |> Enum.uniq() - |> Enum.sort() - end - - @spec send_new_dataset_notifications([Dataset.t()] | []) :: no_return() | :ok - def send_new_dataset_notifications([]), do: :ok - - def send_new_dataset_notifications(datasets) do - # Generated as an integer rather than a UUID because `payload.job_id` - # for other notifications are %Oban.Job.id (bigint). - job_id = Enum.random(1..Integer.pow(2, 63)) - - @new_dataset_reason - |> DB.NotificationSubscription.subscriptions_for_reason_and_role(:reuser) - |> Enum.each(fn %DB.NotificationSubscription{contact: %DB.Contact{} = contact} = subscription -> - contact - |> Transport.UserNotifier.new_datasets(datasets) - |> Transport.Mailer.deliver() - - DB.Notification.insert!(subscription, %{dataset_ids: Enum.map(datasets, & &1.id), job_id: job_id}) - end) - end - - @spec send_outdated_data_notifications(delay_and_records(), integer()) :: delay_and_records() - def send_outdated_data_notifications({delay, records} = payload, job_id) do - Enum.each(records, fn {%DB.Dataset{} = dataset, resources} -> - @expiration_reason - |> DB.NotificationSubscription.subscriptions_for_reason_dataset_and_role(dataset, :producer) - |> Enum.each(fn %DB.NotificationSubscription{contact: %DB.Contact{} = contact} = subscription -> - contact - |> Transport.UserNotifier.expiration_producer(dataset, resources, delay) - |> Transport.Mailer.deliver() - - DB.Notification.insert!(dataset, subscription, %{delay: delay, job_id: job_id}) - end) - end) - - payload - end - - @doc """ - iex> resource_titles([%DB.Resource{title: "B"}]) - "B" - iex> resource_titles([%DB.Resource{title: "B"}, %DB.Resource{title: "A"}]) - "A, B" - """ - def resource_titles(resources) do - resources - |> Enum.sort_by(fn %DB.Resource{title: title} -> title end) - |> Enum.map_join(", ", fn %DB.Resource{title: title} -> title end) - end - - @spec send_outdated_data_mail([delay_and_records()]) :: [delay_and_records()] - defp send_outdated_data_mail([] = _records), do: [] - - defp send_outdated_data_mail(records) do - Transport.AdminNotifier.expiration(records) - |> Transport.Mailer.deliver() - - records - end - # Do nothing if all lists are empty defp send_inactive_datasets_mail([] = _reactivated_datasets, [] = _inactive_datasets, [] = _archived_datasets), do: nil diff --git a/apps/transport/lib/transport/scheduler.ex b/apps/transport/lib/transport/scheduler.ex index 3c5dcfc799..5dc8d56fe2 100644 --- a/apps/transport/lib/transport/scheduler.ex +++ b/apps/transport/lib/transport/scheduler.ex @@ -13,8 +13,6 @@ defmodule Transport.Scheduler do [ # Every day at 4am UTC {"0 4 * * *", {Transport.ImportData, :import_validate_all, []}}, - # Send email for outdated data - {"@daily", {Transport.DataChecker, :outdated_data, []}}, # Set inactive data {"@daily", {Transport.DataChecker, :inactive_data, []}}, # Watch for new comments on datasets diff --git a/apps/transport/test/transport/data_checker_test.exs b/apps/transport/test/transport/data_checker_test.exs index 509c0f3143..c4b9372070 100644 --- a/apps/transport/test/transport/data_checker_test.exs +++ b/apps/transport/test/transport/data_checker_test.exs @@ -4,8 +4,6 @@ defmodule Transport.DataCheckerTest do import DB.Factory import Swoosh.TestAssertions - doctest Transport.DataChecker, import: true - setup :verify_on_exit! setup do @@ -127,237 +125,6 @@ defmodule Transport.DataCheckerTest do end end - test "gtfs_datasets_expiring_on" do - {today, tomorrow, yesterday} = {Date.utc_today(), Date.add(Date.utc_today(), 1), Date.add(Date.utc_today(), -1)} - assert [] == today |> Transport.DataChecker.gtfs_datasets_expiring_on() - - insert_fn = fn %Date{} = expiration_date, %DB.Dataset{} = dataset -> - multi_validation = - insert(:multi_validation, - validator: Transport.Validators.GTFSTransport.validator_name(), - resource_history: insert(:resource_history, resource: insert(:resource, dataset: dataset, format: "GTFS")) - ) - - insert(:resource_metadata, - multi_validation_id: multi_validation.id, - metadata: %{"end_date" => expiration_date} - ) - end - - # Ignores hidden or inactive datasets - insert_fn.(today, insert(:dataset, is_active: false)) - insert_fn.(today, insert(:dataset, is_active: true, is_hidden: true)) - - assert [] == today |> Transport.DataChecker.gtfs_datasets_expiring_on() - - # 2 GTFS resources expiring on the same day for a dataset - %DB.Dataset{id: dataset_id} = dataset = insert(:dataset, is_active: true) - insert_fn.(today, dataset) - insert_fn.(today, dataset) - - assert [ - {%DB.Dataset{id: ^dataset_id}, - [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]} - ] = today |> Transport.DataChecker.gtfs_datasets_expiring_on() - - assert [] == tomorrow |> Transport.DataChecker.gtfs_datasets_expiring_on() - assert [] == yesterday |> Transport.DataChecker.gtfs_datasets_expiring_on() - - insert_fn.(tomorrow, dataset) - - assert [ - {%DB.Dataset{id: ^dataset_id}, - [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]} - ] = today |> Transport.DataChecker.gtfs_datasets_expiring_on() - - assert [ - {%DB.Dataset{id: ^dataset_id}, [%DB.Resource{dataset_id: ^dataset_id}]} - ] = tomorrow |> Transport.DataChecker.gtfs_datasets_expiring_on() - - assert [] == yesterday |> Transport.DataChecker.gtfs_datasets_expiring_on() - - # Multiple datasets - %DB.Dataset{id: d2_id} = d2 = insert(:dataset, is_active: true) - insert_fn.(today, d2) - - assert [ - {%DB.Dataset{id: ^dataset_id}, - [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]}, - {%DB.Dataset{id: ^d2_id}, [%DB.Resource{dataset_id: ^d2_id}]} - ] = today |> Transport.DataChecker.gtfs_datasets_expiring_on() - end - - describe "outdated_data job" do - test "sends email to our team + relevant contact before expiry" do - %DB.Dataset{id: dataset_id} = - dataset = - insert(:dataset, is_active: true, custom_title: "Dataset custom title", custom_tags: ["loi-climat-resilience"]) - - assert DB.Dataset.climate_resilience_bill?(dataset) - # fake a resource expiring today - %DB.Resource{id: resource_id} = - resource = insert(:resource, dataset: dataset, format: "GTFS", title: resource_title = "Super GTFS") - - multi_validation = - insert(:multi_validation, - validator: Transport.Validators.GTFSTransport.validator_name(), - resource_history: insert(:resource_history, resource: resource) - ) - - insert(:resource_metadata, - multi_validation_id: multi_validation.id, - metadata: %{"end_date" => Date.utc_today()} - ) - - assert [{%DB.Dataset{id: ^dataset_id}, [%DB.Resource{id: ^resource_id}]}] = - Date.utc_today() |> Transport.DataChecker.gtfs_datasets_expiring_on() - - %DB.Contact{id: contact_id, email: email} = contact = insert_contact() - - insert(:notification_subscription, %{ - reason: :expiration, - source: :admin, - role: :producer, - contact_id: contact_id, - dataset_id: dataset.id - }) - - # Should be ignored, this subscription is for a reuser - %DB.Contact{id: reuser_id} = insert_contact() - - insert(:notification_subscription, %{ - reason: :expiration, - source: :user, - role: :reuser, - contact_id: reuser_id, - dataset_id: dataset.id - }) - - Transport.DataChecker.outdated_data() - - # a first mail to our team - - assert_email_sent(fn %Swoosh.Email{ - from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, - to: [{"", "contact@transport.data.gouv.fr"}], - subject: "Jeux de données arrivant à expiration", - text_body: nil, - html_body: body - } -> - assert body =~ ~r/Jeux de données périmant demain :/ - - assert body =~ - ~s|
  • #{dataset.custom_title} - ✅ notification automatique ⚖️🗺️ article 122
    • | - end) - - # a second mail to the email address in the notifications config - display_name = DB.Contact.display_name(contact) - - assert_email_sent(fn %Swoosh.Email{ - from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, - to: [{^display_name, ^email}], - subject: "Jeu de données arrivant à expiration", - html_body: html_body - } -> - refute html_body =~ "notification automatique" - refute html_body =~ "article 122" - - assert html_body =~ - ~s(Les données GTFS #{resource_title} associées au jeu de données #{dataset.custom_title} périment demain.) - - assert html_body =~ - ~s(remplaçant la ressource périmée par la nouvelle) - end) - end - - test "outdated_data job with nothing to send should not send email" do - Transport.DataChecker.outdated_data() - assert_no_email_sent() - end - end - - test "send_outdated_data_notifications" do - %{id: dataset_id} = dataset = insert(:dataset) - %DB.Contact{id: contact_id, email: email} = contact = insert_contact() - - %DB.NotificationSubscription{id: ns_id} = - insert(:notification_subscription, %{ - reason: :expiration, - source: :admin, - role: :producer, - contact_id: contact_id, - dataset_id: dataset.id - }) - - job_id = 42 - Transport.DataChecker.send_outdated_data_notifications({7, [{dataset, []}]}, job_id) - - assert_email_sent( - from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, - to: {DB.Contact.display_name(contact), email}, - subject: "Jeu de données arrivant à expiration", - text_body: nil, - html_body: ~r/Bonjour/ - ) - - assert [ - %DB.Notification{ - contact_id: ^contact_id, - email: ^email, - reason: :expiration, - dataset_id: ^dataset_id, - notification_subscription_id: ^ns_id, - role: :producer, - payload: %{"delay" => 7, "job_id" => ^job_id} - } - ] = - DB.Notification |> DB.Repo.all() - end - - describe "send_new_dataset_notifications" do - test "no datasets" do - assert Transport.DataChecker.send_new_dataset_notifications([]) == :ok - end - - test "with datasets" do - %DB.Dataset{id: dataset_id} = dataset = insert(:dataset, type: "public-transit") - - %DB.Contact{id: contact_id, email: email} = contact = insert_contact() - - %DB.NotificationSubscription{id: ns_id} = - insert(:notification_subscription, %{ - reason: :new_dataset, - source: :user, - role: :reuser, - contact_id: contact_id - }) - - Transport.DataChecker.send_new_dataset_notifications([dataset]) - - assert_email_sent( - from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, - to: {DB.Contact.display_name(contact), email}, - subject: "Nouveaux jeux de données référencés", - text_body: nil, - html_body: - ~r|
  • #{dataset.custom_title} - \(Transport public collectif\)
    • | - ) - - assert [ - %DB.Notification{ - contact_id: ^contact_id, - email: ^email, - reason: :new_dataset, - role: :reuser, - dataset_id: nil, - payload: %{"dataset_ids" => [^dataset_id], "job_id" => _}, - notification_subscription_id: ^ns_id - } - ] = - DB.Notification |> DB.Repo.all() - end - end - describe "dataset_status" do test "active" do dataset = %DB.Dataset{datagouv_id: Ecto.UUID.generate()} diff --git a/apps/transport/test/transport/jobs/expiration_admin_producer_notification_job_test.exs b/apps/transport/test/transport/jobs/expiration_admin_producer_notification_job_test.exs new file mode 100644 index 0000000000..c456bace18 --- /dev/null +++ b/apps/transport/test/transport/jobs/expiration_admin_producer_notification_job_test.exs @@ -0,0 +1,172 @@ +defmodule Transport.Test.Transport.Jobs.ExpirationAdminProducerNotificationJobTest do + use ExUnit.Case, async: true + import DB.Factory + import Swoosh.TestAssertions + use Oban.Testing, repo: DB.Repo + + setup do + Ecto.Adapters.SQL.Sandbox.checkout(DB.Repo) + end + + test "sends email to our team + relevant contact before expiry" do + %DB.Dataset{id: dataset_id} = + dataset = + insert(:dataset, is_active: true, custom_title: "Dataset custom title", custom_tags: ["loi-climat-resilience"]) + + assert DB.Dataset.climate_resilience_bill?(dataset) + # fake a resource expiring today + %DB.Resource{id: resource_id} = + resource = insert(:resource, dataset: dataset, format: "GTFS", title: resource_title = "Super GTFS") + + multi_validation = + insert(:multi_validation, + validator: Transport.Validators.GTFSTransport.validator_name(), + resource_history: insert(:resource_history, resource: resource) + ) + + insert(:resource_metadata, + multi_validation_id: multi_validation.id, + metadata: %{"end_date" => Date.utc_today()} + ) + + assert [{%DB.Dataset{id: ^dataset_id}, [%DB.Resource{id: ^resource_id}]}] = + Date.utc_today() |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + %DB.Contact{id: contact_id, email: email} = contact = insert_contact() + + %DB.NotificationSubscription{id: ns_id} = + insert(:notification_subscription, %{ + reason: :expiration, + source: :admin, + role: :producer, + contact_id: contact_id, + dataset_id: dataset.id + }) + + # Should be ignored, this subscription is for a reuser + %DB.Contact{id: reuser_id} = insert_contact() + + insert(:notification_subscription, %{ + reason: :expiration, + source: :user, + role: :reuser, + contact_id: reuser_id, + dataset_id: dataset.id + }) + + assert :ok == perform_job(Transport.Jobs.ExpirationAdminProducerNotificationJob, %{}) + + # a first mail to our team + + assert_email_sent(fn %Swoosh.Email{ + from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, + to: [{"", "contact@transport.data.gouv.fr"}], + subject: "Jeux de données arrivant à expiration", + text_body: nil, + html_body: body + } -> + assert body =~ ~r/Jeux de données périmant demain :/ + + assert body =~ + ~s|
  • #{dataset.custom_title} - ✅ notification automatique ⚖️🗺️ article 122
    • | + end) + + # a second mail to the email address in the notifications config + display_name = DB.Contact.display_name(contact) + + assert_email_sent(fn %Swoosh.Email{ + from: {"transport.data.gouv.fr", "contact@transport.data.gouv.fr"}, + to: [{^display_name, ^email}], + subject: "Jeu de données arrivant à expiration", + html_body: html_body + } -> + refute html_body =~ "notification automatique" + refute html_body =~ "article 122" + + assert html_body =~ + ~s(Les données GTFS #{resource_title} associées au jeu de données #{dataset.custom_title} périment demain.) + + assert html_body =~ + ~s(remplaçant la ressource périmée par la nouvelle) + end) + + # Logs are there + assert [ + %DB.Notification{ + contact_id: ^contact_id, + email: ^email, + reason: :expiration, + dataset_id: ^dataset_id, + notification_subscription_id: ^ns_id, + role: :producer, + payload: %{"delay" => 0, "job_id" => _job_id} + } + ] = + DB.Notification |> DB.Repo.all() + end + + test "outdated_data job with nothing to send should not send email" do + assert :ok == perform_job(Transport.Jobs.ExpirationAdminProducerNotificationJob, %{}) + assert_no_email_sent() + end + + test "gtfs_datasets_expiring_on" do + {today, tomorrow, yesterday} = {Date.utc_today(), Date.add(Date.utc_today(), 1), Date.add(Date.utc_today(), -1)} + assert [] == today |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + insert_fn = fn %Date{} = expiration_date, %DB.Dataset{} = dataset -> + multi_validation = + insert(:multi_validation, + validator: Transport.Validators.GTFSTransport.validator_name(), + resource_history: insert(:resource_history, resource: insert(:resource, dataset: dataset, format: "GTFS")) + ) + + insert(:resource_metadata, + multi_validation_id: multi_validation.id, + metadata: %{"end_date" => expiration_date} + ) + end + + # Ignores hidden or inactive datasets + insert_fn.(today, insert(:dataset, is_active: false)) + insert_fn.(today, insert(:dataset, is_active: true, is_hidden: true)) + + assert [] == today |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + # 2 GTFS resources expiring on the same day for a dataset + %DB.Dataset{id: dataset_id} = dataset = insert(:dataset, is_active: true) + insert_fn.(today, dataset) + insert_fn.(today, dataset) + + assert [ + {%DB.Dataset{id: ^dataset_id}, + [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]} + ] = today |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + assert [] == tomorrow |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + assert [] == yesterday |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + insert_fn.(tomorrow, dataset) + + assert [ + {%DB.Dataset{id: ^dataset_id}, + [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]} + ] = today |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + assert [ + {%DB.Dataset{id: ^dataset_id}, [%DB.Resource{dataset_id: ^dataset_id}]} + ] = tomorrow |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + assert [] == yesterday |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + + # Multiple datasets + %DB.Dataset{id: d2_id} = d2 = insert(:dataset, is_active: true) + insert_fn.(today, d2) + + assert [ + {%DB.Dataset{id: ^dataset_id}, + [%DB.Resource{dataset_id: ^dataset_id}, %DB.Resource{dataset_id: ^dataset_id}]}, + {%DB.Dataset{id: ^d2_id}, [%DB.Resource{dataset_id: ^d2_id}]} + ] = today |> Transport.Jobs.ExpirationAdminProducerNotificationJob.gtfs_datasets_expiring_on() + end +end diff --git a/apps/transport/test/transport/jobs/new_dataset_notifications_job_test.exs b/apps/transport/test/transport/jobs/new_dataset_notifications_job_test.exs index 9da324822c..03c5720053 100644 --- a/apps/transport/test/transport/jobs/new_dataset_notifications_job_test.exs +++ b/apps/transport/test/transport/jobs/new_dataset_notifications_job_test.exs @@ -24,11 +24,10 @@ defmodule Transport.Test.Transport.Jobs.NewDatasetNotificationsJobTest do end test "perform" do - %DB.Dataset{id: dataset_id} = insert(:dataset, inserted_at: hours_ago(23), is_active: true) - %DB.Contact{id: contact_id, email: email} = contact = insert_contact() + {contact, contact_id, email, ns_id} = insert_contact_and_notification_subscription() - %DB.NotificationSubscription{id: ns_id} = - insert(:notification_subscription, %{reason: :new_dataset, source: :admin, role: :reuser, contact_id: contact_id}) + %DB.Dataset{id: dataset_id} = + dataset = insert(:dataset, inserted_at: hours_ago(23), is_active: true, type: "public-transit") assert :ok == perform_job(NewDatasetNotificationsJob, %{}, inserted_at: DateTime.utc_now()) @@ -37,7 +36,8 @@ defmodule Transport.Test.Transport.Jobs.NewDatasetNotificationsJobTest do to: {DB.Contact.display_name(contact), email}, subject: "Nouveaux jeux de données référencés", text_body: nil, - html_body: ~r|

    Bonjour,

    | + html_body: + ~r|
  • #{dataset.custom_title} - \(Transport public collectif\)
    • | ) # Logs have been saved @@ -46,6 +46,7 @@ defmodule Transport.Test.Transport.Jobs.NewDatasetNotificationsJobTest do contact_id: ^contact_id, email: ^email, reason: :new_dataset, + role: :reuser, dataset_id: nil, notification_subscription_id: ^ns_id, payload: %{"dataset_ids" => [^dataset_id]} @@ -54,7 +55,29 @@ defmodule Transport.Test.Transport.Jobs.NewDatasetNotificationsJobTest do DB.Notification |> DB.Repo.all() end + test "no datasets" do + insert_contact_and_notification_subscription() + + assert :ok == perform_job(NewDatasetNotificationsJob, %{}, inserted_at: DateTime.utc_now()) + + assert_no_email_sent() + end + defp hours_ago(hours) when hours > 0 do DateTime.utc_now() |> DateTime.add(-hours * 60 * 60, :second) end + + defp insert_contact_and_notification_subscription do + %DB.Contact{id: contact_id, email: email} = contact = insert_contact() + + %DB.NotificationSubscription{id: ns_id} = + insert(:notification_subscription, %{ + reason: :new_dataset, + source: :user, + role: :reuser, + contact_id: contact_id + }) + + {contact, contact_id, email, ns_id} + end end diff --git a/config/runtime.exs b/config/runtime.exs index af99631d5f..ade7b26480 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -134,7 +134,8 @@ oban_prod_crontab = [ {"0 6 * * 1-5", Transport.Jobs.NewDatagouvDatasetsJob, args: %{check_rules: true}}, {"5 6 * * 1-5", Transport.Jobs.NewDatagouvDatasetsJob}, {"0 6 * * *", Transport.Jobs.NewDatasetNotificationsJob}, - {"30 6 * * *", Transport.Jobs.ExpirationNotificationJob}, + {"30 6 * * *", Transport.Jobs.ExpirationAdminProducerNotificationJob}, + {"45 6 * * *", Transport.Jobs.ExpirationNotificationJob}, {"0 8 * * 1-5", Transport.Jobs.NewCommentsNotificationJob}, {"0 21 * * *", Transport.Jobs.DatasetHistoryDispatcherJob}, # Should be executed after all `DatasetHistoryJob` have been executed From 7783db7964fb388ad1f37850f81484de5c2b6775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Menou?= Date: Mon, 20 Jan 2025 15:30:16 +0100 Subject: [PATCH 5/6] =?UTF-8?q?Registre=20d'arr=C3=AAts=20:=20premi=C3=A8r?= =?UTF-8?q?e=20=C3=A9bauche=20(#4393)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/shared/lib/wrapper/wrapper_req.ex | 25 ++-- apps/transport/lib/gtfs/utils.ex | 87 ++++++++++++++ apps/transport/lib/jobs/gtfs_to_db.ex | 65 ++-------- .../lib/netex/netex_archive_parser.ex | 47 ++++++-- apps/transport/lib/registry/engine.ex | 111 ++++++++++++++++++ apps/transport/lib/registry/extractor.ex | 14 +++ apps/transport/lib/registry/gtfs.ex | 98 ++++++++++++++++ .../lib/registry/model/data_source.ex | 23 ++++ apps/transport/lib/registry/model/stop.ex | 103 ++++++++++++++++ apps/transport/lib/registry/netex.ex | 55 +++++++++ apps/transport/lib/registry/result.ex | 33 ++++++ apps/transport/test/gtfs/utils_test.exs | 4 + .../test/netex/netex_archive_parser_test.exs | 9 ++ apps/transport/test/registry/gtfs_test.exs | 4 + apps/transport/test/registry/model_test.exs | 6 + apps/transport/test/registry/result_test.exs | 29 +++++ scripts/registre-arrets.exs | 1 + 17 files changed, 642 insertions(+), 72 deletions(-) create mode 100644 apps/transport/lib/gtfs/utils.ex create mode 100644 apps/transport/lib/registry/engine.ex create mode 100644 apps/transport/lib/registry/extractor.ex create mode 100644 apps/transport/lib/registry/gtfs.ex create mode 100644 apps/transport/lib/registry/model/data_source.ex create mode 100644 apps/transport/lib/registry/model/stop.ex create mode 100644 apps/transport/lib/registry/netex.ex create mode 100644 apps/transport/lib/registry/result.ex create mode 100644 apps/transport/test/gtfs/utils_test.exs create mode 100644 apps/transport/test/registry/gtfs_test.exs create mode 100644 apps/transport/test/registry/model_test.exs create mode 100644 apps/transport/test/registry/result_test.exs create mode 100644 scripts/registre-arrets.exs diff --git a/apps/shared/lib/wrapper/wrapper_req.ex b/apps/shared/lib/wrapper/wrapper_req.ex index 17a7e9c6ca..b8fbe93ce5 100644 --- a/apps/shared/lib/wrapper/wrapper_req.ex +++ b/apps/shared/lib/wrapper/wrapper_req.ex @@ -35,6 +35,18 @@ defmodule Transport.HTTPClient do """ def get!(url, options) do + {req, options} = setup_cache(options) + + Transport.Req.impl().get!(req, options |> Keyword.merge(url: url)) + end + + def get(url, options) do + {req, options} = setup_cache(options) + + Transport.Req.impl().get(req, options |> Keyword.merge(url: url)) + end + + defp setup_cache(options) do options = Keyword.validate!(options, [ :custom_cache_dir, @@ -48,13 +60,10 @@ defmodule Transport.HTTPClient do {enable_cache, options} = options |> Keyword.pop!(:enable_cache) - req = - if enable_cache do - req |> Transport.Shared.ReqCustomCache.attach() - else - req - end - - Transport.Req.impl().get!(req, options |> Keyword.merge(url: url)) + if enable_cache do + {req |> Transport.Shared.ReqCustomCache.attach(), options} + else + {req, options} + end end end diff --git a/apps/transport/lib/gtfs/utils.ex b/apps/transport/lib/gtfs/utils.ex new file mode 100644 index 0000000000..27ddc49e7e --- /dev/null +++ b/apps/transport/lib/gtfs/utils.ex @@ -0,0 +1,87 @@ +defmodule Transport.GTFS.Utils do + @moduledoc """ + Some helpers for handling GTFS archives. + """ + + def fetch_position(record, field) do + Map.fetch!(record, field) |> convert_text_to_float() + end + + @doc """ + Convert textual values to float. + + iex> convert_text_to_float("") + nil + iex> convert_text_to_float("0") + 0.0 + iex> convert_text_to_float("0.0") + 0.0 + iex> convert_text_to_float("12.7") + 12.7 + iex> convert_text_to_float("-12.7") + -12.7 + iex> convert_text_to_float(" -48.7 ") + -48.7 + """ + def convert_text_to_float(input) do + if input |> String.trim() != "" do + input |> String.trim() |> Decimal.new() |> Decimal.to_float() + else + nil + end + end + + @doc """ + Variant of csv_get_with_default/3 that raises if a mandatory column is missing. + """ + def csv_get_with_default!(map, field, default_value, mandatory_column \\ true) do + value = if mandatory_column, do: Map.fetch!(map, field), else: Map.get(map, field) + + case value do + nil -> default_value + "" -> default_value + v -> v + end + end + + @doc """ + iex> csv_get_with_default(%{}, "field", 0) + 0 + iex> csv_get_with_default(%{"other_field" => 1}, "field", 0) + 0 + iex> csv_get_with_default(%{"field" => 2, "other_field" => 1}, "field", 0) + 2 + iex> csv_get_with_default(%{"field" => "", "other_field" => 1}, "field", 0) + 0 + """ + def csv_get_with_default(map, field, default_value) do + value = Map.get(map, field) + + case value do + nil -> default_value + "" -> default_value + v -> v + end + end + + @doc """ + Transform the stream outputed by Unzip to a stream of maps, each map + corresponding to a row from the CSV. + """ + def to_stream_of_maps(file_stream) do + file_stream + # transform the stream to a stream of binaries + |> Stream.map(fn c -> IO.iodata_to_binary(c) end) + # stream line by line + |> NimbleCSV.RFC4180.to_line_stream() + |> NimbleCSV.RFC4180.parse_stream(skip_headers: false) + # transform the stream to a stream of maps %{column_name1: value1, ...} + |> Stream.transform([], fn r, acc -> + if acc == [] do + {%{}, r |> Enum.map(fn h -> h |> String.replace_prefix("\uFEFF", "") end)} + else + {[acc |> Enum.zip(r) |> Enum.into(%{})], acc} + end + end) + end +end diff --git a/apps/transport/lib/jobs/gtfs_to_db.ex b/apps/transport/lib/jobs/gtfs_to_db.ex index 2b64b38c9f..bc05b4dba7 100644 --- a/apps/transport/lib/jobs/gtfs_to_db.ex +++ b/apps/transport/lib/jobs/gtfs_to_db.ex @@ -3,33 +3,7 @@ defmodule Transport.Jobs.GtfsToDB do Get the content of a GTFS ResourceHistory, store it in the DB """ - @doc """ - Convert textual values to float. - - iex> convert_text_to_float("0") - 0.0 - iex> convert_text_to_float("0.0") - 0.0 - iex> convert_text_to_float("12.7") - 12.7 - iex> convert_text_to_float("-12.7") - -12.7 - iex> convert_text_to_float(" -48.7 ") - -48.7 - """ - def convert_text_to_float(input) do - input |> String.trim() |> Decimal.new() |> Decimal.to_float() - end - - def csv_get_with_default!(map, field, default_value, mandatory_column \\ true) do - value = if mandatory_column, do: Map.fetch!(map, field), else: Map.get(map, field) - - case value do - nil -> default_value - "" -> default_value - v -> v - end - end + alias Transport.GTFS.Utils def import_gtfs_from_resource_history(resource_history_id) do %{id: data_import_id} = %DB.DataImport{resource_history_id: resource_history_id} |> DB.Repo.insert!() @@ -61,16 +35,16 @@ defmodule Transport.Jobs.GtfsToDB do def stops_stream_insert(file_stream, data_import_id) do DB.Repo.transaction(fn -> file_stream - |> to_stream_of_maps() + |> Utils.to_stream_of_maps() # the map is reshaped for Ecto's needs |> Stream.map(fn r -> %{ data_import_id: data_import_id, stop_id: r |> Map.fetch!("stop_id"), stop_name: r |> Map.fetch!("stop_name"), - stop_lat: r |> Map.fetch!("stop_lat") |> convert_text_to_float(), - stop_lon: r |> Map.fetch!("stop_lon") |> convert_text_to_float(), - location_type: r |> csv_get_with_default!("location_type", "0", false) |> String.to_integer() + stop_lat: r |> Utils.fetch_position("stop_lat"), + stop_lon: r |> Utils.fetch_position("stop_lon"), + location_type: r |> Utils.csv_get_with_default!("location_type", "0", false) |> String.to_integer() } end) |> Stream.chunk_every(1000) @@ -79,27 +53,6 @@ defmodule Transport.Jobs.GtfsToDB do end) end - @doc """ - Transform the stream outputed by Unzip to a stream of maps, each map - corresponding to a row from the CSV. - """ - def to_stream_of_maps(file_stream) do - file_stream - # transform the stream to a stream of binaries - |> Stream.map(fn c -> IO.iodata_to_binary(c) end) - # stream line by line - |> NimbleCSV.RFC4180.to_line_stream() - |> NimbleCSV.RFC4180.parse_stream(skip_headers: false) - # transform the stream to a stream of maps %{column_name1: value1, ...} - |> Stream.transform([], fn r, acc -> - if acc == [] do - {%{}, r |> Enum.map(fn h -> h |> String.replace_prefix("\uFEFF", "") end)} - else - {[acc |> Enum.zip(r) |> Enum.into(%{})], acc} - end - end) - end - def fill_calendar_from_resource_history(resource_history_id, data_import_id) do file_stream = file_stream(resource_history_id, "calendar.txt") calendar_stream_insert(file_stream, data_import_id) @@ -108,7 +61,7 @@ defmodule Transport.Jobs.GtfsToDB do def calendar_stream_insert(file_stream, data_import_id) do DB.Repo.transaction(fn -> file_stream - |> to_stream_of_maps() + |> Utils.to_stream_of_maps() |> Stream.map(fn r -> res = %{ data_import_id: data_import_id, @@ -155,7 +108,7 @@ defmodule Transport.Jobs.GtfsToDB do DB.Repo.transaction( fn -> file_stream - |> to_stream_of_maps() + |> Utils.to_stream_of_maps() |> Stream.map(fn r -> %{ data_import_id: data_import_id, @@ -209,7 +162,7 @@ defmodule Transport.Jobs.GtfsToDB do DB.Repo.transaction( fn -> file_stream - |> to_stream_of_maps() + |> Utils.to_stream_of_maps() |> Stream.map(fn r -> %{ data_import_id: data_import_id, @@ -235,7 +188,7 @@ defmodule Transport.Jobs.GtfsToDB do DB.Repo.transaction( fn -> file_stream - |> to_stream_of_maps() + |> Utils.to_stream_of_maps() |> Stream.map(fn r -> %{ data_import_id: data_import_id, diff --git a/apps/transport/lib/netex/netex_archive_parser.ex b/apps/transport/lib/netex/netex_archive_parser.ex index 40b2d95fe8..d95e038425 100644 --- a/apps/transport/lib/netex/netex_archive_parser.ex +++ b/apps/transport/lib/netex/netex_archive_parser.ex @@ -20,16 +20,16 @@ defmodule Transport.NeTEx do # Entry names ending with a slash `/` are directories. Skip them. # https://github.com/akash-akya/unzip/blob/689a1ca7a134ab2aeb79c8c4f8492d61fa3e09a0/lib/unzip.ex#L69 String.ends_with?(file_name, "/") -> - [] + {:ok, []} extension |> String.downcase() == ".zip" -> - raise "Insupported zip inside zip for file #{file_name}" + {:error, "Insupported zip inside zip for file #{file_name}"} extension |> String.downcase() != ".xml" -> - raise "Insupported file extension (#{extension}) for file #{file_name}" + {:error, "Insupported file extension (#{extension}) for file #{file_name}"} true -> - {:ok, state} = + parsing_result = unzip |> Unzip.file_stream!(file_name) |> Stream.map(&IO.iodata_to_binary(&1)) @@ -42,7 +42,21 @@ defmodule Transport.NeTEx do end }) - state.stop_places + case parsing_result do + {:ok, state} -> {:ok, state.stop_places} + {:error, exception} -> {:error, Exception.message(exception)} + {:halt, _state, _rest} -> {:error, "SAX parsing interrupted unexpectedly."} + end + end + end + + @doc """ + Like read_stop_places/2 but raises on errors. + """ + def read_stop_places!(%Unzip{} = unzip, file_name) do + case read_stop_places(unzip, file_name) do + {:ok, stop_places} -> stop_places + {:error, message} -> raise message end end @@ -53,8 +67,14 @@ defmodule Transport.NeTEx do zip_file = Unzip.LocalFile.open(zip_file_name) try do - {:ok, unzip} = Unzip.new(zip_file) - cb.(unzip) + case Unzip.new(zip_file) do + {:ok, unzip} -> + cb.(unzip) + + {:error, message} -> + Logger.error("Error while reading #{zip_file_name}: #{message}") + [] + end after Unzip.LocalFile.close(zip_file) end @@ -67,6 +87,17 @@ defmodule Transport.NeTEx do See tests for actual output. Will be refactored soonish. """ def read_all_stop_places(zip_file_name) do + read_all(zip_file_name, &read_stop_places/2) + end + + @doc """ + Like read_all_stop_places/1 but raises on error. + """ + def read_all_stop_places!(zip_file_name) do + read_all(zip_file_name, &read_stop_places!/2) + end + + defp read_all(zip_file_name, reader) do with_zip_file_handle(zip_file_name, fn unzip -> unzip |> Unzip.list_entries() @@ -75,7 +106,7 @@ defmodule Transport.NeTEx do { metadata.file_name, - read_stop_places(unzip, metadata.file_name) + reader.(unzip, metadata.file_name) } end) end) diff --git a/apps/transport/lib/registry/engine.ex b/apps/transport/lib/registry/engine.ex new file mode 100644 index 0000000000..bd0f72af12 --- /dev/null +++ b/apps/transport/lib/registry/engine.ex @@ -0,0 +1,111 @@ +defmodule Transport.Registry.Engine do + @moduledoc """ + Stream eligible resources and run extractors to produce a raw registry at the end. + """ + + alias Transport.Registry.GTFS + alias Transport.Registry.Model.DataSource + alias Transport.Registry.Model.Stop + alias Transport.Registry.NeTEx + alias Transport.Registry.Result + + import Ecto.Query + + require Logger + + @type option :: {:limit, integer()} | {:formats, [String.t()]} + + @doc """ + execute("/tmp/registre-arrets.csv", formats: ~w(GTFS NeTEx), limit: 100) + """ + @spec execute(output_file :: Path.t(), opts :: [option]) :: :ok + def execute(output_file, opts \\ []) do + limit = Keyword.get(opts, :limit, 1_000_000) + formats = Keyword.get(opts, :formats, ~w(GTFS NeTEx)) + + create_empty_csv_with_headers(output_file) + + enumerate_gtfs_resources(limit, formats) + |> Result.map_result(&prepare_extractor/1) + |> Task.async_stream(&download/1, max_concurrency: 12, timeout: 30 * 60_000) + # one for Task.async_stream + |> Result.cat_results() + # one for download/1 + |> Result.cat_results() + |> Result.map_result(&extract_from_archive/1) + |> dump_to_csv(output_file) + end + + def create_empty_csv_with_headers(output_file) do + headers = NimbleCSV.RFC4180.dump_to_iodata([Stop.csv_headers()]) + File.write(output_file, headers) + end + + def enumerate_gtfs_resources(limit, formats) do + DB.Resource.base_query() + |> DB.ResourceHistory.join_resource_with_latest_resource_history() + |> where([resource: r], r.format in ^formats) + |> preload([resource_history: rh], resource_history: rh) + |> limit(^limit) + |> DB.Repo.all() + end + + def prepare_extractor(%DB.Resource{} = resource) do + data_source_id = "datagouv:resource:#{resource.datagouv_id}" + + case resource.format do + "GTFS" -> {:ok, {GTFS, data_source_id, resource.url}} + "NeTEx" -> {:ok, {NeTEx, data_source_id, resource.url}} + _ -> {:error, "Unsupported format"} + end + end + + def download({extractor, data_source_id, url}) do + Logger.debug("download #{extractor} #{data_source_id} #{url}") + tmp_path = System.tmp_dir!() |> Path.join("#{Ecto.UUID.generate()}.dat") + + safe_error = fn msg -> + File.rm(tmp_path) + Result.error(msg) + end + + http_result = + Transport.HTTPClient.get(url, + decode_body: false, + compressed: false, + into: File.stream!(tmp_path) + ) + + case http_result do + {:error, error} -> + safe_error.("Unexpected error while downloading the resource from #{url}: #{Exception.message(error)}") + + {:ok, %{status: status}} -> + cond do + status >= 200 && status < 300 -> + {:ok, {extractor, data_source_id, tmp_path}} + + status > 400 -> + safe_error.("Error #{status} while downloading the resource from #{url}") + + true -> + safe_error.("Unexpected HTTP error #{status} while downloading the resource from #{url}") + end + end + end + + @spec extract_from_archive({module(), DataSource.data_source_id(), Path.t()}) :: Result.t([Stop.t()]) + def extract_from_archive({extractor, data_source_id, file}) do + Logger.debug("extract_from_archive #{extractor} #{data_source_id} #{file}") + extractor.extract_from_archive(data_source_id, file) + end + + def dump_to_csv(enumerable, output_file) do + enumerable + |> Stream.concat() + |> Stream.map(&Stop.to_csv/1) + |> NimbleCSV.RFC4180.dump_to_stream() + |> Stream.into(File.stream!(output_file, [:append, :utf8])) + |> Stream.run() + end +end diff --git a/apps/transport/lib/registry/extractor.ex b/apps/transport/lib/registry/extractor.ex new file mode 100644 index 0000000000..fa5f0da5a5 --- /dev/null +++ b/apps/transport/lib/registry/extractor.ex @@ -0,0 +1,14 @@ +defmodule Transport.Registry.Extractor do + @moduledoc """ + Interface and utilities for stops extractors. + """ + + require Logger + + alias Transport.Registry.Model.DataSource + alias Transport.Registry.Model.Stop + alias Transport.Registry.Result + + @callback extract_from_archive(data_source_id :: DataSource.data_source_id(), path :: Path.t()) :: + Result.t([Stop.t()]) +end diff --git a/apps/transport/lib/registry/gtfs.ex b/apps/transport/lib/registry/gtfs.ex new file mode 100644 index 0000000000..57351e8ce6 --- /dev/null +++ b/apps/transport/lib/registry/gtfs.ex @@ -0,0 +1,98 @@ +defmodule Transport.Registry.GTFS do + @moduledoc """ + Implementation of a stop extractor for GTFS resources. + """ + + alias Transport.Registry.Model.Stop + alias Transport.Registry.Model.StopIdentifier + alias Transport.Registry.Result + + alias Transport.GTFS.Utils + + require Logger + + @behaviour Transport.Registry.Extractor + @doc """ + Extract stops from GTFS ressource. + """ + def extract_from_archive(data_source_id, archive) do + case file_stream(archive) do + {:error, error} -> + Logger.error(error) + Result.error(error) + + {:ok, {content, zip_file}} -> + Logger.debug("Valid Zip archive") + + try do + content + |> Utils.to_stream_of_maps() + |> Stream.flat_map(&handle_stop(data_source_id, &1)) + |> Enum.to_list() + |> Result.ok() + after + Unzip.LocalFile.close(zip_file) + rescue + e in NimbleCSV.ParseError -> + e + |> Exception.message() + |> Result.error() + end + end + end + + defp handle_stop(data_source_id, record) do + latitude = Utils.fetch_position(record, "stop_lat") + longitude = Utils.fetch_position(record, "stop_lon") + + if latitude != nil && longitude != nil do + [ + %Stop{ + main_id: %StopIdentifier{id: Map.fetch!(record, "stop_id"), type: :main}, + display_name: Map.fetch!(record, "stop_name"), + latitude: latitude, + longitude: longitude, + projection: :utm_wgs84, + stop_type: record |> Utils.csv_get_with_default("location_type", "0") |> to_stop_type(), + data_source_format: :gtfs, + data_source_id: data_source_id + } + ] + else + [] + end + end + + defp to_stop_type("0"), do: :quay + defp to_stop_type("1"), do: :stop + defp to_stop_type(_), do: :other + + defp file_stream(archive) do + zip_file = Unzip.LocalFile.open(archive) + + case Unzip.new(zip_file) do + {:ok, unzip} -> + if has_stops?(unzip) do + content = unzip |> Unzip.file_stream!("stops.txt") + # The zip_file is kept open for now as it's consumed later. + Result.ok({content, zip_file}) + else + Unzip.LocalFile.close(zip_file) + Result.error("Missing stops.txt in #{archive}") + end + + {:error, error} -> + Result.error("Error while unzipping archive #{archive}: #{error}") + end + end + + defp has_stops?(unzip) do + unzip + |> Unzip.list_entries() + |> Enum.any?(&entry_of_name?("stops.txt", &1)) + end + + defp entry_of_name?(name, %Unzip.Entry{file_name: file_name}) do + file_name == name + end +end diff --git a/apps/transport/lib/registry/model/data_source.ex b/apps/transport/lib/registry/model/data_source.ex new file mode 100644 index 0000000000..d985325ce9 --- /dev/null +++ b/apps/transport/lib/registry/model/data_source.ex @@ -0,0 +1,23 @@ +defmodule Transport.Registry.Model.DataSource do + @moduledoc """ + Common attributes describing a data source. + """ + + defstruct [ + :id, + :checksum, + :last_updated_at, + :validity_period + ] + + @type t :: %__MODULE__{ + id: data_source_id(), + checksum: binary(), + last_updated_at: DateTime.t(), + validity_period: date_time_range() + } + + @type data_source_id :: binary() + + @type date_time_range :: binary() +end diff --git a/apps/transport/lib/registry/model/stop.ex b/apps/transport/lib/registry/model/stop.ex new file mode 100644 index 0000000000..3e9c9b84d4 --- /dev/null +++ b/apps/transport/lib/registry/model/stop.ex @@ -0,0 +1,103 @@ +defmodule Transport.Registry.Model.StopIdentifier do + @moduledoc """ + Representation of a Stop ID. + """ + + defstruct [ + :id, + :type + ] + + @type t :: %__MODULE__{ + id: binary(), + type: identifier_type() + } + + @type identifier_type :: :main | :private_code | :stop_code | :other + + def main(id), do: %__MODULE__{type: :main, id: id} + + @doc """ + iex> to_field(%Transport.Registry.Model.StopIdentifier{id: "stop1", type: :main}) + "main:stop1" + iex> to_field(%Transport.Registry.Model.StopIdentifier{id: "FRPLY", type: :private_code}) + "private_code:FRPLY" + iex> to_field(%Transport.Registry.Model.StopIdentifier{id: "PARIS GDL", type: :other}) + "other:PARIS GDL" + """ + def to_field(%__MODULE__{id: id, type: type}) do + "#{type}:#{id}" + end +end + +defmodule Transport.Registry.Model.Stop do + @moduledoc """ + Common attributes describing a stop. + """ + alias Transport.Registry.Model.DataSource + alias Transport.Registry.Model.StopIdentifier + + defstruct [ + :main_id, + :display_name, + :data_source_id, + :data_source_format, + :parent_id, + :latitude, + :longitude, + projection: :utm_wgs84, + stop_type: :stop, + secondary_ids: [] + ] + + @type t :: %__MODULE__{ + main_id: StopIdentifier.t(), + display_name: binary(), + data_source_id: DataSource.data_source_id(), + data_source_format: data_source_format_type(), + parent_id: StopIdentifier.t() | nil, + latitude: float(), + longitude: float(), + projection: projection(), + stop_type: stop_type(), + secondary_ids: [StopIdentifier.t()] + } + + @type data_source_format_type :: :gtfs | :netex + + @type stop_type :: :stop | :quay | :other + + @type projection :: :utm_wgs84 | :lambert93_rgf93 + + def csv_headers do + ~w( + main_id + display_name + data_source_id + data_source_format + parent_id + latitude + longitude + projection + stop_type + ) + end + + def to_csv(%__MODULE__{} = stop) do + [ + StopIdentifier.to_field(stop.main_id), + stop.display_name, + stop.data_source_id, + stop.data_source_format, + maybe(stop.parent_id, &StopIdentifier.to_field/1, ""), + stop.latitude, + stop.longitude, + stop.projection, + stop.stop_type + ] + end + + @spec maybe(value :: any() | nil, mapper :: (any() -> any()), defaultValue :: any()) :: any() | nil + def maybe(nil, _, defaultValue), do: defaultValue + def maybe(value, mapper, _), do: mapper.(value) +end diff --git a/apps/transport/lib/registry/netex.ex b/apps/transport/lib/registry/netex.ex new file mode 100644 index 0000000000..aa8dcf321d --- /dev/null +++ b/apps/transport/lib/registry/netex.ex @@ -0,0 +1,55 @@ +defmodule Transport.Registry.NeTEx do + @moduledoc """ + Implementation of a stop extractor for NeTEx resources. + """ + + alias Transport.Registry.Model.Stop + alias Transport.Registry.Model.StopIdentifier + alias Transport.Registry.Result + + require Logger + + @behaviour Transport.Registry.Extractor + @doc """ + Extract stops from a NeTEx archive. + """ + def extract_from_archive(data_source_id, archive) do + archive + |> Transport.NeTEx.read_all_stop_places() + |> Enum.flat_map(&process_stop_places(data_source_id, &1)) + |> Result.ok() + end + + defp process_stop_places(data_source_id, {_filename, {:ok, stop_places}}) do + stop_places |> Enum.map(&to_stop(data_source_id, &1)) |> Result.cat_results() + end + + defp process_stop_places(_data_source_id, {filename, {:error, message}}) do + Logger.error("Processing of #{filename}, error: #{message}") + [] + end + + defp to_stop(data_source_id, %{id: id, name: name, latitude: latitude, longitude: longitude}) do + %Stop{ + main_id: StopIdentifier.main(id), + display_name: name, + latitude: latitude, + longitude: longitude, + data_source_format: :netex, + data_source_id: data_source_id + } + |> Result.ok() + end + + defp to_stop(_data_source_id, incomplete_record) do + expected_keys = MapSet.new(~w(id name latitude longitude)) + keys = MapSet.new(Map.keys(incomplete_record)) + + missing_keys = MapSet.difference(expected_keys, keys) |> Enum.to_list() + + message = "Can't build stop, missing keys: #{inspect(missing_keys)}" + + Logger.error(message) + Result.error(message) + end +end diff --git a/apps/transport/lib/registry/result.ex b/apps/transport/lib/registry/result.ex new file mode 100644 index 0000000000..e87f61e274 --- /dev/null +++ b/apps/transport/lib/registry/result.ex @@ -0,0 +1,33 @@ +defmodule Transport.Registry.Result do + @moduledoc """ + Type and utilities to represent results. + """ + require Integer + + @type t(positive) :: {:ok, positive} | {:error, binary()} + + def ok(positive), do: {:ok, positive} + + def error(message), do: {:error, message} + + @doc """ + iex> [{:ok, "valid"}, {:error, "invalid"}, {:ok, "relevant"}] |> cat_results() + ["valid", "relevant"] + """ + @spec cat_results(Stream.t(t(term()))) :: Stream.t(term()) + def cat_results(enumerable), do: Stream.flat_map(enumerable, &keep_ok/1) + + defp keep_ok({:ok, result}), do: [result] + defp keep_ok(_), do: [] + + @doc """ + iex> 1..10 |> map_result(fn v -> if Integer.is_odd(v) do {:ok, v} else {:error, "Even Steven"} end end) + [1, 3, 5, 7, 9] + """ + @spec map_result(Stream.t(term()), (term() -> t(term()))) :: Stream.t(term()) + def map_result(enumerable, mapper) do + enumerable + |> Stream.map(mapper) + |> cat_results() + end +end diff --git a/apps/transport/test/gtfs/utils_test.exs b/apps/transport/test/gtfs/utils_test.exs new file mode 100644 index 0000000000..1e232cf18c --- /dev/null +++ b/apps/transport/test/gtfs/utils_test.exs @@ -0,0 +1,4 @@ +defmodule Transport.GTFS.UtilsTest do + use ExUnit.Case, async: true + doctest Transport.GTFS.Utils, import: true +end diff --git a/apps/transport/test/netex/netex_archive_parser_test.exs b/apps/transport/test/netex/netex_archive_parser_test.exs index a619424ad8..1c7c93d641 100644 --- a/apps/transport/test/netex/netex_archive_parser_test.exs +++ b/apps/transport/test/netex/netex_archive_parser_test.exs @@ -43,6 +43,15 @@ defmodule Transport.NeTEx.ArchiveParserTest do # given a zip netex archive containing 1 file, I want the output I expected [{"arrets.xml", data}] = Transport.NeTEx.read_all_stop_places(tmp_file) + assert data == + {:ok, + [ + %{id: "FR:HELLO:POYARTIN:001", latitude: 43.669, longitude: -0.919, name: "Poyartin"} + ]} + + # given a zip netex archive containing 1 file, I want the output I expected + [{"arrets.xml", data}] = Transport.NeTEx.read_all_stop_places!(tmp_file) + assert data == [ %{id: "FR:HELLO:POYARTIN:001", latitude: 43.669, longitude: -0.919, name: "Poyartin"} ] diff --git a/apps/transport/test/registry/gtfs_test.exs b/apps/transport/test/registry/gtfs_test.exs new file mode 100644 index 0000000000..8102b43347 --- /dev/null +++ b/apps/transport/test/registry/gtfs_test.exs @@ -0,0 +1,4 @@ +defmodule Transport.Registry.GTFSTest do + use ExUnit.Case, async: true + doctest Transport.Registry.GTFS, import: true +end diff --git a/apps/transport/test/registry/model_test.exs b/apps/transport/test/registry/model_test.exs new file mode 100644 index 0000000000..23ff7b266a --- /dev/null +++ b/apps/transport/test/registry/model_test.exs @@ -0,0 +1,6 @@ +defmodule Transport.Registry.ModelTest do + use ExUnit.Case, async: true + doctest Transport.Registry.Model.DataSource, import: true + doctest Transport.Registry.Model.Stop, import: true + doctest Transport.Registry.Model.StopIdentifier, import: true +end diff --git a/apps/transport/test/registry/result_test.exs b/apps/transport/test/registry/result_test.exs new file mode 100644 index 0000000000..7c500d3186 --- /dev/null +++ b/apps/transport/test/registry/result_test.exs @@ -0,0 +1,29 @@ +defmodule Transport.Registry.ResultTest do + use ExUnit.Case, async: false + + require Integer + alias Transport.Registry.Result + doctest Result + + test "cat_results" do + assert [] == cat_results([]) + assert [] == cat_results([{:error, "Error message"}]) + assert [1, 3] == cat_results([{:ok, 1}, {:error, "Error message"}, {:ok, 3}]) + end + + test "map_result" do + assert [] == map_result([], &even_is_forbidden/1) + assert [1, 3, 5, 7, 9] == map_result(1..10, &even_is_forbidden/1) + end + + defp cat_results(enumerable) do + enumerable |> Result.cat_results() |> Enum.to_list() + end + + defp map_result(enumerable, mapper) do + enumerable |> Result.map_result(mapper) |> Enum.to_list() + end + + defp even_is_forbidden(i) when Integer.is_odd(i), do: {:ok, i} + defp even_is_forbidden(_), do: {:error, "Even is forbidden"} +end diff --git a/scripts/registre-arrets.exs b/scripts/registre-arrets.exs new file mode 100644 index 0000000000..d02e89cdce --- /dev/null +++ b/scripts/registre-arrets.exs @@ -0,0 +1 @@ +Transport.Registry.Engine.execute("./registre-arrets.csv") From f7e359ade0409c68e01dcb389bf1b98100d1a1fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Pignal?= <162135418+stephane-pignal@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:38:01 +0100 Subject: [PATCH 6/6] Suppression mot clef electrique (#4419) --- apps/transport/lib/jobs/new_datagouv_datasets_job.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/transport/lib/jobs/new_datagouv_datasets_job.ex b/apps/transport/lib/jobs/new_datagouv_datasets_job.ex index 879cab2b64..ea7e5e4f8c 100644 --- a/apps/transport/lib/jobs/new_datagouv_datasets_job.ex +++ b/apps/transport/lib/jobs/new_datagouv_datasets_job.ex @@ -68,8 +68,7 @@ defmodule Transport.Jobs.NewDatagouvDatasetsJob do "infrastructure de recharge", "borne de recharge", "irve", - "sdirve", - "électrique" + "sdirve" ]), formats: MapSet.new([]) }