Send failures alerts (per standard module/policies) on lost runs (#2924)

* close #2921 * remove duplicate steps update for lost runs * Revert "remove duplicate steps update for lost runs" This reverts commit a5954b4. --------- Co-authored-by: Frank Midigo <[email protected]>
OpenFn · Feb 12, 2025 · 30cd216 · 30cd216
1 parent 8ec97c0
commit 30cd216
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 1 deletion.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,9 @@ and this project adheres to
 
 ### Fixed
 
+- Fixed issue where we failed to send failure alerts via email when runs were
+  "lost". [#2921](https://github.com/OpenFn/lightning/issues/2921)
+
 ## [v2.10.14] - 2025-02-06
 
 ### Added

diff --git a/lib/lightning/janitor.ex b/lib/lightning/janitor.ex
@@ -40,9 +40,15 @@ defmodule Lightning.Janitor do
       stream
       |> Stream.each(fn run ->
         Runs.mark_run_lost(run)
+
+        run
+        |> Repo.preload([:log_lines, work_order: [:workflow]])
+        |> Lightning.FailureAlerter.alert_on_failure()
       end)
       |> Stream.run()
 
+      # TODO - this appears to be a duplicate of https://github.com/OpenFn/lightning/blob/main/lib/lightning/runs.ex#L291
+      # but without it, the tests don't pass. @midigofrank, do you have ideas?
       Runs.Query.lost_steps() |> Runs.mark_steps_lost()
     end)
   end

diff --git a/test/lightning/janitor_test.exs b/test/lightning/janitor_test.exs
@@ -10,7 +10,14 @@ defmodule Lightning.JanitorTest do
   describe "find_and_update_lost/0" do
     @tag :capture_log
     test "updates lost runs and their steps" do
-      %{triggers: [trigger]} = workflow = insert(:simple_workflow)
+      project =
+        insert(:project,
+          project_users: [%{user: build(:user), failure_alert: true}]
+        )
+
+      %{triggers: [trigger]} =
+        workflow = insert(:simple_workflow, project: project, name: "pain perdu")
+
       dataclip = insert(:dataclip)
 
       an_hour_in_seconds = 3600
@@ -93,6 +100,9 @@ defmodule Lightning.JanitorTest do
       # should be marked lost, despite having a long runtime
       reloaded_lost_long_run = Repo.get(Run, lost_long_run.id)
       assert reloaded_lost_long_run.state == :lost
+
+      assert_receive {:email, %Swoosh.Email{subject: "\"pain perdu\" failed."}},
+                     1000
     end
 
     test "updates steps whose run has finished but step hasn't" do