From 83443d20e88524ff919ccad31204aa4334205545 Mon Sep 17 00:00:00 2001
From: Rogerio Pontual <44991200+jyeshe@users.noreply.github.com>
Date: Thu, 27 Feb 2025 13:44:19 +0100
Subject: [PATCH] Disable jobs with non retriable steps (#2968)

* Disable jobs with non retriable steps

* Add test case for disabled non retriable jobs

* Changelog

* Pass a disabled set to UI

* Fix the displayed count when some selected work orders are not retriable
---
 CHANGELOG.md                                  |   3 +
 lib/lightning/accounts/user_notifier.ex       |   4 +-
 lib/lightning/work_orders.ex                  |  72 ++-
 lib/lightning_web/live/run_live/index.ex      |   4 -
 .../live/run_live/index.html.heex             |   5 +-
 .../live/run_live/rerun_job_component.ex      |  81 ++-
 lib/lightning_web/utils.ex                    |   3 +
 .../live/run_live/index_test.exs              | 474 ++++++++++++++++++
 .../live/work_order_live_test.exs             | 388 --------------
 9 files changed, 590 insertions(+), 444 deletions(-)
 create mode 100644 test/lightning_web/live/run_live/index_test.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1ad50b379..a8cb87122f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,6 +17,9 @@ and this project adheres to
 
 ### Added
 
+- Disable jobs with non retriable steps
+  [#2925](https://github.com/OpenFn/lightning/issues/2925)
+
 ### Changed
 
 ### Fixed
diff --git a/lib/lightning/accounts/user_notifier.ex b/lib/lightning/accounts/user_notifier.ex
index 26145ec078..7a134d5cd7 100644
--- a/lib/lightning/accounts/user_notifier.ex
+++ b/lib/lightning/accounts/user_notifier.ex
@@ -10,6 +10,7 @@ defmodule Lightning.Accounts.UserNotifier do
     max_attempts: 1
 
   import Swoosh.Email
+  import LightningWeb.Utils, only: [pluralize_with_s: 2]
 
   alias Lightning.Accounts.User
   alias Lightning.Helpers
@@ -413,9 +414,6 @@ defmodule Lightning.Accounts.UserNotifier do
     "THIS LIGHTNING INSTANCE DOES NOT HAVE ALTERNATE STORAGE ENABLED, SO THESE FAILED MESSAGES CANNOT BE RECOVERED WITHOUT MAKING THEM AVAILABLE ON THE KAFKA CLUSTER AGAIN."
   end
 
-  defp pluralize_with_s(1, string), do: string
-  defp pluralize_with_s(_integer, string), do: "#{string}s"
-
   @doc """
   Deliver instructions to confirm a credential transfer.
   """
diff --git a/lib/lightning/work_orders.ex b/lib/lightning/work_orders.ex
index 4687984ba6..d7011d28d1 100644
--- a/lib/lightning/work_orders.ex
+++ b/lib/lightning/work_orders.ex
@@ -402,32 +402,9 @@ defmodule Lightning.WorkOrders do
           | UsageLimiting.error()
           | {:error, :enqueue_error}
   def retry_many([%WorkOrder{} | _rest] = workorders, job_id, opts) do
-    orders_ids = Enum.map(workorders, & &1.id)
-
-    last_runs_query =
-      from(r in Run,
-        where: r.work_order_id in ^orders_ids,
-        group_by: [r.work_order_id],
-        select: %{
-          work_order_id: r.work_order_id,
-          last_inserted_at: max(r.inserted_at)
-        }
-      )
-
-    run_steps_query =
-      from(as in RunStep,
-        join: att in assoc(as, :run),
-        join: wo in assoc(att, :work_order),
-        join: last in subquery(last_runs_query),
-        on:
-          last.work_order_id == att.work_order_id and
-            att.inserted_at == last.last_inserted_at,
-        join: s in assoc(as, :step),
-        on: s.job_id == ^job_id,
-        order_by: [asc: wo.inserted_at]
-      )
-
-    run_steps_query
+    workorders
+    |> Enum.map(& &1.id)
+    |> last_run_steps_query([job_id])
     |> Repo.all()
     |> retry_many(opts)
   end
@@ -506,6 +483,49 @@ defmodule Lightning.WorkOrders do
     {:ok, 0, 0}
   end
 
+  def get_last_runs_steps_with_dataclips(workorders, jobs) do
+    job_ids = Enum.map(jobs, & &1.id)
+
+    workorders
+    |> Enum.map(& &1.id)
+    |> last_run_steps_query(job_ids, non_wiped_dataclip?: true)
+    |> Repo.all()
+  end
+
+  defp last_run_steps_query(workorder_ids, job_ids, opts \\ []) do
+    last_runs_query =
+      from(r in Run,
+        where: r.work_order_id in ^workorder_ids,
+        group_by: [r.work_order_id],
+        select: %{
+          work_order_id: r.work_order_id,
+          last_inserted_at: max(r.inserted_at)
+        }
+      )
+
+    from(rs in RunStep,
+      join: r in assoc(rs, :run),
+      join: s in assoc(rs, :step),
+      join: wo in assoc(r, :work_order),
+      join: lr in subquery(last_runs_query),
+      on:
+        r.work_order_id == lr.work_order_id and
+          r.inserted_at == lr.last_inserted_at,
+      where: s.job_id in ^job_ids,
+      order_by: [asc: wo.inserted_at],
+      select: %{rs | step: s, run: r}
+    )
+    |> then(fn query ->
+      if Keyword.get(opts, :non_wiped_dataclip?) do
+        query
+        |> join(:inner, [rs, _r, s, _wo, _lr], d in assoc(s, :input_dataclip))
+        |> where([_rs, _r, _s, _wo, _lr, d], is_nil(d.wiped_at))
+      else
+        query
+      end
+    end)
+  end
+
   @doc """
   Enqueue multiple runs for retry in the same transaction.
   """
diff --git a/lib/lightning_web/live/run_live/index.ex b/lib/lightning_web/live/run_live/index.ex
index ee76be2801..1d69f187d4 100644
--- a/lib/lightning_web/live/run_live/index.ex
+++ b/lib/lightning_web/live/run_live/index.ex
@@ -575,10 +575,6 @@ defmodule LightningWeb.RunLive.Index do
     |> Enum.count()
   end
 
-  defp selected_workorder_count(selected_orders) do
-    Enum.count(selected_orders)
-  end
-
   defp maybe_humanize_date(date) do
     date && Timex.format!(date, "{D}/{M}/{YY}")
   end
diff --git a/lib/lightning_web/live/run_live/index.html.heex b/lib/lightning_web/live/run_live/index.html.heex
index 43daf6a59d..0692995df3 100644
--- a/lib/lightning_web/live/run_live/index.html.heex
+++ b/lib/lightning_web/live/run_live/index.html.heex
@@ -770,7 +770,7 @@
         pages={@page.total_pages}
         total_entries={@page.total_entries}
         all_selected?={all_selected?(@selected_work_orders, @page.entries)}
-        selected_count={selected_workorder_count(@selected_work_orders)}
+        selected_count={Enum.count(@selected_work_orders)}
         filters={SearchParams.new(@filters)}
         workflows={@workflows}
       />
@@ -780,9 +780,8 @@
         module={LightningWeb.RunLive.RerunJobComponent}
         id="bulk-rerun-from-job-modal"
         total_entries={@page.total_entries}
-        selected_count={selected_workorder_count(@selected_work_orders)}
         all_selected?={all_selected?(@selected_work_orders, @page.entries)}
-        workflow_id={@selected_work_orders |> hd() |> Map.get(:workflow_id)}
+        selected_workorders={@selected_work_orders}
         pages={@page.total_pages}
       />
     </div>
diff --git a/lib/lightning_web/live/run_live/rerun_job_component.ex b/lib/lightning_web/live/run_live/rerun_job_component.ex
index 529829550e..e804693322 100644
--- a/lib/lightning_web/live/run_live/rerun_job_component.ex
+++ b/lib/lightning_web/live/run_live/rerun_job_component.ex
@@ -2,31 +2,55 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
   @moduledoc """
   Rerun job component
   """
-
   use LightningWeb, :live_component
+
+  import LightningWeb.Utils, only: [pluralize_with_s: 2]
+
   alias Lightning.Jobs
   alias Lightning.Workflows
+  alias Lightning.WorkOrders
 
   @impl true
   def update(
         %{
           total_entries: _count,
-          selected_count: _selected_count,
-          workflow_id: workflow_id
+          selected_workorders:
+            [%{workflow_id: workflow_id} | _] = selected_workorders
         } = assigns,
         socket
       ) do
     workflow = Workflows.get_workflow!(workflow_id)
-    jobs = Jobs.list_jobs_for_workflow(workflow)
+    workflow_jobs = Jobs.list_jobs_for_workflow(workflow)
+
+    {retriable_jobs_ids, retriable_count_per_job} =
+      selected_workorders
+      |> WorkOrders.get_last_runs_steps_with_dataclips(workflow_jobs)
+      |> then(fn run_steps ->
+        retriable_jobs_ids = MapSet.new(run_steps, & &1.step.job_id)
+
+        retriable_count_per_job =
+          run_steps
+          |> Enum.group_by(& &1.step.job_id, & &1.run.work_order_id)
+          |> Map.new(fn {job_id, workorder_ids} ->
+            {job_id, Enum.count(workorder_ids)}
+          end)
+
+        {retriable_jobs_ids, retriable_count_per_job}
+      end)
+
+    disabled_jobs_ids =
+      MapSet.difference(MapSet.new(workflow_jobs, & &1.id), retriable_jobs_ids)
 
     {:ok,
      socket
      |> assign(
        show: false,
        workflow: workflow,
-       workflow_jobs: jobs,
-       selected_job: hd(jobs)
+       workflow_jobs: workflow_jobs,
+       disabled_jobs_ids: disabled_jobs_ids,
+       retriable_count_per_job: retriable_count_per_job
      )
+     |> update_selected_job(hd(workflow_jobs).id)
      |> assign(assigns)}
   end
 
@@ -34,12 +58,9 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
   def handle_event(
         "select_job",
         %{"job" => job_id},
-        %{assigns: assigns} = socket
+        socket
       ) do
-    selected_job =
-      Enum.find(assigns.workflow_jobs, fn job -> job.id == job_id end)
-
-    {:noreply, assign(socket, selected_job: selected_job)}
+    {:noreply, update_selected_job(socket, job_id)}
   end
 
   @impl true
@@ -105,10 +126,15 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
                                 do: "checked",
                                 else: false
                             }
+                            disabled={MapSet.member?(@disabled_jobs_ids, job.id)}
                           />
                           <label
+                            id={"jobl_#{job.id}"}
                             for={"job_#{job.id}"}
-                            class="ml-3 block text-sm font-medium leading-6 text-gray-900"
+                            class={[
+                              "ml-3 block text-sm leading-6 font-medium",
+                              "#{if MapSet.member?(@disabled_jobs_ids, job.id), do: "text-slate-500", else: "text-gray-900"}"
+                            ]}
                           >
                             <%= job.name %>
                           </label>
@@ -132,10 +158,10 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
                 phx-disable-with="Running..."
                 class="inline-flex w-full justify-center rounded-md bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 sm:col-start-1"
               >
-                Rerun <%= @selected_count %> selected work order<%= if @selected_count >
-                                                                         1,
-                                                                       do: "s",
-                                                                       else: "" %> from selected job
+                Rerun <%= @retriable_count %> selected work <%= pluralize_with_s(
+                  @retriable_count,
+                  "order"
+                ) %> from selected job
               </button>
               <button
                 id="rerun-all-from-job-trigger"
@@ -179,10 +205,10 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
                 phx-disable-with="Running..."
                 class="inline-flex w-full justify-center rounded-md bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 sm:col-start-2"
               >
-                Rerun <%= @selected_count %> selected work order<%= if @selected_count >
-                                                                         1,
-                                                                       do: "s",
-                                                                       else: "" %> from selected job
+                Rerun <%= @retriable_count %> selected work <%= pluralize_with_s(
+                  @retriable_count,
+                  "order"
+                ) %> from selected job
               </button>
               <button
                 type="button"
@@ -198,4 +224,19 @@ defmodule LightningWeb.RunLive.RerunJobComponent do
     </div>
     """
   end
+
+  defp update_selected_job(socket, job_id) do
+    %{
+      retriable_count_per_job: retriable_count_per_job,
+      workflow_jobs: workflow_jobs
+    } = socket.assigns
+
+    selected_job =
+      Enum.find(workflow_jobs, fn job -> job.id == job_id end)
+
+    assign(socket,
+      selected_job: selected_job,
+      retriable_count: Map.get(retriable_count_per_job, selected_job.id)
+    )
+  end
 end
diff --git a/lib/lightning_web/utils.ex b/lib/lightning_web/utils.ex
index e6c973c66b..9bbec5e488 100644
--- a/lib/lightning_web/utils.ex
+++ b/lib/lightning_web/utils.ex
@@ -11,6 +11,9 @@ defmodule LightningWeb.Utils do
   alias Phoenix.HTML.Form
   alias Plug.Conn.Query
 
+  def pluralize_with_s(n, string) when n <= 1, do: string
+  def pluralize_with_s(_integer, string), do: "#{string}s"
+
   @doc """
   Builds nested parameters for the given `form` field with the specified `value`.
 
diff --git a/test/lightning_web/live/run_live/index_test.exs b/test/lightning_web/live/run_live/index_test.exs
new file mode 100644
index 0000000000..fa9ea0b576
--- /dev/null
+++ b/test/lightning_web/live/run_live/index_test.exs
@@ -0,0 +1,474 @@
+defmodule LightningWeb.RunLive.IndexTest do
+  use LightningWeb.ConnCase, async: true
+
+  import Phoenix.LiveViewTest
+  import Lightning.Factories
+
+  alias Lightning.WorkOrders
+  alias Lightning.WorkOrders.SearchParams
+
+  setup :register_and_log_in_superuser
+  setup :create_project_for_current_user
+  setup :stub_usage_limiter_ok
+
+  setup %{project: project} do
+    %{project: project, triggers: [trigger], jobs: jobs} =
+      workflow = insert(:complex_workflow, project: project) |> with_snapshot()
+
+    dataclip = insert(:dataclip, project: project)
+
+    work_order_1 =
+      insert(:workorder,
+        workflow: workflow,
+        trigger: trigger,
+        dataclip: dataclip,
+        last_activity: DateTime.utc_now()
+      )
+      |> with_run(
+        state: :failed,
+        dataclip: dataclip,
+        starting_trigger: trigger,
+        finished_at: build(:timestamp),
+        steps:
+          jobs
+          |> Enum.map(fn j ->
+            build(:step,
+              job: j,
+              input_dataclip: dataclip,
+              started_at: build(:timestamp),
+              finished_at: build(:timestamp),
+              exit_reason: "failed"
+            )
+          end)
+      )
+
+    dataclip = insert(:dataclip, project: project)
+
+    work_order_2 =
+      insert(:workorder,
+        workflow: workflow,
+        trigger: trigger,
+        dataclip: dataclip,
+        last_activity: DateTime.utc_now()
+      )
+      |> with_run(
+        state: :failed,
+        dataclip: dataclip,
+        starting_trigger: trigger,
+        finished_at: build(:timestamp),
+        steps:
+          jobs
+          |> Enum.map(fn j ->
+            build(:step,
+              job: j,
+              input_dataclip: dataclip,
+              started_at: build(:timestamp),
+              finished_at: build(:timestamp),
+              exit_reason: "failed"
+            )
+          end)
+      )
+
+    %{
+      work_order_1: work_order_1,
+      work_order_2: work_order_2,
+      jobs: jobs,
+      workflow: workflow,
+      selected_workorders: [work_order_1, work_order_2]
+    }
+  end
+
+  describe "bulk rerun from job" do
+    @tag role: :editor
+    test "only selecting workorders from the same workflow shows the rerun button",
+         %{conn: conn, project: project} do
+      trigger = build(:trigger, type: :webhook)
+
+      job_a =
+        build(:job,
+          body: ~s[fn(state => { return {...state, extra: "data"} })],
+          name: "First Job"
+        )
+
+      workflow =
+        build(:workflow, project: project)
+        |> with_job(job_a)
+        |> with_trigger(trigger)
+        |> with_edge({trigger, job_a})
+        |> insert()
+
+      job_a = job_a |> Lightning.Repo.reload()
+      trigger = trigger |> Lightning.Repo.reload()
+
+      dataclip = insert(:dataclip, project: project)
+
+      work_order_3 =
+        insert(:workorder,
+          workflow: workflow,
+          trigger: trigger,
+          dataclip: dataclip,
+          state: :success,
+          last_activity: DateTime.utc_now()
+        )
+        |> with_run(
+          starting_trigger: trigger,
+          dataclip: dataclip,
+          finished_at: build(:timestamp),
+          steps: [
+            %{
+              job: job_a,
+              started_at: build(:timestamp),
+              finished_at: build(:timestamp),
+              exit_reason: "success",
+              input_dataclip: dataclip
+            }
+          ]
+        )
+
+      {:ok, view, html} =
+        live_async(
+          conn,
+          Routes.project_run_index_path(conn, :index, project.id,
+            filters: %{
+              body: true,
+              log: true,
+              success: true,
+              pending: true,
+              crashed: true,
+              failed: true,
+              killed: true,
+              running: true
+            }
+          )
+        )
+
+      refute html =~ "Retry from"
+
+      # All work orders have been selected
+      refute render_change(view, "toggle_all_selections", %{
+               all_selections: true
+             }) =~ "Retry from"
+
+      # uncheck 1 work order
+      view
+      |> form("#selection-form-#{work_order_3.id}")
+      |> render_change(%{selected: false})
+
+      assert render_async(view) =~ "Retry from"
+    end
+
+    @tag role: :viewer
+    test "Project viewers can't rerun runs", %{
+      conn: conn,
+      project: project,
+      jobs: [job_a | _rest]
+    } do
+      {:ok, view, _html} =
+        live(
+          conn,
+          Routes.project_run_index_path(conn, :index, project.id,
+            filters: %{
+              body: true,
+              log: true,
+              success: true,
+              pending: true,
+              crash: true,
+              failure: true
+            }
+          )
+        )
+
+      render_change(view, "toggle_all_selections", %{all_selections: true})
+
+      assert render_click(view, "bulk-rerun", %{type: "all", job: job_a.id}) =~
+               "You are not authorized to perform this action."
+    end
+
+    @tag role: :editor
+    test "Project editors can rerun runs", %{
+      conn: conn,
+      project: project,
+      jobs: [job_a | _rest],
+      work_order_1: work_order_1
+    } do
+      path =
+        Routes.project_run_index_path(conn, :index, project.id,
+          filters: %{
+            body: true,
+            log: true,
+            success: true,
+            pending: true,
+            crash: true,
+            failure: true
+          }
+        )
+
+      {:ok, view, html} = live(conn, path)
+
+      render_async(view)
+
+      refute html =~
+               "Find all runs that include this step and rerun from there"
+
+      assert render_change(view, "toggle_all_selections", %{
+               all_selections: true
+             }) =~ "Find all runs that include this step and rerun from there"
+
+      view
+      |> form("#select-job-for-rerun-form")
+      |> render_change(%{job: job_a.id})
+
+      result = view |> render_click("bulk-rerun", %{type: "all", job: job_a.id})
+
+      {:ok, view, html} = follow_redirect(result, conn)
+
+      assert html =~
+               "New runs enqueued for 2 workorders"
+
+      render_async(view)
+
+      view
+      |> form("#selection-form-#{work_order_1.id}")
+      |> render_change(%{selected: true})
+
+      view
+      |> form("#select-job-for-rerun-form")
+      |> render_change(%{job: job_a.id})
+
+      result =
+        view |> element("#rerun-selected-from-job-trigger") |> render_click()
+
+      {:ok, _view, html} = follow_redirect(result, conn)
+
+      # this is zero because the previous retried run has no steps
+      assert html =~ "New run enqueued for 0 workorder"
+    end
+
+    test "jobs on the modal are updated every time the selected workflow is changed",
+         %{
+           conn: conn,
+           project: project
+         } do
+      scenarios =
+        Enum.map(1..3, fn _n ->
+          %{triggers: [trigger], jobs: jobs} =
+            workflow = insert(:complex_workflow, project: project)
+
+          dataclip = insert(:dataclip, project: project)
+
+          work_order =
+            insert(:workorder,
+              state: :success,
+              workflow: workflow,
+              trigger: trigger,
+              dataclip: dataclip,
+              last_activity: DateTime.utc_now()
+            )
+            |> with_run(
+              state: :success,
+              dataclip: dataclip,
+              starting_trigger: trigger,
+              started_at: build(:timestamp),
+              finished_at: build(:timestamp),
+              steps:
+                Enum.map(jobs, fn j ->
+                  build(:step,
+                    job: j,
+                    input_dataclip: dataclip,
+                    output_dataclip: dataclip,
+                    started_at: build(:timestamp),
+                    finished_at: build(:timestamp),
+                    exit_reason: "success"
+                  )
+                end)
+            )
+
+          %{work_order: work_order, workflow: workflow, jobs: jobs}
+        end)
+
+      path =
+        Routes.project_run_index_path(conn, :index, project.id,
+          filters: %{
+            body: true,
+            log: true,
+            success: true,
+            pending: true,
+            killed: true,
+            running: true,
+            crashed: true,
+            failed: true
+          }
+        )
+
+      {:ok, view, _html} = live_async(conn, path)
+
+      for scenario <- scenarios do
+        for job <- scenario.jobs do
+          refute has_element?(view, "input#job_#{job.id}")
+        end
+
+        view
+        |> form("#selection-form-#{scenario.work_order.id}")
+        |> render_change(%{selected: true})
+
+        for job <- scenario.jobs do
+          assert has_element?(view, "input#job_#{job.id}")
+        end
+
+        view
+        |> form("#selection-form-#{scenario.work_order.id}")
+        |> render_change(%{selected: false})
+      end
+    end
+
+    test "all jobs in the selected workflow are displayed", %{
+      workflow: workflow,
+      selected_workorders: selected_workorders,
+      jobs: jobs
+    } do
+      html =
+        render_component(
+          LightningWeb.RunLive.RerunJobComponent,
+          id: "bulk-rerun-from-start-modal",
+          total_entries: 25,
+          all_selected?: true,
+          selected_workorders: selected_workorders,
+          pages: 2,
+          filters: %SearchParams{},
+          workflow_id: workflow.id
+        )
+
+      for job <- jobs do
+        assert html =~ job.name
+      end
+    end
+
+    test "a job with a latest step having wiped dataclip is shown disabled", %{
+      workflow: workflow,
+      work_order_1: work_order_1,
+      jobs: jobs
+    } do
+      # disabled job
+      last_job = List.last(jobs)
+      second_last_job = jobs |> List.delete(last_job) |> List.last()
+
+      # get latest run step
+      [%{step: step_last_job}] =
+        WorkOrders.get_last_runs_steps_with_dataclips([work_order_1], [last_job])
+
+      [%{step: step_second_last_job}] =
+        WorkOrders.get_last_runs_steps_with_dataclips([work_order_1], [
+          second_last_job
+        ])
+
+      # these 2 last jobs have steps with a wipe dataclip
+      wiped_dataclip = insert(:dataclip, wiped_at: Timex.now())
+
+      step_last_job
+      |> Ecto.Changeset.change(%{input_dataclip_id: wiped_dataclip.id})
+      |> Repo.update!()
+
+      step_second_last_job
+      |> Ecto.Changeset.change(%{input_dataclip_id: wiped_dataclip.id})
+      |> Repo.update!()
+
+      html =
+        render_component(
+          LightningWeb.RunLive.RerunJobComponent,
+          id: "bulk-rerun-from-start-modal",
+          total_entries: 25,
+          all_selected?: true,
+          selected_workorders: [work_order_1],
+          pages: 2,
+          filters: %SearchParams{},
+          workflow_id: workflow.id
+        )
+
+      Enum.each(jobs, fn job ->
+        label =
+          Floki.find(html, "#jobl_#{job.id}")
+          |> Floki.attribute("class")
+          |> List.first()
+
+        disabled =
+          Floki.find(html, "#job_#{job.id}")
+          |> Floki.attribute("disabled")
+          |> List.first()
+
+        if job.id in [last_job.id, second_last_job.id] do
+          assert disabled
+          assert label =~ "text-slate-500"
+        else
+          refute disabled
+          assert label =~ "text-gray-900"
+        end
+      end)
+    end
+
+    test "2 run buttons are present when all entries have been selected", %{
+      workflow: workflow,
+      selected_workorders: selected_workorders
+    } do
+      html =
+        render_component(
+          LightningWeb.RunLive.RerunJobComponent,
+          id: "bulk-rerun-from-start-modal",
+          total_entries: 25,
+          all_selected?: true,
+          selected_workorders: selected_workorders,
+          pages: 2,
+          filters: %SearchParams{},
+          workflow_id: workflow.id
+        )
+
+      assert html =~ "Rerun all 25 matching work orders from selected job"
+
+      assert html =~
+               "Rerun #{length(selected_workorders)} selected work orders from selected job"
+    end
+
+    test "only 1 run button is present when some entries have been selected", %{
+      workflow: workflow,
+      selected_workorders: selected_workorders
+    } do
+      html =
+        render_component(
+          LightningWeb.RunLive.RerunJobComponent,
+          id: "bulk-rerun-from-start-modal",
+          total_entries: 25,
+          all_selected?: false,
+          selected_workorders: selected_workorders,
+          pages: 2,
+          filters: %SearchParams{},
+          workflow_id: workflow.id
+        )
+
+      refute html =~ "Rerun all 25 matching work orders from selected job"
+
+      assert html =~
+               "Rerun #{length(selected_workorders)} selected work orders from selected job"
+    end
+
+    test "only 1 run button is present when total pages is 1", %{
+      workflow: workflow,
+      selected_workorders: selected_workorders
+    } do
+      html =
+        render_component(
+          LightningWeb.RunLive.RerunJobComponent,
+          id: "bulk-rerun-from-start-modal",
+          total_entries: 25,
+          all_selected?: true,
+          selected_workorders: selected_workorders,
+          pages: 1,
+          filters: %SearchParams{},
+          workflow_id: workflow.id
+        )
+
+      refute html =~ "Rerun all 25 matching work orders from selected job"
+
+      assert html =~
+               "Rerun #{length(selected_workorders)} selected work orders from selected job"
+    end
+  end
+end
diff --git a/test/lightning_web/live/work_order_live_test.exs b/test/lightning_web/live/work_order_live_test.exs
index dc3ee5e7a2..f5b93809f3 100644
--- a/test/lightning_web/live/work_order_live_test.exs
+++ b/test/lightning_web/live/work_order_live_test.exs
@@ -2384,394 +2384,6 @@ defmodule LightningWeb.WorkOrderLiveTest do
     string |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string()
   end
 
-  describe "bulk rerun from job" do
-    setup %{project: project} do
-      %{project: project, triggers: [trigger], jobs: jobs} =
-        workflow = insert(:complex_workflow, project: project) |> with_snapshot()
-
-      dataclip = insert(:dataclip, project: project)
-
-      work_order_1 =
-        insert(:workorder,
-          workflow: workflow,
-          trigger: trigger,
-          dataclip: dataclip,
-          last_activity: DateTime.utc_now()
-        )
-        |> with_run(
-          state: :failed,
-          dataclip: dataclip,
-          starting_trigger: trigger,
-          finished_at: build(:timestamp),
-          steps:
-            jobs
-            |> Enum.map(fn j ->
-              build(:step,
-                job: j,
-                input_dataclip: dataclip,
-                started_at: build(:timestamp),
-                finished_at: build(:timestamp),
-                exit_reason: "failed"
-              )
-            end)
-        )
-
-      dataclip = insert(:dataclip, project: project)
-
-      work_order_2 =
-        insert(:workorder,
-          workflow: workflow,
-          trigger: trigger,
-          dataclip: dataclip,
-          last_activity: DateTime.utc_now()
-        )
-        |> with_run(
-          state: :failed,
-          dataclip: dataclip,
-          starting_trigger: trigger,
-          finished_at: build(:timestamp),
-          steps:
-            jobs
-            |> Enum.map(fn j ->
-              build(:step,
-                job: j,
-                input_dataclip: dataclip,
-                started_at: build(:timestamp),
-                finished_at: build(:timestamp),
-                exit_reason: "failed"
-              )
-            end)
-        )
-
-      %{
-        work_order_1: work_order_1,
-        work_order_2: work_order_2,
-        jobs: jobs,
-        workflow: workflow
-      }
-    end
-
-    @tag role: :editor
-    test "only selecting worko rders from the same workflow shows the rerun button",
-         %{conn: conn, project: project} do
-      trigger = build(:trigger, type: :webhook)
-
-      job_a =
-        build(:job,
-          body: ~s[fn(state => { return {...state, extra: "data"} })],
-          name: "First Job"
-        )
-
-      workflow =
-        build(:workflow, project: project)
-        |> with_job(job_a)
-        |> with_trigger(trigger)
-        |> with_edge({trigger, job_a})
-        |> insert()
-
-      job_a = job_a |> Lightning.Repo.reload()
-      trigger = trigger |> Lightning.Repo.reload()
-
-      dataclip = insert(:dataclip, project: project)
-
-      work_order_3 =
-        insert(:workorder,
-          workflow: workflow,
-          trigger: trigger,
-          dataclip: dataclip,
-          state: :success,
-          last_activity: DateTime.utc_now()
-        )
-        |> with_run(
-          starting_trigger: trigger,
-          dataclip: dataclip,
-          finished_at: build(:timestamp),
-          steps: [
-            %{
-              job: job_a,
-              started_at: build(:timestamp),
-              finished_at: build(:timestamp),
-              exit_reason: "success",
-              input_dataclip: dataclip
-            }
-          ]
-        )
-
-      {:ok, view, html} =
-        live_async(
-          conn,
-          Routes.project_run_index_path(conn, :index, project.id,
-            filters: %{
-              body: true,
-              log: true,
-              success: true,
-              pending: true,
-              crashed: true,
-              failed: true,
-              killed: true,
-              running: true
-            }
-          )
-        )
-
-      refute html =~ "Retry from"
-
-      # All work orders have been selected
-      refute render_change(view, "toggle_all_selections", %{
-               all_selections: true
-             }) =~ "Retry from"
-
-      # uncheck 1 work order
-      view
-      |> form("#selection-form-#{work_order_3.id}")
-      |> render_change(%{selected: false})
-
-      assert render_async(view) =~ "Retry from"
-    end
-
-    @tag role: :viewer
-    test "Project viewers can't rerun runs", %{
-      conn: conn,
-      project: project,
-      jobs: [job_a | _rest]
-    } do
-      {:ok, view, _html} =
-        live(
-          conn,
-          Routes.project_run_index_path(conn, :index, project.id,
-            filters: %{
-              body: true,
-              log: true,
-              success: true,
-              pending: true,
-              crash: true,
-              failure: true
-            }
-          )
-        )
-
-      render_change(view, "toggle_all_selections", %{all_selections: true})
-
-      assert render_click(view, "bulk-rerun", %{type: "all", job: job_a.id}) =~
-               "You are not authorized to perform this action."
-    end
-
-    @tag role: :editor
-    test "Project editors can rerun runs", %{
-      conn: conn,
-      project: project,
-      jobs: [job_a | _rest],
-      work_order_1: work_order_1
-    } do
-      path =
-        Routes.project_run_index_path(conn, :index, project.id,
-          filters: %{
-            body: true,
-            log: true,
-            success: true,
-            pending: true,
-            crash: true,
-            failure: true
-          }
-        )
-
-      {:ok, view, html} = live(conn, path)
-
-      render_async(view)
-
-      refute html =~
-               "Find all runs that include this step and rerun from there"
-
-      assert render_change(view, "toggle_all_selections", %{
-               all_selections: true
-             }) =~ "Find all runs that include this step and rerun from there"
-
-      view
-      |> form("#select-job-for-rerun-form")
-      |> render_change(%{job: job_a.id})
-
-      result = view |> render_click("bulk-rerun", %{type: "all", job: job_a.id})
-
-      {:ok, view, html} = follow_redirect(result, conn)
-
-      assert html =~
-               "New runs enqueued for 2 workorders"
-
-      render_async(view)
-
-      view
-      |> form("#selection-form-#{work_order_1.id}")
-      |> render_change(%{selected: true})
-
-      view
-      |> form("#select-job-for-rerun-form")
-      |> render_change(%{job: job_a.id})
-
-      result =
-        view |> element("#rerun-selected-from-job-trigger") |> render_click()
-
-      {:ok, _view, html} = follow_redirect(result, conn)
-
-      # this is zero because the previous retried run has no steps
-      assert html =~ "New run enqueued for 0 workorder"
-    end
-
-    test "jobs on the modal are updated every time the selected workflow is changed",
-         %{
-           conn: conn,
-           project: project
-         } do
-      scenarios =
-        Enum.map(1..3, fn _n ->
-          %{triggers: [trigger], jobs: jobs} =
-            workflow = insert(:complex_workflow, project: project)
-
-          dataclip = insert(:dataclip, project: project)
-
-          work_order =
-            insert(:workorder,
-              state: :success,
-              workflow: workflow,
-              trigger: trigger,
-              dataclip: dataclip,
-              last_activity: DateTime.utc_now()
-            )
-            |> with_run(
-              state: :success,
-              dataclip: dataclip,
-              starting_trigger: trigger,
-              started_at: build(:timestamp),
-              finished_at: build(:timestamp),
-              steps:
-                Enum.map(jobs, fn j ->
-                  build(:step,
-                    job: j,
-                    input_dataclip: dataclip,
-                    output_dataclip: dataclip,
-                    started_at: build(:timestamp),
-                    finished_at: build(:timestamp),
-                    exit_reason: "success"
-                  )
-                end)
-            )
-
-          %{work_order: work_order, workflow: workflow, jobs: jobs}
-        end)
-
-      path =
-        Routes.project_run_index_path(conn, :index, project.id,
-          filters: %{
-            body: true,
-            log: true,
-            success: true,
-            pending: true,
-            killed: true,
-            running: true,
-            crashed: true,
-            failed: true
-          }
-        )
-
-      {:ok, view, _html} = live_async(conn, path)
-
-      for scenario <- scenarios do
-        for job <- scenario.jobs do
-          refute has_element?(view, "input#job_#{job.id}")
-        end
-
-        view
-        |> form("#selection-form-#{scenario.work_order.id}")
-        |> render_change(%{selected: true})
-
-        for job <- scenario.jobs do
-          assert has_element?(view, "input#job_#{job.id}")
-        end
-
-        view
-        |> form("#selection-form-#{scenario.work_order.id}")
-        |> render_change(%{selected: false})
-      end
-    end
-
-    test "all jobs in the selected workflow are displayed", %{
-      workflow: workflow,
-      jobs: jobs
-    } do
-      html =
-        render_component(
-          LightningWeb.RunLive.RerunJobComponent,
-          id: "bulk-rerun-from-start-modal",
-          total_entries: 25,
-          all_selected?: true,
-          selected_count: 5,
-          pages: 2,
-          filters: %SearchParams{},
-          workflow_id: workflow.id
-        )
-
-      for job <- jobs do
-        assert html =~ job.name
-      end
-    end
-
-    test "2 run buttons are present when all entries have been selected", %{
-      workflow: workflow
-    } do
-      html =
-        render_component(
-          LightningWeb.RunLive.RerunJobComponent,
-          id: "bulk-rerun-from-start-modal",
-          total_entries: 25,
-          all_selected?: true,
-          selected_count: 5,
-          pages: 2,
-          filters: %SearchParams{},
-          workflow_id: workflow.id
-        )
-
-      assert html =~ "Rerun all 25 matching work orders from selected job"
-      assert html =~ "Rerun 5 selected work orders from selected job"
-    end
-
-    test "only 1 run button is present when some entries have been selected", %{
-      workflow: workflow
-    } do
-      html =
-        render_component(
-          LightningWeb.RunLive.RerunJobComponent,
-          id: "bulk-rerun-from-start-modal",
-          total_entries: 25,
-          all_selected?: false,
-          selected_count: 5,
-          pages: 2,
-          filters: %SearchParams{},
-          workflow_id: workflow.id
-        )
-
-      refute html =~ "Rerun all 25 matching work orders from selected job"
-      assert html =~ "Rerun 5 selected work orders from selected job"
-    end
-
-    test "only 1 run button is present when total pages is 1", %{
-      workflow: workflow
-    } do
-      html =
-        render_component(
-          LightningWeb.RunLive.RerunJobComponent,
-          id: "bulk-rerun-from-start-modal",
-          total_entries: 25,
-          all_selected?: true,
-          selected_count: 5,
-          pages: 1,
-          filters: %SearchParams{},
-          workflow_id: workflow.id
-        )
-
-      refute html =~ "Rerun all 25 matching work orders from selected job"
-      assert html =~ "Rerun 5 selected work orders from selected job"
-    end
-  end
-
   describe "timestamp" do
     test "default option" do
       now = DateTime.utc_now()