allow users to configure their log levels (#2947)

* allow users to configure their log levels * add test case * update changelog * WIP: review comments * tiny refactor * use radio buttons for log filter * update copy on log filter to mention success * update copy on log filter
OpenFn · Feb 26, 2025 · 1ae154a · 1ae154a
1 parent b849767
commit 1ae154a
Show file tree

Hide file tree

Showing 8 changed files with 313 additions and 33 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,8 @@ and this project adheres to
   [#2820](https://github.com/OpenFn/lightning/issues/2820)
 - Delete unused snapshots on workorders retention cleanup
   [#1832](https://github.com/OpenFn/lightning/issues/1832)
+- Allow users to configure their preferred log levels
+  [#2206](https://github.com/OpenFn/lightning/issues/2206)
 
 ### Changed
 

diff --git a/assets/js/log-viewer/index.ts b/assets/js/log-viewer/index.ts
@@ -27,6 +27,7 @@ export default {
 
     this.store = createLogStore();
     this.store.getState().setStepId(this.el.dataset.stepId);
+    this.store.getState().setDesiredLogLevel(this.el.dataset.logLevel);
 
     this.component = mount(this.viewerEl, this.store);
 
@@ -44,6 +45,7 @@ export default {
 
   updated() {
     this.store.getState().setStepId(this.el.dataset.stepId);
+    this.store.getState().setDesiredLogLevel(this.el.dataset.logLevel);
     // this.el.dispatchEvent(
     //   new CustomEvent('log-viewer:highlight-step', {
     //     detail: { stepId: this.el.dataset.stepId },

diff --git a/assets/js/log-viewer/store.ts b/assets/js/log-viewer/store.ts
@@ -17,9 +17,36 @@ interface LogStore {
   formattedLogLines: string;
   addLogLines: (newLogs: LogLine[]) => void;
   highlightedRanges: { start: number; end: number }[];
+  desiredLogLevel: string;
+  setDesiredLogLevel: (desiredLogLevel: string | undefined) => void;
 }
 
-function findSelectedRanges(logs: LogLine[], stepId: string | undefined) {
+// get score for log level
+function logLevelScore(level: string): number {
+  switch (level) {
+    case 'debug':
+      return 0;
+    case 'info':
+      return 1;
+    case 'warn':
+      return 2;
+    case 'error':
+      return 3;
+    default:
+      return 4;
+  }
+}
+
+// check if a log matches the desired log level
+function matchesLogFilter(log: LogLine, desiredLogLevel: string): boolean {
+  return logLevelScore(log.level) >= logLevelScore(desiredLogLevel);
+}
+
+function findSelectedRanges(
+  logs: LogLine[],
+  stepId: string | undefined,
+  desiredLogLevel: string
+) {
   if (!stepId) return [];
 
   const { ranges } = logs.reduce<{
@@ -33,7 +60,8 @@ function findSelectedRanges(logs: LogLine[], stepId: string | undefined) {
 
       const nextMarker = marker + 1 + newLineCount;
 
-      if (log.step_id !== stepId) {
+      // Skip logs that don't match the step ID or desired log levels
+      if (log.step_id !== stepId || !matchesLogFilter(log, desiredLogLevel)) {
         return { ranges, marker: nextMarker };
       }
 
@@ -96,11 +124,25 @@ function formatLogLine(log: LogLine) {
   return `${source} ${possiblyPrettify(message)}`;
 }
 
+function stringifyLogLines(logLines: LogLine[], desiredLogLevel: string) {
+  const lines = logLines.reduce((formatted, log) => {
+    if (matchesLogFilter(log, desiredLogLevel)) {
+      return formatted + (formatted !== '' ? '\n' : '') + formatLogLine(log);
+    }
+    return formatted;
+  }, '');
+
+  return lines;
+}
+
 export const createLogStore = () => {
   const createStore = create<LogStore>()(
     subscribeWithSelector((set, get) => ({
       stepId: undefined,
       setStepId: (stepId: string | undefined) => set({ stepId }),
+      desiredLogLevel: 'info',
+      setDesiredLogLevel: (desiredLogLevel: string | undefined) =>
+        set({ desiredLogLevel: desiredLogLevel || 'info' }),
       highlightedRanges: [],
       logLines: [],
       stepSetAt: undefined,
@@ -111,28 +153,46 @@ export const createLogStore = () => {
 
         logLines.sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime());
 
+        const desiredLogLevel = get().desiredLogLevel;
+
         set({
-          formattedLogLines: logLines.map(formatLogLine).join('\n'),
+          formattedLogLines: stringifyLogLines(logLines, desiredLogLevel),
           logLines,
         });
       },
     }))
   );
 
   // Subscribe to the store and update the highlighted ranges when the
-  // log lines or step ID changes.
-  createStore.subscribe<[LogLine[], undefined | string]>(
-    state => [state.logLines, state.stepId],
-    ([logLines, stepId], _) => {
-      createStore.setState({
-        highlightedRanges: findSelectedRanges(logLines, stepId),
-      });
+  // log lines or step ID or log levels changes.
+  createStore.subscribe<[LogLine[], undefined | string, string]>(
+    state => [state.logLines, state.stepId, state.desiredLogLevel],
+    (
+      [logLines, stepId, desiredLogLevel],
+      [_prevLogLines, _prevStepId, prevLogLevel]
+    ) => {
+      const state = {
+        highlightedRanges: findSelectedRanges(
+          logLines,
+          stepId,
+          desiredLogLevel
+        ),
+      };
+
+      if (prevLogLevel !== desiredLogLevel) {
+        state.formattedLogLines = stringifyLogLines(logLines, desiredLogLevel);
+      }
+      createStore.setState(state);
     },
     {
-      equalityFn: ([prevLogLines, prevStepId], [nextLogLines, nextStepId]) => {
+      equalityFn: (
+        [prevLogLines, prevStepId, prevLogLevel],
+        [nextLogLines, nextStepId, nextLogLevel]
+      ) => {
         return (
           prevLogLines.length === nextLogLines.length &&
-          prevStepId === nextStepId
+          prevStepId === nextStepId &&
+          prevLogLevel === nextLogLevel
         );
       },
     }

diff --git a/lib/lightning_web/components/viewers.ex b/lib/lightning_web/components/viewers.ex
@@ -40,11 +40,24 @@ defmodule LightningWeb.Components.Viewers do
   attr :logs_empty?, :boolean, required: true
   attr :selected_step_id, :string
 
+  attr :current_user, Lightning.Accounts.User,
+    default: nil,
+    doc: "for checking log filter preference"
+
   attr :class, :string,
     default: nil,
     doc: "Additional classes to add to the log viewer container"
 
   def log_viewer(assigns) do
+    assigns =
+      assign_new(assigns, :selected_log_level, fn ->
+        if assigns[:current_user] do
+          Map.get(assigns.current_user.preferences, "desired_log_level", "info")
+        else
+          "info"
+        end
+      end)
+
     ~H"""
     <%= if @run_state in Lightning.Run.final_states() and @logs_empty? do %>
       <div class={["m-2 relative p-12 text-center col-span-full"]}>
@@ -55,32 +68,124 @@ defmodule LightningWeb.Components.Viewers do
         </span>
       </div>
     <% else %>
-      <div
-        id={@id}
-        class={["flex grow", @class]}
-        phx-hook="LogViewer"
-        phx-update="ignore"
-        data-run-id={@run_id}
-        data-step-id={@selected_step_id}
-        data-loading-el={"#{@id}-nothing-yet"}
-        data-viewer-el={"#{@id}-viewer"}
-      >
-        <div class="relative grow">
-          <div
-            id={"#{@id}-nothing-yet"}
-            class="relative rounded-md p-12 text-center bg-slate-700 font-mono text-gray-200"
-          >
-            <.text_ping_loader>
-              Nothing yet
-            </.text_ping_loader>
+      <div class="relative flex grow">
+        <.log_level_filter
+          :if={@current_user}
+          id={"#{@id}-filter"}
+          selected_level={@selected_log_level}
+        />
+
+        <div
+          id={@id}
+          class={["flex grow", @class]}
+          phx-hook="LogViewer"
+          phx-update="ignore"
+          data-run-id={@run_id}
+          data-step-id={@selected_step_id}
+          data-log-level={@selected_log_level}
+          data-loading-el={"#{@id}-nothing-yet"}
+          data-viewer-el={"#{@id}-viewer"}
+        >
+          <div class="relative grow">
+            <div
+              id={"#{@id}-nothing-yet"}
+              class="relative rounded-md p-12 text-center bg-slate-700 font-mono text-gray-200"
+            >
+              <.text_ping_loader>
+                Nothing yet
+              </.text_ping_loader>
+            </div>
+            <div id={"#{@id}-viewer"} class="hidden absolute inset-0 rounded-md">
+            </div>
           </div>
-          <div id={"#{@id}-viewer"} class="hidden absolute inset-0 rounded-md"></div>
         </div>
       </div>
     <% end %>
     """
   end
 
+  attr :id, :string, required: true
+  attr :selected_level, :string
+
+  defp log_level_filter(assigns) do
+    assigns =
+      assign(assigns,
+        log_levels: [
+          {"debug", "Show all logs"},
+          {"info", "Standard logging, excluding low-level debug logs (default)"},
+          {"warn", "Only show warnings, errors and key logs"},
+          {"error", "Only show errors and key logs"}
+        ]
+      )
+
+    ~H"""
+    <div id={@id} class="absolute top-0 right-4 z-50">
+      <.modal
+        id={"#{@id}-modal"}
+        close_on_click_away={false}
+        close_on_keydown={false}
+      >
+        <:title>
+          <div class="flex justify-between gap-2">
+            <span class="font-bold">
+              Configure log levels
+            </span>
+
+            <button
+              phx-click={hide_modal("#{@id}-modal")}
+              type="button"
+              class="rounded-md bg-white text-gray-400 hover:text-gray-500 focus:outline-none"
+              aria-label={gettext("close")}
+            >
+              <span class="sr-only">Close</span>
+              <.icon name="hero-x-mark" class="h-5 w-5 stroke-current" />
+            </button>
+          </div>
+        </:title>
+        <.form id={"#{@id}-form"} for={to_form(%{})} phx-change="save-log-filter">
+          <fieldset>
+            <p class="mt-1 text-sm/6 text-gray-600">
+              Which logs do you prefer to see?
+            </p>
+            <div class="mt-6 space-y-4">
+              <%= for {level, description} <- @log_levels do %>
+                <div class="relative flex items-start">
+                  <div class="flex h-6 items-center">
+                    <.input
+                      type="radio"
+                      id={"desired_level_#{level}"}
+                      name="desired_log_level"
+                      value={level}
+                      checked={level == @selected_level}
+                      class="relative size-4 appearance-none rounded-full border border-gray-300 bg-white before:absolute before:inset-1 before:rounded-full before:bg-white checked:border-indigo-600 checked:bg-indigo-600 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 disabled:border-gray-300 disabled:bg-gray-100 disabled:before:bg-gray-400 forced-colors:appearance-auto forced-colors:before:hidden [&:not(:checked)]:before:hidden"
+                    />
+                  </div>
+                  <div class="ml-3 text-sm/6">
+                    <.label
+                      for={"desired_level_#{level}"}
+                      class="font-medium text-gray-900 uppercase"
+                    >
+                      <%= level %>
+                    </.label>
+                    <p class="text-gray-500"><%= description %></p>
+                  </div>
+                </div>
+              <% end %>
+            </div>
+          </fieldset>
+        </.form>
+      </.modal>
+      <button
+        phx-click={show_modal("#{@id}-modal")}
+        type="button"
+        class="rounded-bl-md rounded-br-md p-1 pt-0 opacity-70 hover:opacity-100 content-center bg-blue-500 text-blue-900"
+      >
+        <.icon name="hero-cog-6-tooth" class="h-4 w-4 inline-block align-middle" />
+      </button>
+    </div>
+    """
+  end
+
   attr :id, :string, required: true
   attr :dataclip, :map, required: true
 

diff --git a/lib/lightning_web/live/run_live/run_viewer_live.ex b/lib/lightning_web/live/run_live/run_viewer_live.ex
@@ -142,6 +142,7 @@ defmodule LightningWeb.RunLive.RunViewerLive do
                       run_state={@run.result.state}
                       logs_empty?={@log_lines_empty?}
                       selected_step_id={@selected_step_id}
+                      current_user={@current_user}
                     />
                   </div>
                 </div>
@@ -274,7 +275,8 @@ defmodule LightningWeb.RunLive.RunViewerLive do
        selected_step_id: nil,
        job_id: Map.get(session, "job_id"),
        selected_step: nil,
-       steps: []
+       steps: [],
+       current_user: project_user.user
      )
      |> assign(:input_dataclip, nil)
      |> assign(:output_dataclip, nil)

diff --git a/lib/lightning_web/live/run_live/show.ex b/lib/lightning_web/live/run_live/show.ex
@@ -222,14 +222,15 @@ defmodule LightningWeb.RunLive.Show do
                     can_edit_data_retention={@can_edit_data_retention}
                   />
                 </:panel>
-                <:panel hash="log" class="flex-grow">
+                <:panel hash="log" class="flex h-full">
                   <Viewers.log_viewer
                     id={"run-log-#{run.id}"}
                     class="h-full"
                     run_id={run.id}
                     run_state={@run.result.state}
                     logs_empty?={@log_lines_empty?}
                     selected_step_id={@selected_step_id}
+                    current_user={@current_user}
                   />
                 </:panel>
                 <:panel hash="output" class="flex-1">

diff --git a/lib/lightning_web/live/run_live/streaming.ex b/lib/lightning_web/live/run_live/streaming.ex
@@ -99,6 +99,8 @@ defmodule LightningWeb.RunLive.Streaming do
       unquote(helpers())
       unquote(handle_infos())
       @impl true
+      unquote(handle_events())
+      @impl true
       unquote(handle_asyncs())
     end
   end
@@ -205,6 +207,25 @@ defmodule LightningWeb.RunLive.Streaming do
     end
   end
 
+  defp handle_events do
+    quote do
+      def handle_event(
+            "save-log-filter",
+            %{"desired_log_level" => log_level},
+            socket
+          ) do
+        {:ok, updated_user} =
+          Lightning.Accounts.update_user_preference(
+            socket.assigns.current_user,
+            "desired_log_level",
+            log_level
+          )
+
+        {:noreply, assign(socket, current_user: updated_user)}
+      end
+    end
+  end
+
   defp helpers do
     quote do
       import unquote(__MODULE__)