Skip to content

Commit

Permalink
Add alert resolution CRUD
Browse files Browse the repository at this point in the history
will be used to persist resolution info in alerts, whcih can then be vector indexed to aid in further debugging efforts.
  • Loading branch information
michaeljguarino committed Feb 26, 2025
1 parent 4ace586 commit e0721a4
Show file tree
Hide file tree
Showing 8 changed files with 130 additions and 3 deletions.
1 change: 1 addition & 0 deletions lib/console/deployments/events.ex
Original file line number Diff line number Diff line change
Expand Up @@ -77,5 +77,6 @@ defmodule Console.PubSub.StackStateInsight, do: use Piazza.PubSub.Event
defmodule Console.PubSub.AlertInsight, do: use Piazza.PubSub.Event

defmodule Console.PubSub.AlertCreated, do: use Piazza.PubSub.Event
defmodule Console.PubSub.AlertResolutionCreated, do: use Piazza.PubSub.Event

defmodule Console.PubSub.ScmWebhook, do: use Piazza.PubSub.Event
49 changes: 47 additions & 2 deletions lib/console/deployments/observability.ex
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@ defmodule Console.Deployments.Observability do
alias Console.Deployments.Settings
alias Console.PubSub
alias Console.Schema.{
User,
Alert,
Cluster,
Project,
Service,
AlertResolution,
DeploymentSettings,
ObservabilityProvider,
ObservabilityWebhook,
User,
ServiceComponent,
Cluster
}

@cache Console.conf(:cache_adapter)
Expand Down Expand Up @@ -53,6 +56,9 @@ defmodule Console.Deployments.Observability do
@decorate cacheable(cache: @cache, key: {:obs_webhook, id}, opts: [ttl: @ttl])
def get_webhook_by_ext_id(id), do: Repo.get_by!(ObservabilityWebhook, external_id: id)

@spec get_alert!(binary) :: Alert.t | nil
def get_alert!(id), do: Repo.get!(Alert, id)

@doc """
Create or update a provider, must inclue name in attrs
"""
Expand Down Expand Up @@ -101,6 +107,41 @@ defmodule Console.Deployments.Observability do
|> when_ok(:delete)
end

@doc """
Determines if an individual alert is accessible from a user to perform the given action
"""
@spec accessible(Alert.t, User.t, atom) :: {:ok, any} | error
def accessible(%Alert{} = alert, %User{} = user, action) do
case Repo.preload(alert, [:service, :cluster, :project]) do
%Alert{service: %Service{} = svc} -> allow(svc, user, action)
%Alert{cluster: %Cluster{} = cluster} -> allow(cluster, user, action)
%Alert{project: %Project{} = project} -> allow(project, user, action)
_ -> {:error, :forbidden}
end
end

@doc """
Persists the resolution associated with this alert, cannot be done if the user doesn't have write access
"""
@spec set_resolution(map, binary, User.t) :: {:ok, AlertResolution.t} | Console.error
def set_resolution(attrs, alert_id, %User{} = user) do
start_transaction()
|> add_operation(:alert, fn _ ->
get_alert!(alert_id)
|> accessible(user, :write)
end)
|> add_operation(:upsert, fn _ ->
case Repo.get_by(AlertResolution, alert_id: alert_id) do
%AlertResolution{} = res -> res
nil -> %AlertResolution{alert_id: alert_id}
end
|> AlertResolution.changeset(attrs)
|> Repo.insert_or_update()
end)
|> execute(extract: :upsert)
|> notify(:create)
end

@doc """
Saves a set of parsed alert data straight to the db
"""
Expand Down Expand Up @@ -187,4 +228,8 @@ defmodule Console.Deployments.Observability do
defp find(%DeploymentSettings{loki_connection: loki}, :loki), do: loki
defp find(%DeploymentSettings{prometheus_connection: prometheus}, :prometheus), do: prometheus
defp find(_, _), do: nil

defp notify({:ok, %AlertResolution{} = res}, :create),
do: handle_notify(PubSub.AlertResolutionCreated, res)
defp notify(pass, _), do: pass
end
5 changes: 4 additions & 1 deletion lib/console/schema/alert.ex
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ defmodule Console.Schema.Alert do
Service,
Tag,
ObservabilityWebhook,
AiInsight
AiInsight,
AlertResolution
}

defenum Severity, low: 0, medium: 1, high: 2, critical: 3, undefined: 4
Expand All @@ -31,6 +32,8 @@ defmodule Console.Schema.Alert do
belongs_to :cluster, Cluster
belongs_to :service, Service

has_one :resolution, AlertResolution

has_many :tags, Tag, on_replace: :delete

timestamps()
Expand Down
21 changes: 21 additions & 0 deletions lib/console/schema/alert_resolution.ex
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
defmodule Console.Schema.AlertResolution do
use Piazza.Ecto.Schema
alias Console.Schema.{Alert}
schema "alert_resolutions" do
field :resolution, :string

belongs_to :alert, Alert

timestamps()
end

@valid ~w(alert_id resolution)a

def changeset(model, attrs \\ %{}) do
model
|> cast(attrs, @valid)
|> foreign_key_constraint(:alert_id)
|> unique_constraint(:alert_id)
|> validate_required([:resolution])
end
end
Binary file modified priv/agent-chart.tgz
Binary file not shown.
15 changes: 15 additions & 0 deletions priv/repo/migrations/20250226224105_add_alert_resolutions.exs
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
defmodule Console.Repo.Migrations.AddAlertResolutions do
use Ecto.Migration

def change do
create table(:alert_resolutions, primary_key: false) do
add :id, :uuid, primary_key: true
add :alert_id, references(:alerts, type: :uuid, on_delete: :delete_all)
add :resolution, :binary

timestamps()
end

create unique_index(:alert_resolutions, [:alert_id])
end
end
35 changes: 35 additions & 0 deletions test/console/deployments/observability_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,41 @@ defmodule Console.Deployments.ObservabilityTest do
end
end

describe "#set_resolution/3" do
test "it can set a resolution for an alert" do
service = insert(:service)
alert = insert(:alert, service: service)

{:ok, res} = Observability.set_resolution(%{
resolution: "resolved"
}, alert.id, admin_user())

assert res.alert_id == alert.id
assert res.resolution == "resolved"
end

test "it can update a resolution for an alert" do
service = insert(:service)
alert = insert(:alert, service: service)
resolution = insert(:alert_resolution, alert: alert)

{:ok, res} = Observability.set_resolution(%{
resolution: "updated"
}, alert.id, admin_user())

assert res.id == resolution.id
assert res.alert_id == alert.id
assert res.resolution == "updated"
end

test "non accessible users cannot update alerts" do
service = insert(:service)
alert = insert(:alert, service: service)

{:error, _} = Observability.set_resolution(%{resolution: "updated"}, alert.id, insert(:user))
end
end

describe "#delete_webhook/2" do
test "it can delete a webhook by id" do
webhook = insert(:observability_webhook)
Expand Down
7 changes: 7 additions & 0 deletions test/support/factory.ex
Original file line number Diff line number Diff line change
Expand Up @@ -854,6 +854,13 @@ defmodule Console.Factory do
}
end

def alert_resolution_factory do
%Schema.AlertResolution{
alert: build(:alert),
resolution: "resolved",
}
end

def setup_rbac(user, repos \\ ["*"], perms) do
role = insert(:role, repositories: repos, permissions: Map.new(perms))
insert(:role_binding, role: role, user: user)
Expand Down

0 comments on commit e0721a4

Please sign in to comment.