From 2859fdc68124008181c5f8557367857a101aeda9 Mon Sep 17 00:00:00 2001
From: Chris Smith <1979423+chris13524@users.noreply.github.com>
Date: Thu, 16 Nov 2023 17:21:44 -0500
Subject: [PATCH] fix: alarm immediately (#213)

---
 .../panels/app/relay_incomming_message_server_errors.libsonnet | 3 ++-
 .../panels/app/relay_outgoing_message_failures.libsonnet       | 3 ++-
 terraform/monitoring/panels/lb/error_5xx.libsonnet             | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/terraform/monitoring/panels/app/relay_incomming_message_server_errors.libsonnet b/terraform/monitoring/panels/app/relay_incomming_message_server_errors.libsonnet
index dae54826..6f607418 100644
--- a/terraform/monitoring/panels/app/relay_incomming_message_server_errors.libsonnet
+++ b/terraform/monitoring/panels/app/relay_incomming_message_server_errors.libsonnet
@@ -18,9 +18,10 @@ local targets   = grafana.targets;
       message       = '%(env)s - Failed to process incomming relay message' % { env: vars.environment },
       notifications = vars.notifications,
       noDataState   = 'no_data',
+      period        = '0m',
       conditions    = [
         grafana.alertCondition.new(
-          evaluatorParams = [ 1 ],
+          evaluatorParams = [ 0 ],
           evaluatorType   = 'gt',
           operatorType    = 'or',
           queryRefId      = 'RelayIncommingMessagesServerErrors',
diff --git a/terraform/monitoring/panels/app/relay_outgoing_message_failures.libsonnet b/terraform/monitoring/panels/app/relay_outgoing_message_failures.libsonnet
index 8979e345..794a54a2 100644
--- a/terraform/monitoring/panels/app/relay_outgoing_message_failures.libsonnet
+++ b/terraform/monitoring/panels/app/relay_outgoing_message_failures.libsonnet
@@ -18,9 +18,10 @@ local targets   = grafana.targets;
       message       = '%(env)s - Failed to publish message to relay' % { env: vars.environment },
       notifications = vars.notifications,
       noDataState   = 'no_data',
+      period        = '0m',
       conditions    = [
         grafana.alertCondition.new(
-          evaluatorParams = [ 1 ],
+          evaluatorParams = [ 0 ],
           evaluatorType   = 'gt',
           operatorType    = 'or',
           queryRefId      = 'RelayOutgoingMessagePermenantFailures',
diff --git a/terraform/monitoring/panels/lb/error_5xx.libsonnet b/terraform/monitoring/panels/lb/error_5xx.libsonnet
index 43c7abbc..0ead551b 100644
--- a/terraform/monitoring/panels/lb/error_5xx.libsonnet
+++ b/terraform/monitoring/panels/lb/error_5xx.libsonnet
@@ -4,7 +4,7 @@ local defaults  = import '../../grafonnet-lib/defaults.libsonnet';
 local panels    = grafana.panels;
 local targets   = grafana.targets;
 
-local threshold = 1;
+local threshold = 0;
 
 local _configuration = defaults.configuration.timeseries
   .withSoftLimit(
@@ -23,6 +23,7 @@ local _alert(namespace, env, notifications) = grafana.alert.new(
   message       = '%(env)s - Notify - 5XX alert'  % { env: grafana.utils.strings.capitalize(env) },
   notifications = notifications,
   noDataState   = 'no_data',
+  period        = '0m',
   conditions    = [
     grafana.alertCondition.new(
       evaluatorParams = [ threshold ],