BFD-3565: Simplify BFD Server error alerting infrastructure (#2534)

CMSgov · Jan 30, 2025 · cbec656 · cbec656
1 parent 36bb743
commit cbec656
Show file tree

Hide file tree

Showing 8 changed files with 72 additions and 498 deletions.
diff --git a/.github/scripts/pre-commit.sh b/.github/scripts/pre-commit.sh
@@ -15,8 +15,8 @@ checkSecretFilesForPlainText() {
   IFS=$'\n' read -d '' -r -a commits <"$tmpfile"
 
   # for all files to be committed which are a secret file, grep for
-  #   1. "<<SECURE>> or <</SECURE>>" 
-  #   2. "CIPHER ENCRYPTED FILE HEADER; SHOULD BE ENCRYPTED" 
+  #   1. "<<SECURE>> or <</SECURE>>"
+  #   2. "CIPHER ENCRYPTED FILE HEADER; SHOULD BE ENCRYPTED"
   # if either is found, this indicates that secrets are possibly unencrypted and the file should not
   # be committed. additionally, the file header preamble is grepped for to ensure that the header
   # was not removed unintentionally
@@ -62,7 +62,7 @@ runShellCheckForCommitFiles() {
 
       # Skip binary formats
       case "$extension" in
-        "zip" | "p12" | "pfx" | "cer" | "pem")
+        "zip" | "p12" | "pfx" | "cer" | "pem" | "png" | "jpg")
           continue ;;
         *) ;;
       esac
@@ -73,7 +73,7 @@ runShellCheckForCommitFiles() {
         # run shellcheck with severity level warning, and suppress warnings about invalid hashbangs (allows it to ignore other types of scripts, e.g. python)
         if ! shellcheck -e SC1071,SC2239 -S warning "$file"; then
           echo "Please fix errors before continuing."
-          exit 1 
+          exit 1
         fi
       fi
     done

diff --git a/ops/terraform/services/server/modules/bfd_server_error_alerts/README.md b/ops/terraform/services/server/modules/bfd_server_error_alerts/README.md
@@ -6,7 +6,7 @@
 
 ## Overview
 
-This module implements Slack alerts for _any_ `500`s (errors) logged by BFD Server in a particular environment (`terraform workspace`). Whenever a particular environment's `server` logs a `500`, an alert will be sent within 10 to 15 seconds, and then additional alerts will be sent periodically at per-environment intervals until the errors cease in the given environment.
+This module implements Slack alerts for _any_ `500`s (errors) logged by BFD Server in a particular environment (`terraform workspace`). Whenever a particular environment's `server` logs a `500`, an alert will be sent within 10 to 15 seconds.
 
 As it is possible that _many_ errors occur in a short period, these alerts summarize the number of errors that have been returned during a "lookback" period (defined per-environment). These summaries include a count of `500`s per-partner, per-API-operation, along with a total number of errors returned within the "lookback" period.
 
@@ -23,45 +23,37 @@ Along with the summary, the alert includes various links to Log Insights queries
 No requirements.
 
 <!-- GENERATED WITH `terraform-docs .`
-Manually updating the README.md will be overwritten.
-For more details, see the file '.terraform-docs.yml' or
-https://terraform-docs.io/user-guide/configuration/
+     Manually updating the README.md will be overwritten.
+     For more details, see the file '.terraform-docs.yml' or
+     https://terraform-docs.io/user-guide/configuration/
 -->
 
 ## Inputs
 
 No inputs.
 
 <!-- GENERATED WITH `terraform-docs .`
-Manually updating the README.md will be overwritten.
-For more details, see the file '.terraform-docs.yml' or
-https://terraform-docs.io/user-guide/configuration/
+     Manually updating the README.md will be overwritten.
+     For more details, see the file '.terraform-docs.yml' or
+     https://terraform-docs.io/user-guide/configuration/
 -->
 
 ## Resources
 
 | Name | Type |
 |------|------|
-| [aws_cloudwatch_metric_alarm.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_metric_alarm) | resource |
 | [aws_iam_policy.invoke_alerter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.logs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.read_log](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
-| [aws_iam_policy.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
-| [aws_iam_role.lambda_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.alerter_lambda_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.scheduler_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.invoke_alerter_to_scheduler_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_iam_role_policy_attachment.logs_to_lambda_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_iam_role_policy_attachment.read_log_to_alerter_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_iam_role_policy_attachment.scheduler_to_alert_scheduler_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
-| [aws_lambda_function.alert_lambda_scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
-| [aws_lambda_function.alerting_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
-| [aws_lambda_permission.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_iam_role_policy_attachment.logs_to_alerter_lambda_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.read_log_to_alerter_lambda_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_lambda_function.alerter_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_function) | resource |
+| [aws_scheduler_schedule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule) | resource |
 | [aws_scheduler_schedule_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/scheduler_schedule_group) | resource |
-| [aws_sns_topic.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic) | resource |
-| [aws_sns_topic_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_policy) | resource |
-| [aws_sns_topic_subscription.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/sns_topic_subscription) | resource |
-| [archive_file.alert_lambda_scheduler_src](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
-| [archive_file.alerting_lambda_src](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
+| [archive_file.alerter_lambda_src](https://registry.terraform.io/providers/hashicorp/archive/latest/docs/data-sources/file) | data source |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [aws_kms_key.cmk](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source |
 | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
@@ -70,9 +62,9 @@ https://terraform-docs.io/user-guide/configuration/
 | [aws_ssm_parameters_by_path.nonsensitive_service](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameters_by_path) | data source |
 
 <!-- GENERATED WITH `terraform-docs .`
-Manually updating the README.md will be overwritten.
-For more details, see the file '.terraform-docs.yml' or
-https://terraform-docs.io/user-guide/configuration/
+     Manually updating the README.md will be overwritten.
+     For more details, see the file '.terraform-docs.yml' or
+     https://terraform-docs.io/user-guide/configuration/
 -->
 
 ## Outputs

diff --git a/ops/terraform/services/server/modules/bfd_server_error_alerts/data-sources.tf b/ops/terraform/services/server/modules/bfd_server_error_alerts/data-sources.tf
@@ -19,22 +19,12 @@ data "aws_ssm_parameter" "alerter_slack_webhook" {
   with_decryption = true
 }
 
-data "archive_file" "alert_lambda_scheduler_src" {
+data "archive_file" "alerter_lambda_src" {
   type        = "zip"
-  output_path = "${path.module}/lambda_src/${local.alert_lambda_scheduler_src}.zip"
+  output_path = "${path.module}/lambda_src/${local.alerter_lambda_src}.zip"
 
   source {
-    content  = file("${path.module}/lambda_src/${local.alert_lambda_scheduler_src}.py")
-    filename = "${local.alert_lambda_scheduler_src}.py"
-  }
-}
-
-data "archive_file" "alerting_lambda_src" {
-  type        = "zip"
-  output_path = "${path.module}/lambda_src/${local.alerting_lambda_src}.zip"
-
-  source {
-    content  = file("${path.module}/lambda_src/${local.alerting_lambda_src}.py")
-    filename = "${local.alerting_lambda_src}.py"
+    content  = file("${path.module}/lambda_src/${local.alerter_lambda_src}.py")
+    filename = "${local.alerter_lambda_src}.py"
   }
 }
diff --git a/...terraform/services/server/modules/bfd_server_error_alerts/docs/diagram_src/gen_diagram.py b/...terraform/services/server/modules/bfd_server_error_alerts/docs/diagram_src/gen_diagram.py
@@ -4,8 +4,8 @@
 
 from diagrams import Cluster, Diagram, Edge
 from diagrams.aws.compute import LambdaFunction
-from diagrams.aws.integration import Eventbridge, SimpleNotificationServiceSnsTopic
-from diagrams.aws.management import Cloudwatch, CloudwatchAlarm
+from diagrams.aws.integration import Eventbridge
+from diagrams.aws.management import Cloudwatch
 from diagrams.saas.chat import Slack
 
 graph_attrs = {"direction": "RL"}
@@ -23,41 +23,24 @@ def _solid_cluster(
     )
 
 
-with Diagram("bfd-2728", graph_attr=graph_attrs, outformat="png", show=False):
+with Diagram(
+    "BFD Server Error Alerts Infastructure", graph_attr=graph_attrs, outformat="png", show=False
+):
     with _solid_cluster(label="server Terraservice"):
-        metrics = Cloudwatch("Log metrics for 500s")
         with _solid_cluster(label="bfd_server_error_alerts Terraform module"):
-            err_alarm = CloudwatchAlarm("Alarm for any 500")
-            sns_topic = SimpleNotificationServiceSnsTopic("Scheduler Lambda Topic")
-            scheduler_lambda = LambdaFunction("Scheduler Lambda")
-            event_bridge = Eventbridge("EventBridge Scheduler")
+            event_bridge = Eventbridge("EventBridge Scheduler\nrate(5 minutes) Schedule")
             with _solid_cluster(
                 label="Alerter Lambda", direction="TB", graph_attr={"rankdir": "TB"}
             ):
                 log_insights = Cloudwatch("Log Insights\naccess.json")
                 insights_slack_lambda = LambdaFunction("Alerter Lambda\nFunction")
 
-    metrics >> err_alarm
-    err_alarm >> Edge(label="OK -> ALARM") >> sns_topic
-    err_alarm >> Edge(label="ALARM -> OK") >> sns_topic
-    sns_topic >> Edge(label="Event indicating\nOK -> ALARM") >> scheduler_lambda
-    sns_topic >> Edge(label="Event indicating\nALARM -> OK") >> scheduler_lambda
-    (
-        scheduler_lambda
-        >> Edge(label="rate every 5 mins\n+ start in 10 sec\n+ remove any orphan scheds")
-        >> event_bridge
-    )
-    (
-        scheduler_lambda
-        >> Edge(label="remove rate every 5 mins\n+ start in 10 sec\n+ remove any orphan scheds")
-        >> event_bridge
-    )
     (
         log_insights
         >> Edge(label="past 5:30 mins of 500s", reverse=True, forward=False)
         >> insights_slack_lambda
     )
-    (event_bridge >> Edge(label="rate(5 minutes),\none-time sched") >> insights_slack_lambda)
+    (event_bridge >> insights_slack_lambda)
     (
         insights_slack_lambda
         >> Edge(

diff --git a/...ules/bfd_server_error_alerts/docs/resources/bfd_server_error_alerts_diagram.png b/...ules/bfd_server_error_alerts/docs/resources/bfd_server_error_alerts_diagram.png
diff --git a/ops/terraform/services/server/modules/bfd_server_error_alerts/iam.tf b/ops/terraform/services/server/modules/bfd_server_error_alerts/iam.tf
@@ -1,10 +1,8 @@
 resource "aws_iam_policy" "logs" {
-  for_each = toset(["${local.alert_lambda_scheduler_name}", "${local.alerting_lambda_name}"])
-
-  name = "${each.key}-logs"
+  name = "${local.alerter_lambda_name}-logs"
   description = join("", [
-    "Permissions for the ${each.key} Lambda to write to its corresponding CloudWatch Log Group ",
-    "and Log Stream"
+    "Permissions for the ${local.alerter_lambda_name} Lambda to write to its corresponding ",
+    "CloudWatch Log Group and Log Stream"
   ])
 
   policy = jsonencode(
@@ -20,52 +18,18 @@ resource "aws_iam_policy" "logs" {
           Effect = "Allow"
           Action = ["logs:CreateLogStream", "logs:PutLogEvents"]
           Resource = [
-            "arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/${each.key}:*"
+            "arn:aws:logs:${local.region}:${local.account_id}:log-group:/aws/lambda/${local.alerter_lambda_name}:*"
           ]
         }
       ]
     }
   )
 }
 
-resource "aws_iam_policy" "scheduler" {
-  name = "${local.alert_lambda_scheduler_name}-scheduler"
-  description = join("", [
-    "Permissions for ${local.alert_lambda_scheduler_name} to create and delete schedules within ",
-    "the ${aws_scheduler_schedule_group.this.name} schedule group, list all schedules, and pass ",
-    "the ${aws_iam_role.scheduler_assume_role.name} role to EventBridge Scheduler"
-  ])
-
-  policy = jsonencode(
-    {
-      Version = "2012-10-17"
-      Statement = [
-        {
-          Effect   = "Allow"
-          Action   = ["scheduler:CreateSchedule", "scheduler:DeleteSchedule"]
-          Resource = ["arn:aws:scheduler:${local.region}:${local.account_id}:schedule/${aws_scheduler_schedule_group.this.name}/*"]
-        },
-        # Unfortunately, ListSchedules does not support any resource-level restrictions
-        # See https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazoneventbridgescheduler.html#amazoneventbridgescheduler-schedule
-        {
-          Effect   = "Allow"
-          Action   = ["scheduler:ListSchedules"]
-          Resource = ["*"]
-        },
-        {
-          Effect   = "Allow"
-          Action   = ["iam:PassRole"]
-          Resource = [aws_iam_role.scheduler_assume_role.arn]
-        }
-      ]
-    }
-  )
-}
-
 resource "aws_iam_policy" "read_log" {
-  name = "${local.alerting_lambda_name}-read-logs"
+  name = "${local.alerter_lambda_name}-read-logs"
   description = join("", [
-    "Permissions for ${local.alerting_lambda_name} to start and retrieve the results of a Log ",
+    "Permissions for ${local.alerter_lambda_name} to start and retrieve the results of a Log ",
     "Insights query against the ${local.access_json_log_group_name} CloudWatch Log Group"
   ])
 
@@ -74,8 +38,8 @@ resource "aws_iam_policy" "read_log" {
       Version = "2012-10-17"
       Statement = [
         {
-          Effect   = "Allow"
-          Action   = ["logs:StartQuery"]
+          Effect = "Allow"
+          Action = ["logs:StartQuery"]
           Resource = [
             "arn:aws:logs:${local.region}:${local.account_id}:log-group:${local.access_json_log_group_name}",
             "arn:aws:logs:${local.region}:${local.account_id}:log-group:${local.access_json_log_group_name}:log-stream:*",
@@ -93,12 +57,11 @@ resource "aws_iam_policy" "read_log" {
   )
 }
 
-
 resource "aws_iam_policy" "invoke_alerter" {
   name = "${local.name_prefix}-scheduler-assumee-allow-lambda-invoke"
   description = join("", [
     "Permissions for EventBridge Scheduler assumed role to invoke the ",
-    "${local.alerting_lambda_name} Lambda"
+    "${local.alerter_lambda_name} Lambda"
   ])
 
   policy = jsonencode(
@@ -108,20 +71,17 @@ resource "aws_iam_policy" "invoke_alerter" {
         {
           Effect   = "Allow"
           Action   = "lambda:InvokeFunction"
-          Resource = aws_lambda_function.alerting_lambda.arn
+          Resource = aws_lambda_function.alerter_lambda.arn
         }
       ]
     }
   )
 }
 
-
-resource "aws_iam_role" "lambda_roles" {
-  for_each = toset(["${local.alert_lambda_scheduler_name}", "${local.alerting_lambda_name}"])
-
-  name        = each.key
+resource "aws_iam_role" "alerter_lambda_role" {
+  name        = local.alerter_lambda_name
   path        = "/"
-  description = "Role for ${each.key} Lambda"
+  description = "Role for ${local.alerter_lambda_name} Lambda"
 
   assume_role_policy = jsonencode(
     {
@@ -142,11 +102,11 @@ resource "aws_iam_role" "lambda_roles" {
 }
 
 resource "aws_iam_role" "scheduler_assume_role" {
-  name = "${local.name_prefix}-scheduler-assumee"
+  name = "${local.name_prefix}-scheduler-assume"
   path = "/"
   description = join("", [
     "Role for EventBridge Scheduler to assume allowing permissions to invoke the ",
-    "${local.alerting_lambda_name} Lambda"
+    "${local.alerter_lambda_name} Lambda"
   ])
 
   assume_role_policy = jsonencode(
@@ -167,23 +127,16 @@ resource "aws_iam_role" "scheduler_assume_role" {
   force_detach_policies = true
 }
 
-resource "aws_iam_role_policy_attachment" "logs_to_lambda_roles" {
-  for_each = toset(["${local.alert_lambda_scheduler_name}", "${local.alerting_lambda_name}"])
-
-  role       = aws_iam_role.lambda_roles[each.key].name
-  policy_arn = aws_iam_policy.logs[each.key].arn
+resource "aws_iam_role_policy_attachment" "logs_to_alerter_lambda_role" {
+  role       = aws_iam_role.alerter_lambda_role.name
+  policy_arn = aws_iam_policy.logs.arn
 }
 
-resource "aws_iam_role_policy_attachment" "read_log_to_alerter_role" {
-  role       = aws_iam_role.lambda_roles[local.alerting_lambda_name].name
+resource "aws_iam_role_policy_attachment" "read_log_to_alerter_lambda_role" {
+  role       = aws_iam_role.alerter_lambda_role.name
   policy_arn = aws_iam_policy.read_log.arn
 }
 
-resource "aws_iam_role_policy_attachment" "scheduler_to_alert_scheduler_role" {
-  role       = aws_iam_role.lambda_roles[local.alert_lambda_scheduler_name].name
-  policy_arn = aws_iam_policy.scheduler.arn
-}
-
 resource "aws_iam_role_policy_attachment" "invoke_alerter_to_scheduler_assume_role" {
   role       = aws_iam_role.scheduler_assume_role.name
   policy_arn = aws_iam_policy.invoke_alerter.arn