cloudwatch dashboard for singer taps

catalog/aws/singer-taps/cloudwatch.tf

@@ -0,0 +1,120 @@
+locals {
+  cloudwatch_query_text = <<EOF
+filter @message like /(Beginning|Completed) running/
+| filter @message not like /running discovery/
+| filter @message not like /\s--discover\s/
+| parse @message "catalog/${var.taps[0].id}-*-catalog.json" as tablename
+| parse @message " (* elapsed)" as elapsed
+| fields @timestamp, @message
+| sort tablename desc, @timestamp desc
+EOF
+  dashboard_markdown    = <<EOF
+## Data Pipeline (${var.taps[0].id}-to-${local.target.id})
+
+Additional Actions:
+
+ - [View Running ECS Tasks](https://console.aws.amazon.com/ecs/home?region=${var.environment.aws_region}#/clusters/${module.ecs_cluster.ecs_cluster_name}/tasks)
+ - [View ECS CloudWatch Logs](${module.ecs_tap_sync_task.ecs_logging_url})
+
+EOF
+}
+
+resource "aws_cloudwatch_dashboard" "main" {
+  dashboard_name = "${var.taps[0].id}-to-${local.target.id}-v${var.pipeline_version_number}-dashboard"
+  dashboard_body = <<EOF
+{
+  "periodOverride": "auto",
+  "widgets": [
+    {
+      "type": "text",
+      "x": 0,
+      "y": 0,
+      "width": 24,
+      "height": 3,
+      "properties": {
+        "markdown": "${
+  replace(replace(replace(local.dashboard_markdown, "\\", "\\\\"), "\n", "\\n"), "\"", "\\\"")
+  }"
+      }
+    },
+    {
+      "type": "metric",
+      "x": 0,
+      "y": 4,
+      "width": 12,
+      "height": 6,
+      "properties": {
+        "metrics": [
+          [
+            "ECS/ContainerInsights", "TaskCount",
+            "ClusterName", "${module.ecs_cluster.ecs_cluster_name}"
+          ]
+        ],
+        "period": 60,
+        "stat": "Average",
+        "region": "${var.environment.aws_region}",
+        "title": "Running ECS Task",
+        "yAxis": {
+          "left": {
+            "min": 0,
+            "showUnits": false
+          }
+        }
+      }
+    },
+    {
+      "type": "metric",
+      "x": 12,
+      "y": 4,
+      "width": 12,
+      "height": 6,
+      "properties": {
+        "metrics": [
+          [ "ECS/ContainerInsights", "CpuReserved", "ClusterName", "${module.ecs_cluster.ecs_cluster_name}", { "id": "m1", "yAxis": "right", "visible": false  } ],
+          [ ".", "CpuUtilized", ".", ".", { "id": "m2", "yAxis": "right" } ],
+          [ ".", "MemoryReserved", ".", ".", { "id": "m3", "yAxis": "right", "visible": false  } ],
+          [ ".", "MemoryUtilized", ".", ".", { "id": "m4", "yAxis": "right" } ],
+          [ { "expression": "100*(m2/m1)", "label": "CpuUtilization", "id": "e1" } ],
+          [ { "expression": "100*(m4/m3)", "label": "MemoryUtilization", "id": "e2" } ]
+        ],
+        "view": "timeSeries",
+        "stacked": false,
+        "region": "${var.environment.aws_region}",
+        "stat": "Average",
+        "period": 300,
+        "yAxis": {
+          "left": {
+            "min": 0,
+            "max": 100,
+            "label": "Percentage Usage"
+          },
+          "right": {
+            "label": "Reserved | Utilized",
+            "showUnits": false,
+            "min": 0
+          }
+        },
+        "legend": {
+          "position": "bottom"
+        }
+      }
+    },
+    {
+      "type": "log",
+      "x": 0,
+      "y": 9,
+      "width": 24,
+      "height": 12,
+      "properties": {
+        "query": "SOURCE '${module.ecs_tap_sync_task.cloudwatch_log_group_name}' | ${
+  replace(replace(replace(local.cloudwatch_query_text, "\\", "\\\\"), "\n", "\\n"), "\"", "\\\"")
+}",
+        "region": "${var.environment.aws_region}",
+        "stacked": "false",
+        "view": "table"
+      }
+    }
+  ]
+}
+EOF
+}

catalog/aws/singer-taps/main.tf

@@ -82,8 +82,9 @@ module "ecs_tap_sync_task" {
   use_fargate         = true
   environment_vars = merge(
     {
-      "TAP_CONFIG_DIR" : "${var.data_lake_metadata_path}/tap-snapshot-${local.unique_hash}",
-      "TAP_STATE_FILE" : "${coalesce(var.data_lake_storage_path, var.data_lake_metadata_path)}/${var.state_file_naming_scheme}",
+      TAP_CONFIG_DIR          = "${var.data_lake_metadata_path}/tap-snapshot-${local.unique_hash}",
+      TAP_STATE_FILE          = "${coalesce(var.data_lake_storage_path, var.data_lake_metadata_path)}/${var.state_file_naming_scheme}",
+      PIPELINE_VERSION_NUMBER = var.pipeline_version_number
     },
     {
       for k, v in var.taps[0].settings :

catalog/aws/singer-taps/variables.tf

@@ -123,3 +123,13 @@ variable "container_ram_gb" {
   description = "Optional. Specify the amount of RAM to be available to the container."
   default     = 1
 }
+variable "pipeline_version_number" {
+  description = <<EOF
+Optional. (Default="1") Specify a pipeline version number when there are breaking changes which require
+isolation. Note if you want to avoid overlap between versions, be sure to (1) cancel the
+previous version and (2) specify a `start_date` on the new version which is not duplicative
+of the previously covered time period.
+EOF
+  type        = string
+  default     = "1"
+}

components/aws/ecs-task/outputs.tf

@@ -1,3 +1,7 @@
+output "cloudwatch_log_group_name" {
+  description = "Name of Cloudwatch log group used for this task."
+  value       = aws_cloudwatch_log_group.cw_log_group.name
+}
 output "ecs_checklogs_cli" {
   description = "Command-ling string used to print Cloudwatch logs locally."
   value       = "aws logs get-log-events --log-group-name ${aws_cloudwatch_log_group.cw_log_group.name} --limit 100"