init

README.md

@@ -1,2 +1,26 @@
-# cloud-storage-monitoring
+# Terraform Module: cloud-storage-monitoring
 Basic module to monitor object count in buckets.
+
+```terraform
+module "cloud_storage_monitor" {
+  source = "./modules/cloud-storage-monitoring"
+
+  name       = "storage-monitor-v1"
+  bucket_paths = [
+    "<BUCKET>/some/folder",
+  ]
+  cron_schedule         = "*/30 * * * *"
+  cron_time_zone        = "Australia/Melbourne"
+  region                = "us-east1"
+  threshold             = "1000"
+  timeout               = "7200s"
+  service_account       = data.google_compute_default_service_account.default.email
+  notification_channels = [
+    # Add notification channel ids here
+  ]
+}
+
+data "google_compute_default_service_account" "default" {}
+
+data "google_project" "current" {}
+```

data.tf

@@ -0,0 +1,5 @@
+data "google_service_account" "current" {
+  account_id = var.service_account
+}
+
+data "google_project" "current" {}

google_cloud_build_trigger.tf

@@ -0,0 +1,41 @@
+resource "google_cloudbuild_trigger" "monitor" {
+  for_each        = local.bucket_paths
+
+  name            = format("%s-%s", var.name, each.key)
+  description     = format("monitoring %s", each.value)
+  location        = var.region
+  disabled        = false
+  tags            = ["monitoring", "storage"]
+  service_account = data.google_service_account.current.id
+
+  build {
+    timeout = var.timeout
+    substitutions = {
+      _BUCKET_PATHS : "${each.key}"
+      _ALERT_THRESHOLD : var.threshold
+      _CLOUD_LOGGING_SEVERITY : "WARNING"
+      _SERVICE_NAME : var.name
+    }
+
+    step {
+      name       = "gcr.io/cloud-builders/gsutil"
+      entrypoint = "bash"
+      args = [
+        "-c",
+        file("${path.module}/script.sh")
+      ]
+    }
+
+    options {
+      substitution_option   = "ALLOW_LOOSE"
+      dynamic_substitutions = true
+      logging               = "CLOUD_LOGGING_ONLY"
+    }
+  }
+
+  # Required but can be left blank
+  trigger_template {
+    branch_name = ""
+    repo_name   = ""
+  }
+}

google_cloud_scheduler.tf

@@ -0,0 +1,29 @@
+resource "google_cloud_scheduler_job" "monitor" {
+  for_each = local.bucket_paths
+
+  name             = format("%s-%s", var.name, each.key)
+  description      = format("monitoring %s", each.value)
+  schedule         = var.cron_schedule
+  time_zone        = var.cron_time_zone
+  attempt_deadline = "320s"
+  region           = var.region
+
+  retry_config {
+    retry_count = 3
+  }
+
+  http_target {
+    http_method = "POST"
+    uri = format(
+      "https://cloudbuild.googleapis.com/v1/projects/%s/locations/%s/triggers/%s:run",
+      data.google_project.current.project_id,
+      google_cloudbuild_trigger.monitor[each.key].location,
+      google_cloudbuild_trigger.monitor[each.key].trigger_id,
+    )
+    oauth_token {
+      service_account_email = data.google_service_account.current.email
+    }
+
+    body = base64encode("")
+  }
+}

google_monitoring_alert_policy.tf

@@ -0,0 +1,22 @@
+resource "google_monitoring_alert_policy" "object_gt_threshold" {
+  display_name          = "(Terraform) Bucket Object Count Greater Than Threshold"
+  combiner              = "OR"
+  notification_channels = var.notification_channels
+
+  conditions {
+    display_name = "Log match condition"
+    condition_matched_log {
+      filter = <<-EOT
+            severity=WARNING
+            logName="projects/${data.google_project.current.project_id}/${var.name}"
+            jsonPayload.object_count >= ${var.threshold}
+        EOT
+    }
+  }
+  alert_strategy {
+    notification_rate_limit {
+      period = "3600s"
+    }
+    auto_close = "604800s"
+  }
+}

locals.tf

@@ -0,0 +1,3 @@
+locals {
+  bucket_paths = { for p in var.bucket_paths : index(var.bucket_paths, p) => p }
+}

script.sh

@@ -0,0 +1,16 @@
+IFS=' ' read -ra ITEMS <<< "$_BUCKET_PATHS"
+
+for ITEM in "$${ITEMS[@]}"; do
+
+    subfolders=$(gsutil ls -d gs://$$ITEM/*/ | grep "/$")
+    while IFS= read -r subfolder; do
+
+        object_count=$(gsutil ls -r $$subfolder** | wc -l)
+        echo "Path: $${subfolder}, Object Count: $${object_count}"
+
+        if [[ "$$object_count" -gt "$_ALERT_THRESHOLD" ]]; then
+            gcloud logging write --payload-type=json --severity="$_CLOUD_LOGGING_SEVERITY" "$_SERVICE_NAME"  "{\"message\": \"object_count_gt_threshold\", \"build_id\":\"$BUILD_ID\", \"threshold\": $_ALERT_THRESHOLD, \"subfolder_path\": \"$$subfolder\", \"object_count\": $$object_count}"
+        fi
+
+    done <<< "$$subfolders"
+done

variables.tf

@@ -0,0 +1,58 @@
+variable "name" {
+  type        = string
+  description = "service name"
+}
+
+variable "bucket_paths" {
+  type        = list(string)
+  description = "list of bucket paths"
+
+  validation {
+    condition     = length(var.bucket_paths) >= 1
+    error_message = "length of 'bucket_paths' cannot be 0"
+  }
+}
+
+variable "cron_schedule" {
+  type        = string
+  description = "cron schedule (e.g. * * * * *)"
+}
+
+variable "cron_time_zone" {
+  type        = string
+  description = "cron time zone (e.g. Australia/Melbourne)"
+}
+
+variable "region" {
+  type        = string
+  description = "deployment region (e.g. asia-east1, us-east1)"
+
+  validation {
+    condition = !contains(
+      ["global"],
+      var.region
+    )
+    error_message = "Err: region '${var.region}' is not supported."
+  }
+}
+
+variable "threshold" {
+  type        = string
+  description = "object threshold before log is created"
+}
+
+variable "timeout" {
+  type        = string
+  description = "timeout before build fails (e.g. 300s, 7200s)"
+}
+
+variable "service_account" {
+  type        = string
+  description = "service account email"
+}
+
+variable "notification_channels" {
+  type        = list(string)
+  description = "notication channel ids (['projects/[PROJECT_ID]/notificationChannels/[CHANNEL_ID]'])"
+}
+