chore: Added Datadog heartbeat metrics

edx · Sep 4, 2024 · 23b5090 · 23b5090
1 parent 21066af
commit 23b5090
Show file tree

Hide file tree

Showing 10 changed files with 85 additions and 0 deletions.
diff --git a/dataeng/resources/opsgenie-enable-heartbeat.sh b/dataeng/resources/opsgenie-enable-heartbeat.sh
@@ -13,6 +13,7 @@
 
 
 OPSGENIE_HEARTBEAT_API_URL="https://api.opsgenie.com/v2/heartbeats"
+PAGERDUTY_HEARTBEAT_API_URL=""
 
 if [ -n "$OPSGENIE_HEARTBEAT_NAME" ] && \
    [ -n "$OPSGENIE_HEARTBEAT_CONFIG_KEY" ] &&  \

diff --git a/devops/jobs/CheckASGLifeCycleHooks.groovy b/devops/jobs/CheckASGLifeCycleHooks.groovy
@@ -56,6 +56,7 @@ class CheckASGLifeCycleHooks {
                         credentialsBinding {
                             string('ROLE_ARN', "check-lifecycle-hooks-${deployment}-role-arn")
                             string("GENIE_KEY", "opsgenie_heartbeat_key")
+                            string("DD_KEY", "datadog_heartbeat_key")
                         }
                     }
 
@@ -104,6 +105,21 @@ class CheckASGLifeCycleHooks {
                         if (opsgenie_heartbeat_name) {
                              shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                         }
+                        String datadog_heartbeat_name = configuration.get('DATADOG_HEARTBEAT_NAME', '')
+                        if (datadog_heartbeat_name) {
+                            String DD_JSON = """
+                                {
+                                    "series": [{
+                                        "metric": "${datadog_heartbeat_name}",
+                                        "points": [['\$(date +%s)', 1]],
+                                        "type": "gauge",
+                                        "tags": ["env:${deployment}"]
+                                    }]
+                                }
+                                """
+
+                                shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                        }
                     }
                 }
             }

diff --git a/devops/jobs/CheckTableSize.groovy b/devops/jobs/CheckTableSize.groovy
@@ -25,6 +25,7 @@ class CheckTableSize {
                             def variable = "${deployment}-table-size-monitoring"
                             string("ROLE_ARN", variable)
                             string("GENIE_KEY", "opsgenie_heartbeat_key")
+                            string("DD_KEY", "datadog_heartbeat_key")
                         }
                     }
 

diff --git a/devops/jobs/CreateSandboxCI.groovy b/devops/jobs/CreateSandboxCI.groovy
@@ -32,6 +32,7 @@ class CreateSandboxCI {
         wrappers {
             credentialsBinding {
                 string("GENIE_KEY", "opsgenie_heartbeat_key")
+                string("DD_KEY", "datadog_heartbeat_key")
             }
         }
 
@@ -124,6 +125,18 @@ class CreateSandboxCI {
             }
           }
           shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/SandboxCI'+type+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
+          def metricName = "SandboxCI${type}.heartbeat"
+          String DD_JSON = """
+                          {
+                              "series": [{
+                                  "metric": "${metricName}",
+                                  "points": [['\$(date +%s)', 1]],
+                                  "type": "gauge"
+                              }]
+                          }
+                          """
+
+          shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
         }
 
         triggers {

diff --git a/devops/jobs/Janitor.groovy b/devops/jobs/Janitor.groovy
@@ -42,6 +42,7 @@ class Janitor {
                     credentialsBinding {
                         string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")
                         string("GENIE_KEY", "opsgenie_heartbeat_key")
+                        string("DD_KEY", "datadog_heartbeat_key")
                     }
                 }
 
@@ -102,6 +103,7 @@ class Janitor {
                     env('AWS_CLEANER', configuration.get('aws_cleaner'))
                     env('DENY_LIST', configuration.get('deny_list'))
                     env('NOOP', extraVars.get('NOOP', false))
+                    env('DEPLOYMENT', deployment)
                 }
 
 

diff --git a/devops/jobs/JenkinsHeartbeat.groovy b/devops/jobs/JenkinsHeartbeat.groovy
@@ -19,6 +19,7 @@ class JenkinsHeartbeat{
             wrappers {
                 credentialsBinding {
                     string("GENIE_KEY", "opsgenie_heartbeat_key")
+                    string("DD_KEY", "datadog_heartbeat_key")
                 }
             }
 
@@ -30,6 +31,20 @@ class JenkinsHeartbeat{
                 if (opsgenie_heartbeat_name) {
                     shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                 }
+                String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '')
+                if (datadog_heartbeat_name) {
+                    String DD_JSON = """
+                        {
+                            "series": [{
+                                "metric": "${datadog_heartbeat_name}",
+                                "points": [['\$(date +%s)', 1]],
+                                "type": "gauge"
+                            }]
+                        }
+                        """
+
+                        shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                    }
             }
 
         }

diff --git a/devops/jobs/ProspectusJanitor.groovy b/devops/jobs/ProspectusJanitor.groovy
@@ -41,6 +41,7 @@ class ProspectusJanitor {
                     credentialsBinding {
                         string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn")
                         string("GENIE_KEY", "opsgenie_heartbeat_key")
+                        string("DD_KEY", "datadog_heartbeat_key")
                     }
                 }
 
@@ -100,6 +101,7 @@ class ProspectusJanitor {
                     env('AWS_REGION', configuration.get('aws_region', 'us-east-1'))
                     env('AWS_CLEANER', configuration.get('aws_cleaner'))
                     env('NOOP', extraVars.get('NOOP', false))
+                    env('DEPLOYMENT', deployment)
                 }
 
 

diff --git a/devops/jobs/SandboxTermination.groovy b/devops/jobs/SandboxTermination.groovy
@@ -37,6 +37,7 @@ class SandboxTermination{
                     string('ROLE_ARN', "launch-sandboxes-role-arn")
                     string('EDX_GIT_BOT_TOKEN', "edx_git_bot_token")
                     string("GENIE_KEY", "opsgenie_heartbeat_key")
+                    string("DD_KEY", "datadog_heartbeat_key")
                 }
             }
 
@@ -102,6 +103,20 @@ class SandboxTermination{
                 if (opsgenie_heartbeat_name) {
                     shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"')
                 }
+                String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '')
+                if (datadog_heartbeat_name) {
+                    String DD_JSON = """
+                        {
+                            "series": [{
+                                "metric": "${datadog_heartbeat_name}",
+                                "points": [['\$(date +%s)', 1]],
+                                "type": "gauge"
+                            }]
+                        }
+                        """
+
+                        shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'')
+                    }
             }
 
             if (extraVars.get('NOTIFY_ON_FAILURE')){

diff --git a/devops/resources/janitor.sh b/devops/resources/janitor.sh
@@ -43,3 +43,13 @@ else
 fi
 
 curl -X GET 'https://api.opsgenie.com/v2/heartbeats/'${JOB_NAME##*/}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY}
+curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \
+-H "Content-Type: application/json" \
+-d '{
+      "series" : [{
+          "metric": '${JOB_NAME##*/}".heartbeat"',
+          "points": [['"$(date +%s)"', 1]],
+          "type": "gauge",
+          "tags": ["env:'${DEPLOYMENT}'"]
+      }]
+  }'
diff --git a/devops/resources/table-size-monitoring.sh b/devops/resources/table-size-monitoring.sh
@@ -32,3 +32,13 @@ fi
 python check_table_size.py  --threshold ${THRESHOLD} ${RDSTHRESHOLD} ${RDSIGNORE}
 
 curl -X GET 'https://api.opsgenie.com/v2/heartbeats/table-size-monitoring-'${DEPLOYMENT}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY}
+curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \
+-H "Content-Type: application/json" \
+-d '{
+      "series" : [{
+          "metric": "table-size-monitoring-'${DEPLOYMENT}'.heartbeat",
+          "points": [['"$(date +%s)"', 1]],
+          "type": "gauge",
+          "tags": ["env:'${DEPLOYMENT}'"]
+      }]
+  }'