diff --git a/dataeng/resources/opsgenie-enable-heartbeat.sh b/dataeng/resources/opsgenie-enable-heartbeat.sh index 47b040344..b497b1ab9 100644 --- a/dataeng/resources/opsgenie-enable-heartbeat.sh +++ b/dataeng/resources/opsgenie-enable-heartbeat.sh @@ -13,6 +13,7 @@ OPSGENIE_HEARTBEAT_API_URL="https://api.opsgenie.com/v2/heartbeats" +PAGERDUTY_HEARTBEAT_API_URL="" if [ -n "$OPSGENIE_HEARTBEAT_NAME" ] && \ [ -n "$OPSGENIE_HEARTBEAT_CONFIG_KEY" ] && \ diff --git a/devops/jobs/CheckASGLifeCycleHooks.groovy b/devops/jobs/CheckASGLifeCycleHooks.groovy index f36052cd3..4e7549a45 100644 --- a/devops/jobs/CheckASGLifeCycleHooks.groovy +++ b/devops/jobs/CheckASGLifeCycleHooks.groovy @@ -56,6 +56,7 @@ class CheckASGLifeCycleHooks { credentialsBinding { string('ROLE_ARN', "check-lifecycle-hooks-${deployment}-role-arn") string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -104,6 +105,21 @@ class CheckASGLifeCycleHooks { if (opsgenie_heartbeat_name) { shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"') } + String datadog_heartbeat_name = configuration.get('DATADOG_HEARTBEAT_NAME', '') + if (datadog_heartbeat_name) { + String DD_JSON = """ + { + "series": [{ + "metric": "${datadog_heartbeat_name}", + "points": [['\$(date +%s)', 1]], + "type": "gauge", + "tags": ["env:${deployment}"] + }] + } + """ + + shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'') + } } } } diff --git a/devops/jobs/CheckTableSize.groovy b/devops/jobs/CheckTableSize.groovy index 654313831..595ec3703 100644 --- a/devops/jobs/CheckTableSize.groovy +++ b/devops/jobs/CheckTableSize.groovy @@ -25,6 +25,7 @@ class CheckTableSize { def variable = "${deployment}-table-size-monitoring" string("ROLE_ARN", variable) string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } diff --git a/devops/jobs/CreateSandboxCI.groovy b/devops/jobs/CreateSandboxCI.groovy index dba7ed1a1..c63177ca6 100644 --- a/devops/jobs/CreateSandboxCI.groovy +++ b/devops/jobs/CreateSandboxCI.groovy @@ -32,6 +32,7 @@ class CreateSandboxCI { wrappers { credentialsBinding { string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -124,6 +125,18 @@ class CreateSandboxCI { } } shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/SandboxCI'+type+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"') + def metricName = "SandboxCI${type}.heartbeat" + String DD_JSON = """ + { + "series": [{ + "metric": "${metricName}", + "points": [['\$(date +%s)', 1]], + "type": "gauge" + }] + } + """ + + shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'') } triggers { diff --git a/devops/jobs/Janitor.groovy b/devops/jobs/Janitor.groovy index b618365c7..65a66817b 100644 --- a/devops/jobs/Janitor.groovy +++ b/devops/jobs/Janitor.groovy @@ -42,6 +42,7 @@ class Janitor { credentialsBinding { string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn") string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -102,6 +103,7 @@ class Janitor { env('AWS_CLEANER', configuration.get('aws_cleaner')) env('DENY_LIST', configuration.get('deny_list')) env('NOOP', extraVars.get('NOOP', false)) + env('DEPLOYMENT', deployment) } diff --git a/devops/jobs/JenkinsHeartbeat.groovy b/devops/jobs/JenkinsHeartbeat.groovy index dd72dcf51..297ffb8dc 100644 --- a/devops/jobs/JenkinsHeartbeat.groovy +++ b/devops/jobs/JenkinsHeartbeat.groovy @@ -19,6 +19,7 @@ class JenkinsHeartbeat{ wrappers { credentialsBinding { string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -30,6 +31,20 @@ class JenkinsHeartbeat{ if (opsgenie_heartbeat_name) { shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"') } + String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '') + if (datadog_heartbeat_name) { + String DD_JSON = """ + { + "series": [{ + "metric": "${datadog_heartbeat_name}", + "points": [['\$(date +%s)', 1]], + "type": "gauge" + }] + } + """ + + shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'') + } } } diff --git a/devops/jobs/ProspectusJanitor.groovy b/devops/jobs/ProspectusJanitor.groovy index fbca7a5e2..435eca579 100644 --- a/devops/jobs/ProspectusJanitor.groovy +++ b/devops/jobs/ProspectusJanitor.groovy @@ -41,6 +41,7 @@ class ProspectusJanitor { credentialsBinding { string('ROLE_ARN', "tools-jenkins-janitor-${deployment}-role-arn") string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -100,6 +101,7 @@ class ProspectusJanitor { env('AWS_REGION', configuration.get('aws_region', 'us-east-1')) env('AWS_CLEANER', configuration.get('aws_cleaner')) env('NOOP', extraVars.get('NOOP', false)) + env('DEPLOYMENT', deployment) } diff --git a/devops/jobs/SandboxTermination.groovy b/devops/jobs/SandboxTermination.groovy index 7e1e28148..c8a641528 100644 --- a/devops/jobs/SandboxTermination.groovy +++ b/devops/jobs/SandboxTermination.groovy @@ -37,6 +37,7 @@ class SandboxTermination{ string('ROLE_ARN', "launch-sandboxes-role-arn") string('EDX_GIT_BOT_TOKEN', "edx_git_bot_token") string("GENIE_KEY", "opsgenie_heartbeat_key") + string("DD_KEY", "datadog_heartbeat_key") } } @@ -102,6 +103,20 @@ class SandboxTermination{ if (opsgenie_heartbeat_name) { shell('curl -X GET "https://api.opsgenie.com/v2/heartbeats/'+opsgenie_heartbeat_name+'/ping" -H "Authorization: GenieKey ${GENIE_KEY}"') } + String datadog_heartbeat_name = extraVars.get('DATADOG_HEARTBEAT_NAME', '') + if (datadog_heartbeat_name) { + String DD_JSON = """ + { + "series": [{ + "metric": "${datadog_heartbeat_name}", + "points": [['\$(date +%s)', 1]], + "type": "gauge" + }] + } + """ + + shell('curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" -H "Content-Type: application/json" -d \'' + DD_JSON + '\'') + } } if (extraVars.get('NOTIFY_ON_FAILURE')){ diff --git a/devops/resources/janitor.sh b/devops/resources/janitor.sh index b84caa927..609cd7dd7 100644 --- a/devops/resources/janitor.sh +++ b/devops/resources/janitor.sh @@ -43,3 +43,13 @@ else fi curl -X GET 'https://api.opsgenie.com/v2/heartbeats/'${JOB_NAME##*/}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY} +curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \ +-H "Content-Type: application/json" \ +-d '{ + "series" : [{ + "metric": '${JOB_NAME##*/}".heartbeat"', + "points": [['"$(date +%s)"', 1]], + "type": "gauge", + "tags": ["env:'${DEPLOYMENT}'"] + }] + }' diff --git a/devops/resources/table-size-monitoring.sh b/devops/resources/table-size-monitoring.sh index 24f306966..db284ffa8 100644 --- a/devops/resources/table-size-monitoring.sh +++ b/devops/resources/table-size-monitoring.sh @@ -32,3 +32,13 @@ fi python check_table_size.py --threshold ${THRESHOLD} ${RDSTHRESHOLD} ${RDSIGNORE} curl -X GET 'https://api.opsgenie.com/v2/heartbeats/table-size-monitoring-'${DEPLOYMENT}'/ping' -H 'Authorization: GenieKey '${GENIE_KEY} +curl -X POST "https://api.datadoghq.com/api/v1/series?api_key=${DD_KEY}" \ +-H "Content-Type: application/json" \ +-d '{ + "series" : [{ + "metric": "table-size-monitoring-'${DEPLOYMENT}'.heartbeat", + "points": [['"$(date +%s)"', 1]], + "type": "gauge", + "tags": ["env:'${DEPLOYMENT}'"] + }] + }'