notebooks/grafana/configure-lambda-error-alerts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Grafana\n",
    "##### Required Data & Folder List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import botoplus.organization as _org\n",
    "\n",
    "accounts = _org.accounts()\n",
    "\n",
    "account_list = []\n",
    "\n",
    "regions = []\n",
    "regions.append('us-east-1')\n",
    "regions.append('us-east-2')\n",
    "regions.append('us-west-2')\n",
    "\n",
    "for account in accounts:\n",
    "    for region in regions:\n",
    "        temp = {}\n",
    "        temp['awsalias'] = account['awsalias']\n",
    "        temp['awsaccount'] = str(account['awsaccount'])\n",
    "        temp['name'] = account['awsalias']+' - '+str(account['awsaccount']+' - '+region)\n",
    "        temp['region'] = region\n",
    "        account_list.append(temp)\n",
    "        print(temp)\n",
    "\n",
    "print('Count: '+str(len(account_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Cloud Watch Source List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/datasources?query=json'\n",
    "response = requests.get(url, headers=headers)\n",
    "j = response.json()\n",
    "\n",
    "data_sources = []\n",
    "\n",
    "for source in j:\n",
    "    \n",
    "    if source['type'] == 'cloudwatch' and len(source['jsonData']) > 0:\n",
    "        temp = {}\n",
    "        temp['uid'] = source['uid']\n",
    "        temp['name'] = source['name']\n",
    "        temp['region'] = source['jsonData']['defaultRegion']\n",
    "        data_sources.append(temp)\n",
    "        print(temp)\n",
    "\n",
    "print('Count: '+str(len(data_sources)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### New Cloud Watch Sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/datasources?query=json'\n",
    "\n",
    "for account in account_list:\n",
    "    found = False\n",
    "    for source in data_sources:\n",
    "        if account['name'] == source['name']:\n",
    "            found = True\n",
    "            break\n",
    "    if not found:\n",
    "\n",
    "        j = {}\n",
    "        j['orgId'] = 1\n",
    "        j['name'] = account['name']\n",
    "        j['type'] = 'cloudwatch'\n",
    "        j['typeName'] = 'CloudWatch'\n",
    "        j['typeLogoUrl'] = 'public/app/plugins/datasource/cloudwatch/img/amazon-web-services.png'\n",
    "        j['access'] = 'proxy'\n",
    "        j['basicAuth'] = False\n",
    "        j['isDefault'] = False\n",
    "        j['jsonData'] = {}\n",
    "        j['jsonData']['assumeRoleArn'] = 'arn:aws:iam::'+str(account['awsaccount'])+':role/grafana'\n",
    "        j['jsonData']['authType'] = 'grafana_assume_role'\n",
    "        j['jsonData']['defaultRegion'] = account['region']\n",
    "\n",
    "        response = requests.post(url, headers=headers, json=j)\n",
    "        print(response.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Unused Cloud Watch Sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "\n",
    "for source in data_sources:\n",
    "    found = False\n",
    "    for account in account_list:\n",
    "        if account['name'] == source['name']:\n",
    "            found = True\n",
    "            break\n",
    "    if not found:\n",
    "        url = grafana_url+'/api/datasources/uid/'+source['uid']\n",
    "        response = requests.delete(url, headers=headers)\n",
    "        print(response.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Existing Folder Lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/folders?query=json'\n",
    "response = requests.get(url, headers=headers)\n",
    "j = response.json()\n",
    "\n",
    "existing_folder_names = []\n",
    "folder_list = []\n",
    "\n",
    "for folder in j:\n",
    "    if folder['title'] != 'GrafanaCloud' and folder['title'] != 'OnCall Insights':\n",
    "        existing_folder_names.append(folder['title'])\n",
    "        parsed = folder['title'].split(' - ')\n",
    "        if len(parsed) == 3:\n",
    "            temp = {}\n",
    "            temp['uid'] = folder['uid']\n",
    "            temp['awsalias'] = parsed[0]\n",
    "            temp['awsaccount'] = str(parsed[1])\n",
    "            temp['region'] = parsed[2]\n",
    "            folder_list.append(temp)\n",
    "            print(temp)\n",
    "\n",
    "print('Count: '+str(len(folder_list)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### New Folder Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/folders?query=json'\n",
    "\n",
    "for account in account_list:\n",
    "    if account['name'] not in existing_folder_names:\n",
    "        data = {\n",
    "            \"title\": account['name']\n",
    "        }\n",
    "        r = requests.post(url, headers=headers, json=data)\n",
    "        print(r.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Unused Folder Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "\n",
    "for folder in folder_list:\n",
    "    found = False\n",
    "    for account in account_list:\n",
    "        if account['awsaccount'] == folder['awsaccount'] and account['awsalias'] == folder['awsalias'] and account['region'] == folder['region']:\n",
    "            found = True\n",
    "            break\n",
    "    if not found:\n",
    "        url = url = grafana_url+'/api/folders/'+folder['uid']+'?query=json'\n",
    "        response = requests.delete(url, headers=headers)\n",
    "        print(response.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Provisioning Format Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/v1/provisioning/alert-rules/export?query=json'\n",
    "\n",
    "response = requests.get(url, headers=headers)\n",
    "j = response.json()\n",
    "\n",
    "update_value = 'CHANGEIT'\n",
    "\n",
    "for item in j['groups']:\n",
    "    for rule in item['rules']:\n",
    "\n",
    "        url = grafana_url+'/api/v1/provisioning/alert-rules/'+rule['uid']+'/export'\n",
    "        response = requests.get(url, headers=headers)\n",
    "        j = response.json()\n",
    "\n",
    "        j = j['groups'][0]['rules'][0]\n",
    "        j.pop('uid', None)\n",
    "        j['orgID'] = 1\n",
    "        j['data'][0]['model']['logGroups'] = []\n",
    "\n",
    "        j['title'] = update_value\n",
    "        j['folderUID'] = update_value\n",
    "        j['ruleGroup'] = 'Alerts'\n",
    "        j['data'][0]['datasourceUid'] = update_value\n",
    "        j['data'][0]['model']['datasource']['uid'] = update_value\n",
    "        j['data'][0]['model']['dimensions']['FunctionName'] = update_value\n",
    "        j['data'][0]['model']['sql']['where']['expressions'][0]['operator']['value'] = update_value\n",
    "        j['data'][0]['model']['sqlExpression'] = \"SELECT SUM(Errors) FROM \\\"AWS/Lambda\\\" WHERE FunctionName = '\"+update_value+\"'\"\n",
    "\n",
    "        with open(update_value+'.json', 'w') as f:\n",
    "            json.dump(j, f)\n",
    "\n",
    "        print(json.dumps(j, indent=4))\n",
    "        break\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Lambda Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import botoplus.organization as _org\n",
    "\n",
    "display_denied = False\n",
    "\n",
    "accounts = _org.accounts()\n",
    "\n",
    "lambda_functions = []\n",
    "\n",
    "for account in accounts:\n",
    "    print(str(account['awsaccount'])+' '+account['awsalias'])\n",
    "    sessions = _org.sessions(account['awsaccount'])\n",
    "    ec2 = sessions.client('ec2')\n",
    "    regions = ec2.describe_regions()\n",
    "\n",
    "    for region in regions['Regions']:\n",
    "        try:\n",
    "            lambda_region = sessions.client('lambda', region_name = region['RegionName'])\n",
    "            paginator = lambda_region.get_paginator('list_functions')\n",
    "            response_iterator = paginator.paginate()\n",
    "            for page in response_iterator:\n",
    "                if len(page['Functions']) > 0:\n",
    "                    for item in page['Functions']:\n",
    "                        temp = {}\n",
    "                        temp['account'] = str(account['awsaccount'])\n",
    "                        temp['alias'] = account['awsalias']\n",
    "                        temp['name'] = item['FunctionName']\n",
    "                        temp['region'] = region['RegionName']\n",
    "                        lambda_functions.append(temp)\n",
    "        except:\n",
    "            if display_denied == True:\n",
    "                print(' - '+region['RegionName']+' DENIED')\n",
    "            pass\n",
    "\n",
    "print('Count: '+str(len(lambda_functions)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Alert List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/v1/provisioning/alert-rules?query=json'\n",
    "\n",
    "response = requests.get(url, headers=headers)\n",
    "j = response.json()\n",
    "\n",
    "alert_rules = []\n",
    "\n",
    "for alert in j:\n",
    "    temp = {}\n",
    "    temp['uid'] = alert['uid']\n",
    "    temp['title'] = alert['title']\n",
    "    temp['ruleGroup'] = alert['ruleGroup']\n",
    "    temp['folderUID'] = alert['folderUID']\n",
    "    alert_rules.append(temp)\n",
    "    print(temp)\n",
    "\n",
    "print('Count: '+str(len(alert_rules)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### New Alert Creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "def alertuid(title, region, folderuid):\n",
    "    for alert in alert_rules:\n",
    "        if alert['title'] == title and alert['ruleGroup'] == region and alert['folderUID'] == folderuid:\n",
    "            return alert['uid']\n",
    "    return None\n",
    "\n",
    "def folderuid(awsaccount, region):\n",
    "    for folder in folder_list:\n",
    "        if folder['awsaccount'] == awsaccount and folder['region'] == region:\n",
    "            return folder['uid']\n",
    "    return None\n",
    "\n",
    "def sourceuid(awsaccount, region):\n",
    "    for source in data_sources:\n",
    "        parsed = source['name'].split(' - ')\n",
    "        if parsed[1] == awsaccount and parsed[2] == region:\n",
    "            return source['uid']\n",
    "    return None\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "\n",
    "for lambda_function in lambda_functions:\n",
    "\n",
    "    source = sourceuid(str(lambda_function['account']), lambda_function['region'])\n",
    "    folder = folderuid(str(lambda_function['account']), lambda_function['region'])\n",
    "    notify = alertuid(lambda_function['name'], lambda_function['region'], folder)\n",
    "\n",
    "    if notify == None and folder != None:\n",
    "\n",
    "        with open('lambda-error-alert.json') as f:\n",
    "            j = json.load(f)\n",
    "\n",
    "        j['title'] = lambda_function['name']\n",
    "        j['folderUID'] = folder\n",
    "        j['data'][0]['datasourceUid'] = source\n",
    "        j['data'][0]['model']['datasource']['uid'] = source\n",
    "        j['data'][0]['model']['dimensions']['FunctionName'] = lambda_function['name']\n",
    "        j['data'][0]['model']['sql']['where']['expressions'][0]['operator']['value'] = lambda_function['name']\n",
    "        j['data'][0]['model']['sqlExpression'] = \"SELECT SUM(Errors) FROM \\\"AWS/Lambda\\\" WHERE FunctionName = '\"+lambda_function['name']+\"'\"\n",
    "\n",
    "        url = grafana_url+'/api/v1/provisioning/alert-rules?query=json'\n",
    "        response = requests.post(url, headers=headers, json=j)\n",
    "        print(response.json())\n",
    "\n",
    "        config = {}\n",
    "        config['interval'] = 900\n",
    "        url = grafana_url+'/api/v1/provisioning/folder/'+folder+'/rule-groups/Alerts?query=json'\n",
    "        response = requests.put(url, headers=headers, json=config)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Unused Alert Removal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "\n",
    "for alert in alert_rules:\n",
    "    found = False\n",
    "    for lambda_function in lambda_functions:\n",
    "        ### SUSPENDED ACCOUNTS ###\n",
    "        if alert['folderUID'] == '<ENTER FOLDER UID HERE>':\n",
    "            continue\n",
    "        elif alert['title'] == lambda_function['name']:\n",
    "            found = True\n",
    "            break\n",
    "    if not found:\n",
    "        print(alert)\n",
    "        url = grafana_url+'/api/v1/provisioning/alert-rules/'+alert['uid']+'?query=json'\n",
    "        response = requests.delete(url, headers=headers)\n",
    "        print(response.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Delete Every Alert"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import botoplus.organization as _org\n",
    "\n",
    "selected = _org.selected()\n",
    "\n",
    "ssm = selected.client('ssm')\n",
    "grafana_url = ssm.get_parameter(Name='/grafana/url', WithDecryption=False)['Parameter']['Value']\n",
    "grafana_token = ssm.get_parameter(Name='/grafana/token', WithDecryption=True)['Parameter']['Value']\n",
    "\n",
    "headers = {\n",
    "    \"Accept\": \"application/json\",\n",
    "    \"Content-Type\": \"application/json\",\n",
    "    \"Authorization\": \"Bearer \"+grafana_token\n",
    "}\n",
    "url = grafana_url+'/api/v1/provisioning/alert-rules?query=json'\n",
    "\n",
    "response = requests.get(url, headers=headers)\n",
    "j = response.json()\n",
    "\n",
    "alert_rules = []\n",
    "\n",
    "for alert in j:\n",
    "    url = grafana_url+'/api/v1/provisioning/alert-rules/'+alert['uid']+'?query=json'\n",
    "    response = requests.delete(url, headers=headers)\n",
    "    print(response.text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}