Add AWSHealth Integration (#10119)

* Add AWSHealth dataset
elastic · Jul 12, 2024 · d4dcb0a · d4dcb0a
1 parent f0d3fd9
commit d4dcb0a
Show file tree

Hide file tree

Showing 17 changed files with 2,191 additions and 2 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -27,6 +27,7 @@
 /packages/aws/changelog.yml @elastic/obs-ds-hosted-services @elastic/security-service-integrations @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/apigateway_logs @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/apigateway_metrics @elastic/obs-infraobs-integrations
+/packages/aws/data_stream/awshealth @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/billing @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/cloudfront_logs @elastic/obs-infraobs-integrations
 /packages/aws/data_stream/cloudtrail @elastic/obs-infraobs-integrations

diff --git a/packages/aws/_dev/build/docs/awshealth.md b/packages/aws/_dev/build/docs/awshealth.md
@@ -0,0 +1,51 @@
+# AWS Health
+
+AWS Health metrics provide insights into the health of your AWS environment by monitoring various aspects such as open issues, scheduled maintenance events, security advisories, compliance status, notification counts, and service disruptions. These metrics help you proactively identify and address issues impacting your AWS resources, ensuring the reliability, security, and compliance of your infrastructure.
+
+## Data streams
+
+The AWS Health integration collects one type of data: metrics.
+
+Metrics provide insight into the operational health of your AWS environment, including the status of AWS services, scheduled changes, and notifications about potential issues that could impact your resources. Metrics are gathered with the [AWS Health API](https://docs.aws.amazon.com/health/latest/APIReference/Welcome.html)
+
+See more details in the [Metrics reference](#metrics-reference).
+
+
+## Requirements
+
+You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it.
+You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware.
+
+Before using any AWS integration you will need:
+
+* **AWS Credentials** to connect with your AWS account.
+* **AWS Permissions** to make sure the user you're using to connect has permission to share the relevant data.
+
+For more details about these requirements, please take a look at the [AWS integration documentation](https://docs.elastic.co/integrations/aws#requirements).
+
+To collect AWS Health metrics, you would need specific AWS permissions to access the necessary data. Here's a list of permissions required for an IAM user to collect AWS Health metrics:
+
+- `health:DescribeAffectedEntities`
+- `health:DescribeEventDetails`
+- `health:DescribeEvents`
+
+
+## Setup
+
+If you want to collect data from two or more AWS services, consider using the **AWS** integration. When you configure the AWS integration, you can collect data from as many AWS services as you'd like.
+
+For step-by-step instructions on how to set up an integration, see the
+{{ url "getting-started-observability" "Getting started" }} guide.
+
+### Data stream specific configuration notes
+
+`Period`:: (_string_) Reporting interval. Recommended value is `24h`.
+
+## Metrics reference
+
+The `awshealth` data stream collects AWS Health metrics from AWS.
+
+
+{{event "awshealth"}}
+
+{{fields "awshealth"}}
diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.18.0"
+  changes:
+    - description: Add AWS Health integration.
+      type: enhancement
+      link: https://github.com/elastic/integrations/issues/10119
 - version: "2.17.0"
   changes:
     - description: ECS version updated to 8.11.0. Update the kibana constraint to ^8.13.0. Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

diff --git a/packages/aws/data_stream/awshealth/agent/stream/stream.yml.hbs b/packages/aws/data_stream/awshealth/agent/stream/stream.yml.hbs
@@ -0,0 +1,39 @@
+metricsets: ["awshealth"]
+period: {{period}}
+{{#if access_key_id}}
+access_key_id: {{access_key_id}}
+{{/if}}
+{{#if secret_access_key}}
+secret_access_key: {{secret_access_key}}
+{{/if}}
+{{#if session_token}}
+session_token: {{session_token}}
+{{/if}}
+{{#if credential_profile_name}}
+credential_profile_name: {{credential_profile_name}}
+{{/if}}
+{{#if shared_credential_file}}
+shared_credential_file: {{shared_credential_file}}
+{{/if}}
+{{#if role_arn}}
+role_arn: {{role_arn}}
+{{/if}}
+{{#if default_region}}
+default_region: {{default_region}}
+{{/if}}
+{{#if regions}}
+regions:
+{{#each regions as |region i|}}
+- {{region}}
+{{/each}}
+{{/if}}
+{{#if tags_filter}}
+tags_filter: {{tags_filter}}
+{{/if}}
+{{#if proxy_url }}
+proxy_url: {{proxy_url}}
+{{/if}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
diff --git a/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/awshealth/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,31 @@
+---
+description: Pipeline for AWS Health metrics
+processors:
+- script:
+    if: "ctx.aws != null && ctx.aws.awshealth != null && ctx.aws.awshealth.end_time == '0001-01-01T00:00:00.000Z'"
+    "lang": "painless"
+    "source": "ctx.aws.awshealth.end_time = null"
+
+- script:
+    lang: painless
+    source: |-
+        boolean drop(Object o) {
+            if (o == null || o == '') {
+            return true;
+            } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+            } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+            }
+            return false;
+        }
+        drop(ctx);
+    description: Drops null/empty values recursively.
+
+on_failure:
+- set:
+    field: error.message
+    value: "{{ _ingest.on_failure_message }}"
+
diff --git a/packages/aws/data_stream/awshealth/fields/base-fields.yml b/packages/aws/data_stream/awshealth/fields/base-fields.yml
@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
diff --git a/packages/aws/data_stream/awshealth/fields/ecs.yml b/packages/aws/data_stream/awshealth/fields/ecs.yml
@@ -0,0 +1,9 @@
+- external: ecs
+  name: cloud.account.id
+  dimension: true
+- external: ecs
+  name: cloud.region
+  dimension: true
+- name: agent.id
+  external: ecs
+  dimension: true
diff --git a/packages/aws/data_stream/awshealth/fields/fields.yml b/packages/aws/data_stream/awshealth/fields/fields.yml
@@ -0,0 +1,118 @@
+- name: aws
+  type: group
+  fields:
+    - name: awshealth
+      type: group
+      description: >
+        AWS Health metrics.
+      fields:
+        - name: affected_entities_others
+          type: float
+          metric_type: gauge
+          description: >
+            The number of affected resources related to the event whose status cannot be verified.
+
+        - name: affected_entities_pending
+          type: float
+          metric_type: gauge
+          description: >
+            The number of affected resources that may require action.
+
+        - name: affected_entities_resolved
+          type: float
+          metric_type: gauge
+          description: >
+            The number of affected resources that do not require any action.
+
+        - name: end_time
+          type: date
+          description: >
+            The date and time when the event ended. Some events may not have an end date.
+
+        - name: event_arn
+          type: keyword
+          dimension: true
+          description: >
+            The unique identifier for the event. The event ARN has the format `arn:aws:health:<event-region>::event/<SERVICE>/<EVENT_TYPE_CODE>/<EVENT_TYPE_PLUS_ID>`. For example, `arn:aws:health:us-east-1::event/EC2/EC2_INSTANCE_RETIREMENT_SCHEDULED/EC2_INSTANCE_RETIREMENT_SCHEDULED_ABC123-DEF456`.
+
+        - name: event_scope_code
+          type: keyword
+          description: >
+            The `event_scope_code` parameter specifies whether the Health event is a public Amazon Web Service event or an account-specific event. Allowed values are `PUBLIC`, `ACCOUNT_SPECIFIC`, or `NONE`."
+
+        - name: event_type_category
+          type: keyword
+          description: >
+            The event type category code. The possible values are `issue`, `accountNotification`, `investigation` or `scheduledChange`.
+
+        - name: event_type_code
+          type: keyword
+          description: >
+            The unique identifier for the event type. The format is `AWS_<SERVICE_DESCRIPTION>`. For example, `AWS_EC2_SYSTEM_MAINTENANCE_EVENT`.
+
+        - name: last_updated_time
+          type: date
+          description: >
+            The most recent date and time when the event was updated.
+
+        - name: region
+          type: keyword
+          description: >
+            The Amazon Web Services Region name of the event.
+
+        - name: service
+          type: keyword
+          description: >
+            The Amazon Web Service affected by the event. For example, EC2 or RDS.
+
+        - name: start_time
+          type: date
+          description: >
+            The date and time when the event began.
+
+        - name: status_code
+          type: keyword
+          description: >
+            The most recent status of the event. Possible values are `open`, `closed`, and `upcoming`.
+
+        - name: event_description
+          type: text
+          description: >
+            The detailed description of the event.
+
+        - name: affected_entities
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            The details of the affected entities related to the event.
+
+        - name: affected_entities.aws_account_id
+          type: keyword
+          description: >
+            The Amazon Web Services account number that contains the affected entity.
+
+        - name: affected_entities.entity_url
+          type: keyword
+          description: >
+            The URL of the affected entity.
+
+        - name: affected_entities.entity_value
+          type: keyword
+          description: >
+            The ID of the affected entity.
+
+        - name: affected_entities.last_updated_time
+          type: date
+          description: >
+            The last updated time of the affected entity.
+
+        - name: affected_entities.status_code
+          type: keyword
+          description: >
+            The most recent status of the entity affected by the event. The possible values include `IMPAIRED`, `UNIMPAIRED`, `UNKNOWN`, `PENDING`, `RESOLVED`.
+
+        - name: affected_entities.entity_arn
+          type: keyword
+          description: >
+            The unique identifier for the affected entities. The entity ARN has the format: `arn:aws:health:<entity-region>:<aws-account>:entity/<entity-id>`. For example, `arn:aws:health:us-east-1:111222333444:entity/AVh5GGT7ul1arKr1sE1K`.
diff --git a/packages/aws/data_stream/awshealth/fields/package-fields.yml b/packages/aws/data_stream/awshealth/fields/package-fields.yml
@@ -0,0 +1,18 @@
+- name: aws
+  type: group
+  fields:
+    - name: tags
+      type: flattened
+      description: |
+        Tag key value pairs from aws resources.
+    - name: linked_account
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          description: |
+            ID used to identify linked account.
+        - name: name
+          type: keyword
+          description: |
+            Name or alias used to identify linked account.
diff --git a/packages/aws/data_stream/awshealth/manifest.yml b/packages/aws/data_stream/awshealth/manifest.yml
@@ -0,0 +1,51 @@
+title: "AWS Health"
+type: metrics
+streams:
+  - input: aws/metrics
+    vars:
+      - name: period
+        type: text
+        title: Collection Period
+        multi: false
+        required: true
+        show_user: true
+        default: 24h
+      - name: data_granularity
+        type: text
+        title: Data Granularity
+        multi: false
+        required: false
+        show_user: false
+      - name: regions
+        type: text
+        title: Regions
+        multi: true
+        required: false
+        show_user: true
+      - name: tags_filter
+        type: yaml
+        title: Tags Filter
+        multi: false
+        required: false
+        show_user: false
+        default: |
+          # - key: "created-by"
+            # value: "foo"
+      - name: include_linked_accounts
+        type: bool
+        title: Include Linked Accounts
+        multi: false
+        required: false
+        show_user: false
+        default: true
+        description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the events are shipped. See [Processors](https://www.elastic.co/guide/en/fleet/current/elastic-agent-processor-configuration.html) for details.
+    title: AWS Health metrics
+    description: Collect AWS Health metrics