Add support for AWS Access Point ARN in compatible integrations (elas…

…tic#12154) Add support for the new setting access_point_arn for the integrations that rely on the AWS S3 input.
taylor-swanson · Jan 3, 2025 · c447829 · c447829
1 parent 15418df
commit c447829
Show file tree

Hide file tree

Showing 155 changed files with 976 additions and 161 deletions.
diff --git a/packages/amazon_security_lake/_dev/build/docs/README.md b/packages/amazon_security_lake/_dev/build/docs/README.md
@@ -61,7 +61,7 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
       ![Role ARN and External ID](../img/role_arn_and_external_id.png)
 
 6. If you want to collect logs via AWS S3, then you have to put the following details:
-    - bucket arn
+    - bucket ARN or access point ARN
     - role ARN
     - external id
 

diff --git a/packages/amazon_security_lake/changelog.yml b/packages/amazon_security_lake/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.3.0"
+  changes:
+    - description: Add support for Access Point ARN when collecting logs via the AWS S3 Bucket.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12154
 - version: "2.2.0"
   changes:
     - description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".

diff --git a/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs b/packages/amazon_security_lake/data_stream/event/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/amazon_security_lake/data_stream/event/manifest.yml b/packages/amazon_security_lake/data_stream/event/manifest.yml
@@ -61,7 +61,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: ARN of the AWS S3 bucket that will be polled for list operation. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: ARN of the AWS S3 bucket that will be polled for list operation. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: ARN of the AWS S3 Access Point that will be polled for list operation. Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/amazon_security_lake/docs/README.md b/packages/amazon_security_lake/docs/README.md
@@ -61,7 +61,7 @@ The Amazon Security Lake integration collects logs from both [Third-party servic
       ![Role ARN and External ID](../img/role_arn_and_external_id.png)
 
 6. If you want to collect logs via AWS S3, then you have to put the following details:
-    - bucket arn
+    - bucket ARN or access point ARN
     - role ARN
     - external id
 

diff --git a/packages/amazon_security_lake/manifest.yml b/packages/amazon_security_lake/manifest.yml
@@ -1,13 +1,13 @@
 format_version: "3.0.3"
 name: amazon_security_lake
 title: Amazon Security Lake
-version: "2.2.0"
+version: "2.3.0"
 description: Collect logs from Amazon Security Lake with Elastic Agent.
 type: integration
 categories: ["aws", "security"]
 conditions:
   kibana:
-    version: "^8.13.0"
+    version: "^8.16.2"
   elastic:
     subscription: basic
 screenshots:

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.38.0"
+  changes:
+    - description: Add support for Access Point ARN when collecting logs via the AWS S3 Bucket.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12154
 - version: "2.37.0"
   changes:
     - description: Map aws.dimensions as object instead of flattened in CloudWatch metrics.

diff --git a/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/apigateway_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/apigateway_logs/manifest.yml b/packages/aws/data_stream/apigateway_logs/manifest.yml
@@ -21,7 +21,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudfront_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/cloudfront_logs/manifest.yml b/packages/aws/data_stream/cloudfront_logs/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/cloudtrail/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/cloudtrail/manifest.yml b/packages/aws/data_stream/cloudtrail/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/ec2_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/ec2_logs/manifest.yml b/packages/aws/data_stream/ec2_logs/manifest.yml
@@ -21,7 +21,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/elb_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/elb_logs/manifest.yml b/packages/aws/data_stream/elb_logs/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/emr_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/emr_logs/manifest.yml b/packages/aws/data_stream/emr_logs/manifest.yml
@@ -21,7 +21,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/firewall_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/firewall_logs/manifest.yml b/packages/aws/data_stream/firewall_logs/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/guardduty/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/guardduty/manifest.yml b/packages/aws/data_stream/guardduty/manifest.yml
@@ -136,7 +136,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/route53_resolver_logs/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/route53_resolver_logs/manifest.yml b/packages/aws/data_stream/route53_resolver_logs/manifest.yml
@@ -143,7 +143,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/s3access/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/s3access/manifest.yml b/packages/aws/data_stream/s3access/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/vpcflow/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/vpcflow/manifest.yml b/packages/aws/data_stream/vpcflow/manifest.yml
@@ -21,7 +21,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: queue_url
         type: text
         title: "[SQS] Queue URL"

diff --git a/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs b/packages/aws/data_stream/waf/agent/stream/aws-s3.yml.hbs
@@ -2,6 +2,8 @@
 
 {{#if bucket_arn}}
 bucket_arn: {{bucket_arn}}
+{{else if access_point_arn}}
+access_point_arn: {{access_point_arn}}
 {{/if}}
 {{#if number_of_workers}}
 number_of_workers: {{number_of_workers}}

diff --git a/packages/aws/data_stream/waf/manifest.yml b/packages/aws/data_stream/waf/manifest.yml
@@ -20,7 +20,14 @@ streams:
         multi: false
         required: false
         show_user: true
-        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket.
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set an Access Point ARN. In case both configurations are added, this one takes precedence.
+      - name: access_point_arn
+        type: text
+        title: "[S3] Access Point ARN"
+        multi: false
+        required: false
+        show_user: true
+        description: Mandatory if the "Collect logs via S3 Bucket" switch is on. It is a required parameter for collecting logs via the AWS S3 Bucket unless you set a Bucket ARN.
       - name: bucket_list_prefix
         type: text
         title: "[S3] Bucket Prefix"

diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml
@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.37.0
+version: 2.38.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:
@@ -10,7 +10,7 @@ conditions:
   elastic:
     subscription: basic
   kibana:
-    version: "^8.16.0"
+    version: "^8.16.2"
 screenshots:
   - src: /img/metricbeat-aws-overview.png
     title: metricbeat aws overview