From f8e5c2a74ac81e963e366c1aa8ecc273714b4f20 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:59:45 -0400 Subject: [PATCH 01/11] build(deps): bump github.com/elastic/go-sysinfo from 1.13.1 to 1.14.0 (#38970) * build(deps): bump github.com/elastic/go-sysinfo from 1.13.1 to 1.14.0 Bumps [github.com/elastic/go-sysinfo](https://github.com/elastic/go-sysinfo) from 1.13.1 to 1.14.0. - [Release notes](https://github.com/elastic/go-sysinfo/releases) - [Commits](https://github.com/elastic/go-sysinfo/compare/v1.13.1...v1.14.0) --- updated-dependencies: - dependency-name: github.com/elastic/go-sysinfo dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Update NOTICE.txt --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: dependabot[bot] --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 498631ae75fe..8a1f36ebc510 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -15011,11 +15011,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v -------------------------------------------------------------------------------- Dependency : github.com/elastic/go-sysinfo -Version: v1.13.1 +Version: v1.14.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.13.1/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sysinfo@v1.14.0/LICENSE.txt: Apache License diff --git a/go.mod b/go.mod index 8b253b1e81b3..43430ffbf1f9 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,7 @@ require ( github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595 github.com/elastic/go-seccomp-bpf v1.4.0 github.com/elastic/go-structform v0.0.10 - github.com/elastic/go-sysinfo v1.13.1 + github.com/elastic/go-sysinfo v1.14.0 github.com/elastic/go-ucfg v0.8.8 github.com/elastic/gosigar v0.14.3 github.com/fatih/color v1.15.0 diff --git a/go.sum b/go.sum index b6600e837a2c..22b28557aaa7 100644 --- a/go.sum +++ b/go.sum @@ -711,8 +711,8 @@ github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1 h1:KS+lvT/rUS8Z4++ github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1/go.mod h1:/FB/tWFyF33vmdjwIwqAKu9QMVFVEjeoWi9V6eUcQEQ= github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= -github.com/elastic/go-sysinfo v1.13.1 h1:U5Jlx6c/rLkR72O8wXXXo1abnGlWGJU/wbzNJ2AfQa4= -github.com/elastic/go-sysinfo v1.13.1/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ= +github.com/elastic/go-sysinfo v1.14.0 h1:dQRtiqLycoOOla7IflZg3aN213vqJmP0lpVpKQ9lUEY= +github.com/elastic/go-sysinfo v1.14.0/go.mod h1:FKUXnZWhnYI0ueO7jhsGV3uQJ5hiz8OqM5b3oGyaRr8= github.com/elastic/go-ucfg v0.8.8 h1:54KIF/2zFKfl0MzsSOCGOsZ3O2bnjFQJ0nDJcLhviyk= github.com/elastic/go-ucfg v0.8.8/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT2QGAEKA= github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0= @@ -2884,4 +2884,4 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZa sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= \ No newline at end of file +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= From c2a2b6b55944bdd320897ca3ae00f321be1aaeee Mon Sep 17 00:00:00 2001 From: Agi K Thomas <101976829+agithomas@users.noreply.github.com> Date: Wed, 17 Apr 2024 01:53:27 +0530 Subject: [PATCH 02/11] [AWS] AWS Health metricbeat module (#38370) * AWS Health assists in effectively managing ongoing events by offering continuous insight into the performance of your resources and the availability of your AWS services and accounts. By leveraging AWS Health events, users obtain valuable insights into how service and resource modifications may impact their applications hosted on AWS. --------- Co-authored-by: subham sarkar --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 212 ++++++++++++ go.mod | 1 + go.sum | 2 + metricbeat/docs/fields.asciidoc | 217 ++++++++++++ metricbeat/docs/modules/aws.asciidoc | 4 + .../docs/modules/aws/awshealth.asciidoc | 30 ++ metricbeat/docs/modules_list.asciidoc | 3 +- x-pack/metricbeat/include/list.go | 1 + x-pack/metricbeat/module/aws/_meta/config.yml | 1 + .../module/aws/awshealth/_meta/data.json | 44 +++ .../module/aws/awshealth/_meta/docs.asciidoc | 21 ++ .../module/aws/awshealth/_meta/fields.yml | 91 +++++ .../module/aws/awshealth/awshealth.go | 312 ++++++++++++++++++ .../awshealth/awshealth_integration_test.go | 39 +++ .../module/aws/awshealth/awshealth_test.go | 247 ++++++++++++++ x-pack/metricbeat/module/aws/fields.go | 2 +- x-pack/metricbeat/modules.d/aws.yml.disabled | 1 + 18 files changed, 1227 insertions(+), 2 deletions(-) create mode 100644 metricbeat/docs/modules/aws/awshealth.asciidoc create mode 100644 x-pack/metricbeat/module/aws/awshealth/_meta/data.json create mode 100644 x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/aws/awshealth/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/aws/awshealth/awshealth.go create mode 100644 x-pack/metricbeat/module/aws/awshealth/awshealth_integration_test.go create mode 100644 x-pack/metricbeat/module/aws/awshealth/awshealth_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 311b8dd20193..4eac716b12c0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -237,6 +237,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Parse more fields from Elasticsearch slowlogs {pull}38295[38295] - Update CEL mito extensions to v1.10.0 to add keys/values helper. {pull}38504[38504] - Add support for Active Directory an entity analytics provider. {pull}37919[37919] +- Add AWS AWSHealth metricset. {pull}38370[38370] - Add debugging breadcrumb to logs when writing request trace log. {pull}38636[38636] - added benchmark input {pull}37437[37437] - added benchmark input and discard output {pull}37437[37437] diff --git a/NOTICE.txt b/NOTICE.txt index 8a1f36ebc510..88d816cd4a3a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -7374,6 +7374,218 @@ Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/servi limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/aws/aws-sdk-go-v2/service/health +Version: v1.17.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-sdk-go-v2/service/health@v1.17.0/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-sdk-go-v2/service/iam Version: v1.18.4 diff --git a/go.mod b/go.mod index 43430ffbf1f9..8a848c2c409e 100644 --- a/go.mod +++ b/go.mod @@ -197,6 +197,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.20.4 + github.com/aws/aws-sdk-go-v2/service/health v1.17.0 github.com/aws/aws-sdk-go-v2/service/kinesis v1.15.8 github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 diff --git a/go.sum b/go.sum index 22b28557aaa7..0e904f3eb5e2 100644 --- a/go.sum +++ b/go.sum @@ -344,6 +344,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.36.1 h1:FS8Ja6LuLDVHcX+rmoNpOXqYb52N github.com/aws/aws-sdk-go-v2/service/ec2 v1.36.1/go.mod h1:KOy1O7Fc2+GRgsbn/Kjr15vYDVXMEQALBaPRia3twSY= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.4 h1:ZBYifRGfN3dOKzvk0+XJiUKOFzqoJddYqCVsN5quCh4= github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2 v1.18.4/go.mod h1:9wKR88sRRyxrUAw5iVSDTfcCz90BLEFcAiyzP4v39uY= +github.com/aws/aws-sdk-go-v2/service/health v1.17.0 h1:DlG9888p6n8Fizx8Vuw1qalBOBtjoDk70UzqyilQ7+s= +github.com/aws/aws-sdk-go-v2/service/health v1.17.0/go.mod h1:z7JTQWRaBIdYYxK8TqDi4MKYYl04uI+jvTJuMEKIsL0= github.com/aws/aws-sdk-go-v2/service/iam v1.18.4 h1:E41guA79mjEbwJdh0zXz1d8+Zt4zxRr+b1ipiVbKXzs= github.com/aws/aws-sdk-go-v2/service/iam v1.18.4/go.mod h1:FpNvAfCZyIQ3qeNJUOw4CShKvdizHblXqAvSk0qmyL4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 40c8928113e5..f5eee7474847 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -1541,6 +1541,223 @@ type: keyword Name or alias used to identify linked account. +type: keyword + +-- + +[float] +=== awshealth + +AWS Health metrics + + + +*`aws.awshealth.affected_entities_others`*:: ++ +-- +The number of affected resources related to the event whose status cannot be verified. + + +type: float + +-- + +*`aws.awshealth.affected_entities_pending`*:: ++ +-- +The number of affected resources that may require action. + + +type: float + +-- + +*`aws.awshealth.affected_entities_resolved`*:: ++ +-- +The number of affected resources that do not require any action. + + +type: float + +-- + +*`aws.awshealth.end_time`*:: ++ +-- +The date and time when the event ended. Some events may not have an end date. + + +type: date + +-- + +*`aws.awshealth.event_arn`*:: ++ +-- +The unique identifier for the event. The event ARN has the format arn:aws:health:event-region::event/SERVICE/EVENT_TYPE_CODE/EVENT_TYPE_PLUS_ID. + + +type: keyword + +-- + +*`aws.awshealth.event_scope_code`*:: ++ +-- +This parameter specifies whether the Health event is a public Amazon Web Service event or an account-specific event. Allowed values are PUBLIC, ACCOUNT_SPECIFIC, or NONE. + + +type: keyword + +-- + +*`aws.awshealth.event_type_category`*:: ++ +-- +The event type category code. Possible values are issue, accountNotification, or scheduledChange. + + +type: keyword + +-- + +*`aws.awshealth.event_type_code`*:: ++ +-- +The unique identifier for the event type. The format is AWS_SERVICE_DESCRIPTION. + + +type: keyword + +-- + +*`aws.awshealth.last_updated_time`*:: ++ +-- +The most recent date and time when the event was updated. + + +type: date + +-- + +*`aws.awshealth.region`*:: ++ +-- +The Amazon Web Services Region name of the event. + + +type: keyword + +-- + +*`aws.awshealth.service`*:: ++ +-- +The Amazon Web Service affected by the event. For example, EC2 or RDS. + + +type: keyword + +-- + +*`aws.awshealth.start_time`*:: ++ +-- +The date and time when the event began. + + +type: date + +-- + +*`aws.awshealth.status_code`*:: ++ +-- +The most recent status of the event. Possible values are open, closed, and upcoming. + + +type: keyword + +-- + +*`aws.awshealth.event_description`*:: ++ +-- +The detailed description of the event. + + +type: text + +-- + +*`aws.awshealth.affected_entities`*:: ++ +-- +Information about an entity affected by a AWS Health event. + + +type: array + +-- + +*`aws.awshealth.affected_entities.aws_account_id`*:: ++ +-- +The Amazon Web Services account number that contains the affected entity. + + +type: keyword + +-- + +*`aws.awshealth.affected_entities.entity_url`*:: ++ +-- +The URL of the affected entity. + + +type: keyword + +-- + +*`aws.awshealth.affected_entities.entity_value`*:: ++ +-- +The ID of the affected entity. + + +type: keyword + +-- + +*`aws.awshealth.affected_entities.last_updated_time`*:: ++ +-- +The most recent time that the entity was updated. + + +type: date + +-- + +*`aws.awshealth.affected_entities.status_code`*:: ++ +-- +The most recent status of the event. Possible values are open, closed, and upcoming. + + +type: keyword + +-- + +*`aws.awshealth.affected_entities.entity_arn`*:: ++ +-- +The unique identifier for the entity. The entity ARN has the format: arn:aws:health:entity-region:aws-account:entity/entity-id. + + type: keyword -- diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index ee2a73e17dc6..47edfa1888fa 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -439,6 +439,8 @@ metricbeat.modules: The following metricsets are available: +* <> + * <> * <> @@ -473,6 +475,8 @@ The following metricsets are available: * <> +include::aws/awshealth.asciidoc[] + include::aws/billing.asciidoc[] include::aws/cloudwatch.asciidoc[] diff --git a/metricbeat/docs/modules/aws/awshealth.asciidoc b/metricbeat/docs/modules/aws/awshealth.asciidoc new file mode 100644 index 000000000000..35a8cecde3c8 --- /dev/null +++ b/metricbeat/docs/modules/aws/awshealth.asciidoc @@ -0,0 +1,30 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// +:edit_url: https://github.com/elastic/beats/edit/main/x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc + + +[[metricbeat-metricset-aws-awshealth]] +[role="xpack"] +=== AWS awshealth metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +:edit_url: + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/aws/awshealth/_meta/data.json[] +---- +:edit_url!: \ No newline at end of file diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2a77f4d38cd3..da0f7525e0fe 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -16,7 +16,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .1+| .1+| |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.17+| .17+| |<> beta[] +.18+| .18+| |<> beta[] +|<> beta[] |<> |<> beta[] |<> diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index dc482961ebcb..492e4e7d4d01 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -11,6 +11,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/activemq" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/airflow" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/awshealth" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/billing" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/cloudwatch" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/awsfargate" diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 6adf3af2fcd5..dacfadb9f312 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -46,6 +46,7 @@ period: 24h metricsets: - s3_daily_storage + - awshealth - module: aws period: 1m latency: 5m diff --git a/x-pack/metricbeat/module/aws/awshealth/_meta/data.json b/x-pack/metricbeat/module/aws/awshealth/_meta/data.json new file mode 100644 index 000000000000..ec0ffd5ad67b --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/_meta/data.json @@ -0,0 +1,44 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "aws": { + "awshealth": { + "affected_entities": [ + { + "aws_account_id": "12301234013123", + "entity_url": "", + "entity_value": "arn:aws:eks:us-east-2:627286350134:cluster/XXXXXXXXXXXXX", + "last_updated_time": "2024-04-12T12:56:29.7Z", + "status_code": "PENDING", + "entity_arn": "arn:aws:health:us-east-2:627286350134:entity/YYYYYYYYYYYYYYYYYYYY" + } + ], + "affected_entities_others": 0, + "affected_entities_pending": 1, + "affected_entities_resolved": 0, + "end_time": "0001-01-01T00:00:00Z", + "event_arn": "arn:aws:health:us-east-2::event/EKS/AWS_EKS_PLANNED_LIFECYCLE_EVENT/AWS_EKS_PLANNED_LIFECYCLE_EVENT_a7e64e77680080d19971a80f0131ff2239909cdbe7647dd57710b764b988f476", + "event_description": "On May 1, 2024, standard support for Kubernetes version 1.25 in Amazon EKS will end. From May 2, 2024 all Amazon EKS clusters running on 1.25 will enter extended support and will remain in extended support until May 1, 2025.\n\nAfter May 1, 2025, Kubernetes 1.25 will no longer be supported on Amazon EKS, and all Amazon EKS clusters running on 1.25 will be automatically updated to Kubernetes version 1.26.\n\nYou are receiving this message because you currently have 1 or more Amazon EKS clusters running on Kubernetes version 1.25. A list of your impacted clusters can be found in the \"Affected resources\" tab.\n\nExtended support is currently in free preview and is available to all customers. Effective April 1 2024, your Amazon EKS clusters running on a Kubernetes version in extended support will be charged at $0.60 per cluster hour.\n\nIf you do not want to use extended support, we recommend that you update your 1.25 clusters to Kubernetes version 1.26 or newer before May 1, 2024. To learn more about the extended support for Kubernetes versions pricing, see our announcement [1]. For instructions on how to update your cluster(s), see the Amazon EKS service 'Updating an Amazon EKS cluster Kubernetes version' documentation [2].\n\nTo learn more on Kubernetes version support, see the 'Amazon EKS Kubernetes versions' documentation [3].\n\nFor any questions or assistance, please contact AWS Support [4].\n\n\n[1] https://aws.amazon.com/blogs/containers/amazon-eks-extended-support-for-kubernetes-versions-pricing/\n[2] https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html\n[3] https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html\n[4] https://aws.amazon.com/support", + "event_scope_code": "ACCOUNT_SPECIFIC", + "event_type_category": "scheduledChange", + "event_type_code": "AWS_EKS_PLANNED_LIFECYCLE_EVENT", + "last_updated_time": "2024-04-12T13:12:39.273Z", + "region": "us-east-2", + "service": "EKS", + "start_time": "2024-05-01T07:00:00Z", + "status_code": "upcoming" + } + }, + "cloud.provider": "aws", + "event": { + "dataset": "aws.awshealth", + "duration": 115000, + "module": "aws" + }, + "metricset": { + "name": "awshealth", + "period": 10000 + }, + "service": { + "type": "aws-health" + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc new file mode 100644 index 000000000000..a95db3fc2e75 --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/_meta/docs.asciidoc @@ -0,0 +1,21 @@ +AWS Health metrics provide insights into the health of your AWS environment by monitoring various aspects such as open issues, scheduled maintenance events, security advisories, compliance status, notification counts, and service disruptions. These metrics help you proactively identify and address issues impacting your AWS resources, ensuring the reliability, security, and compliance of your infrastructure. + +[float] +=== AWS Permissions +To collect AWS Health metrics using Elastic Metricbeat, you would need specific AWS permissions to access the necessary data. Here's a list of permissions required for an IAM user to collect AWS Health metrics: +---- +health:DescribeAffectedEntities +health:DescribeEventDetails +health:DescribeEvents +---- + +[float] +=== Configuration example +[source,yaml] +---- + +- module: aws + period: 24h + metricsets: + - awshealth +---- diff --git a/x-pack/metricbeat/module/aws/awshealth/_meta/fields.yml b/x-pack/metricbeat/module/aws/awshealth/_meta/fields.yml new file mode 100644 index 000000000000..358e74210e4a --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/_meta/fields.yml @@ -0,0 +1,91 @@ +- name: awshealth + type: group + release: beta + description: > + AWS Health metrics + fields: + - name: affected_entities_others + type: float + description: > + The number of affected resources related to the event whose status cannot be verified. + - name: affected_entities_pending + type: float + description: > + The number of affected resources that may require action. + - name: affected_entities_resolved + type: float + description: > + The number of affected resources that do not require any action. + - name: end_time + type: date + description: > + The date and time when the event ended. Some events may not have an end date. + - name: event_arn + type: keyword + description: > + The unique identifier for the event. The event ARN has the format arn:aws:health:event-region::event/SERVICE/EVENT_TYPE_CODE/EVENT_TYPE_PLUS_ID. + - name: event_scope_code + type: keyword + description: > + This parameter specifies whether the Health event is a public Amazon Web Service event or an account-specific event. Allowed values are PUBLIC, ACCOUNT_SPECIFIC, or NONE. + - name: event_type_category + type: keyword + description: > + The event type category code. Possible values are issue, accountNotification, or scheduledChange. + - name: event_type_code + type: keyword + description: > + The unique identifier for the event type. The format is AWS_SERVICE_DESCRIPTION. + - name: last_updated_time + type: date + description: > + The most recent date and time when the event was updated. + - name: region + type: keyword + description: > + The Amazon Web Services Region name of the event. + - name: service + type: keyword + description: > + The Amazon Web Service affected by the event. For example, EC2 or RDS. + - name: start_time + type: date + description: > + The date and time when the event began. + - name: status_code + type: keyword + description: > + The most recent status of the event. Possible values are open, closed, and upcoming. + - name: event_description + type: text + description: > + The detailed description of the event. + - name: affected_entities + type: array + description: > + Information about an entity affected by a AWS Health event. + + - name: affected_entities.aws_account_id + type: keyword + description: > + The Amazon Web Services account number that contains the affected entity. + - name: affected_entities.entity_url + type: keyword + description: > + The URL of the affected entity. + - name: affected_entities.entity_value + type: keyword + description: > + The ID of the affected entity. + - name: affected_entities.last_updated_time + type: date + description: > + The most recent time that the entity was updated. + - name: affected_entities.status_code + type: keyword + description: > + The most recent status of the event. Possible values are open, closed, and upcoming. + - name: affected_entities.entity_arn + type: keyword + description: > + The unique identifier for the entity. The entity ARN has the format: arn:aws:health:entity-region:aws-account:entity/entity-id. diff --git a/x-pack/metricbeat/module/aws/awshealth/awshealth.go b/x-pack/metricbeat/module/aws/awshealth/awshealth.go new file mode 100644 index 000000000000..a86ac1b5da4f --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/awshealth.go @@ -0,0 +1,312 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awshealth + +import ( + "context" + "fmt" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/health" + "github.com/aws/aws-sdk-go-v2/service/health/types" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const metricsetName = "awshealth" + +var ( + locale = "en" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host is defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet(aws.ModuleName, metricsetName, New, + mb.DefaultMetricSet(), + ) +} + +type AffectedEntityDetails struct { + AwsAccountId string `json:"aws_account_id"` + EntityUrl string `json:"entity_url"` + EntityValue string `json:"entity_value"` + LastUpdatedTime time.Time `json:"last_updated_time"` + StatusCode string `json:"status_code"` + EntityArn string `json:"entity_arn"` +} + +type AWSHealthMetric struct { + EventArn string `json:"event_arn"` + EndTime time.Time `json:"end_time"` + EventScopeCode string `json:"event_scope_code"` + EventTypeCategory string `json:"event_type_category"` + EventTypeCode string `json:"event_type_code"` + LastUpdatedTime time.Time `json:"last_updated_time"` + Region string `json:"region"` + Service string `json:"service"` + StartTime time.Time `json:"start_time"` + StatusCode string `json:"status_code"` + AffectedEntitiesPending int32 `json:"affected_entities_pending"` + AffectedEntitiesResolved int32 `json:"affected_entities_resolved"` + AffectedEntitiesOthers int32 `json:"affected_entities_others"` + AffectedEntities []AffectedEntityDetails `json:"affected_entities"` + EventDescription string `json:"event_description"` +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + *aws.MetricSet + logger *logp.Logger + Config Config `config:"aws_health_config"` +} + +// Config holds the configuration specific for aws-awshealth metricset +type Config struct { + EventARNPattern []string `config:"event_arns_pattern"` +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + + logger := logp.NewLogger(metricsetName) + metricSet, err := aws.NewMetricSet(base) + if err != nil { + return nil, fmt.Errorf("error creating aws metricset: %w", err) + } + + cfgwarn.Beta("The aws:awshealth metricset is beta.") + + config := struct { + Config Config `config:"aws_health_config"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + MetricSet: metricSet, + logger: logger, + Config: config.Config, + }, nil +} + +// Fetch method implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(ctx context.Context, report mb.ReporterV2) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var config aws.Config + if err := m.Module().UnpackConfig(&config); err != nil { + return err + } + + awsConfig := m.MetricSet.AwsConfig.Copy() + + health_client := health.NewFromConfig(awsConfig, func(o *health.Options) { + if config.AWSConfig.FIPSEnabled { + o.EndpointOptions.UseFIPSEndpoint = awssdk.FIPSEndpointStateEnabled + } + }) + events := m.getEventDetails(ctx, health_client) + for _, event := range events { + report.Event(event) + } + + return nil +} + +// getEventDetails retrieves AWS health events and their details using the provided AWS Health client. +// It returns a list of Metricbeat events containing relevant AWS health information. +func (m *MetricSet) getEventDetails( + ctx context.Context, + awsHealth *health.Client, +) []mb.Event { + // Define event filter to fetch only upcoming and open events + eventFilter := types.EventFilter{ + EventStatusCodes: []types.EventStatusCode{ + types.EventStatusCodeUpcoming, + types.EventStatusCodeOpen, + }, + } + var ( + deEvents []types.Event + affPage health.DescribeAffectedEntitiesPaginator + healthDetails []AWSHealthMetric + healthDetailsTemp []AWSHealthMetric + affEntityTemp AffectedEntityDetails + affInputParams health.DescribeAffectedEntitiesInput + ) + + // Create an instance of DescribeEventsInput with desired parameters + deInputParams := health.DescribeEventsInput{ + Filter: &eventFilter, + } + + // Define options for DescribeEventsPaginator + deOptions := &health.DescribeEventAggregatesPaginatorOptions{ + Limit: 10, + StopOnDuplicateToken: true, + } + + // Function option to apply options to the paginator + deOptFn := func(options *health.DescribeEventsPaginatorOptions) { + // Apply the provided options + options.Limit = deOptions.Limit + options.StopOnDuplicateToken = deOptions.StopOnDuplicateToken + } + // Define options for DescribeAffectedEntitiesPaginator + affOptions := &health.DescribeAffectedEntitiesPaginatorOptions{ + Limit: 10, + StopOnDuplicateToken: true, + } + // Function option to apply options to the paginator + affOptFn := func(options *health.DescribeAffectedEntitiesPaginatorOptions) { + // Apply the provided options + options.Limit = affOptions.Limit + options.StopOnDuplicateToken = affOptions.StopOnDuplicateToken + } + // Create DescribeEventsPaginator with AWS Health client and options + dePage := health.NewDescribeEventsPaginator(awsHealth, &deInputParams, deOptFn) + + for dePage.HasMorePages() { + healthDetailsTemp = []AWSHealthMetric{} + + // Perform actions for the current page + currentPage, err := dePage.NextPage(ctx) + if err != nil { + m.Logger().Errorf("[AWS Health] DescribeEvents failed with : %w", err) + break + } + deEvents = currentPage.Events + eventArns := make([]string, len(deEvents)) + // Iterate through events to extract relevant information + for i, de := range deEvents { + healthDetailsTemp = append(healthDetailsTemp, AWSHealthMetric{ + EventArn: awssdk.ToString(de.Arn), + EndTime: awssdk.ToTime(de.EndTime), + EventScopeCode: string(de.EventScopeCode), + EventTypeCategory: string(de.EventTypeCategory), + EventTypeCode: awssdk.ToString(de.EventTypeCode), + LastUpdatedTime: awssdk.ToTime(de.LastUpdatedTime), + Region: awssdk.ToString(de.Region), + Service: awssdk.ToString(de.Service), + StartTime: awssdk.ToTime(de.StartTime), + StatusCode: string(de.StatusCode), + }) + eventArns[i] = awssdk.ToString(de.Arn) + } + // Fetch event details for the current page of events + eventDetails, err := awsHealth.DescribeEventDetails(ctx, &health.DescribeEventDetailsInput{ + EventArns: eventArns, + Locale: &locale, + }) + if err != nil { + m.Logger().Errorf("[AWS Health] DescribeEventDetails failed with : %w", err) + break + } + // Fetch event description for the current page of events + successSet := eventDetails.SuccessfulSet + for x := range successSet { + for y := range healthDetailsTemp { + if awssdk.ToString(successSet[x].Event.Arn) == healthDetailsTemp[y].EventArn { + healthDetailsTemp[y].EventDescription = awssdk.ToString(successSet[x].EventDescription.LatestDescription) + } + } + } + // Fetch affected entities related to event ARNs in the current page + affInputParams = health.DescribeAffectedEntitiesInput{ + Filter: &types.EntityFilter{ + EventArns: eventArns, + }, + } + affPage = *health.NewDescribeAffectedEntitiesPaginator( + awsHealth, + &affInputParams, + affOptFn, + ) + + for affPage.HasMorePages() { + // Fetch current page of affected entities + affCurrentPage, err := affPage.NextPage(ctx) + if err != nil { + m.Logger().Errorf("[AWS Health] DescribeAffectedEntitie failed with : %w", err) + break + } + // Extract relevant details of affected entities and match them with event details + for _, ace := range affCurrentPage.Entities { + affEntityTemp = AffectedEntityDetails{ + AwsAccountId: awssdk.ToString(ace.AwsAccountId), + EntityUrl: awssdk.ToString(ace.EntityUrl), + EntityValue: awssdk.ToString(ace.EntityValue), + LastUpdatedTime: awssdk.ToTime(ace.LastUpdatedTime), + StatusCode: string(ace.StatusCode), + EntityArn: awssdk.ToString(ace.EntityArn), + } + for l, hd := range healthDetailsTemp { + if awssdk.ToString(ace.EventArn) == hd.EventArn { + healthDetailsTemp[l].AffectedEntities = append(healthDetailsTemp[l].AffectedEntities, affEntityTemp) + switch string(ace.StatusCode) { + case "PENDING": + healthDetailsTemp[l].AffectedEntitiesPending++ + case "RESOLVED": + healthDetailsTemp[l].AffectedEntitiesResolved++ + case "": + // Do Nothing + default: + healthDetailsTemp[l].AffectedEntitiesOthers++ + + } + } + } + } + } + // Append current page's health details to the overall list + healthDetails = append(healthDetails, healthDetailsTemp...) + } + // Convert health details to Metricbeat events + var events = make([]mb.Event, 0, len(healthDetails)) + for _, detail := range healthDetails { + event := mb.Event{ + MetricSetFields: mapstr.M{ + "event_arn": detail.EventArn, + "end_time": detail.EndTime, + "event_scope_code": detail.EventScopeCode, + "event_type_category": detail.EventTypeCategory, + "event_type_code": detail.EventTypeCode, + "last_updated_time": detail.LastUpdatedTime, + "region": detail.Region, + "service": detail.Service, + "start_time": detail.StartTime, + "status_code": detail.StatusCode, + "affected_entities": detail.AffectedEntities, + "event_description": detail.EventDescription, + "affected_entities_pending": detail.AffectedEntitiesPending, + "affected_entities_resolved": detail.AffectedEntitiesResolved, + "affected_entities_others": detail.AffectedEntitiesOthers, + }, + RootFields: mapstr.M{ + "cloud.provider": "aws", + }, + Service: "aws-health", + } + events = append(events, event) + } + return events +} diff --git a/x-pack/metricbeat/module/aws/awshealth/awshealth_integration_test.go b/x-pack/metricbeat/module/aws/awshealth/awshealth_integration_test.go new file mode 100644 index 000000000000..3a9f99de077c --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/awshealth_integration_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build aws && integration && awshealth + +package awshealth + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws/mtest" +) + +// TODO +// There seems to be problem with Flatten() function of type M map[string]interface{} +// The Flatten function returns aws.awshealth.affected_entities instead of aws.awshealth.affected_entities.aws_account_id, needed for nested type. + +func TestFetch(t *testing.T) { + config := mtest.GetConfigForTest(t, "awshealth", "24h") + metricSet := mbtest.NewReportingMetricSetV2WithContext(t, config) + events, errs := mbtest.ReportingFetchV2WithContext(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + config := mtest.GetConfigForTest(t, "awshealth", "1h") + config["dataset_id"] = "master_aws_awshealth" + + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/aws/awshealth/awshealth_test.go b/x-pack/metricbeat/module/aws/awshealth/awshealth_test.go new file mode 100644 index 000000000000..ee285cfccf4a --- /dev/null +++ b/x-pack/metricbeat/module/aws/awshealth/awshealth_test.go @@ -0,0 +1,247 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package awshealth + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/service/health" + "github.com/stretchr/testify/assert" + + aws "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/health/types" +) + +// HealthClient interface defines the methods used by the MetricSet +type HealthClient interface { + DescribeEvents(ctx context.Context, input *health.DescribeEventsInput, optFns ...func(*health.Options)) (*health.DescribeEventsOutput, error) + DescribeEventDetails(ctx context.Context, input *health.DescribeEventDetailsInput, optFns ...func(*health.Options)) (*health.DescribeEventDetailsOutput, error) + DescribeAffectedEntities(ctx context.Context, input *health.DescribeAffectedEntitiesInput, optFns ...func(*health.Options)) (*health.DescribeAffectedEntitiesOutput, error) +} + +// MockAWSHealthClient implements the HealthClient interface +type MockAWSHealthClient struct{} + +func (m *MockAWSHealthClient) DescribeEvents(ctx context.Context, input *health.DescribeEventsInput, optFns ...func(*health.Options)) (*health.DescribeEventsOutput, error) { + // Mock implementation of DescribeEvents method + output := &health.DescribeEventsOutput{ + Events: []types.Event{ + { + Arn: aws.String("mock-event-arn-1"), + EndTime: aws.Time(time.Now()), + EventScopeCode: MapScopeCode("PUBLIC"), + EventTypeCategory: MapEventTypeCategory("issue"), + EventTypeCode: aws.String("mock-event-type-1"), + LastUpdatedTime: aws.Time(time.Now()), + Region: aws.String("mock-region-1"), + Service: aws.String("mock-service-1"), + StartTime: aws.Time(time.Now()), + StatusCode: MapEventStatusCode("open"), + }, + // add more mock events as needed + }, + } + return output, nil +} + +func (m *MockAWSHealthClient) DescribeEventDetails(ctx context.Context, input *health.DescribeEventDetailsInput, optFns ...func(*health.Options)) (*health.DescribeEventDetailsOutput, error) { + // Mock implementation of DescribeEventDetails method + ev_desc := "mock-event-description" + event_arn := "mock-entity-arn-1" + output := &health.DescribeEventDetailsOutput{ + SuccessfulSet: []types.EventDetails{ + { + Event: &types.Event{ + Arn: &event_arn, + }, + EventDescription: &types.EventDescription{ + LatestDescription: &ev_desc, + }, + }, + // add more successful items as needed + }, + } + return output, nil +} + +func (m *MockAWSHealthClient) DescribeAffectedEntities(ctx context.Context, input *health.DescribeAffectedEntitiesInput, optFns ...func(*health.Options)) (*health.DescribeAffectedEntitiesOutput, error) { + // Mock implementation of DescribeAffectedEntities method + output := &health.DescribeAffectedEntitiesOutput{ + Entities: []types.AffectedEntity{ + { + AwsAccountId: aws.String("mock-account-id-1"), + EntityUrl: aws.String("mock-entity-url-1"), + EntityValue: aws.String("mock-entity-value-1"), + LastUpdatedTime: aws.Time(time.Now()), + StatusCode: MapStatusCode("PENDING"), + EntityArn: aws.String("mock-entity-arn-1"), + }, + // add more affected entities as needed + }, + } + return output, nil +} + +// ConvertToHealthClient converts MockAWSHealthClient to *health.Client +func (m *MockAWSHealthClient) ConvertToHealthClient() *health.Client { + return &health.Client{ + // initialize with required options + } +} + +// MapEventStatusCode maps a string status code to its corresponding EventStatusCode enum value +func MapEventStatusCode(eventStatusCode string) types.EventStatusCode { + switch eventStatusCode { + case "open": + return types.EventStatusCodeOpen + case "closed": + return types.EventStatusCodeClosed + default: + return types.EventStatusCodeUpcoming // Or any default value you prefer + } +} + +// MapEventTypeCategory maps a string status code to its corresponding EventTypeCategory enum value +func MapEventTypeCategory(eventTypeCategory string) types.EventTypeCategory { + switch eventTypeCategory { + case "issue": + return types.EventTypeCategoryIssue + case "accountNotification": + return types.EventTypeCategoryAccountNotification + case "scheduledChange": + return types.EventTypeCategoryScheduledChange + default: + return types.EventTypeCategoryInvestigation // Or any default value you prefer + } +} + +// MapScopeCode maps a string status code to its corresponding EventScopeCode enum value +func MapScopeCode(scopeCode string) types.EventScopeCode { + switch scopeCode { + case "PUBLIC": + return types.EventScopeCodePublic + case "ACCOUNT_SPECIFIC": + return types.EventScopeCodeAccountSpecific + default: + return types.EventScopeCodeNone // Or any default value you prefer + } +} + +// MapStatusCode maps a string status code to its corresponding EntityStatusCode enum value +func MapStatusCode(statusCode string) types.EntityStatusCode { + switch statusCode { + case "PENDING": + return types.EntityStatusCodeImpaired + case "RESOLVED": + return types.EntityStatusCodeUnimpaired + default: + return types.EntityStatusCodeUnknown // Or any default value you prefer + } +} + +func TestGetEventDetails(t *testing.T) { + // Mock context + ctx := context.Background() + + // Create a mock AWSHealth client + awsHealth := &MockAWSHealthClient{} + // Call DescribeEvents + eventsOutput, err := awsHealth.DescribeEvents(ctx, &health.DescribeEventsInput{}) + assert.NoError(t, err) + // Validate eventsOutput.Events is not empty + assert.NotEmpty(t, eventsOutput.Events) + + // Create a slice to store AWSHealthMetrics + var awsHealthMetrics = make([]AWSHealthMetric, 0, len(eventsOutput.Events)) + + for _, event := range eventsOutput.Events { + // Create a new instance of AWSHealthMetric + + awsHealthMetric := AWSHealthMetric{ + EventArn: *event.Arn, + EndTime: *event.EndTime, + EventScopeCode: string(event.EventScopeCode), + EventTypeCategory: string(event.EventTypeCategory), + EventTypeCode: *event.EventTypeCode, + LastUpdatedTime: *event.LastUpdatedTime, + Region: *event.Region, + Service: *event.Service, + StartTime: *event.StartTime, + StatusCode: string(event.StatusCode), + } + // Call DescribeEventDetails for the current event + eventDetailsOutput, err := awsHealth.DescribeEventDetails(ctx, &health.DescribeEventDetailsInput{ + EventArns: []string{*event.Arn}, + }) + assert.NoError(t, err) + + // Validate eventDetailsOutput.SuccessfulSet is not empty + assert.NotEmpty(t, eventDetailsOutput.SuccessfulSet) + + // Update EventDescription in awsHealthMetric + if len(eventDetailsOutput.SuccessfulSet) > 0 { + awsHealthMetric.EventDescription = *eventDetailsOutput.SuccessfulSet[0].EventDescription.LatestDescription + } + + // Call DescribeAffectedEntities for the current event + affectedEntitiesOutput, err := awsHealth.DescribeAffectedEntities(ctx, &health.DescribeAffectedEntitiesInput{ + Filter: &types.EntityFilter{ + EventArns: []string{*event.Arn}, + }, + }) + assert.NoError(t, err) + + // Validate affectedEntitiesOutput.Entities is not empty + assert.NotEmpty(t, affectedEntitiesOutput.Entities) + + // Count affected entities by status + var pending, resolved, others int32 + for j, entity := range affectedEntitiesOutput.Entities { + switch aws.ToString((*string)(&affectedEntitiesOutput.Entities[j].StatusCode)) { + case "PENDING": + pending++ + case "RESOLVED": + resolved++ + default: + others++ + } + awsHealthMetric.AffectedEntities = append(awsHealthMetric.AffectedEntities, + AffectedEntityDetails{ + AwsAccountId: *entity.AwsAccountId, + EntityUrl: *entity.EntityUrl, + EntityValue: *entity.EntityValue, + LastUpdatedTime: *entity.LastUpdatedTime, + StatusCode: string(entity.StatusCode), + EntityArn: *entity.EntityArn, + }, + ) + } + + // Update affected entities counts in awsHealthMetric + awsHealthMetric.AffectedEntitiesPending = pending + awsHealthMetric.AffectedEntitiesResolved = resolved + awsHealthMetric.AffectedEntitiesOthers = others + + // Append awsHealthMetric to the slice + awsHealthMetrics = append(awsHealthMetrics, awsHealthMetric) + } + for _, metric := range awsHealthMetrics { + assert.NotEmpty(t, metric.EventArn) + assert.NotEmpty(t, metric.EventScopeCode) + assert.NotEmpty(t, metric.EventTypeCategory) + assert.NotEmpty(t, metric.EventTypeCode) + assert.NotEmpty(t, metric.Region) + assert.NotEmpty(t, metric.Service) + assert.NotEmpty(t, metric.StatusCode) + assert.NotEmpty(t, metric.LastUpdatedTime) + assert.NotEmpty(t, metric.StartTime) + assert.NotEmpty(t, metric.EndTime) + assert.NotEmpty(t, metric.EventDescription) + assert.NotEmpty(t, metric.AffectedEntities) + assert.GreaterOrEqual(t, (metric.AffectedEntitiesOthers + metric.AffectedEntitiesPending + metric.AffectedEntitiesResolved), int32(0)) + } +} diff --git a/x-pack/metricbeat/module/aws/fields.go b/x-pack/metricbeat/module/aws/fields.go index 0af54980375b..ce27511a9e60 100644 --- a/x-pack/metricbeat/module/aws/fields.go +++ b/x-pack/metricbeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded zlib format compressed contents of module/aws. func AssetAws() string { - return "eJzsfVtzI7fx77s/BSov3k1pFXvXTp3yw6nSzbFOtFpZ1MZ+Y8CZJokIA8wCGGnpyoc/hQYwg7nxOkPJ//rvS2KRBH7daDS6G43ud+QRVj8R+qy/IcQww+En8pez3yZ/+YaQFHSiWG6YFD+R//sNIYT8mz7rf5NMpgUHkkjOITGanP02IZkUzEjFxIJkYBRLNJkrmeFnF1wW6TM1yfL0G0IUcKAafiIL+g0hcwY81T/h6O+IoBkENPafWeX2i0oWuf9LB6j6IPFAhi706V/LP4fx5Ow/kJjoz+4PU/fpI6yepUq7P55mNM+ZWPjv/uWvf4m+14nN/XugCzsweaK8AJJTpjx/6LMmCrQsVAL6tEWB/nA6K5JHMKf2v1uUtLGuwXBLMyByTiiZfCB+1NaEKctAaCbFK2HcRxSmGFYL8rd/PfUid/rX079+uyPqVBYzDmOA1sQsqSEKTKEEpG69q71Azu6uyZcC1KpNEmfiEdIpTRJZCNOiKN4QpEP+46FYWvtzv+RsoMn+u74khYaUGElYCsKw+cpDJR7qaSeGhuweiMLJsSKUM6q3BxTAzBjnTCw2MnUNin/7Mf5NEikMZcIuNRDQhmXUQEqSJVUL0GQuFVnJQqEa9IgIEw2NGP6VmnEGhm65vFdhzgs3ZSebuazR26LuI/3KsiLrIcBjX7O+F4VSIJLVvmt81Zo38SOSQrCeSSegnlgCtwfIlh8CB0RS7SpmfczohnGWSWXYH5BeSG06gTQFq29J41Fp1tj49SFbSquTvBIaSaQ2fWOGKS2neyfsZuamGVtDhrnOOYj0NbLMAzsaw2rz9bLrVqqMcsvXz5ou4KwL1wszroJICovxGMzrmbOfj5/F7LUKXgntaKLXmLGfaZa1vxZUGGa6NfzLMQ1X/YvHdhSm1WfsZZo2VJlpSs3+Z5MdgdgR8GRS1qSEJ+tf2fPYLpnunBlEetC8VyLdY1YUgWkKcyaYHWcwOXmEpsxtoqZF0cMSiDbomnqDPFegQRhNKDplllJKdA4JmzNIO3FGTuUq7xLMoSBZE8SOZT21NpA6v2ermpNG+l0estlZIzv4Pi2C2la6VbEEvuZcKlAOL5mtKidYtwzzpDSKD7LNq2Ea5nkWe2XPoIDoRNE8eGZlpOI39M6elyxZVgN0xDfselmSUjafg7L/YenQOU3qtmI94BH+rTPqy3GGc5qsxJXDRrL+vAThvNCI/4TmrCM0sBI0k+nsoNUJgxxpbewPL3HKy/NDXS0/9mCqbVIkCWg9L/g9fClAmxtqrM9zSp+a3hrZ8WBsrz/xMkCfQNkjjLu5rJbRJQ6iHBBtHerANutqn2X0DymqP02MApo1WeGBFF6vxWJmWAYkB8Vkero7QzL6dTSGBHfvNTLkk+BMwLVI4esdqASEoQu4U3KhQOtRxSQvp7MMSWSWc7C/cfqCEgHPZMHljHKiIZEipWpFmAVKmCYzsATTNHWhGUoMnXHop/NOySemmRSQ/qaYgQua04SZ1WfBzLh0iiKbgbI05hUG8mxBkMSjQCtPeysBKcHoUw/9W1F5DzR9aSIV0HRwGi+k0EV2bAKDUqsI7SIu8diIfALVvx1POqfRkqxkQRIqiFE0eSRL+UyyIlna2TDEF/PWLJUsFsu8MHY7FBrWbPJ+luki62VZR0hvB4bpIvuTcunI+qEtWZ264c/HtNFl68/Ep3vIOUuopeyYNhhwmutA+QzMM9izVZAiTzHuzAxkhOY5UDQgmECOlTaHRpvD6uzOmaQA61dawpxGPyFUpM7Cbo9MhTRLUOUv/GRe/284vzv4dwyT7X8M/x4UFZomlu4LKeacJWY0ATzzwqfAuvqeS+84PEFk7aYFWMPNVLgot5sXoemS14kU7qKmK65GquGkY4amGeB0ejdWjKSrpKH8tbLhzF239ZmMhnH2B+63oyiqujewyYgsEB2k1v+29HZeDa8ntn5gvRpqO8+0ncmdrLSB7EopqcY8h3d0XZ1iW4AA1Y4eu39UkF8eHu7Ij999R7ShprAHegoHOLgXUqTM7auLJSSPP1PGrag75CMyp7Ln5jglocZAljtu5aDmUmV2Xwd0bunXbNg7ECkTi+gkvEApOAYJeBq5Q88vI1WAiA0IS1D7KOscdVYY9/MlfQIipCErMGRmVVw02IGWAk0flkoaw+HqCcRoi3zfJf1IHHxNAO1D6NdknUMO5CIH8scW8505EFnMnGXMdEezpCC0zFgjb7S1v6musUQ4Frzt5wHq99cpB3UdP6Yg+GPvI/1qd4VeazIfpiqCwbw+PoJcsd7VDFxW1Wxl17L3PHOjM+2khaQSNCoNmud85dTOuxQyNJotl7RlUzeT1mnWik0PdpQba6K9YoZVEuFI7XZlGzFTOY85TX6Wqs08U7E6obmOc5V6zE6aBiPAA95CXJGeQq9R4V3r8Zs7HY+5IJ222OteEQd51CV51Qvx8rrkI/0aeRkov31+1ZgBjMP8qSVbLKGVv+T+tcZqyP4GOd+Fcb0+2stwrimG3UyLf7Jmj+7JtTIHZxbbTrtfksNMH/F+/Op8cli6wtAX4/+SvMhwY56vrDY73OkPQS/N/kDBAZos3f6QufV3mRSxF+uj0Ggi5saavE8ISVs3kSbLcK15y4yS72bUKjgmtKEigRPyvLTrY6KIQiO9J/y5Iwi+yWF2rMGtNypv3Db4UzLHys2nfAjOWIVjMErYsANLvmgM/bYg2i8alq05saN1HA9rYxEPBPtrAQXcgFiY5UB4G1y1h3tT7sog1jNlBiVQWpPCJySgZB1A0kPp8VbpFQPRVj+orv/2KV6HHJQ/Usib6093k7ckBc6eQIGDXq6l/bB2ys2df+1jeFfnE7/5Tslnu8+emVnGeQZugMnkstyjUvDVJrbEN9KjiKjP016z8Jq8EVV2t5Hk/Y9//2fDMHpbXSeul4JheHNeKG3OKbd6bABuVJj+gTFXTu4KlUsNCOnNIn//9oRUAko+5YZlyI1fLi/JG22+f+supC4kD39Lvn9bJ8bRm4Ld+nPLT9xUdCYx0tclpYmC1Bqdb6ykWRAEn8WUMGqfa/M9QsCJFWSUieiibWYZ1npo2C1yeBmDwUG7YOtCQfurQ7fjtJUTZ/xQzlv63DkuA6kXC8CFuo5MVWs3DUnWdcqPQdBajC4PTUi/fqpNsTOSi1nGjInv/ksbPXl/mI2evD+mjX7x/jAbPcmLU+T0ad5KDHfE64RySKdzLmnzC1vkFtc1CeVcJngHf3XxHuWuMBCHBqgC/8bPcOtUkUJDuB8NxmL3eztLiFNCU3z000nLpgePPfnRpQxe3H0uNV25sWJseBDbbxWR47sJ78wdHqMgBopvjGPgjtGiwryk2vqsqoCUaGb/wgx5pppwWgg03FGnU2WayTIxMbpQOS/09AhE+anqFOHlFF5KVSpPkEJg5CjyNZyKsD+7uPt8gSP409u/wmea/AFKbkupnrp3oN3vqQ8mFWnpJNjuFSENySlLSSqfhSW5vd7OGnBqxSwLq0CTAq1FmpbXmI6EnlfaYJ6lejxl4jSn9tDe7zFxN6VNLe9nIAoSYE9W9ASeXB4EYcKAmtMEdGvrMRFKT1hjpssp7KdomoOaakhG0IBt2iIzH3W5tbq2JnM9RbIwR1yk3dHvsUgRSf9TVomJ09nKbP8o35noP5GuH+2xfDjM0XYYznaUlXN0Reu2O4mbRfHlF+5ou+4FV26oHZcy/cjkqfUGjrdyuGphk1Fv5lsqyvXQRiqo4qNPlHG8WTBy33VrETrSup1XZEXLtTeFa4lB3+1Fli1OazrKukWkjrpwgbBo7fakcbMYNmsUrV24rRanClQ0wzPH3mJI29qV2p3Gi17qhthpu8R2OoVzzOVsx6WOu/HGXc4WdYfvvn1W06XmniZLSB6nLr11IFLvIZfKaOtZYwpoDemSapJTjcke0izrH4Z0YYvJP6MAojERuv6Zjx1zqg3JmCjM9kRO3XhHpnUMQsI8L0BK94ptS0x5aCRSrdMk1rxbQPPdze4hOql8jbLNJ1b5KcvoAk6HLItngV1fhps7HL8sS+dCa7vgqyLBp3YNBqwBcS1SlmCSeJCEFIxLf4/Cz0wTEFYX9SjUEmiu2BM1cJoKPR22wh8GlN3o5PJ24uqzefa2PIQtUbJmFoqXxOafd4B2fff0A6FpqkBrQrWWCcOYN97q7YW1mHGWjMVQHLzFzy2l0kMbkIuBcR7HlVUuLCHXd+UnbyyD35KZLNwBug9LcQudJjLt5ubeigjHbfLwxGXCf//3dzNmSCE0WwiMSOMkWyEdft07kZI3uXuwQv5LVCGE+396WRjDxOIdRpn/SwyojAmU6f9aiwULAoX/C+nbDRSZpTVwnaNjVfVYR4GfB82tcCx0XPjxwyrXAD9m0Zqrm+56NS+WlHdOk0cQ6YUUwlndAz1gqy9lUg4fs1VIExVl4SsC2tAZZ3ppjU3/ChMNFElT4m+kVGlnKlgwbTC7JsjmmhzhXx4e7i5kClNP8fT9778PTCW+onv/++9Egc6l0ODe0YXHd5i0eiDoD+OA/jAq6B/GAf3DqKB/HAf0j6OAvro5H5PLCWdWh4FVDQha11G39uiWkEfksQb1BGoQyP6t2TAPP5sJkj4PsoqlINxKW2a07yUumkpPlK95kZwzzuUTqOGgt/Nmwzu8UquXT+9nkNBCu6xgXSgssAnugt6q+zUyApSb5eoXGZh+6LuXOtOXbvhqg8W7Do18rDqypXRMLGVxEu0QYHvZ/AYFnFu0AtTbprS8ebiIPy3zDIJVqGQR0m1piw/9NH4WIy9JIYZdlOHKvVSrgflpvjbJCWEiZLSdOLMQs3vtV9oGCxqApnq779jfoepJIQzjrYCNMi7ooKG0fPwBsgSaglpzQpQl2M9uzs8Sw56gsvTcQg7Doqqqes3o87lgxIplLKcUoTjGucNFB0+wbeuV7K1/ZL9P1QLMluSH9Oebi89DpT13UV0H2Xjz9ebm4vPb+OXcWV4WFiA39pfnG2U7pukWno+3ngKeWwsZW+zHW807Ja3TAIM9JOoj2V9sh+m2X7SyHnb11UMd1fpQR/RZI3JfnfvardPGsHRegTa7wLEfbia3sJCG0dJdH8M0fbiZ1IjECuCx9eydApS4lKXozZfqgFCiQWssLRrCpnWCfREmihOhmb7eaZj+zL5COr33R990DJrndop35elKWxGLKlqxAew9pExBYkaBqfzggwD8rPj0hmXMTK+wcgakR8ScyIKn4ltTf/wVOw6f72/CNVW5LpiEbkXLmT/WoeB27yg7qCD/559bup8ffv99FFqjkIoj2mJ1PihSLRVbYPy1Rxls7/CPB7/H7R8S/49j4u+JAQyK/7vvRsT/3XcjAn8/JvD3IwL/MCbwDyMC/2FM4D8MCfz67unvDQN7DHuqw7RuGwn4WtwCWg93xAidHb4Kv5QZybtFEDvctDFY+uIO2msTmx+QoPXyc+/DlWMs0KYLsM5QaZ2UJVZ7cvUXmNEdhXqioV82hl0tyk78LzhcPVFeuOS6ocEVfLO4LNgTuPJ3LjynrNr0BSs8MVSQpSzWbPERokt7xZR2iZKOHNT16iJ6GSqFZilGPH249wVDzuvQleHodkDHJ6ocGsyphjliIOfWTfpKgzg/c/k8ZAhzTQBnzuWzJm/qlydv2+fjpvOuAXz6cHE3Pnh7wo9GwM3kCATcTEYj4PPlEVbg8+VwK/BnPDeOEIdsct/KzJKKVC/pY3BxfIlnfzkuKixV04AQwrBmiIs0hsvRtcZ6pYrGMtN7xGette4PLB8N26oQd0wLbu7R3I7+PT00Ta/EyTghTCS8wGv1h4u7v13fbb6NrUMfbUE64Meiv65NA67Hn2JnxxT5/e2kaQ11F3dTp7um96BhyOB8O2FDgyFv7icPb+vP7d0DsPLyRG4J++rm/EUw75szZTE7YXpxVjv2OlY7tv+vRzSkR/TIBGh2WGVUP8axfCFXZu+fbtJOX+gF+4f+A8w9JFKlejpUesMujdDCw3fsEA1RONCzy/dXOiEZUF2ojU2/+gU6IvTaWC0j1dkCPjLOmU+tGpf0Rfl8Ah/AKcSCLzM5j8CRhHLuEzHpwsqWIXQ4bth/ZwvMirS/Cl2HE6hlwAfXA4dyBYFAtLB7chrYsRZU9CTeuC7VQLOt1uYYrdNc4hx99E/bIwLKZ7fDCpz/36P0cVGelI49pZdUpcNS5rvtHoWyqKNu15L5l9JD6YtrkciMicX4WrFVsiV+pJIXJuyiuhLYRJirhO+OC+88YB0UOwNKxF3heYg7vPyvmKObuRMk+zj8CbI9Joe8csNnx0Nwqvz6Ec7XllPWy5pChxT/iriqYPK+e6ai9QXUeAchA6iBiqSg6o7XMTtSeK5zmHMrBjeNKol+QRtwJ1nVQwrrcYyOQHef1A5rfETEbSO3hx3RTR+5U0P6ov3UEOv0GN9ODvcryjmkkJ4gS8aUb18iemxzrB018AeXtaqxGkunKMfUp9TQUVgwKbXKMc3SSJcFZrwwH0KbyJcwzX0ihs9Kxrd1gnKMRBYKXpw1UXfBl+eOcWBCi+Rjs+UeaBp3LSqr2IcE5hEVa8WcVoDAhCUqK9dsZ/T20jkpZhbTDB7kxPqJ03tqYHQaIwNcE3Bl1l20geJNj3aocJTQXhW3iY6zmOy5whXQdGWHwaZK+Eai9msfUsbGyb7ngyJSEYZ93+NOjtE1LPI6qsBFOZfPJdNZJII7cHbsE7liathTcdGmJpw6l56p7q2SsD2J2I1zC2NyqO3RKFI7eMSjmz4XPDyHJROpNSH1+mSSw4gdIlTXWnqwdOwTsetmyMscF8dd9ONt3kgjwhOoVVhjv2pMu2pMeNUdb9lTco2fSmG3b6xTUVV+26ch+zmB7Ude/hDcwkLY7TDsjgDFw+0c/gks4zSbpfSgayo3xBEz9m5wwteVrXctnvz7q+ETlqwoaJeLNC9E9XAKd95XSArj3t6HxIvIh3Efu/eYIo3/ExdGgS649/TKoTc8O/SlkIYmklUM3B/bJdD0BowBNRjKn6UiVK9EslRSSOw0EYCeNKwwt05OOms99rFEQakQMTiWAk3fcYTqC4DMCm8xriNPGyZw7kvXQ231s3fFXjOlJeitaCy8nTqMgFXd13wZCmq6dpLOQaRldpDdRIGINdkc3rUZcy80SslQbCfpPao1tx7VxfFAcuHW0/fBzyjWxiv3aagG7w4z7YSl/yLZ/mkDay/KtNKrUmMNzuVSAqrqKlVpmEoQwHXs78X6WSjAN5XpSKh/9m0Qf5uQe1h07EaHsAI/A4u7lukWaPXfSqX41jcXCuCrVN5kTbpNZFx1Ujts7s1OK0SkqHWP3JueZOsGz9tQhKtHnkDhs3/7H5xRv0dcayY538RWkrInllbpZs3Gkj1kV53JdmVAbM0M+/RoG1tmwJUsi329PEVWglOq6ivkQkic9y4h075hXMejG2oW1MAzXR1kvlfD9JjwqNvRWH9GY916MoomjwRb0lkW3J49ED+GNcWpK/fvTotXl0mG0Z5r8bOSWWRPDSwUjUCP37cxn8owQGQf9Ut3BHqCbH0ZvCFJnQkn8P+6u9iA+VNhHuTYfC4b6/jerS3wPlq0PasR9oic9sXP1qLdidnVQ90zZ46P+163MvoxA7CHkm3gXlVx27GfGMdvLnZGjA7lnVTmjIdKK6McJE1hwGIwWEYonOaEBkM8l2qNER1a18piZGHwVplRVGjsvRhHOUMADwtzh4YfKfd/WXOeu6T1SyXzMdCHnPhUYXXvDo23EdrYZ0ira+TBp0gN+CjabWvMOyk3j3vks6TVAHKI0ySGPirHBz9RuovIjVHHNXpE6rVFs17HRm0dQKv0sCcXKj3acwvrzl9ODotiY9f50JV4n2bXobWcT+j7Zs3KrW2F7e7+e1r7O5jvXHP8WuP9shVUd3OFg/p4H0Rao9u1K9fn3HVFxcLXD/rObqzv+7H/b+vuF2rdnVJDZ1TDNFIio5ATJmoUhaw77DVks7Lh1SlVohPUXt1P/POte+92k1uawZuz+9u3KAJAk6VVjptBJZzqbl7tBesi1qUiasUT+uRTkZIMMqlW1VN8xBC+eHm+qT1jhJ6lIAybs1aPlSFIoHZZ1Ttd5DlnkFaLX83qr2qrP4QXTIVgXwqwAJy8l9+ww+5EoutVNhx5E3/lrGt5GlEjHZ+qZintQcf04xQvsaYp5GbZie3gdrqyMBhCs4fp9SdN3iig6d9cG8JwS/KWPFNW1nLHW1D/fkw/dmMP3fS+8KkrjDilCxBm+h85G0dj+Ffck19vyMRVYjyzExI7YdzSYGP7ubkCsEfn1O2eo/ZwrkLPVTNHRUUqs8B1D6oX+VQbqejiiJ1we2B7HETnvf21fHWxaaEhnaKX68q1Tlk6pIyEImbRDOT60qkLeyTOAPBgSU9d8Wn3zuJOarNQMPn1phu85NZPmSooy1VPNZdmyuniNJsNCJ/TxQKzD9gfpZL3s5afoUEtNd7qG1AZKvnfzm5cLmxwGneiDzvXMpl3r8SeWqf9+MNqEHfVae3XzranffiQBcjvHfqwBklP/XX4IcLu8rEwQ5jc+xWJjhy7Ola6liy0KnUWRHwqxSvycTX59eaEfKSK0cvzE5dtVK5SbZoee0M/09xZxS+0/S0At+NdqR8pWqZGM8UNI3Cl1rA2VaXCu6mMNQWXCz31ZSPaq3nItkPBjEixDkCkQOzEO+0n1zv4SBvKnd477qgvBSi2vdDshc7PUV3gbQKVAk25TB7HhVXOEvIoShN0Ez7XfBmPsJfac/6grdXWOiuUVDVthE1kcLJ1hJyuV/vD0xFd4DDOQzvzhuSWZWwKbUB5qCf2CJDYrYYa8uM7Z9OVjarWk7lhN45Jp9ubuE0bZJbRxMPJRFOQy4TyFzYIg3TWVbyBLJeKqhUx9m8uidKq1E1SyuWCiWl4JzWqTvAOBc5YXctt0gemTI0+TWSWse7w2mDa3s2xi5aPAKbAoac19HDHEc5R6v1d0KV8XGiXlzdROeEdgGUjA2NCgzL6hBR5Sg1oZwo6Tu6E1A10DLD7LLCvSjsovFLvhBbP1XxkJs2y8VwEm7taq86/kjCyvMuZrVxwr7TmvWXgT1Y01u356rV1pbj2YMHUoxqSFcxX6CBv7t3gbyueKDqfs6TDOo+z3ZFdSaGNzEBVBlH4sWVdiI1eTso/oxViVXx0RWO/GvnJW3MlrMyQbJGFWUhky4Mf/c/DF2sajbGZm7ncZXGCtpGyEaMGDj13SoOpHDfHPirHKdRx0bk59kGHluG44JyGih774RJvwsh9fYwdLZohYy0eAm6hltGDyjeLS7mtJWMXy2IsGjAwl8IcG51JQTgVi8Ku1ZvLy5u3pV2yK2U7mCZjUbbWetmRnh0NmHFJClt6Rxp20toDUDCUUg/4d9ToY61BXenvuAY76v2xaKgfDTvSsNvp8AoFaUd3czTNW/NIt1wEvIr1kXWGYecXiqdEYWmZJEXOXNBvxgRVKwyhBPM1o9Yvad8wuAibWnuREJHbvOAa9nKrI8oeTUjshGTOOOwWa4/gNy8LRod/0CVB9GN96nL2Ro1xldUjonnDY2axsJJERfB4q6yM4BFvNG1jamZcJo/QfRIORU6NjGYkv3rG55BsvnqIkkPS2dQ7+tMxUmH2TG4JkWJf3SChnDsd5x3Q6hbAf3MzoUq2Hk4eQNflObEDasLZI5Df7q8fru6JVOT+6uzy6v5kSOAgFkzA1H4wHP4rmixrV7qqEJ73br4TR1nz6ja6tsXyACbpJoAinVN/pEyjO+0h90nzwlpVd9VBglQhhN/xnveYmekOjERmOTVsxjgzqzW32mvXypO64HJG+TSdlQcLpFM0baZM7nambiD9OlZe/8BpyaVXBs2Xvp33pRXA6jlArlhmD9rq0XD3rY0rr+C0S/37W3LHqi0XAJuDOjJfKoFRkEp7ijl3NcBRMUecmdFgyEGkxxYHZtMMRXl48L0V6Zwu3CvSEo5YBJd2nTxsaVB6qv3gpyPS6VNGDqOvdou8D3XTjH4djsI4ratOUlwbqwne6WKr0tvX48FcaET09yOViYFJZeI1kDqjySO+UJ4mSyoWMPUFmU4TBW67qj4v+9DsznJq4qYua0Hh1KHI15w9gc/t1GhOYC7EppOplyxtpBrWYk1MUe+d1EdWLZljewKemUjl86mbZ1A/p7P4nG92U1Hh5nfXahW9zc+3pYL3xf4OlabwIpSadTCtFa4zynlo9b+O5DmWo3C9XV11sTBRT6qey4vwiUM0eSzyqQJj7Xsppr4+2ZDH/kNHfQs3b5mjUd5govQZSXSR51I5JuWSCfOOiXdoRCrAzUHmQE2hAK3F+gVpJbTf6jBRSeBaQaixRgua66U0L8YLXyoUdyPlPJAXcDk9QztcFkysZylgI+WdGJDQZAnTJTNTNEVPZ4XdfQPSXi8z3y6F5CvX+DdPbnqHajvArsTYVMOQ23c30PcIQYNZh9v7jEWO+3SHLOLdva5S2dReY2Hqufe98BBee/6qVE+NnHqLI3c+pv7Cp3vmQu8YQl1EAHcwHe8vJ7E/XNJvJJFYlFbIFMpwzcaDrshDRtvUZQxO3UvGl9IPdvu716krWbj4kktkjE+ELeMZfmV9TimHuRmJOAUZZejwRw82MIwZCmg2kxDLaqrt/LxSb3+YppTxVVifb5pYd3kw3Bys8XoYPysXY8y3xJMPhz4lTh7BnGr2x0ulYKLvXsqrM2pdfMJj68TtrKWpnE/l7D+QmOH3VvQEzc3Qgc3tIs7LpcYnjD3S58+EQ+XODxNJXGjV/5rlLByI7mH3iIv1y8PDXXX8ujI1Ei0gF7udfPBrd0IULKhKOfhHp6u85xwusS8GtRgamP9x9dDAbYUryB4TXTRswJsXI+K9+zw43jVXsINAvry6uXq4Ghr1si+DYhDMv1ydXW4lz5tkQeoxheHTpCkNe6Fck81xKM4KyeTq5urigXzCRcd33lbRDSwVjpKpTqgQR35808ynC4esx+LuTrZmxyHUKzCFei3kBzDHoJ+zMXdb3bu0c/naCggdKV5vPaXyWXBJ05dZGbcsFQbcbNsd2a5Ll3tjrHMp8L7fF8anZCbTnrfnRf7S5AYEbs282YW3nc54s9hPdtec4Aqe//C1WZ9pQHH74evX0KMdpyOuFoUrd7rNurkdR6vCt8DQtf6OSNVXSMcT9uOYhP349auLy6gjEhbyzeYMizitDOxwG3N41lkO6l2QOQz9lBGRRGY55p6VIokFpeOabl0sMLJq6lJuSizTgzlFMygV73p+oCEfvJujsgQ4zbXLuOlhDa4VbuSKHf5iHQt24Cc61L5ft3dLf1AcVrFMi2NWLJvcdlcse8F6vneuucyE/TFEsXsrBqGCRQZa0wVE/Wv6q+VNPk58x517aoYConwNnqihx+TjJOAiqesBwZqPUGNct6joPs0/elruSlKGLUPY5pXdAe6Nt98DtxNiZM6SLdDeSoPpVpjg4ltdjAe51kcsDbO53dJNgbt0cm2XsAK9SPHeaVfSXG/HsehCBRBh9y+FjQxE7oqWcWM586kYuhprHTLqrLJ50mxV7/w2RxQkl5wlW4l+Hw3vrsUT5Sw9M0axWWFg6MrwB1AVdw0sx/mW0BIqRvCZI4C8czXevlJ7bp/Uflv+gvy/yadbV04+kUpBYlwqY0bN2gYBG7l4K71u+dPw0TW+EDJi547030Oq2BOIB3nJv4xKLULF67dMemOjo3nQXmrnQXoyxqcCi1iLb60luScdk4+Tj1KY5YO8pAYmOQjzeXI5COhkSdXCtXBw7K7XnnRt2qgyZeVCn4yeUA4ipfhU1iz94x9Xny46pbuuAL4caPJ9OarJ9+uBRWp9BTLPjyld7HSFPcDzmjxX8ivLsHR61ZXIwSJCincu3JyWhpW/4+0QycqI9YubAqer4ZKvejZRDKjKJPBzYxpTuzyVAoqyyLIMUkYN8J6QSEmLkGb6xDRrW6fDuNp1neAOMDLnbLHsiWmUyI6Cqsk+oxg8UV45f1vKgxWlcZEGed0JWfBXx4VWxlZnq9ABWvpEF5ze2wrEvX7ZAFm36zYPveZpGg6jNTyELDerUPxinLKgDfac3V2XDbSpISlzO9xxl9BAQE9iGohK3R79Qr/lPW/HY/fRsM9iJr9OvM6sjVt798UGaaJUH2rvRkp+mD9dM6WjdCNqMKffVgwdfMbr3FMq3q0xla03jtRwY1dgw7Or1phiP1Rl85dzTpPHpeRjNc8ou8BU3uKKZHaTWvOKzML0RMlWPeY1sG/lPX7/iKDDSYHgCd0EGLfKyHj9Ld/gaMcSijV4txCJ8qJRH5paiCPsfZRomQH6b6/27LignI/R2qnqoW6tqHrJQWvbuMw9DOzSJEEAvRhDO4UxcNb7vZfLVL5xbYL0GbHha84DnDNRKf2UZSC06/attUwYGg94NVkJT1tUn3JxkKA+5WJvMf3X3e3rt3IeCiGAT8xwNztRewUgBoc/xfeQ9gOWWLboE/IdYSLFp72aXH767RY9/e+jP36+c786/8ed/0n86dXk4ez85nryy9Ul/vI7wnRV4I1y7hPbEcyaEKgj/5IausF82Z7+hoUX93eyEuE5sgWiTXbLrpBabbRiOP8/AAD//+8XbTo=" + return "eJzsvVtz4zbyN3yfT4Ham8xs2U4ySbbeysVbZcvOxs96PI5lJ3muuBDZkrAmAQ4A2qPUfvin0ABI8KQjKTv/+s/NbiwJ+PUBjUaj0X1KnmD1E6Ev6itCNNMp/ET+dv779G9fEZKAiiXLNRP8J/L/f0UIIf+mL+rfJBNJkQKJRZpCrBU5/31KMsGZFpLxBclASxYrMpciw88mqSiSF6rj5dlXhEhIgSr4iSzoV4TMGaSJ+glHPyWcZuDRmH96lZsvSlHk7i8doOqDhANpulBnfy//7McTs/9ArIM/2z9E9tMnWL0ImXR/HGU0zxlfuO/+7e9/C77Xic3+e6ALMzB5pmkBJKdMOv7QF0UkKFHIGNRZiwL1/dmsiJ9An5n/blHSxroGwy3NgIg5oWT6PXGjtiZMWAZcMcHfCOM+ojKFsFqQv/77mVO5s7+f/f3rHVEnopilMAZoRfSSaiJBF5JDYuVdrQVyfndNPhcgV22SUsafIIloHIuC6xZF4YIgHfofDsWS2p/7NWcDTebf9SUpFCREC8IS4JrNVw4qcVDPOjE0dPdAFFaPJaEpo2p7QIF1WQJN9XIjW0tLNQNNt5O8MXe/4OjeDG4pJjqfQ6whiQwVmoGKhF6CVJ1sm6eC6t2Y9rAEwotsBhJNgJutMj2GWKotK/USCDwD1+RlKRQQpakuFIkp50KTGZBnkGzOIOmWdpuUHHjC+OJotOC6y+iKSPhcMAmExmasbeGagdJn6F44o+FNBDHsLSHz1VrYwJNIs551lVDd/KCOkXSCND8jlCfEDExelsADXQCeQHJGpiJzf1HIYoN5SZ/N78xXcIwexOZHEZV8OFNgQBecfS7Ar38GksyFrHCf4ZcsCef3t2RJFX44FzKjmlDJf6Iv6idrEn7C751KWJgJ7X99M726/+16cvXN1W9Xtw/Rw/+9u4omny5r/3138ziNri/X0a1ikUMUi2RAS/iwZIrkVNIMNEiicogNB5QRnbEeSKgzR5YDTBFK8mKWspicZ/RPwcnvMCNTkM8s9mwytpV7A3rqRo09O8/TVLxAYl0ZRagEcvd4cXM9OSHnk8mnx9uHaHp3Nbn+2fxFSHL76fbqjKzhDO60MdWwEHI1rG5YeswgxE9AjAjOyJ1Qis1SCMlgShVw4gm/FUadYmrGR0JUvATj+CaTJeWLtUpuSRpY1htVHYe3+u60m6FzHjkNji6vppP767uH60+33ehTqnRU5GYNH2JdOsFnQhnrFhugaw3Ni9nVLYZulHZ5Dsva9mJQ5B7nwUmN4a5MSicqZX82Nqxq/5itQiv3s5AEvtAsT+GEXE0+GI29v5z2YNVU6qEFvFaoM1jQnp3MuhcjLJdQ45wPUxNjpxEQOfATEqdCQXKC5BR5LDLGF+sWfICmkwYNX/ZwGhLQlKWQhN/dQhNbHk0nJColbZrbTf4/t3bFwKAzUWi76WumVzW1pKEf3AC6HeIz+qL88Sca8gjTt9bdXN5PQ48sFlxTxq2/UNJn6d2S82f221Eh02FpeLy/8ZpwGDJU/WGxXV8eCO1Y2xAaKhQ1riiryBu3nzbev7YF61WNQ1z17gPGGgfGKoj12qwg2t76Ty13Hb/p/XX6ok7dOnaffOO+wJJ2HGDG0rR+KO2OAqwR07/dGP+umwpQmmV4mI6XVC5AIZUrUUi0i97UMN6IjPp/fXGHdfGDKz/nxE7ZKbhUtA7hNeo+0i8sK7IeAhz2NXGeSSEl8HhvJ/6qNW/sRjSa0zOpM+G3B8SYvGuFnp7XyKyPGd0wzjMhNfsTkolQza2+W7H6RBqOSrNGALA+ZCt42UleCY3Exp70jOmnNJzunbCbmZtmbA3p57pIMarw9ljmgB2NYbX5etl1a6xgavj6qOgCzrtwvTLjKoikMBiPwbyeOfv5+Mhnb1XxSmhHU73GjP1MM6z9taC4o74xpqHUPztsR2FafcZeptmDdoenuv3eZEawZ2uzM0nQksEz4wvcj43IVPe5lCcHzXvlYrm7zYoqECUwZ5z1Hob30ZMnaOrcJmpaFBnXUmm8onYXc7kEhYFsipezhlJaRlGTTpzB5fIq71LMoSAZFwQDl2LeAaTO79mqdllL+q8+yeZLW7LDHWiLoPYxxphYAl/yVEiQFi+ZrarLcNVyzOPSKT7IN6+GabjnWXg7+wISiIolzf0NbZmx8Dve0r4sWbysBujIc1AY31qRhM3nIM1/GDpUTuO6r1hPfPD/1jn15TjDnirLYQNdx3Ad3kYH/Cc0Zx0pAitOM5HMDpKOH+RIsjE/vMQpLy8OPWq1L3b7WdA3VjjetIhjUGpepPfwuQClb6g2Z54z+tw8rZEdN8a2/InTAfoM0mxhqZ3LWBlV4sBLSFDGComSbcQcAW3krPzTVEugWZMVDkjh7FqoZhhxyUEy0QiubMWQjH4ZjSH+uPcWGfKJp4zDNU/gyx3IGLimC7iTYiFBqVHVJC+nMwyJRZangOFftBeUcHghi1TMaEoUxIInVK4IM0AJU2QGhmCaJDavgBJNZyn003knxTNTTHBIfpdMw4TmNGZ69ciZHpfO6no+rzCQFwOCxA4FennKeQlICd6U9tC/FZX3QJPXJlICTQancSK4KrJjE+iNWkVoF3Gxw0bEs7si71yOJ53TKEFWoiAx5URLGj+RpXghWREvzWwY4gt5q5dSFItlXuBVbKGaIeXtWKaKrJdlHSG9HRimiuwvyqUj24e2ZnXahr8e00bXrb8Sn+4hT13GxzF9MEhprjzlM9AvYPZW7m+hCNOQEZrnQNGBYPZevfQ5FPocxmZ3ziQ4mHOlIcxadHsnhB52e2TKMe+w/IWbzNn/Dft3B/+O4bL9j+Hfg6Rc2aS/ieDzlMV6NAU8d8on4T/2itbQcprCMwTeblIAJoRWuGhqFi9CUyWvY8HtRU1XXI1UwwnLDEUzwOnUbqwYyVYJTdO3yoZze93W5zJqlrI/cb0dxVDVTwObnMgC0VWJSp0p4uuJrW9Yb4bazj1tZ3KnK6Uhu5JSyDH34R2PrtawLYCDbEeP7T/KyS8PD3fkx2+/LXPDRQIHHHAngifMrqvJEuKnnzHtyZ3+x2RO5c/NbaYV1Rqy3HIrBzkXMjPr2qOzol+zYO9sqnuwE05QC45BAu5GdtNzYqQSELEGbghqb2Wdo84KbX+Oed1caLICTWbGxAWDHegp0ORhKYXWKVxhMvlIHLrv0n4kDr7EgP4h9FuyziEHOiJ78sdW8505EHjMKcuY7o5mCU5o+YSBvFPG/6aqxhJuWfC+nwdo39+mHtRt/JiK4La9j/SLWRVqrct8mKnwDvP6+IjNfaSczMC+rpqtgvcAbTqIf4mA2kISAQqNBs3zdGXNzmkCGTrNhkvKsKmbSessa8WmBzPKjXHR3jDDKo2wpHYfZRsxUzEPOY053S3m6YrVMc1VmKvU43bSxDsBDvAW6or0FGqNCe+Sx+92dzymQDp9sbctEQt5VJG8aUG8vi35SL8EpwzU375z1ZgBjMPOU0u2WEIrf8n+a43V0P0Ner4L43rPaK/DuaYadjMt/MmaNbon18ocnFnoO+1+SQ4zdcT78auL6WHpCkNfjP8m0iLDhXmxMtbs8EO/D3op9icqDtB4adeHyM15lwkenmJdFBpdxFwbl/cZISlzTKTx0l9r3jItxemMGgPHuNKUx3BCXpZGPjqIKDTSe/yfO4Lgmw7MljW49EbljV0Gf0nmGL35lA/BGWNwNEYJG35gyReFod8WRHyswbI1O3Ygx/GwNoR4INhfCyjgBvhCLwfC2+Cq2dybelcGsV4o06iBwrgULiEBNesAkh7KE2+VXjEQbfWN6vqbT6EccpBuSyHvrj/dTd+TBFL2DBIS/87HytJ8WNvl5vZ87WJ4VxdTt/jOyKNZZy9ML8M8AzvAdHpZrlHB09UmtoQ30qOoqMvTXiN4Rd7xKrtbC/Lhx3/8q+EYva+uE9drwTC8uSik0hc0NXZsAG5UmP6JMdeU3BUyFwoQ0rtF/uH9CakUlHzKNcuQG79cXpJ3Sn/33l5ITUTq/xZ/975OjKU3AbP0m084daeWxhIS43S+M5pmQBB8FlPCqH2u9HcIASeWkFHGg4u2mWFYq+BQt8rhZQwGB43A1oWC9jeHdsUpoyfW+aFp2rLn7gHoMObFALChriNT1VpNQ5J1naTHIGgtRpuHxoWTn2xTbJ3kYpYxXXvRWfro8YfDfPT4wzF99MmHw3z0OC/OkNNneSsx3BKvYppCEu1Z56ZuSWiaihjv4K8mH1DvCg1haIBKcG/8dGoOVaRQ4O9HvbPY/d7OEGKNUISPfjpp2fTgcVOtnsndY2npyoUVYsON2HyrCA6+m/DO7OYxCmKgWGssBG4ZzSvMS4rv3WUBCVHM/IXZqhspLTg67mjTqex9Am2IUYXM00JFRyDKTVWnCC+n8FKqMnmcFBwjR8FZw5oI87PJ3eMER3C7t6vGxxT5E6TYllIV2Xeg3UUJDiYVaekk2KwVLjTJKUtIIl64Ibktb+sNWLOil4UxoHFhK/ok5TWmJaGnWhvoFyGfzhg/y6nZtPd7TNxNadPKuxnwkTt7NqrHcedyIAjjGuScxqBaS49xX4LSODNdh8J+iqIcZKQgHsECtmkL3Hy05cbr2prM9RSJQh9RSLuj30NIAUn/U6TE+Nlspbd/lO+rHXT9aA/x4TBHW2E421EkZ+kK5LY7iZtV8fUFd7RV94qSG2rFJUw9MXFmTgPHkxxKzS8y6tx8Q0UpD6WFhCo++kxZijcLrtjmHnJrETqS3C4qsgJx7U3hWmLw7PYqYgvTmo4it4DUUQXnCQtktyeNm9WwWat4reC2Ek4VqGiGZ469xGwhsnWS2p3GSS91Q6y0XWI7nco5pjjbcanjLrxxxdmi7vDVt480bWruWbyE+Cmy6a0DkXoPuZC6Xma2Fl3IqcJkD6GX9Q99urDB5J5RAFGYCF3/zMWOU6o0yRgv+ooKdxAZ2fGOTOsYhPh5XoGUboltS0y5acRCrrMkxr1bQPPdze4hOiFdjbLNO1b5KcvoAs6Gri1Z1T7E8cvy9Da0tgu+KhKM9QUHrAFxzRMWY5K414Sy0GgQfmaKADe2qMeglkBzyZ6phrOEq2jYSv8YULajk8vbaa0Sb+uEsCVK1sxCcZrY/PMO0K7vnn8gNEkkKEWoUiJmGPPGW729sGKB7rEYaqt/N/m5pVY6aANy0TPO4bgyxoXF5Pqu/OSdYfB7MhMFL/sT7MpSXEJnvRU69zZEOG6Thyc2E/67f5zOmCYFV2zBMSKNk2yFdHi5dyIl71xvBvJfIgvO7f9Ty0JrxhenGGX+L9EgM8ZRp/9rPBYsCOT/LyTvN1Ckl8bBtQcdY6rH2grcPOhu+W2h48IvPaxyDaTHLFpzddNdr+bVkvIuaPwEPJkIzq3XPdADtroo43L4kK1c6KAoS7oioDSdpUwtjbPpXmGigyJoQtyNlCz9TAkLpjRm13jdXJMj/MvDw91EJBA5iqMPf/wxMJX4iu7DH38QCSoXXIF9R+cf32HS6oGgvx8H9Pejgv5hHNA/jAr6x3FA/zgK6KubizG5HKcMW8YY04CgVR11a41uCXlEHiuQzyAHgezemg3z8LOZIOnyIKtYCsKtrGVG+17ioqv0TNM1L5JzlqbiGeRw0Nt5s/4dXmnVy6f3M4hpoWxWsCokFtgEe0FvzP0aHcFq4KtfhGf6oe9e6ky3xcZX1QILVx06+Vh1ZEvtmBrKwiTaIcD2svkdKnhq0HKQ75va8u5hEn5a5hl4r1CKwqfb0hYf+ml85COLpODDCmW4ci+VNDA/zdUmOSGM+4y2E+sWYnavbUDQdFjQAdTV233L/g5TTwquWdoK2EjXyE1B6fm4DWQJNAG5ZocoS7Cf31ycx5o9Q+XpWUEOw6KqqnrN6XO5YMSoZainFKFYxtnNRfmTYNvXK9lb/8h8n8oF6C3J9+nPN5PHodKeu6iug2y8+Xp3M3l8H76cO8/LwgLkxvzyYqNuhzTdwsvx5MnhpSXI0GM/njTvpDCHBhjsIVEfye5i20+3vdDKetjVVw89qNaHOuKZNSD3zR1fu23aGJ7OG7BmExz74WZ6CwuhGS2P62O4pg830xqRWAE89J7docA2u2QJnuZLc0AoUaAUlhb1YdM6wa4IE8WJ0E1ff2iIfmZfIInu3dYXjUHz3ExxWu6utBWxqKIVG8DeQ8IkxHoUmNINPgjAR5lGNyxjOrrCyhmQHBFzLIo04V/r+uOv8ODweH/jr6lKuWASulEt6/6YA0Vq1g72+eLk//vXlsfP7//4YxRag5CKJdpgtWdQpFpItsD4a48x2P7APx78nmP/kPh/HBN/TwxgUPzffjsi/m+/HRH4hzGBfxgR+PdjAv9+ROA/jAn8hyGBX989/6PhYI/hT3W41m0nAV+LG0Dr4Y4YoTPDV+GXMiN5twhixzFtDJa++gHtranND0jQev25d+HKMQS06QKsM1RaJ2WJ1Z5s/QWGPdqbhXqCoV83hl0JZSf+FylcPdO0sMl1Q4Mr0s3qsmDPYMvf2fCcxM7HtmCFI4ZyshTFmiU+QnRpr5jSLlHSkYO6zlwEL0MFVyzBiKcL975iyHkdujIc3Q7ouESVQ4M51TBHDOTc2knfaBDn51S8DBnCXBPAmafiRZF39cuT9+39cdN+1wAePUzuxgdvdvjRCLiZHoGAm+loBDxeHkECj5fDSeCvuG8cIQ7Z5L7RmSXliVrSJ3/EcSWe3eU4r7BUTQN8CMO4ITbS6C9H1zrrlSkay03vUZ+13rrbsFw0bKtC3CEtuLhHO3b0r+mhaXojh4wTwnicFnit/jC5++b6bvNtbB36aALpgB+q/ro2DSiPv8TKDily69tq0xrqJneRtV3RPSgYMjjfTthQoMm7++nD+/pze/sArLw8EVvCvrq5eBXM++ZMGcxWmV6d1Za9ltWW7f97IhryRPTEOCh2WGVUN8axzkK2zN6/7KSdZ6FX7B/6T9D3EAuZqGio9IZdGqH5h+/YIRqCcKBjl+uvdEIyoKqQG5t+9St0QOi1NlZGyPMFfGRpylxq1bikL8rnE/gATiIWfJmZpgE4EtM0dYmYdGF0SxM6HDfMv/MFZkWaX/muwzHUMuD90QOHsgWBgLewO3Ia2LEWVPAkXtsu1UCzrWRzjNZpNnGOPrmn7QEB5bPbYRXO/e9R+rhIR0rHmlJLKpNhKXPddo9CWdBRt0tk7qX0UPbimsciY3wxvlVslWwJH6nkhfarqG4ENhFmK+Hb7cIdHrAOipkBNeKucDzEFV7+V8jRzdzxmn0c/njdHpNDzrjhs+MhOFV+/Qj7a+tQ1suaQvkU/4q4qmDyvmumovUVzHgHIQOYgYokb+qO1zE7MHi2c5g9VgzuGlUa/Yo+4E66qoZU1uM4HZ7uPq0d1vkIiNtGbw/boptn5E4L6Yr2U03MoUe7dnK4XlHPIYHkBFkypn67EtFju2PtqIHbuIxXjdVYOlU5pD6hmo7CgmlpVY7plga2zDPjlfng20S+hmvuEjFcVjK+reM0xUhkIeHVWRN0F3x97mgLxrdIPjZb7oEmYdeisoq9T2Ae0bBWzGkFCLQXUVm5Zjunt5fOaTEzmGbwIKbmnBjdUw2j0xg44IqALbNuow0Ub3qURYWj+PaquExUmMVk9pVUAk1WZhhsqoRvJGq/diFlbJzsej5IIiRh2Pc97OQYXMMir4MKXDRNxUvJdBao4A6cHXtHrpjq11RYtKkJp86lF6p6qyRsTyJ249zCmRxqeTSK1A4e8eimzwYPL2DJeGJcSLU+meQwYocI1bVED4aOfSJ23Qx5ne3iuEI/3uINLCI8g1x5GTupMWWrMeFVd7hkz8g1fiq4Wb6hTUVT+XWfheznBLYfef1NcAsPYbfNsDsCFA63c/jHsyyl2SyhB11T2SGOmLF3gxO+rWy9a/7s3l8Nn7BkVEHZXKR5wauHU7jyvkBcaPv23ideBGcY+7F9j8mT8D9RMBJUkbqTXjn0hmeHrhTS0ESyioH7Y7sEmtyA1iAHQ/mzkISqFY+XUnCBnSY80JOGF2blZLWz1mMfSxSUBhGDYwnQ5DRFqK4AyKxwHuM68pRmHOe+tD3UVj+7o9hbprQEvRWNhfNTh1GwqvuaK0NBdddKUjnwpMwOMovIE7Emm8MdbcZcC41SMhTbSboT1Zpbj+rieCC9sPJ0ffAzirXxynXqq8HbzUxZZem/SDZ/2sDaSZlWelVarMG5XGpAVV2lKg1TKQLYjv29WB+5BHxTmYyE+mfXBvH3KbmHRcdqtAgr8DMwuGuZbp5W961E8K9dcyEPvkrljdek2wTOVSe1w+be7CQhInite+Te9MRbN3jehiKUHnkGic/+zX+kjLo1YlszifkmtpKEPbOkSjdrNpbsIbvqTLYrA0JvZtinR9v4MgNKsiz29foUGQ1OqKxLyIaQ0rRXhEy5hnEdj26oXlANL3R1kPteDdPjwqNtR2f9BZ11c5KRNH4i2JLOsOD2/IG4MYwrTm25f7tbvLlMMoz2XPOfpcgCf2pgpWgEety6DflUhgEC/6hfuwPQU2Tr6+D1SeqMW4X/7W6yAfOnQj+IsflcNtZxvVtb4F20aHtWI+wROe2Kn61FuxOzq4e659YdH/e9buX0YwZgDyXbwL2q4rZjPzEO31zsjBgPlHdC6vPUV1oZZSNpKgMWg8EyQn43J9Q74rmQa5xo37pWFCMrg/PKtKRcYe/FMMrpA3hYmNs3/EhS95c1+7lNWr+UIh8Dvc+JTyRW9+6weBuhjb2HtLpGHryL1ICPYt22xryTcXO4R95LWg0gh9hNQuijcnzwHaW7iNwYdVyDR6TOWjTrdWy01h60TA57ciGToz23MMf5y+lhUWzsOu+7Eu/T7Nq3lnMJfV+tkdzaVtj27r+ntb+FeWqb49ca75etoLqbKxzUx/sg0hrdrm25Pntcl5QvXP2gb83C+q4f+/+27n6l1t0J1XRGFUSBERmFHD9Royhk/cBeQzYrG16dUck7Qe3V/cQ937p3x25ySzN4d35/+x5VAGi8NMZxM6g4paqbV3vBmoS2lAeteHyffMoTkkEm5Kp6io8Y/BcvLza1ZwzQswS4ZnPW6rEyBAnUiFWeqiLPUwZJJfxqVndVW/3Bv2AqOPtcgAFg9b38hhl2JxJtr7LhyJu6K2dVy9MIGum4VDVDaQ86pp4ivMSKEsj1shPbwe10RaExhGY20+tPiryTQJNvbBtCf0vynrxQVtZyx1tQ935MPXVj9930PqeRLYwY0QVwHf1HzMaxGO4V9/TXGzK1lRjPzYTETBi2NNjYfm4uAczWGdnVc9QezlXouWrmKClPROa57kD1Io+UFpIujtgJtwe2w0FU3ttfy1UXiwoFSYSnXFuuNWLJkDrii5gFM5DrS2suzJY4A8CNJTmzxaftO4s7ofRCwvTXm27wIjXnlEhCWa46UqnQUUoXZ9lsQPgpXSww+4D9WRp5N2v5GTrUQuGtvgaZoZH//fzG5sL6Q+NO9GHnWibybknsaXXajz+MBbFXncZ/7Wx72ocPWYD83qEPq9f0xF2HH6LsNh8LM4TJvZNIsOUY6RjtWjLfqtR6EOGuFErk42r6680J+Uglo5cXJzbbqJRSbZoef0O90Nx6xa+0/A0Au+JtqR/BW65GM8UNI3Cl1TA+VWXCu6kMLUUqFipyZSPa0jxk2aFiBqSYA0BgQMzEO60n2zv4SAvK7t47rqjPBUi2vdLshc7NUV3gbQKVAE1SET+NC6ucxedRlC7oJny2+TJuYa+15txGW6utdV5IIWvWCJvI4GTrCDlbb/aHpyO4wGFp6tuZNzS3LGNTKA3SQT0xW4DAbjVUkx9PrU9XNqpaT+aG1TgmnXZt4jJtkFlGEw8nE13BVMQ0fWWH0Gtn3cRryHIhqVwRbf5mkyiNSd2kpalYMB75d1Kj2gR3oMAZq2u5TfZAl6nRZ7HIMtYdXhvM2ts5drHyAcAEUuhpDT3cdoRzlHZ/F3RJOi60y8uboJzwDsCykYExrkBqdUKKPKEalHUFLSd3QmoHOgbYfQTsqtIOCq+0O77FczUfmQm9bDwXweauxqtzryS0KO9yZisb3Cu9eecZuJ0VnXWzvzprXRmuPVgQOVRDsoK5Ch3k3b0d/H3FE0nncxZ3eOdhtjuyKy6UFhnIyiHyPzas87HRy2n5Z/RCjIkPrmjMV4Nz8tZc8ZIZki2i0AuBbHlwo/91+GJcozEWczOXuyxO0HZSNmJUkELPndJgJsfOsY/JsQZ1XHR2jn3QoWc4LjhroYLHfijiTRhTVx9jR49myFiLg4BLqOX0oPHNwlJua8nYxbMYiwYMzCUwx0ZngpOU8kVhZPXu8vLmfemX7ErZDq7JWJSt9V52pGdHB2ZckvyS3pGGnaz2ABQMZdQ9/h0t+lgyqBv9HWWwo90fi4b61rAjDbvtDm9QkXY8bo5meWsn0i2FgFexLrLOMOz8SvGUICwt4rjImQ36zRincoUhFO++ZtScS9o3DDbCJtdeJATkNi+4hr3c6oiyBxMSMyGZsxR2i7UH8JuXBaPDP+iSIPixOrM5e6PGuMrqEcG8/jEzXxhNotyfeKusDH8i3ujahtTMUhE/QfdOOBQ5NTKakfzqGZ9FsvnqIUgOSWaRO+hHY6TC7Jnc4iPFrrpBTNPU2jh3AK1uAdw3NxMqRevh5AF0XV4QM6AiKXsC8vv99cPVPRGS3F+dX17dnwwJHPiCcYjMB8Phv6LxsnalKwvueG/nO7GUNa9ug2tbLA+g424CKNIZuS0lCu60h1wnzQtrWd1Vew2SBeduxTveY2am3TBikeVUsxlLmV6tudVeKytH6iIVM5pGyazcWCCJ0LWJmNhtT91A+nVovP6J05JLZwyaL30770srgNVzgFyyzGy01aPh7lsbW17BWpf697fkjjFbNgA2B3lkvlQKIyERZhezx1UPR4YcsW5GgyEHkR56HJhNMxTl/sH3VqSndGFfkZZw+MIfadfpw5YOpaPaDX42Ip0uZeQw+mq3yPtQF2X0y3AUhmlddZLC2lhN8NYWG5Pevh737kIjor8fqYwPTCrjb4HUGY2f8IVyFC8pX0DkCjKdxRLscpV9p+xDszvLqYmduqwFhVP7Il9z9gwut1OhO4G5EJt2pl6ylBZyWI811kW9d1IfWbVkju0JeGE8ES9ndp5Bzzmdxedcs5uKCju/vVar6G1+vi0VaV/s71Bt8i9CqV4H03jhKqNp6lv9ryN5juUobG9XW13MT9STqmfzIlziEI2fijySoI1/L3jk6pMNue0/dNS3sPOWORrlDSZqnxZEFXkupGVSLhjXp4yfohMpARcHmQPVhQT0FusXpJXSfq38RCWBaxWhxhrFaa6WQr8aL1ypUFyNNE09eR6XtTO048iCifUsAWykvBMDYhovIVoyHaErejYrzOobkPZ6mfl2KSRXuca9ebLTW1TbAbYlxiIFQy7f3UDfIwQFeh1ud2YsclynO2QR737qKo1N7TUWpp67sxduwmv3X5moSIvIeRy5PWOqz2m0Zy70jiHURQBwB9fx/nIanodL+rUgAovScpFAGa7ZuNEVuc9oi2zGYGRfMr6WfTDL375OXYnCxpdsImO4I2wZz3CSdTmlKcz1SMRJyCjDA3/wYAPDmL6AZjMJsaym2s7PK+3291FCWbry8vmqiXWXB8PNwRqvh/GzUhhjviWefn/oU+L4CfSZYn++Vgomnt1LfbVOrY1POGyduK23FIl5JGb/gVgPv7aCJ2h2hg5sdhWlaSlqfMLYo31uTzhU79wwgcb5Vv1vWc/8hmgfdo8orF8eHu6q7deWqRHoAdnY7fR7J7sTImFBZZKCe3S6ynv24RL7YlCPoYH5n1cPDdxGubzuMd5Fwwa8eTEi3rvHwfGuuYIdBPLl1c3Vw9XQqJd9GRSDYP7l6vxyK33epAtCjakMn6ZNbdgL5ZpsjkNxVkimVzdXkwfyCYWO77yNoRtYKywlkYop50d+fNPMp/ObrMNi7062Zsch1EvQhXwr5Hswx6A/ZWOutvrp0szlaisgdKR4vfeUiBeeCpq8jmSsWCoMuNi227Jtly77xljlguN9vyuMT8lMJD1vz4v8tcn1CKzMnNuFt53WeTPYT3a3nGALnv/wpVmfaUB1++HLF9+jHacjthaFLXe6jdzsiqNV4VtgeLT+lgjZV0jHEfbjmIT9+OWLjcvIIxLm883mDIs4rTTscBtzeNZZDvLU6xyGfsqISCyyHHPPSpXEgtJhTbcuFmhRNXUpFyWW6cGcohmUhnc9P9CR96ebo7IEUporm3HTwxqUFS7kih3uYh0LduAnyte+X7d2y/MgP6ximeLHrFg2ve2uWPaK9XzvbHOZKftziGL3Rg18BYsMlKILCPrX9FfLm36cuo4791QPBUS6GjxBQ4/px6nHRRLbA4I1H6GGuG7R0H2af3S03JWkDFuGsM0rswLsG2+3Bm6nRIucxVugvRUa060wwcW1uhgPcq2PWOJns6ulmwJ76WTbLmEFep7gvdOupNnejmPRhQYgwO5eCmvhidwVLUu14cynYuhqrHXIaLPK5kmzVb3z2xxRkFykLN5K9ftoOL3mzzRlybnWks0KDUNXhj+AqrBrYDnO14SWUDGCzywB5NTWePtCzb59Uvtt+Qvyf6afbm05+VhICbG2qYwZ1WsbBGzk4q1wtuUvw0fb+IKLgJ070n8PiWTPwB/EZfp5VGoRKl6/ZcI5Gx3Ng/YyOw/CkTE+FVjEmn9tPMk96Zh+nH4UXC8fxCXVMM2B68fp5SCg4yWVC9vCwbK7XnvStmmjUpeVC10yekxT4AnFp7J66R7/2Pp0wS7ddQXw+UCX7/NRXb5fDyxS6yqQOX5EdLHTFfYAz2vyXIovLMPS6VVXIguLcMFPbbg5KR0rd8fboZKVE+uEm0BKV8MlX/UsohBQlUng5sY0pnZ5KgkUdZFlGSSMakh7QiIlLVzo6Jkp1vZOhzlq122C3cDIPGWLZU9Mo0R2FFRN9mnJ4Jmm1eFvS30wqjQuUq+vOyHz59VxoZWx1dnKd4AWLtEFp3e+ArGvXzZAVu26zUPLPEn8ZrSGh5DleuWLX4xTFrTBnvO767KBNtUkYXaFW+4S6gnoSUwDXpnbo1/ot07P2/HYfjTss5jpr1NnM2vj1t59sUGaKNWH2ruRkhvmL9dM6SjdiBrM6fcVfQef8Tr3lIZ3a0xl640jNdzYFdjw7Ko1ptgPVdn85SKl8dNSpGM1zyi7wFSnxRXJzCI17hWZ+emJFK16zGtg34p7/P4RQfudAsETugkwLpWR8bpbvsHRjqUUa/BuoRLlRaM6NLUQR9h7K1EiAzy/vdm9Y0LTdIzWTlUPdeNF1UsOGt/GZu5hYJfGMQLoxejbKYyBs97vvRRT+ca1CdJlxPqv2RPgnPHK6CcsA65st2+lRMzQecCryUp52qr6nPODFPU553ur6W93t2/fy3koOId0qoe72QnaKwDROPwZvoc0H7DYsEWdkG8J4wk+7VXk8tPvt3jS/y744+Od/dXFP+/cT8JPr6YP5xc319Nfri7xl98SpqoCbzRNXWI7glkTArXkX1JNN7gv29Pf8PDC/k5GIxxHtkC0yW/ZFVKrjVYI5/8FAAD//1iMAS8=" } diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index ddd36a4c3269..28b6a2bd60ab 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -49,6 +49,7 @@ period: 24h metricsets: - s3_daily_storage + - awshealth - module: aws period: 1m latency: 5m From aae918507c36e1de05fda238b2ec08579bbf2d6e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 17 Apr 2024 09:09:51 +0930 Subject: [PATCH 03/11] x-pack/filebeat/input/{cel,httpjson}: fix oauth2 config validation (#38962) The logic for validation assumed that client.id and client.secret must be present, but this is not the case for password grant, so relax the requirement. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/cel/config_auth.go | 6 +++--- x-pack/filebeat/input/cel/config_test.go | 10 ++++++++++ x-pack/filebeat/input/httpjson/config_auth.go | 6 +++--- x-pack/filebeat/input/httpjson/config_test.go | 10 ++++++++++ 5 files changed, 27 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 4eac716b12c0..bd82b306d17c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488] - [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] - Prevent GCP Pub/Sub input blockage by increasing default value of `max_outstanding_messages` {issue}35029[35029] {pull}38985[38985] +- Fix config validation for CEL and HTTPJSON inputs when using password grant authentication and `client.id` or `client.secret` are not present. {pull}38962[38962] *Heartbeat* diff --git a/x-pack/filebeat/input/cel/config_auth.go b/x-pack/filebeat/input/cel/config_auth.go index ac187f4ffa1e..02998fffaba9 100644 --- a/x-pack/filebeat/input/cel/config_auth.go +++ b/x-pack/filebeat/input/cel/config_auth.go @@ -263,12 +263,12 @@ func (o *oAuth2Config) Validate() error { case oAuth2ProviderOkta: return o.validateOktaProvider() case oAuth2ProviderDefault: - if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == nil { - return errors.New("both token_url and client credentials must be provided") - } if (o.User != "" && o.Password == "") || (o.User == "" && o.Password != "") { return errors.New("both user and password credentials must be provided") } + if o.TokenURL == "" || ((o.ClientID == "" || o.ClientSecret == nil) && (o.User == "" || o.Password == "")) { + return errors.New("both token_url and client credentials must be provided") + } default: return fmt.Errorf("unknown provider %q", o.getProvider()) } diff --git a/x-pack/filebeat/input/cel/config_test.go b/x-pack/filebeat/input/cel/config_test.go index 0a686df099c1..dfc1b82a954a 100644 --- a/x-pack/filebeat/input/cel/config_test.go +++ b/x-pack/filebeat/input/cel/config_test.go @@ -291,6 +291,16 @@ var oAuth2ValidationTests = []struct { }, }, }, + { + name: "if_password_is_set_credentials_may_be_missing_for_user-password_authentication", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "user": "a_client_user", + "password": "a_client_password", + "token_url": "localhost", + }, + }, + }, { name: "must_fail_with_an_unknown_provider", wantErr: errors.New("unknown provider \"unknown\" accessing 'auth.oauth2'"), diff --git a/x-pack/filebeat/input/httpjson/config_auth.go b/x-pack/filebeat/input/httpjson/config_auth.go index b25bab03dd39..f9d3e16300f2 100644 --- a/x-pack/filebeat/input/httpjson/config_auth.go +++ b/x-pack/filebeat/input/httpjson/config_auth.go @@ -227,12 +227,12 @@ func (o *oAuth2Config) Validate() error { case oAuth2ProviderOkta: return o.validateOktaProvider() case oAuth2ProviderDefault: - if o.TokenURL == "" || o.ClientID == "" || o.ClientSecret == nil { - return errors.New("both token_url and client credentials must be provided") - } if (o.User != "" && o.Password == "") || (o.User == "" && o.Password != "") { return errors.New("both user and password credentials must be provided") } + if o.TokenURL == "" || ((o.ClientID == "" || o.ClientSecret == nil) && (o.User == "" || o.Password == "")) { + return errors.New("both token_url and client credentials must be provided") + } default: return fmt.Errorf("unknown provider %q", o.getProvider()) } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 910510b6e9cc..2be99ba68b95 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -222,6 +222,16 @@ func TestConfigOauth2Validation(t *testing.T) { }, }, }, + { + name: "if password is set credentials may be missing for user-password authentication", + input: map[string]interface{}{ + "auth.oauth2": map[string]interface{}{ + "user": "a_client_user", + "password": "a_client_password", + "token_url": "localhost", + }, + }, + }, { name: "must fail with an unknown provider", expectedErr: "unknown provider \"unknown\" accessing 'auth.oauth2'", From 3f06575996bc91402a5c330c18b0e7451c43785b Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 17 Apr 2024 03:56:25 +0200 Subject: [PATCH 04/11] [CI] Jenkins package pipeline for agentbeat (#38951) * [CI] Jenkins package pipeline for auditbeat * Jenkins packaging: install msitools before packaging agentbeat * Install msiutils * Removed packaging-arm for agentbeat * Install msitools for agentbeat PR packaging * Install msitools for agentbeat PR packaging * Fix missing close ) * Fix osquerybeat packaging buildkite * Fix ispackaging * Improved changesets for agentbeat * made install-msitools.sh executable * Fix osquerybeat packaging for buildkite * Cleanup * Install tools without sudo * test: moved packaging before build&tests for check * test: packaging pipeline * added sudo * Cleanup * Cleanup --------- Co-authored-by: Blake Rouse Co-authored-by: Craig MacKenzie --- .buildkite/scripts/install-msitools.sh | 5 ++ .../x-pack/pipeline.xpack.osquerybeat.yml | 1 + .ci/packaging.groovy | 4 + Jenkinsfile | 3 + Jenkinsfile.yml | 1 + x-pack/agentbeat/Jenkinsfile.yml | 39 ++++++++ x-pack/osquerybeat/internal/distro/distro.go | 2 +- x-pack/osquerybeat/internal/msiutil/expand.go | 21 +++++ x-pack/osquerybeat/magefile.go | 89 +------------------ x-pack/osquerybeat/scripts/mage/distro.go | 36 +++++--- 10 files changed, 102 insertions(+), 99 deletions(-) create mode 100755 .buildkite/scripts/install-msitools.sh create mode 100644 x-pack/agentbeat/Jenkinsfile.yml create mode 100644 x-pack/osquerybeat/internal/msiutil/expand.go diff --git a/.buildkite/scripts/install-msitools.sh b/.buildkite/scripts/install-msitools.sh new file mode 100755 index 000000000000..682a7da68f93 --- /dev/null +++ b/.buildkite/scripts/install-msitools.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +sudo apt-get update -y +DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends --yes msitools \ No newline at end of file diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 874f8802263d..8c9137cb423e 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -199,6 +199,7 @@ steps: - label: ":linux: Packaging Linux" key: "packaging-linux" command: | + .buildkite/scripts/install-msitools.sh cd x-pack/osquerybeat mage package agents: diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 3fd9b148330e..7b44fa6a4ca2 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -204,6 +204,7 @@ def generateSteps() { 'metricbeat', 'packetbeat', 'winlogbeat', + 'x-pack/agentbeat' 'x-pack/auditbeat', 'x-pack/dockerlogbeat', 'x-pack/filebeat', @@ -280,6 +281,9 @@ def generateLinuxStep(beat) { withEnv(["HOME=${env.WORKSPACE}", "PLATFORMS=${linuxPlatforms()}", "BEATS_FOLDER=${beat}"]) { withGithubNotify(context: "Packaging Linux ${beat}") { deleteDir() + if (beat.equals('x-pack/agentbeat') || beat.equals('x-pack/osquerybeat')) { + sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') + } release('snapshot') dir("${BASE_DIR}"){ pushCIDockerImages(arch: 'amd64') diff --git a/Jenkinsfile b/Jenkinsfile index 9733b7b74c94..23a4f3798dfe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -608,6 +608,9 @@ def targetWithoutNode(Map args = [:]) { } } withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) { + if (isPackaging && (directory.equals('x-pack/agentbeat') || directory.equals('x-pack/osquerybeat'))) { + sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') + } // make commands use -C while mage commands require the dir(folder) // let's support this scenario with the location variable. dir(isMage ? directory : '') { diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 811b4ae985be..399e0be48d11 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -7,6 +7,7 @@ projects: - "metricbeat" - "packetbeat" - "winlogbeat" + - "x-pack/agentbeat" - "x-pack/auditbeat" - "x-pack/dockerlogbeat" - "x-pack/filebeat" diff --git a/x-pack/agentbeat/Jenkinsfile.yml b/x-pack/agentbeat/Jenkinsfile.yml new file mode 100644 index 000000000000..1686cb776cb2 --- /dev/null +++ b/x-pack/agentbeat/Jenkinsfile.yml @@ -0,0 +1,39 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/agentbeat/.*" + - "^auditbeat/.*" + - "^filebeat/.*" + - "^heartbeat/.*" + - "^metricbeat/.*" + - "^osquerybeat/.*" + - "^packetbeat/.*" + - "@oss" + - "@xpack" + comments: ## when PR comment contains any of those entries + - "/test agentbeat" + labels: ## when PR labels matches any of those entries + - "agentbeat" + parameters: ## when parameter was selected in the UI. + - "agentbeat" + tags: true ## for all the tags +platform: "immutable && ubuntu-22" ## default label for all the stages +stages: + packaging-linux: + packaging-linux: "mage package" + e2e: + enabled: false + stage: packaging + when: + branches: false ## Only on a PR basis for the time being + tags: false ## packaging on branches/tags is already in place with the downstream build. + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/agentbeat/.*" + - "^auditbeat/.*" + - "^filebeat/.*" + - "^heartbeat/.*" + - "^metricbeat/.*" + - "^osquerybeat/.*" + - "^packetbeat/.*" + - "@oss" ## special token regarding the changeset for the oss + - "@xpack" diff --git a/x-pack/osquerybeat/internal/distro/distro.go b/x-pack/osquerybeat/internal/distro/distro.go index f560cdb23be5..87d04d3079a7 100644 --- a/x-pack/osquerybeat/internal/distro/distro.go +++ b/x-pack/osquerybeat/internal/distro/distro.go @@ -178,7 +178,7 @@ var specs = map[OSArch]Spec{ {"linux", "arm64"}: {"_1.linux_aarch64.tar.gz", osqueryDistroLinuxARMSHA256, true}, {"darwin", "amd64"}: {osqueryPkgExt, osqueryDistroDarwinSHA256, true}, {"darwin", "arm64"}: {osqueryPkgExt, osqueryDistroDarwinSHA256, true}, - {"windows", "amd64"}: {osqueryMSIExt, osqueryDistroWindowsSHA256, false}, + {"windows", "amd64"}: {osqueryMSIExt, osqueryDistroWindowsSHA256, true}, } func GetSpec(osarch OSArch) (spec Spec, err error) { diff --git a/x-pack/osquerybeat/internal/msiutil/expand.go b/x-pack/osquerybeat/internal/msiutil/expand.go new file mode 100644 index 000000000000..3af5c24441ab --- /dev/null +++ b/x-pack/osquerybeat/internal/msiutil/expand.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package msiutil + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command" +) + +// Expand runs msiextract to extract the MSI. +func Expand(msiFile, dstDir string) error { + output, err := command.Execute(context.Background(), "msiextract", "--directory", dstDir, msiFile) + if err != nil { + return fmt.Errorf("failed to run msiextract: %w (output: %s)", err, output) + } + return nil +} diff --git a/x-pack/osquerybeat/magefile.go b/x-pack/osquerybeat/magefile.go index 7a21869ffd14..ae5c4c9874be 100644 --- a/x-pack/osquerybeat/magefile.go +++ b/x-pack/osquerybeat/magefile.go @@ -7,21 +7,16 @@ package main import ( - "context" - "errors" "fmt" "os" "path/filepath" "runtime" - "strings" "time" "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" "github.com/elastic/beats/v7/dev-tools/mage/target/build" - "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command" - "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/distro" osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage" // mage:import @@ -92,92 +87,10 @@ func Clean() error { return devtools.Clean(paths) } -func extractFromMSI() error { - if os.Getenv("GOOS") != "windows" { - return nil - } - - ctx := context.Background() - - execCommand := func(name string, args ...string) error { - ps := strings.Join(append([]string{name}, args...), " ") - fmt.Println(ps) - output, err := command.Execute(ctx, name, args...) - if err != nil { - fmt.Println(ps, ", failed: ", err) - return err - } - fmt.Print(output) - return err - } - - osArchs := osquerybeat.OSArchs(devtools.Platforms) - - for _, osarch := range osArchs { - if osarch.OS != "windows" { - continue - } - spec, err := distro.GetSpec(osarch) - if err != nil { - if errors.Is(err, distro.ErrUnsupportedOS) { - continue - } else { - return err - } - } - dip := distro.GetDataInstallDir(osarch) - msiFile := spec.DistroFilepath(dip) - - // MSI extract - err = execCommand("msiextract", "--directory", dip, msiFile) - if err != nil { - return err - } - - fmt.Println("copy certs.pem from MSI") - err = devtools.Copy(filepath.Join(dip, distro.OsquerydCertsWindowsDistroPath()), distro.OsquerydCertsPath(dip)) - if err != nil { - return err - } - - fmt.Println("copy osqueryd.exe from MSI") - dp := distro.OsquerydPathForOS(osarch.OS, dip) - err = devtools.Copy(filepath.Join(dip, "osquery", "osqueryd", "osqueryd.exe"), dp) - if err != nil { - fmt.Println("copy osqueryd.exe from MSI failed: ", err) - return err - } - // Chmod set to the same as other executables in the final package - if err = os.Chmod(dp, 0755); err != nil { - return err - } - } - - return nil -} - // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { - // This is to fix a defect in the field where msiexec fails to extract the osqueryd.exe - // from bundled osquery.msi, with error code 1603 - // https://docs.microsoft.com/en-us/troubleshoot/windows-server/application-management/msi-installation-error-1603 - // SDH: https://github.com/elastic/sdh-beats/issues/1575 - // Currently we can't reproduce this is issue, but here we can eliminate the need for calling msiexec - // if extract the osqueryd.exe binary during the build. - // - // The cross build is currently called for two binaries osquerybeat and osqquery-extension - // Only extract osqueryd.exe during osquerybeat build on windows - args := devtools.DefaultGolangCrossBuildArgs() - - if !strings.HasPrefix(args.Name, "osquery-extension-") { - // Extract osqueryd.exe from MSI - if err := extractFromMSI(); err != nil { - return err - } - } - - return devtools.GolangCrossBuild(args) + return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs()) } // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). diff --git a/x-pack/osquerybeat/scripts/mage/distro.go b/x-pack/osquerybeat/scripts/mage/distro.go index c5fca545ee65..0263eac829da 100644 --- a/x-pack/osquerybeat/scripts/mage/distro.go +++ b/x-pack/osquerybeat/scripts/mage/distro.go @@ -18,6 +18,7 @@ import ( "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fetch" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fileutil" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/hash" + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/msiutil" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/pkgutil" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/tar" ) @@ -139,6 +140,7 @@ func checkCacheAndFetch(osarch distro.OSArch, spec distro.Spec) (fetched bool, e const ( suffixTarGz = ".tar.gz" suffixPkg = ".pkg" + suffixMsi = ".msi" ) func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error { @@ -157,7 +159,7 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error { return devtools.Copy(src, dst) } - if !strings.HasSuffix(src, suffixTarGz) && !strings.HasSuffix(src, suffixPkg) { + if !strings.HasSuffix(src, suffixTarGz) && !strings.HasSuffix(src, suffixPkg) && !strings.HasSuffix(src, suffixMsi) { return fmt.Errorf("unsupported file: %s", src) } tmpdir, err := os.MkdirTemp(distro.DataDir, "") @@ -188,7 +190,6 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error { return err } } - if strings.HasSuffix(src, suffixPkg) { log.Printf("Extract .pkg from %v", src) @@ -204,6 +205,19 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error { return err } } + if strings.HasSuffix(src, suffixMsi) { + log.Printf("Extract .msi from %v", src) + + osdp = filepath.Join("osquery", "osqueryd", "osqueryd.exe") + osdcp = distro.OsquerydCertsWindowsDistroPath() + distp = distro.OsquerydPathForOS(osarch.OS, dir) + + // Msiutil expand full + err = msiutil.Expand(src, tmpdir) + if err != nil { + return err + } + } // Copy over certs directory certsDir := filepath.Dir(distro.OsquerydCertsPath(dir)) @@ -217,14 +231,16 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error { } // Copy over lenses directory - lensesDir := distro.OsquerydLensesDir(dir) - err = os.MkdirAll(lensesDir, 0750) - if err != nil { - return err - } - err = devtools.Copy(filepath.Join(tmpdir, osdlp), lensesDir) - if err != nil { - return err + if osdlp != "" { + lensesDir := distro.OsquerydLensesDir(dir) + err = os.MkdirAll(lensesDir, 0750) + if err != nil { + return err + } + err = devtools.Copy(filepath.Join(tmpdir, osdlp), lensesDir) + if err != nil { + return err + } } // Copy over the osqueryd binary or osquery.app dir From d756f3566362715cbc554cf7a815713c778ef980 Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Tue, 16 Apr 2024 22:08:36 -0400 Subject: [PATCH 05/11] Adding missing comma in packaging.groovy, add agentbeat for arm. (#38998) --- .ci/packaging.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 7b44fa6a4ca2..480fce0df70d 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -204,7 +204,7 @@ def generateSteps() { 'metricbeat', 'packetbeat', 'winlogbeat', - 'x-pack/agentbeat' + 'x-pack/agentbeat', 'x-pack/auditbeat', 'x-pack/dockerlogbeat', 'x-pack/filebeat', @@ -222,6 +222,7 @@ def generateSteps() { 'heartbeat', 'metricbeat', 'packetbeat', + 'x-pack/agentbeat', 'x-pack/auditbeat', 'x-pack/dockerlogbeat', 'x-pack/filebeat', From 5b24b7d28afb35583233b3849128f8b2f1d18abc Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 17 Apr 2024 08:16:21 +0200 Subject: [PATCH 06/11] [CI] xpack/auditbeat monorepo migration (#38828) This commit switched x-pack/auditbeat to a static pipeline using the monorepo plugin. Relates: https://github.com/elastic/ingest-dev/issues/3072 --------- Co-authored-by: Dimitrios Liappis --- .buildkite/pipeline.yml | 27 ++ .buildkite/scripts/common.sh | 19 +- .../generate_xpack_auditbeat_pipeline.sh | 246 ---------------- .../x-pack/pipeline.xpack.auditbeat.yml | 278 +++++++++++++++--- 4 files changed, 262 insertions(+), 308 deletions(-) delete mode 100755 .buildkite/scripts/generate_xpack_auditbeat_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eb8c5e5bd171..7f7c1f107f49 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -258,3 +258,30 @@ steps: env: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger x-pack/auditbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/auditbeat/ + - .buildkite/x-pack/pipeline.xpack.auditbeat.yml + - .buildkite/scripts/ + - .buildkite/hooks/ + # x-pack + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + - x-pack/libbeat/ + config: + trigger: "beats-xpack-auditbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ed529576f101..ba076194bb74 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -18,29 +18,23 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" -[ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" -[ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" [ -z "${run_xpack_filebeat_arm_tests+x}" ] && run_xpack_filebeat_arm_tests="$(buildkite-agent meta-data get run_xpack_filebeat_arm_tests --default "false")" [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_auditbeat_macos_tests+x}" ] && run_xpack_auditbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_macos_tests --default "false")" [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" -# define if needed run Windows platform-specific tests for the particular beat -[ -z "${run_auditbeat_win_tests+x}" ] && run_auditbeat_win_tests="$(buildkite-agent meta-data get run_auditbeat_win_tests --default "false")" - # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" @@ -58,10 +52,6 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) -xpack_auditbeat_changeset=( - "^x-pack/auditbeat/.*" - ) - xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) @@ -127,9 +117,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; - "beats-xpack-auditbeat") - BEAT_CHANGESET_REFERENCE=${xpack_auditbeat_changeset[@]} - ;; "beats-xpack-filebeat") BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]} ;; @@ -384,7 +371,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -395,7 +382,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -571,7 +558,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-auditbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh deleted file mode 100755 index f1b40d992d75..000000000000 --- a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.xpack-auditbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - - label: ":linux: Ubuntu Unit (MODULE) Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - env: - MODULE: $MODULE - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit (MODULE) Tests" - - - label: ":rhel: RHEL9 Unit Tests" - key: "mandatory-rhel9-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - -## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index c0c92cc3ae13..36fcb9bebd99 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -3,10 +3,11 @@ name: "beats-xpack-auditbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" @@ -18,54 +19,239 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - - #Deps + # Other deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "x-pack/auditbeat Mandatory Tests" + key: "x-pack-auditbeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit (MODULE) Tests" + key: "mandatory-linux-unit-test" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet x-pack/auditbeat + echo "~~~ Will run tests with env var MODULE=$$MODULE" + cd x-pack/auditbeat + mage update build test + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Ubuntu Unit (MODULE) Tests" + + - label: ":rhel: RHEL9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: | + cd x-pack/auditbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: RHEL Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/auditbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Windows 2022 Unit Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/auditbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Windows 2016 Unit Tests" + + - group: "x-pack/auditbeat Extended Windows Tests" + key: "x-pack-auditbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/auditbeat + mage build unitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Windows 2019 Extended Tests" - - input: "Input Parameters" - key: "force-run-stages" - fields: - - select: "Auditbeat - run_xpack_auditbeat" - key: "run_xpack_auditbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Auditbeat - run_xpack_auditbeat_macos_tests" - key: "run_xpack_auditbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Auditbeat - run_xpack_auditbeat_arm_tests" - key: "run_xpack_auditbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - if: "build.source == 'ui'" + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/auditbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Windows 10 Extended Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/auditbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Windows 11 Extended Tests" + + - group: "x-pack/auditbeat MacOS Extended Tests" + key: "x-pack-auditbeat-extended-tests-macos" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS x86_64 Unit Tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/auditbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: macOS x86_64 Extended Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/auditbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: macOS arm64 Extended Tests" + + - group: "x-pack/auditbeat Linux arm Extended Tests" + key: "x-pack-auditbeat-extended-tests-linux-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":linux: Ubuntu ARM Unit Tests" + command: | + cd x-pack/auditbeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/auditbeat/build/*.xml" + - "x-pack/auditbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Linux arm64 Extended Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack auditbeat pipeline" - key: "xpack-auditbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - step: "x-pack-auditbeat-mandatory-tests" + + - group: "x-pack/auditbeat Packaging" + key: "x-pack-auditbeat-packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd x-pack/auditbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Packaging Linux Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd x-pack/auditbeat + mage package + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "x-pack/auditbeat: Packaging Linux ARM" From dc826eb5db404d714d1217dd6f8484a5073ff3be Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 17 Apr 2024 09:29:01 +0200 Subject: [PATCH 07/11] Added an empty ironbank target to agentbeat (#39004) --- x-pack/agentbeat/magefile.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/agentbeat/magefile.go b/x-pack/agentbeat/magefile.go index 571f87f74455..874c79bf7a30 100644 --- a/x-pack/agentbeat/magefile.go +++ b/x-pack/agentbeat/magefile.go @@ -134,6 +134,14 @@ func TestPackages() error { return devtools.TestPackages() } +// Package packages the Beat for IronBank distribution. +// +// Use SNAPSHOT=true to build snapshots. +func Ironbank() error { + fmt.Println(">> Ironbank: this module is not subscribed to the IronBank releases.") + return nil +} + // Update is an alias for running fields, dashboards, config. func Update() { callForEachBeat("update") From 5bc43ff59e65d23fc1f712e93b8b08f2f45fbf9e Mon Sep 17 00:00:00 2001 From: ShourieG <105607378+ShourieG@users.noreply.github.com> Date: Wed, 17 Apr 2024 14:13:02 +0530 Subject: [PATCH 08/11] [filebeat][websocket] - Updated Input title to align with existing inputs (#39006) * Updated input title to align with existing inputs --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/docs/inputs/input-websocket.asciidoc | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bd82b306d17c..16af0b1a6526 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -133,6 +133,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917] - Prevent GCP Pub/Sub input blockage by increasing default value of `max_outstanding_messages` {issue}35029[35029] {pull}38985[38985] - Fix config validation for CEL and HTTPJSON inputs when using password grant authentication and `client.id` or `client.secret` are not present. {pull}38962[38962] +- Updated Websocket input title to align with existing inputs {pull}39006[39006] *Heartbeat* diff --git a/x-pack/filebeat/docs/inputs/input-websocket.asciidoc b/x-pack/filebeat/docs/inputs/input-websocket.asciidoc index 8ee2da2b42ad..9e08060a22b5 100644 --- a/x-pack/filebeat/docs/inputs/input-websocket.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-websocket.asciidoc @@ -8,6 +8,10 @@ === Websocket Input experimental[] +++++ +Websocket +++++ + The `websocket` input reads messages from a websocket server or api endpoint. This input uses the `CEL engine` and the `mito` library interally to parse and process the messages. Having support for `CEL` allows you to parse and process the messages in a more flexible way. It has many similarities with the `cel` input as to how the `CEL` programs are written but deviates in the way the messages are read and processed. The `websocket` input is a `streaming` input and can only be used to read messages from a websocket server or api endpoint. This input supports: From 1ade6d21421fb5f750bcc70f7d141ae228132eea Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Wed, 17 Apr 2024 12:51:54 +0200 Subject: [PATCH 09/11] Fix msitools installation for packaging pipeline (#39005) * Test packaging * Moved msitool to earlier install phase * Moved msitool to earlier install phase * fix: run within the withBeatsEnv context * Cleanup * Inline the msitool script * Removed arm docker build step from agentbeat --------- Co-authored-by: Victor Martinez --- .ci/packaging.groovy | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 480fce0df70d..ea8f8f7daefe 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -222,7 +222,6 @@ def generateSteps() { 'heartbeat', 'metricbeat', 'packetbeat', - 'x-pack/agentbeat', 'x-pack/auditbeat', 'x-pack/dockerlogbeat', 'x-pack/filebeat', @@ -282,9 +281,6 @@ def generateLinuxStep(beat) { withEnv(["HOME=${env.WORKSPACE}", "PLATFORMS=${linuxPlatforms()}", "BEATS_FOLDER=${beat}"]) { withGithubNotify(context: "Packaging Linux ${beat}") { deleteDir() - if (beat.equals('x-pack/agentbeat') || beat.equals('x-pack/osquerybeat')) { - sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') - } release('snapshot') dir("${BASE_DIR}"){ pushCIDockerImages(arch: 'amd64') @@ -382,6 +378,16 @@ def release(type){ withEnv([ "DEV=${!type.equals('staging')}" ]) { + dir("${BASE_DIR}"){ + if (env.BEATS_FOLDER.equals('x-pack/agentbeat') || env.BEATS_FOLDER.equals('x-pack/osquerybeat')) { + // sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh') + sh '''#!/usr/bin/env bash + set -euo pipefail + sudo apt-get update -y + DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends --yes msitools + ''' + } + } dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") dir("${env.BEATS_FOLDER}") { sh(label: "mage package ${type} ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package') @@ -451,4 +457,4 @@ def notifyStatus(def args = [:]) { to: "${env.NOTIFY_TO}", subject: subject, body: "Build: (<${env.RUN_DISPLAY_URL}|here>).\n ${body}") -} +} \ No newline at end of file From 4a5d33364442e6d4b3e514d387db03bd8fe76984 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 17:55:03 +0300 Subject: [PATCH 10/11] Remove BK PR Bot trigger for x-pack/heartbeat (#39014) In preparation for the migration to a static pipeline (#38845), this commit removes the Buildkite PR Bot trigger for the pipeline. --- .buildkite/pull-requests.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 5b631c82254a..2bb553ab6cd4 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -95,22 +95,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-heartbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/heartbeat$", - "always_trigger_comment_regex": "^/test x-pack/heartbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } From ab27a657e4f15976c181cf44c529bba6159f2c64 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 17 Apr 2024 13:18:53 -0400 Subject: [PATCH 11/11] chore: Update snapshot.yml (#39017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index b6034789e584..83b403ab2f43 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-918a815f-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.14.0-799c1dc0-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.14.0-918a815f-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.14.0-799c1dc0-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.14.0-918a815f-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.14.0-799c1dc0-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing"