Add Integration tests to Auditbeat CI pipeline (#37892) (#41451)

Refactor the CI integration test steps for the Auditbeat pipeline to use build matrices for Linux tests. By doing this, it will be easier to add additional VM types to the integration tests in the future.

With this, the integration tests will run on  Ubuntu 20.04, 22.04, 24.04 for both x86_64 and arm64, and RHEL 9 x86_64.

(cherry picked from commit efbc4ff)

Co-authored-by: Michael Wolf <[email protected]>
Co-authored-by: Andrew Kroh <[email protected]>

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -3,7 +3,6 @@ name: "beats-auditbeat"
 
 env:
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
-  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
@@ -12,7 +11,12 @@ env:
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9"
-  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  IMAGE_UBUNTU_2004_X86_64: "family/platform-ingest-beats-ubuntu-2004"
+  IMAGE_UBUNTU_2004_ARM64: "platform-ingest-beats-ubuntu-2004-aarch64"
+  IMAGE_UBUNTU_2204_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  IMAGE_UBUNTU_2204_ARM64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_UBUNTU_2404_X86_64: "family/platform-ingest-beats-ubuntu-2404"
+  IMAGE_UBUNTU_2404_ARM64: "platform-ingest-beats-ubuntu-2404-aarch64"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
   IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
@@ -86,7 +90,7 @@ steps:
             - limit: 1
         agents:
           provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          image: "${IMAGE_UBUNTU_2204_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
@@ -162,7 +166,7 @@ steps:
             - limit: 1
         agents:
           provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          image: "${IMAGE_UBUNTU_2204_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
@@ -173,7 +177,7 @@ steps:
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" ||  build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/
 
     steps:
-      - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests"
+      - label: ":ubuntu: Auditbeat: Ubuntu x86_64 Integration Tests -- {{matrix.image}}"
         key: "auditbeat-extended-integ-tests"
         if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
         command: |
@@ -185,16 +189,22 @@ steps:
             - limit: 1
         agents:
           provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
+          image: "{{matrix.image}}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Ubuntu x86_64 Integration Tests"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_UBUNTU_2004_X86_64}"
+              - "${IMAGE_UBUNTU_2204_X86_64}"
+              - "${IMAGE_UBUNTU_2404_X86_64}"
 
-      - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests"
+      - label: ":ubuntu: Auditbeat: Ubuntu arm64 Integration Tests -- {{matrix.image}}"
         key: "auditbeat-extended-arm64-integ-tests"
         if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/
         command: |
@@ -206,14 +216,20 @@ steps:
             - limit: 1
         agents:
           provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "{{matrix.image}}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
         notify:
           - github_commit_status:
               context: "auditbeat: Ubuntu arm64 Integration Tests"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_UBUNTU_2004_ARM64}"
+              - "${IMAGE_UBUNTU_2204_ARM64}"
+              - "${IMAGE_UBUNTU_2404_ARM64}"
 
       - label: ":ubuntu: Auditbeat: Ubuntu arm64 Unit Tests"
         key: "auditbeat-extended-arm64-unit-tests"
@@ -227,7 +243,7 @@ steps:
             - limit: 1
         agents:
           provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${IMAGE_UBUNTU_2204_ARM64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
@@ -375,7 +391,7 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: gcp
-          image: "${IMAGE_UBUNTU_X86_64}"
+          image: "${IMAGE_UBUNTU_2204_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         notify:
           - github_commit_status:
@@ -396,7 +412,7 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${IMAGE_UBUNTU_2204_ARM64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         notify:
           - github_commit_status: