Merge branch 'main' into add_cloud_metadata_env

elastic · Apr 16, 2024 · 2925254 · 2925254
2 parents 067f6f5 + 11bc06c
commit 2925254
Show file tree

Hide file tree

Showing 470 changed files with 33,416 additions and 581,759 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -2,11 +2,8 @@
 name: "beats-auditbeat"
 
 env:
-  BEATS_PROJECT_NAME: "auditbeat"
-
-  ASDF_MAGE_VERSION: 1.15.0
-
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
 
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
@@ -23,52 +20,47 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
-  #Packaging
-  PACKAGING_ARM_PLATFORMS: "linux/arm64"
-  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-
-notify:
-  - github_commit_status:
-      context: "$BEATS_PROJECT_NAME"
+  # Other deps
+  ASDF_MAGE_VERSION: 1.15.0
 
 steps:
   - group: "Auditbeat Mandatory Testing"
     key: "auditbeat-mandatory-tests"
 
     steps:
       - label: ":ubuntu: Auditbeat Unit Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
-        artifact_paths:
-          - "auditbeat/build/*.xml"
-          - "auditbeat/build/*.json"
-
-      - label: ":ubuntu: Auditbeat Unit Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        command: |
+          cd auditbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Ubuntu Unit Tests"
 
       - label: ":rhel: Auditbeat Unit Tests"
-        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        command: |
+          cd auditbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: RHEL Unit Tests"
 
       - label: ":windows: Auditbeat Win-2016 Unit Tests"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path auditbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -79,10 +71,13 @@ steps:
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Windows 2016 Unit Tests"
 
       - label: ":windows: Auditbeat Win-2022 Unit Tests"
         command: |
-          Set-Location -Path $BEATS_PROJECT_NAME
+          Set-Location -Path auditbeat
           mage build unitTest
         agents:
           provider: "gcp"
@@ -93,109 +88,164 @@ steps:
         artifact_paths:
           - "auditbeat/build/*.xml"
           - "auditbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Windows 2022 Unit Tests"
 
       - label: ":linux: Auditbeat Crosscompile"
-        command: "make -C $BEATS_PROJECT_NAME crosscompile"
+        command: |
+          make -C auditbeat crosscompile
         env:
           GOX_FLAGS: "-arch amd64"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Cross compile"
 
-  - group: "Auditbeat ARM Tests"
-    key: "auditbeat-extended-tests-arm"
+  - group: "Auditbeat Linux arm64 Tests"
+    key: "auditbeat-extended-tests-linux-arm64"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
     steps:
-      - label: ":arm: ARM64 Unit Tests"
+      - label: ":linux: Auditbeat Linux arm64 Unit Tests"
         key: "auditbeat-extended-arm64-unit-tests"
-        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+        command: |
+          set -euo pipefail
+          cd auditbeat
+          mage unitTest
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "metricbeat: Linux arm64 tests"
 
   - group: "Auditbeat MacOS Extended"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
     steps:
-      - label: ":mac: MacOS Unit Tests"
-        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+      - label: ":mac: Auditbeat macOS x86_64 Unit Tests"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd auditbeat
+          mage unitTest
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Extended MacOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS ARM Unit Tests"
-        command: "cd ${BEATS_PROJECT_NAME} && mage unitTest"
+      - label: ":mac: Auditbeat macOS arm64 Unit Tests"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd auditbeat
+          mage unitTest
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_ARM}"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Extended MacOS arm64 Unit Tests"
 
   - group: "Auditbeat Windows Extended Testing"
     key: "auditbeat-extended-tests-win"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
     steps:
       - label: ":windows: Auditbeat Win-2019 Unit Tests"
         key: "auditbeat-extended-win-2019-unit-tests"
-        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        command: |
+          Set-Location -Path auditbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Extended Windows 2019 Unit Tests"
 
       - label: ":windows: Auditbeat Win-10 Unit Tests"
         key: "auditbeat-extended-win-10-unit-tests"
-        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        command: |
+          Set-Location -Path auditbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_10}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Extended Windows 10 Unit Tests"
 
       - label: ":windows: Auditbeat Win-11 Unit Tests"
         key: "auditbeat-extended-win-11-unit-tests"
-        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        command: |
+          Set-Location -Path auditbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_11}"
           machine_type: "${GCP_WIN_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+        artifact_paths: "auditbeat/build/*.*"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Extended Windows 11 Unit Tests"
+
+  - wait: ~
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "auditbeat-mandatory-tests"
 
   - group: "Auditbeat Packaging"
     key: "auditbeat-packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "auditbeat-mandatory-tests"
-
     steps:
-      - label: ":ubuntu: Auditbeat/Packaging Linux X86"
+      - label: ":ubuntu: Auditbeat/Packaging Linux"
         key: "auditbeat-package-linux-x86"
         env:
-          PLATFORMS: "${PACKAGING_PLATFORMS}"
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-        command: "cd $BEATS_PROJECT_NAME && mage package"
+        command: |
+          cd auditbeat
+          mage package
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Packaging Linux"
 
-      - label: ":linux: Auditbeat/Packaging Linux ARM"
-        key: "auditbeat-package-linux-arm"
+      - label: ":linux: Auditbeat/Packaging Linux arm64"
+        key: "auditbeat-package-linux-arm64"
         env:
-          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-        command: "cd $BEATS_PROJECT_NAME && mage package"
+        command: |
+          cd auditbeat
+          mage package
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "auditbeat: Packaging Linux arm64"
diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -4,24 +4,20 @@ env:
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
 
-notify:
-  - github_commit_status:
-      context: "Deploy/K8S"
-
 steps:
-  - group: "Depoy/K8S"
+  - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/
 
     steps:
     - label: "Checks"
       command: ".buildkite/deploy/kubernetes/scripts/make.sh"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-checks"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-checks"
 
     - label: "K8S Test/K8S version: v1.29.0"
       key: "k8s-test-129"
@@ -30,13 +26,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.29.0"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.29.0"
 
     - label: "K8S Test/K8S version: v1.28.0"
       key: "k8s-test-128"
@@ -45,13 +41,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.28.0"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.28.0"
 
     - label: "K8S Test/K8S version: v1.27.3"
       key: "k8s-test-1273"
@@ -60,13 +56,13 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.27.3"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.27.3"
 
     - label: "K8S Test/K8S version: v1.26.6"
       key: "k8s-test-1266"
@@ -75,10 +71,10 @@ steps:
       commands:
         - "MODULE=kubernetes make -C metricbeat integration-tests"
         - "make -C deploy/kubernetes test"
-      notify:
-        - github_commit_status:
-            context: "Deploy/k8s-test v1.26.6"
       agents:
         provider: "gcp"
         image: "${IMAGE_UBUNTU_X86_64}"
         machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      notify:
+        - github_commit_status:
+            context: "Deploy/k8s-test v1.26.6"