[8.13](backport #38825) Remove old steps from filebeat-pipeline.yaml (#…

…38935) This commit migrate the filebeat pipeline into the main one Signed-off-by: Alexandros Sapranidis <[email protected]> (cherry picked from commit 0f118b1) Co-authored-by: Alexandros Sapranidis <[email protected]>
elastic · Apr 15, 2024 · b9db63a · b9db63a
1 parent fb0985d
commit b9db63a
Show file tree

Hide file tree

Showing 5 changed files with 296 additions and 554 deletions.
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -4,7 +4,6 @@ name: "beats-filebeat"
 env:
   AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
-  BEATS_PROJECT_NAME: "filebeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
@@ -17,60 +16,276 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
-  #Packaging
-  PACKAGING_ARM_PLATFORMS: "linux/arm64"
-  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
+  # Integration Tests
+  K8S_VERSION: "v1.29.0"
+  ASDF_KIND_VERSION: "0.20.0"
 
 steps:
-  - input: "Input Parameters"
-    key: "run_filebeat"
-    fields:
-      - select: "filebeat - run_filebeat"
-        key: "run_filebeat"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "filebeat - run_filebeat_macos_tests"
-        key: "run_filebeat_macos_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "filebeat - run_filebeat_arm_tests"
-        key: "run_filebeat_arm_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-      - select: "filebeat - run_filebeat_win_tests"
-        key: "run_filebeat_win_tests"
-        options:
-          - label: "True"
-            value: "true"
-          - label: "False"
-            value: "false"
-        default: "false"
-    if: "build.source == 'ui'"
+  - group: "Filebeat Mandatory Tests"
+    key: "filebeat-mandatory-tests"
+    steps:
+      - label: ":ubuntu: Ubuntu Unit Tests"
+        command: |
+          cd filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Ubuntu Unit Tests"
+
+      - label: ":ubuntu: Ubuntu Go Integration Tests"
+        command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet filebeat
+          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          # TODO move this section to base image / pre-command hook
+          echo "~~~ Installing kind"
+          asdf plugin add kind
+          asdf install kind $ASDF_KIND_VERSION
+          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
+
+          export KUBECONFIG="$$PWD/kubecfg"
+          echo "~~~ Running tests"
+          cd filebeat
+          mage goIntegTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Ubuntu Go Integration Tests"
+
+      - label: ":ubuntu: Ubuntu Python Integration Tests"
+        command: |
+          set -euo pipefail
+          # defines the MODULE env var based on what's changed in a PR
+          source .buildkite/scripts/changesets.sh
+          defineModuleFromTheChangeSet filebeat
+          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          # TODO move this section to base image / pre-command hook
+          echo "~~~ Installing kind"
+          asdf plugin add kind
+          asdf install kind $ASDF_KIND_VERSION
+          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
+
+          export KUBECONFIG="$$PWD/kubecfg"
+          echo "~~~ Running tests"
+          cd filebeat
+          mage pythonIntegTest
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Python Integration Tests"
+
+      - label: ":windows: Windows 2016 Unit Tests"
+        key: "windows-2016-unit-tests"
+        command: |
+          Set-Location -Path filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2016}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Windows 2016 Unit Tests"
+
+      - label: ":windows: Windows 2022 Unit Tests"
+        key: "windows-2022-unit-tests"
+        command: |
+          Set-Location -Path filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Windows 2022 Unit Tests"
+
+  - group: "Filebeat Extended Tests"
+    key: "filebeat-extended-tests"
+    steps:
+      - label: ":mac: MacOS x64_64 Unit Tests"
+        key: "macos-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd filebeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended MacOS Unit Tests"
+
+      - label: ":mac: MacOS arm64 Unit Tests"
+        key: "macos-arm64-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd filebeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended MacOS ARM Unit Tests"
+
+      - label: ":linux: Ubuntu ARM Unit Tests"
+        key: "extended-arm64-unit-test"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command: |
+          cd filebeat
+          mage build unitTest
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended Ubuntu ARM Unit Tests"
+
+  - group: "Extended Windows Tests"
+    key: "filebeat-extended-win-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+    steps:
+      - label: ":windows: Windows 2019 Unit Tests"
+        key: "windows-extended-2019"
+        command: |
+          Set-Location -Path filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended Windows 2019 Unit Tests"
+
+      - label: ":windows: Windows 11 Unit Tests"
+        key: "windows-extended-11"
+        command: |
+          Set-Location -Path filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_11}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended Windows 11 Unit Tests"
+
+      - label: ":windows: Windows 10 Unit Tests"
+        key: "windows-extended-10"
+        command: |
+          Set-Location -Path filebeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_10}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "filebeat/build/*.xml"
+          - "filebeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Extended Windows 10 Unit Tests"
 
   - wait: ~
-    if: "build.source == 'ui'"
-    allow_dependency_failure: false
-
-  - label: ":linux: Load dynamic filebeat pipeline"
-    key: "filebeat-pipeline"
-    command: ".buildkite/scripts/generate_filebeat_pipeline.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
-    notify:
-      - github_commit_status:
-          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
+    # with PRs, we want to run packaging only if mandatory tests succeed
+    # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
+    # this allows building DRA artifacts even if there is flakiness in mandatory tests
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "filebeat-mandatory-tests"
+
+  - group: "Filebeat Packaging"
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: |
+          cd filebeat
+          mage package
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Packaging"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: |
+          cd filebeat
+          mage package
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        notify:
+          - github_commit_status:
+              context: "filebeat: Packaging ARM"