Merge branch 'main' into azure-eventhub-dependencies

elastic · Apr 16, 2024 · d01d9e4 · d01d9e4
2 parents c401221 + 3f1d32b
commit d01d9e4
Show file tree

Hide file tree

Showing 473 changed files with 32,985 additions and 581,281 deletions.
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -43,19 +43,6 @@ steps:
 
       - label: ":ubuntu: Ubuntu Go Integration Tests"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet filebeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
-          # TODO move this section to base image / pre-command hook
-          echo "~~~ Installing kind"
-          asdf plugin add kind
-          asdf install kind $ASDF_KIND_VERSION
-          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
-
-          export KUBECONFIG="$$PWD/kubecfg"
-          echo "~~~ Running tests"
           cd filebeat
           mage goIntegTest
         agents:
@@ -71,19 +58,6 @@ steps:
 
       - label: ":ubuntu: Ubuntu Python Integration Tests"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet filebeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
-          # TODO move this section to base image / pre-command hook
-          echo "~~~ Installing kind"
-          asdf plugin add kind
-          asdf install kind $ASDF_KIND_VERSION
-          .buildkite/deploy/kubernetes/scripts/kind-setup.sh
-
-          export KUBECONFIG="$$PWD/kubecfg"
-          echo "~~~ Running tests"
           cd filebeat
           mage pythonIntegTest
         agents:

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -57,7 +57,7 @@ fi
 ENABLED_BEATS_PIPELINES_SLUGS=(
   "auditbeat"
   "filebeat"
-  "beats-libbeat"
+  "beats-metricbeat"
   "beats-packetbeat"
   "beats-winlogbeat"
   "beats-winlogbeat"
@@ -67,8 +67,8 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "beats-xpack-dockerlogbeat"
   "beats-xpack-auditbeat"
   "beats-xpack-filebeat"
+  "beats-xpack-metricbeat"
   "beats-xpack-heartbeat"
-  "beats-xpack-osquerybeat"
   "deploy-k8s"
 )
 

diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -4,50 +4,117 @@ name: "beats-libbeat"
 env:
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
   AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
-  BEATS_PROJECT_NAME: "libbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
-  #Packaging
-  PACKAGING_ARM_PLATFORMS: "linux/arm64"
-  PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
 steps:
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          mage build unitTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths:
+          - "libbeat/build/*.xml"
+          - "libbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Ununtu Unit Tests"
+
+      - label: ":go: Go Integration Tests"
+        key: "mandatory-int-test"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          mage goIntegTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "libbeat/build/*.xml"
+          - "libbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Go Integration Tests"
+
+      - label: ":python: Python Integration Tests"
+        key: "mandatory-python-int-test"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          mage pythonIntegTest
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "libbeat/build/*.xml"
+          - "libbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Python Integration Tests"
+
+      - label: ":negative_squared_cross_mark: Cross compile"
+        key: "mandatory-cross-compile"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          make crosscompile
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - "libbeat/build/*.xml"
+          - "libbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Cross compile"
+
+      - label: ":testengine: Stress Tests"
+        key: "mandatory-stress-test"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "libbeat/libbeat-stress-test.xml"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Stress Tests"
 
-  - input: "Input Parameters"
-    key: "input-run-all-stages"
-    fields:
-    - select: "Libbeat - run_libbeat"
-      key: "run_libbeat"
-      options:
-        - label: "True"
-          value: "true"
-        - label: "False"
-          value: "false"
-      default: "false"
-    - select: "Libbeat - run_libbeat_arm_tests"
-      key: "run_libbeat_arm_tests"
-      options:
-        - label: "True"
-          value: "true"
-        - label: "False"
-          value: "false"
-      default: "false"
-    if: "build.source == 'ui'"
-
-  - wait: ~
-    if: "build.source == 'ui'"
-    allow_dependency_failure: false
-
-  - label: ":linux: Load dynamic Libbeat pipeline"
-    key: "libbeat-pipeline"
-    command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
-    notify:
-      - github_commit_status:
-          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
+  - group: "Extended Tests"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+    steps:
+      - label: ":linux: Ubuntu ARM64 Unit Tests"
+        key: "extended-arm64-unit-tests"
+        command: |
+          set -euo pipefail
+          cd libbeat
+          mage build unitTest
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - "libbeat/build/*.xml"
+          - "libbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "libbeat: Ubuntu ARM64 Unit Tests"
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -159,6 +159,35 @@ steps:
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
+  - label: "Trigger x-pack/osquerybeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/osquerybeat/**
+                - .buildkite/x-pack/pipeline.xpack.metricbeat.yml
+                - .buildkite/scripts/**
+                - .buildkite/hooks/**
+                # x-pack
+                - libbeat/**
+                - x-pack/libbeat/**
+                #OSS
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/**
+                - testing/**
+              config:
+                trigger: "beats-xpack-osquerybeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
   - label: "Trigger Xpack/Winlogbeat"
     plugins:
       - monorepo-diff#v1.0.1:
@@ -205,3 +234,27 @@ steps:
                     - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - label: "Trigger Libbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - libbeat/
+                - .buildkite/libbeat/pipeline.libbeat.yml
+                - .buildkite/scripts
+                - .buildkite/hooks
+                #OSS
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - testing/
+              config:
+                trigger: "beats-libbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json
@@ -16,22 +16,6 @@
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": [ ]
         },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-libbeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test libbeat$|^/test filebeat",
-            "always_trigger_comment_regex": "^/test libbeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
-        },
         {
             "enabled": true,
             "pipelineSlug": "beats-packetbeat",
@@ -112,38 +96,6 @@
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
         },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-xpack-auditbeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test x-pack/auditbeat$",
-            "always_trigger_comment_regex": "^/test x-pack/auditbeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
-        },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-xpack-filebeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test x-pack/filebeat$",
-            "always_trigger_comment_regex": "^/test x-pack/filebeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
-        },
         {
             "enabled": true,
             "pipelineSlug": "beats-xpack-heartbeat",
@@ -159,22 +111,6 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
-        },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-xpack-osquerybeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test x-pack/osquerybeat$",
-            "always_trigger_comment_regex": "^/test x-pack/osquerybeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
         }
     ]
 }