From 1ecc48109da24b7da2f068adb1d3e41f2754ef67 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 16 Apr 2024 12:20:19 +0300 Subject: [PATCH 01/14] migrate x-pack/libbeat to static --- .buildkite/hooks/pre-command | 1 - .buildkite/pipeline.yml | 24 ++ .buildkite/pull-requests.json | 251 ++++++++++++------ .buildkite/scripts/common.sh | 2 - .../generate_xpack_libbeat_pipeline.sh | 188 ------------- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 207 ++++++++++++--- 6 files changed, 364 insertions(+), 309 deletions(-) delete mode 100755 .buildkite/scripts/generate_xpack_libbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f3e74714e947..1b7de73ebc3f 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -61,7 +61,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-packetbeat" "beats-winlogbeat" "beats-winlogbeat" - "beats-xpack-libbeat" "beats-xpack-packetbeat" "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 35c6e327523b..5a49cc59caa1 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -205,3 +205,27 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger x-pack/libbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/libbeat/** + - .buildkite/x-pack/pipeline.xpack.libbeat.yml + - .buildkite/scripts + - .buildkite/hooks + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - testing/** + config: + trigger: "beats-xpack-libbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index d787efb5a693..5fc141a4214e 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -4,177 +4,274 @@ "enabled": true, "pipelineSlug": "beats", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ "mergify[bot]" ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "mergify[bot]" + ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/(?:test)", "always_trigger_comment_regex": "^/(?:test)", - "skip_ci_labels": ["skip-ci"], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": [ ] + "skip_ci_labels": [ + "skip-ci" + ], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [] }, { "enabled": true, "pipelineSlug": "beats-libbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test libbeat$|^/test filebeat", "always_trigger_comment_regex": "^/test libbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-packetbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test packetbeat$", "always_trigger_comment_regex": "^/test packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^packetbeat/.*", + ".buildkite/packetbeat/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test elastic-agent$", "always_trigger_comment_regex": "^/test elastic-agent$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], - "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^x-pack/elastic-agent/README.md", + "^x-pack/elastic-agent/docs/.*", + "^x-pack/elastic-agent/devtools/.*" + ], + "always_require_ci_on_changed": [ + "^x-pack/elastic-agent/.*", + ".buildkite/x-pack/elastic-agent/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-winlogbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test winlogbeat$", "always_trigger_comment_regex": "^/test winlogbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^winlogbeat/.*", + ".buildkite/winlogbeat/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-packetbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/packetbeat$", "always_trigger_comment_regex": "^/test x-pack/packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-libbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/libbeat$", - "always_trigger_comment_regex": "^/test x-pack/libbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^x-pack/packetbeat/.*", + "^.buildkite/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*", + "^x-pack/libbeat/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-auditbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/auditbeat$", "always_trigger_comment_regex": "^/test x-pack/auditbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^x-pack/auditbeat/.*", + "^.buildkite/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*", + "^x-pack/libbeat/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-filebeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/filebeat$", "always_trigger_comment_regex": "^/test x-pack/filebeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^x-pack/filebeat/.*", + "^.buildkite/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*", + "^x-pack/libbeat/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-heartbeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/heartbeat$", "always_trigger_comment_regex": "^/test x-pack/heartbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^x-pack/heartbeat/.*", + "^.buildkite/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*", + "^x-pack/libbeat/.*" + ] }, { "enabled": true, "pipelineSlug": "beats-xpack-osquerybeat", "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/osquerybeat$", "always_trigger_comment_regex": "^/test x-pack/osquerybeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [], + "always_require_ci_on_changed": [ + "^x-pack/osquerybeat/.*", + "^.buildkite/.*", + "^go.mod", + "^pytest.ini", + "^dev-tools/.*", + "^libbeat/.*", + "^testing/.*", + "^x-pack/libbeat/.*" + ] } ] -} +} \ No newline at end of file diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d69ae12db457..99fabc352ac2 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -15,7 +15,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" [ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" -[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" @@ -28,7 +27,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" [ -z "${run_xpack_filebeat_arm_tests+x}" ] && run_xpack_filebeat_arm_tests="$(buildkite-agent meta-data get run_xpack_filebeat_arm_tests --default "false")" -[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh deleted file mode 100755 index a99564e8613b..000000000000 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.libbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":go: Go Integration Tests" - key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Python Integration Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - -### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":linux: Ubuntu ARM64 Unit Tests" - key: "extended-arm64-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index c029951f0107..6930579db2ad 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -3,13 +3,9 @@ name: "beats-xpack-libbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/libbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" @@ -18,45 +14,174 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: | + set -euo pipefail + cd x-pack/libbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Ubuntu Unit Tests" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: | + set -euo pipefail + cd x-pack/libbeat + mage goIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Go Integration Tests" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: | + set -euo pipefail + cd x-pack/libbeat + mage pythonIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Python Integration Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2022 Unit Tests" + + ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2019 Unit Tests" - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Libbeat - run_xpack_libbeat" - key: "run_xpack_libbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Libbeat - run_xpack_libbeat_arm_tests" - key: "run_xpack_libbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack libbeat pipeline" - key: "libbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Ubuntu ARM64 Unit Tests" + key: "extended-arm64-unit-tests" + command: | + set -euo pipefail + cd x-pack/libbeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Ubuntu ARM64 Unit Tests" From 1c2d56ad5fcba853ff971f86f4e84678907fac55 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 16 Apr 2024 12:25:38 +0300 Subject: [PATCH 02/14] Update pull-requests.json --- .buildkite/pull-requests.json | 235 +++++++++------------------------- 1 file changed, 61 insertions(+), 174 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 5fc141a4214e..8025682e9fd7 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -4,274 +4,161 @@ "enabled": true, "pipelineSlug": "beats", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [ - "mergify[bot]" - ], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ "mergify[bot]" ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/(?:test)", "always_trigger_comment_regex": "^/(?:test)", - "skip_ci_labels": [ - "skip-ci" - ], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [] + "skip_ci_labels": ["skip-ci"], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ ] }, { "enabled": true, "pipelineSlug": "beats-libbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test libbeat$|^/test filebeat", "always_trigger_comment_regex": "^/test libbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, "pipelineSlug": "beats-packetbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test packetbeat$", "always_trigger_comment_regex": "^/test packetbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^packetbeat/.*", - ".buildkite/packetbeat/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test elastic-agent$", "always_trigger_comment_regex": "^/test elastic-agent$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [ - "^x-pack/elastic-agent/README.md", - "^x-pack/elastic-agent/docs/.*", - "^x-pack/elastic-agent/devtools/.*" - ], - "always_require_ci_on_changed": [ - "^x-pack/elastic-agent/.*", - ".buildkite/x-pack/elastic-agent/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": ["^x-pack/elastic-agent/README.md", "^x-pack/elastic-agent/docs/.*", "^x-pack/elastic-agent/devtools/.*" ], + "always_require_ci_on_changed": ["^x-pack/elastic-agent/.*", ".buildkite/x-pack/elastic-agent/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, "pipelineSlug": "beats-winlogbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test winlogbeat$", "always_trigger_comment_regex": "^/test winlogbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^winlogbeat/.*", - ".buildkite/winlogbeat/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-packetbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/packetbeat$", "always_trigger_comment_regex": "^/test x-pack/packetbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^x-pack/packetbeat/.*", - "^.buildkite/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*", - "^x-pack/libbeat/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-auditbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/auditbeat$", "always_trigger_comment_regex": "^/test x-pack/auditbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^x-pack/auditbeat/.*", - "^.buildkite/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*", - "^x-pack/libbeat/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-filebeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/filebeat$", "always_trigger_comment_regex": "^/test x-pack/filebeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^x-pack/filebeat/.*", - "^.buildkite/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*", - "^x-pack/libbeat/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-heartbeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/heartbeat$", "always_trigger_comment_regex": "^/test x-pack/heartbeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^x-pack/heartbeat/.*", - "^.buildkite/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*", - "^x-pack/libbeat/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, { "enabled": true, "pipelineSlug": "beats-xpack-osquerybeat", "allow_org_users": true, - "allowed_repo_permissions": [ - "admin", - "write" - ], - "allowed_list": [], + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^/test x-pack/osquerybeat$", "always_trigger_comment_regex": "^/test x-pack/osquerybeat$", - "skip_ci_labels": [], - "skip_target_branches": [], - "skip_ci_on_only_changed": [], - "always_require_ci_on_changed": [ - "^x-pack/osquerybeat/.*", - "^.buildkite/.*", - "^go.mod", - "^pytest.ini", - "^dev-tools/.*", - "^libbeat/.*", - "^testing/.*", - "^x-pack/libbeat/.*" - ] + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] -} \ No newline at end of file +} From 0a2a955cd5784c046fdf7d25042260a1796d8601 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 16 Apr 2024 12:32:41 +0300 Subject: [PATCH 03/14] change paths --- .buildkite/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 5a49cc59caa1..187b0c9c5a54 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -212,15 +212,15 @@ steps: diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD" watch: - path: - - x-pack/libbeat/** + - x-pack/libbeat/ - .buildkite/x-pack/pipeline.xpack.libbeat.yml - .buildkite/scripts - .buildkite/hooks #OSS - go.mod - pytest.ini - - dev-tools/** - - testing/** + - dev-tools/ + - testing/ config: trigger: "beats-xpack-libbeat" build: From 98879dfd68004979a6af0a1bf368abdedaf91d57 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 16 Apr 2024 16:47:25 +0300 Subject: [PATCH 04/14] Update pull-requests.json --- .buildkite/pull-requests.json | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index f6d1d912dd0e..b1160a272b4b 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -79,23 +79,7 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-auditbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/auditbeat$", - "always_trigger_comment_regex": "^/test x-pack/auditbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/auditbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, + }, { "enabled": true, "pipelineSlug": "beats-xpack-heartbeat", From b964c7d27e974808410a8ded53c36b393ee066f2 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 16 Apr 2024 17:23:48 +0300 Subject: [PATCH 05/14] change conditions for group-steps --- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 6930579db2ad..e941305e04ce 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -111,6 +111,7 @@ steps: ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" steps: - label: ":windows: Windows 10 Unit Tests" command: | @@ -168,6 +169,7 @@ steps: - group: "Extended Tests" key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" From 6a5b50761ec4863b0802e3a517ab72c676a561fa Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 17 Apr 2024 09:59:51 +0300 Subject: [PATCH 06/14] change the pipeline.yml --- .buildkite/pipeline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7f2285a50ac3..60ffffc0e5bf 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -238,7 +238,7 @@ steps: - label: "Trigger Libbeat" plugins: - monorepo-diff#v1.0.1: - diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD" + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" watch: - path: - libbeat/ @@ -263,7 +263,7 @@ steps: - label: "Trigger x-pack/libbeat" plugins: - monorepo-diff#v1.0.1: - diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD" + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" watch: - path: - x-pack/libbeat/ From 976c177743f7e8c774640bffce2feb31c6e3d0eb Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 11:29:03 +0300 Subject: [PATCH 07/14] work with remarks --- .buildkite/pipeline.yml | 2 ++ .buildkite/x-pack/pipeline.xpack.libbeat.yml | 19 +++++++------------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 60ffffc0e5bf..ffb22683984f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -275,6 +275,8 @@ steps: - pytest.ini - dev-tools/ - testing/ + # x-pack + - libbeat/ config: trigger: "beats-xpack-libbeat" build: diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index e941305e04ce..6cf47dcd23f1 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -18,13 +18,12 @@ env: ASDF_MAGE_VERSION: 1.15.0 steps: - - group: "Mandatory Tests" - key: "mandatory-tests" + - group: "x-pack/libbeat Mandatory Tests" + key: "x-pack-libbeat-mandatory-tests" steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" command: | - set -euo pipefail cd x-pack/libbeat mage build unitTest agents: @@ -41,7 +40,6 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: | - set -euo pipefail cd x-pack/libbeat mage goIntegTest agents: @@ -58,7 +56,6 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: | - set -euo pipefail cd x-pack/libbeat mage pythonIntegTest agents: @@ -108,10 +105,9 @@ steps: - github_commit_status: context: "x-pack/libbeat: Windows 2022 Unit Tests" - ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" + - group: "x-pack/libbeat Extended Windows Tests" + key: "x-pack-libbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - label: ":windows: Windows 10 Unit Tests" command: | @@ -167,14 +163,13 @@ steps: - github_commit_status: context: "x-pack/libbeat: Windows 2019 Unit Tests" - - group: "Extended Tests" - key: "extended-tests" + - group: "x-pack/libbeat Linux arm Extended Tests" + key: "x-pack-libbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" command: | - set -euo pipefail cd x-pack/libbeat mage build unitTest agents: From ad232549d2d1532ededf63ebb85df584f35fd268 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 11:47:13 +0300 Subject: [PATCH 08/14] work with remarks --- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 6cf47dcd23f1..575abdfa1dca 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -164,7 +164,7 @@ steps: context: "x-pack/libbeat: Windows 2019 Unit Tests" - group: "x-pack/libbeat Linux arm Extended Tests" - key: "x-pack-libbeat-extended-tests" + key: "x-pack-libbeat-extended-tests-linux-arm" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - label: ":linux: Ubuntu ARM64 Unit Tests" From 90e465fe826a2c5735f1270671220ae066712588 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 12:39:51 +0300 Subject: [PATCH 09/14] remove libbeat and x-pack/libbeat changesets from the common.sh --- .buildkite/scripts/common.sh | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index fd343f3d54b5..5cc4f1640a0f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -44,10 +44,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" -libbeat_changeset=( - "^libbeat/.*" - ) - packetbeat_changeset=( "^packetbeat/.*" ) @@ -72,10 +68,6 @@ xpack_filebeat_changeset=( "^x-pack/filebeat/.*" ) -xpack_libbeat_changeset=( - "^x-pack/libbeat/.*" - ) - xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) @@ -101,7 +93,6 @@ oss_changeset=( ) xpack_changeset=( - "${xpack_libbeat_changeset[@]}" "${oss_changeset[@]}" ) @@ -116,8 +107,6 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-libbeat") - BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} ;; "beats-packetbeat") BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} @@ -134,9 +123,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-heartbeat") BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]} ;; - "beats-xpack-libbeat") - BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -382,7 +368,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi From cd81faa4705b758cf7ca9052d9da5e44fa04f9c4 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 12:47:20 +0300 Subject: [PATCH 10/14] remove libbeat and x-pack/libbeat changesets from the common.sh --- .buildkite/scripts/common.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 622001b72d67..08dae1d3f1a4 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -50,10 +50,6 @@ xpack_filebeat_changeset=( "^x-pack/filebeat/.*" ) -xpack_libbeat_changeset=( - "^x-pack/libbeat/.*" - ) - xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) From 08a89693eb3e34d329d7dba335757edb5f85ff07 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 13:42:55 +0300 Subject: [PATCH 11/14] fix merge issues --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 08dae1d3f1a4..f00b735a741e 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -340,7 +340,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi From ae6fb756d2f3dd2dc5580ecc4564307292d78933 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:43:31 +0300 Subject: [PATCH 12/14] Update .buildkite/x-pack/pipeline.xpack.libbeat.yml Co-authored-by: Dimitrios Liappis --- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 575abdfa1dca..14316a3ecd70 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -181,4 +181,4 @@ steps: - "x-pack/libbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/libbeat: Ubuntu ARM64 Unit Tests" + context: "x-pack/libbeat: Ubuntu ARM64 Extended Unit Tests" From ca82305401f00f54b30c3e2185b74ad42d89047b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 13:49:03 +0300 Subject: [PATCH 13/14] apply common.sh from main and cleanup it --- .buildkite/scripts/common.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index f00b735a741e..db2fa2b0558c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -42,14 +42,6 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) -xpack_heartbeat_changeset=( - "^x-pack/heartbeat/.*" - ) - -xpack_filebeat_changeset=( - "^x-pack/filebeat/.*" - ) - xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) @@ -532,4 +524,4 @@ if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_K defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi -check_and_set_beat_vars +check_and_set_beat_vars \ No newline at end of file From ff1e84e44d8bd914b11db878e7542d7c3ecba748 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 19 Apr 2024 16:04:49 +0300 Subject: [PATCH 14/14] change paths --- .buildkite/pipeline.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index b972384ad3d7..69e24a41f213 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -297,13 +297,12 @@ steps: - .buildkite/x-pack/pipeline.xpack.libbeat.yml - .buildkite/scripts - .buildkite/hooks - #OSS + # x-pack - go.mod - pytest.ini - dev-tools/ - - testing/ - # x-pack - libbeat/ + - testing/ config: trigger: "beats-xpack-libbeat" build: