From 2250273c77116f409d3d2243dc7a441dc2fb6d4f Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Fri, 19 Apr 2024 10:04:53 +0300 Subject: [PATCH 01/12] Moved winlogbeat pipeline to central pipeline (#38926) * updated common.sh * updated cnetral pipleine with winlogbeat * fixed typo --- .buildkite/pipeline.yml | 26 +++ .buildkite/scripts/common.sh | 1 - .buildkite/winlogbeat/pipeline.winlogbeat.yml | 162 ++++++++++++++---- 3 files changed, 159 insertions(+), 30 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 63acda710aa6..eea60dcca412 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -339,3 +339,29 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Winlogbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - winlogbeat/ + - .buildkite/winlogbeat/pipeline.winlogbeat.yml + - .buildkite/scripts + - .buildkite/hooks + #OSS + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + config: + trigger: "beats-winlogbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index abc30997f08b..faee83486d3b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,7 +14,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" -[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 254370db030b..c71858b45b0a 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,10 +2,10 @@ name: "beats-winlogbeat" env: - BEATS_PROJECT_NAME: "winlogbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -13,37 +13,141 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - - #Deps + # Other deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "Winlogbeat Mandatory Tests" + key: "winlogbeat-mandatory-tests" + + steps: + - label: ":ubuntu: Winlogbeat Crossccompile" + key: "mandatory-cross-compile" + command: "make -C winlogbeat crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Crosscompile" + + - label: ":windows: Winlogbeat Win-2016 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2016 Unit Tests" + + - label: ":windows: Winlogbeat Win-2019 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2019 Unit Tests" - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Winlogbeat - run_winlogbeat" - key: "run_winlogbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" + - label: ":windows: Winlogbeat Win-2022 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2022 Unit Tests" + + - group: "Winlogbeat Extended Windows Tests" + key: "winlogbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + + steps: + - label: ":windows: Winlogbeat Win-10 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-10 Unit Tests" + + - label: ":windows: Winlogbeat Win-11 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-11 Unit Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic winlogbeat pipeline" - key: "winlogbeat-pipeline" - command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "winlogbeat-mandatory-tests" + + - group: "Winlogbeat Packaging" + key: "winlogbeat-packaging" + steps: + - label: ":ubuntu: Winlogbeat Packaging Ubuntu x86_64" + key: "packaging-linux" + command: | + cd winlogbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "Winlogbeat: Packaging Ubuntu x86_64" From 8738b6cde971378e31ca910e266b40ed4c0de6cf Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 19 Apr 2024 17:15:36 +0300 Subject: [PATCH 02/12] Improve conditionals for x-pack/metricbeat BK CI (#39071) PR#38913 had a bug for the extended tests group: it wouldn't allow to run steps in the extended group when not using a PR (e.g. merge commits). This commit allows all steps in the extended group to run when the trigger is not a PR. --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 3b79117e46b2..317b9069c556 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -180,7 +180,7 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|aws).*/ steps: - label: ":mac: MacOS x86_64 Unit Tests" - if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -197,7 +197,7 @@ steps: - label: ":mac: MacOS arm64 Unit Tests" skip: "https://github.com/elastic/beats/issues/33036" - if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.**/ command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -215,6 +215,7 @@ steps: - label: ":linux: Cloud (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test" skip: "doesn't belong in a stage in Jenkins, thus skipped" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ # see link in Jenkins: https://github.com/elastic/beats/blob/ccd7b135df70358f8a02393d9bd8b716428b8048/x-pack/metricbeat/Jenkinsfile.yml#L39 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 command: | @@ -244,10 +245,10 @@ steps: - label: ":linux: Cloud AWS (MODULE) Tests" key: "x-pack-metricbeat-extended-cloud-test-aws" - if: build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ skip: "https://github.com/elastic/beats/issues/36425" # see commented out section in Jenkins: https://github.com/elastic/beats/blob/main/x-pack/metricbeat/Jenkinsfile.yml#L41-L52 # additionally skipping due to https://github.com/elastic/ingest-dev/issues/3170 + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*aws.*/ command: | set -euo pipefail # defines the MODULE env var based on what's changed in a PR From 0d726a60be840c73725c8a3d84695b42099f5e6f Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Fri, 19 Apr 2024 17:33:07 +0300 Subject: [PATCH 03/12] Remove BK PR Bot trigger for packetbeat (#39077) In preparation for the migration to a static pipeline for packetbeat, this commit removes the Buildkite PR Bot trigger. Relates elastic/ingest-dev#3072 --- .buildkite/pull-requests.json | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2bb553ab6cd4..64713cb6e103 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -16,22 +16,6 @@ "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ ] }, - { - "enabled": true, - "pipelineSlug": "beats-packetbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test packetbeat$", - "always_trigger_comment_regex": "^/test packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^packetbeat/.*", ".buildkite/packetbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, { "enabled": true, "pipelineSlug": "beats-xpack-elastic-agent", From b0690b5fb9556e670e86be63d5d40fdf44f78849 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Fri, 19 Apr 2024 18:01:59 +0300 Subject: [PATCH 04/12] migrate x-pack/libbeat to static (#38964) * migrate x-pack/libbeat to static * Update pull-requests.json * change paths * Update pull-requests.json * change conditions for group-steps * change the pipeline.yml * work with remarks * work with remarks * remove libbeat and x-pack/libbeat changesets from the common.sh * remove libbeat and x-pack/libbeat changesets from the common.sh * fix merge issues * Update .buildkite/x-pack/pipeline.xpack.libbeat.yml Co-authored-by: Dimitrios Liappis * apply common.sh from main and cleanup it * change paths --------- Co-authored-by: Dimitrios Liappis --- .buildkite/hooks/pre-command | 1 - .buildkite/pipeline.yml | 29 ++- .buildkite/pull-requests.json | 18 +- .buildkite/scripts/common.sh | 23 +- .../generate_xpack_libbeat_pipeline.sh | 188 ---------------- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 204 ++++++++++++++---- 6 files changed, 196 insertions(+), 267 deletions(-) delete mode 100755 .buildkite/scripts/generate_xpack_libbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8fc0e9142787..cd6cc499c47c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -62,7 +62,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-packetbeat" "beats-winlogbeat" "beats-winlogbeat" - "beats-xpack-libbeat" "beats-xpack-packetbeat" "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index eea60dcca412..69e24a41f213 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -265,7 +265,7 @@ steps: - label: "Trigger Libbeat" plugins: - monorepo-diff#v1.0.1: - diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD" + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" watch: - path: - libbeat/ @@ -284,6 +284,33 @@ steps: branch: "${BUILDKITE_BRANCH}" env: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger x-pack/libbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/libbeat/ + - .buildkite/x-pack/pipeline.xpack.libbeat.yml + - .buildkite/scripts + - .buildkite/hooks + # x-pack + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + config: + trigger: "beats-xpack-libbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - label: "Trigger x-pack/auditbeat" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 64713cb6e103..c98c1f07578d 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -63,22 +63,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-libbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/libbeat$", - "always_trigger_comment_regex": "^/test x-pack/libbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] -} +} \ No newline at end of file diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index faee83486d3b..db2fa2b0558c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,14 +14,14 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" -[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" + # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" -[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" + # define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" @@ -30,10 +30,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" -libbeat_changeset=( - "^libbeat/.*" - ) - packetbeat_changeset=( "^packetbeat/.*" ) @@ -46,10 +42,6 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) -xpack_libbeat_changeset=( - "^x-pack/libbeat/.*" - ) - xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) @@ -75,7 +67,6 @@ oss_changeset=( ) xpack_changeset=( - "${xpack_libbeat_changeset[@]}" "${oss_changeset[@]}" ) @@ -90,18 +81,12 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-libbeat") - BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} - ;; "beats-packetbeat") BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} ;; "beats-winlogbeat") BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} ;; - "beats-xpack-libbeat") - BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -347,7 +332,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -539,4 +524,4 @@ if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_K defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi -check_and_set_beat_vars +check_and_set_beat_vars \ No newline at end of file diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh deleted file mode 100755 index a99564e8613b..000000000000 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.libbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":go: Go Integration Tests" - key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Python Integration Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - -### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage -w reader\etw build goUnitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":linux: Ubuntu ARM64 Unit Tests" - key: "extended-arm64-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index c029951f0107..14316a3ecd70 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -3,13 +3,9 @@ name: "beats-xpack-libbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/libbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" @@ -18,45 +14,171 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "x-pack/libbeat Mandatory Tests" + key: "x-pack-libbeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: | + cd x-pack/libbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Ubuntu Unit Tests" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: | + cd x-pack/libbeat + mage goIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Go Integration Tests" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: | + cd x-pack/libbeat + mage pythonIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Python Integration Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2022 Unit Tests" + + - group: "x-pack/libbeat Extended Windows Tests" + key: "x-pack-libbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/libbeat + mage -w reader\etw build goUnitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Windows 2019 Unit Tests" - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Libbeat - run_xpack_libbeat" - key: "run_xpack_libbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Libbeat - run_xpack_libbeat_arm_tests" - key: "run_xpack_libbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack libbeat pipeline" - key: "libbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - group: "x-pack/libbeat Linux arm Extended Tests" + key: "x-pack-libbeat-extended-tests-linux-arm" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":linux: Ubuntu ARM64 Unit Tests" + key: "extended-arm64-unit-tests" + command: | + cd x-pack/libbeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/libbeat/build/*.xml" + - "x-pack/libbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/libbeat: Ubuntu ARM64 Extended Unit Tests" From 243f9c33d8aab9e5aba7584d7b3eb64cb28bc18c Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Mon, 22 Apr 2024 11:50:23 +0300 Subject: [PATCH 05/12] Buildkite Xpack/Winlogbeat pipeline fixes (#39007) * updated common.sh * fixed typo * updated common.sh * added changeset ps1 script and updated xpack-winlogbeat pipeline * pr fixes * handle backslashes in changeset directories --- .buildkite/scripts/changesets.psm1 | 72 +++++++++++++++++++ .buildkite/scripts/common.sh | 11 +-- .../x-pack/pipeline.xpack.winlogbeat.yml | 38 +++++----- 3 files changed, 94 insertions(+), 27 deletions(-) create mode 100644 .buildkite/scripts/changesets.psm1 diff --git a/.buildkite/scripts/changesets.psm1 b/.buildkite/scripts/changesets.psm1 new file mode 100644 index 000000000000..15fabd3eba5d --- /dev/null +++ b/.buildkite/scripts/changesets.psm1 @@ -0,0 +1,72 @@ +function ArePathsChanged($patterns) { + $changedlist = @() + foreach ($pattern in $patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD | Select-String -Pattern $pattern -SimpleMatch + if ($changedFiles) { + $changedlist += $changedFiles + } + } + if ($changedlist) { + Write-Host "--- Files changed: $changedlist" + return $true + } + else { + Write-Host "--- No files changed within specified changeset: $patterns" + return $false + } +} + +function AreChangedOnlyPaths($patterns) { + $changedFiles = & git diff --name-only "HEAD@{1}" HEAD + Write-Host "--- Git Diff result:" + Write-Host "$changedFiles" + + $matchedFiles = @() + foreach ($pattern in $patterns) { + $matched = $changedFiles | Select-String -Pattern $pattern -SimpleMatch + if ($matched) { + $matchedFiles += $matched + } + } + if (($matchedFiles.Count -eq $changedFiles.Count) -or ($changedFiles.Count -eq 0)) { + return $true + } + return $false +} + +# This function sets a `MODULE` env var, required by IT tests, containing a comma separated list of modules for a given beats project (specified via the first argument). +# The list is built depending on directories that have changed under `modules/` excluding anything else such as asciidoc and png files. +# `MODULE` will empty if no changes apply. +function DefineModuleFromTheChangeSet($projectPath) { + $projectPathTransformed = $projectPath -replace '/', '\\' + $projectPathExclusion = "((?!^$projectPathTransformed\\\/).)*\$" + $exclude = @("^($projectPathExclusion|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + + $changedModules = '' + + $moduleDirs = Get-ChildItem -Directory "$projectPath\module" + foreach($moduleDir in $moduleDirs) { + if((ArePathsChanged($moduleDir)) -and !(AreChangedOnlyPaths($exclude))) { + if(!$changedModules) { + $changedModules = $moduleDir.Name + } + else { + $changedModules += ',' + $moduleDir.Name + } + } + } + + # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved + if(!$changedModules) { + if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') { + $Env:MODULE = "aws" + } + else { + $Env:MODULE = "kubernetes" + } + } + else { + # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export + $Env:MODULE = $changedModules + } +} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index db2fa2b0558c..a925ec913cf6 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -46,10 +46,6 @@ xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) -xpack_winlogbeat_changeset=( - "^x-pack/winlogbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -84,9 +80,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-packetbeat") BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} ;; - "beats-winlogbeat") - BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -519,9 +512,9 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" fi -check_and_set_beat_vars \ No newline at end of file +check_and_set_beat_vars diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index e1e2ec3e045d..c07e537adf09 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -15,13 +15,16 @@ env: ASDF_MAGE_VERSION: 1.15.0 steps: - - group: "Mandatory Tests" - key: "xpack-winlogbeat-mandatory-tests" + - group: "x-pack/Winlogbeat Mandatory Tests" + key: "x-pack-winlogbeat-mandatory-tests" steps: - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" key: "mandatory-win-2019-module-unit-tests" command: | + Import-Module ./.buildkite/scripts/changesets.psm1 + defineModuleFromTheChangeSet 'x-pack/winlogbeat' + Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE" Set-Location -Path x-pack/winlogbeat mage build unitTest env: @@ -37,9 +40,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2019 Unit (MODULE) Tests" + context: "x-pack/Winlogbeat Win-2019 Unit (MODULE) Tests" - - label: ":windows: Xpack/Winlogbeat Win-2016 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2016 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -55,9 +58,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2016 Unit Tests" + context: "x-pack/Winlogbeat Win-2016 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2022 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2022 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -73,14 +76,14 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-2022 Unit Tests" + context: "x-pack/Winlogbeat Win-2022 Unit Tests" - group: "Extended Windows Tests" key: "extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Xpack/Winlogbeat Win-10 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-10 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -96,9 +99,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-10 Unit Tests" + context: "x-pack/Winlogbeat Win-10 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-11 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-11 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -114,9 +117,9 @@ steps: - "x-pack/winlogbeat/build/*.json" notify: - github_commit_status: - context: "Xpack/Winlogbeat Win-11 Unit Tests" + context: "x-pack/Winlogbeat Win-11 Unit Tests" - - label: ":windows: Xpack/Winlogbeat Win-2019 Unit Tests" + - label: ":windows: x-pack/Winlogbeat Win-2019 Unit Tests" command: | Set-Location -Path x-pack/winlogbeat mage build unitTest @@ -140,10 +143,10 @@ steps: # this allows building DRA artifacts even if there is flakiness in mandatory tests if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - "xpack-winlogbeat-mandatory-tests" + - "x-pack-winlogbeat-mandatory-tests" - - group: "Xpack/Winlogbeat Packaging" - key: "xpack-winlogbeat-packaging" + - group: "x-pack/Winlogbeat Packaging" + key: "x-pack-winlogbeat-packaging" steps: - label: ":ubuntu: Packaging Linux" @@ -159,5 +162,4 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: - context: "Xpack/Winlogbeat Packaging" - + context: "x-pack/Winlogbeat Packaging" From 8f8f3137f2ed1a78a9db4d48831d6578831619f8 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Mon, 22 Apr 2024 12:40:42 +0300 Subject: [PATCH 06/12] Migrate packetbeat to static Buildkite pipeline (#39087) This commit refactors the packetbeat pipeline to the pipeline selector approach as laid out in #38783. Relates https://github.com/elastic/ingest-dev/issues/3072 --- .buildkite/hooks/pre-command | 1 - .buildkite/packetbeat/pipeline.packetbeat.yml | 268 +++++++++++++++--- .buildkite/pipeline.yml | 26 ++ .buildkite/scripts/common.sh | 23 +- .../scripts/generate_packetbeat_pipeline.sh | 241 ---------------- 5 files changed, 249 insertions(+), 310 deletions(-) delete mode 100755 .buildkite/scripts/generate_packetbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index cd6cc499c47c..a014542c9c38 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -59,7 +59,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "heartbeat" "filebeat" "beats-metricbeat" - "beats-packetbeat" "beats-winlogbeat" "beats-winlogbeat" "beats-xpack-packetbeat" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 82630978bd3a..c0f5c1e1a735 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -2,12 +2,12 @@ name: "beats-packetbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" - BEATS_PROJECT_NAME: "packetbeat" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" @@ -17,52 +17,228 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 steps: - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Packetbeat - run_packetbeat" - key: "run_packetbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_packetbeat_arm_tests" - key: "run_packetbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_packetbeat_macos_tests" - key: "run_packetbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" + - group: "packetbeat Mandatory Tests" + key: "packetbeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + command: | + cd packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Ubuntu Unit Tests" + + - label: ":rhel: RHEL9 Unit Tests" + command: | + cd packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: RHEL9 Unit Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Windows 2022 Unit Tests" + + - group: "Extended Windows Tests" + key: "packetbeat-extended-windows-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path packetbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path packetbeat + mage build unitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended Windows 2019 Unit Tests" + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS x86_64 Unit Tests" + key: "macos-x86-64-unit-tests-extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended MacOS x86_64 Unit Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + key: "macos-arm64-unit-tests-extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended MacOS arm64 Unit Tests" + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "linux-arm64-unit-tests-extended" + command: "cd packetbeat && mage build unitTest" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "packetbeat/build/*.xml" + - "packetbeat/build/*.json" + notify: + - github_commit_status: + context: "packetbeat: Extended Ubuntu ARM Unit Tests" + - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic packetbeat pipeline" - key: "packetbeat-pipeline" - command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "packetbeat-mandatory-tests" + + - group: "Packetbeat Packaging" + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd packetbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "packetbeat: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd packetbeat + mage package + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "packetbeat: Packaging Linux ARM" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 69e24a41f213..7deb787b6afd 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -392,3 +392,29 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Packetbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - packetbeat/ + - .buildkite/packetbeat/pipeline.packetbeat.yml + - .buildkite/scripts/ + - .buildkite/hooks/ + #OSS + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + config: + trigger: "beats-packetbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a925ec913cf6..2aebe4b5638c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -13,27 +13,20 @@ OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" -[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" -[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" -packetbeat_changeset=( - "^packetbeat/.*" - ) - winlogbeat_changeset=( "^winlogbeat/.*" ) @@ -77,9 +70,6 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "beats-packetbeat") - BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} - ;; "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; @@ -323,20 +313,9 @@ are_conditions_met_mandatory_tests() { return 1 } -are_conditions_met_arm_tests() { - if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then - return 0 - fi - fi - fi - return 1 -} - are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh deleted file mode 100755 index 2a5011e46ac9..000000000000 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ /dev/null @@ -1,241 +0,0 @@ -#!/bin/bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.packetbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":rhel: RHEL9 Unit Tests" - key: "mandatory-rhel9-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - - - group: "Extended Windowds Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - -YAML -fi - - -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName From 8e9a2761d765bb3e106b897bb19bd6624974b723 Mon Sep 17 00:00:00 2001 From: Taylor Swanson <90622908+taylor-swanson@users.noreply.github.com> Date: Mon, 22 Apr 2024 08:32:47 -0500 Subject: [PATCH 07/12] [libbeat] Fix parsing of RFC 3164 process IDs in syslog processor (#38982) - The pattern for parsing process IDs was too relaxed and would match everything between the first opening and the last closing square bracket in a message. If the message included multiple closing square brackets, the process ID would be set to not only the process ID, but also whatever leads up to the last closing square bracket. - The pattern has now been locked down to only digits. - Added test case. --- CHANGELOG.next.asciidoc | 1 + libbeat/reader/syslog/parser/rfc3164.rl | 2 +- libbeat/reader/syslog/rfc3164_gen.go | 60 +++---------------------- libbeat/reader/syslog/rfc3164_test.go | 13 ++++++ 4 files changed, 20 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c9d0f8127736..c28b56a184ee 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -91,6 +91,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Change cache processor documentation from `write_period` to `write_interval`. {pull}38561[38561] - Fix cache processor expiries heap cleanup on partial file writes. {pull}38561[38561] - Fix cache processor expiries infinite growth when large a large TTL is used and recurring keys are cached. {pull}38561[38561] +- Fix parsing of RFC 3164 process IDs in syslog processor. {issue}38947[38947] {pull}38982[38982] *Auditbeat* - Set field types to correctly match ECS in sessionmd processor {issue}38955[38955] {pull}38994[38994] diff --git a/libbeat/reader/syslog/parser/rfc3164.rl b/libbeat/reader/syslog/parser/rfc3164.rl index 0dac77314040..709c049eb06e 100644 --- a/libbeat/reader/syslog/parser/rfc3164.rl +++ b/libbeat/reader/syslog/parser/rfc3164.rl @@ -16,7 +16,7 @@ hostname = graph+ >tok %set_hostname; tag = (print -- [ :\[])+ >tok %set_tag; - content_value = print+ >tok %set_content; + content_value = digit+ >tok %set_content; content = '[' content_value ']'; msg = (tag content? ':' sp)? any+ >tok %set_msg; }%% diff --git a/libbeat/reader/syslog/rfc3164_gen.go b/libbeat/reader/syslog/rfc3164_gen.go index 852ec066f597..b4c4146ba82b 100644 --- a/libbeat/reader/syslog/rfc3164_gen.go +++ b/libbeat/reader/syslog/rfc3164_gen.go @@ -80,10 +80,6 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { goto st_case_29 case 30: goto st_case_30 - case 31: - goto st_case_31 - case 32: - goto st_case_32 case 7: goto st_case_7 case 8: @@ -317,7 +313,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { goto _test_eof28 } st_case_28: - if 32 <= data[p] && data[p] <= 126 { + if 48 <= data[p] && data[p] <= 57 { goto tr37 } goto st24 @@ -334,7 +330,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { if data[p] == 93 { goto tr39 } - if 32 <= data[p] && data[p] <= 126 { + if 48 <= data[p] && data[p] <= 57 { goto st29 } goto st24 @@ -342,56 +338,16 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { m.setContent(data[tok:p]) - goto st30 - tr42: - - m.setContent(data[tok:p]) - - tok = p - goto st30 st30: if p++; p == pe { goto _test_eof30 } st_case_30: - switch data[p] { - case 58: - goto st31 - case 93: - goto tr39 - } - if 32 <= data[p] && data[p] <= 126 { - goto st29 - } - goto st24 - st31: - if p++; p == pe { - goto _test_eof31 - } - st_case_31: - switch data[p] { - case 32: - goto st32 - case 93: - goto tr39 - } - if 33 <= data[p] && data[p] <= 126 { - goto st29 + if data[p] == 58 { + goto st26 } goto st24 - st32: - if p++; p == pe { - goto _test_eof32 - } - st_case_32: - if data[p] == 93 { - goto tr42 - } - if 32 <= data[p] && data[p] <= 126 { - goto tr37 - } - goto tr11 st7: if p++; p == pe { goto _test_eof7 @@ -816,12 +772,6 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { _test_eof30: cs = 30 goto _test_eof - _test_eof31: - cs = 31 - goto _test_eof - _test_eof32: - cs = 32 - goto _test_eof _test_eof7: cs = 7 goto _test_eof @@ -879,7 +829,7 @@ func parseRFC3164(data string, loc *time.Location) (message, error) { } if p == eof { switch cs { - case 24, 25, 26, 27, 28, 29, 30, 31, 32: + case 24, 25, 26, 27, 28, 29, 30: m.setMsg(data[tok:p]) diff --git a/libbeat/reader/syslog/rfc3164_test.go b/libbeat/reader/syslog/rfc3164_test.go index d1c75fe574eb..a2c80e8c263f 100644 --- a/libbeat/reader/syslog/rfc3164_test.go +++ b/libbeat/reader/syslog/rfc3164_test.go @@ -88,6 +88,19 @@ func TestParseRFC3164(t *testing.T) { msg: "message", }, }, + "ok-procid-with-square-brackets-msg": { + in: "<114>Apr 12 13:30:01 aaaaaa001.adm.domain aaaaaa001[25259]: my.some.domain 10.11.12.13 - USERNAME [12/Apr/2024:13:29:59.993 +0200] /skodas \"GET /skodas/group/pod-documentation/aaa HTTP/1.1\" 301 301 290bytes 1 10327", + want: message{ + timestamp: mustParseTime(time.Stamp, "Apr 12 13:30:01", time.Local), + priority: 114, + facility: 14, + severity: 2, + hostname: "aaaaaa001.adm.domain", + process: "aaaaaa001", + pid: "25259", + msg: "my.some.domain 10.11.12.13 - USERNAME [12/Apr/2024:13:29:59.993 +0200] /skodas \"GET /skodas/group/pod-documentation/aaa HTTP/1.1\" 301 301 290bytes 1 10327", + }, + }, "err-pri-not-a-number": { in: "Oct 11 22:14:15 test-host this is the message", want: message{ From b04f48b102f7c3253bed3050e7a469f849ceb969 Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Mon, 22 Apr 2024 16:42:01 +0200 Subject: [PATCH 08/12] Packaging pipeline resource (#39081) * Packaging pipieline resource * Revetred agentbeat pipeline deletion * Cleanup --- .buildkite/packaging.pipeline.yml | 0 catalog-info.yaml | 36 +++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .buildkite/packaging.pipeline.yml diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/catalog-info.yaml b/catalog-info.yaml index 92af68f805f2..c9fe5b4f05c3 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -1016,3 +1016,39 @@ spec: access_level: BUILD_AND_READ everyone: access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: beats-packaging-pipeline + description: Buildkite pipeline for packaging and publishing to DRA + links: + - title: Pipeline + url: https://buildkite.com/elastic/beats-packaging-pipeline +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: beats-packaging-pipeline + description: Pipeline for Beats packaging and publishing DRA artifacts + spec: + repository: elastic/beats + pipeline_file: ".buildkite/packaging.pipeline.yml" + branch_configuration: "main 8.* 7.17" + cancel_intermediate_builds: false + skip_intermediate_builds: false + provider_settings: + trigger_mode: none + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + release-eng: + access_level: BUILD_AND_READ + everyone: + access_level: READ_ONLY From b34334fd0c9f210ffa13e21361098fada26da77c Mon Sep 17 00:00:00 2001 From: Michael Wolf Date: Mon, 22 Apr 2024 10:03:08 -0700 Subject: [PATCH 09/12] Remove ReplaceFields config from add_session_metadata processor (#39134) The ReplaceFields config option was used to support compatibility with session viewer in Kibana that didn't support auditbeat fields. Kibana has now been updated, and this config option isn't needed. --- .../sessionmd/add_session_metadata.go | 45 +------------------ .../sessionmd/add_session_metadata_test.go | 21 +++------ .../auditbeat/processors/sessionmd/config.go | 10 ++--- 3 files changed, 13 insertions(+), 63 deletions(-) diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go index efcfdc96778f..5b934980494d 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -11,7 +11,6 @@ import ( "fmt" "reflect" "strconv" - "time" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" @@ -113,8 +112,8 @@ func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { } func (p *addSessionMetadata) String() string { - return fmt.Sprintf("%v=[backend=%s, pid_field=%s, replace_fields=%t]", - processorName, p.config.Backend, p.config.PIDField, p.config.ReplaceFields) + return fmt.Sprintf("%v=[backend=%s, pid_field=%s]", + processorName, p.config.Backend, p.config.PIDField) } func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { @@ -148,12 +147,6 @@ func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { return nil, fmt.Errorf("merging enriched fields with event: %w", err) } result.Fields["process"] = m - - if p.config.ReplaceFields { - if err := p.replaceFields(result); err != nil { - return nil, fmt.Errorf("replace fields: %w", err) - } - } return result, nil } @@ -184,40 +177,6 @@ func pidToUInt32(value interface{}) (pid uint32, err error) { return pid, nil } -// replaceFields replaces event fields with values suitable user with the session viewer in Kibana -// The current version of session view in Kibana expects different values than what are used by auditbeat -// for some fields. This function converts these field to have values that will work with session view. -// -// This function is temporary, and can be removed when this Kibana issue is completed: https://github.com/elastic/kibana/issues/179396. -func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { - kind, err := ev.Fields.GetValue("event.kind") - if err != nil { - return err - } - isAuditdEvent, err := ev.Fields.HasKey("auditd") - if err != nil { - return err - } - if kind == "event" && isAuditdEvent { - // process start - syscall, err := ev.Fields.GetValue("auditd.data.syscall") - if err != nil { - return nil //nolint:nilerr // processor can be called on unsupported events; not an error - } - switch syscall { - case "execveat", "execve": - ev.Fields.Put("event.action", []string{"exec", "fork"}) - ev.Fields.Put("event.type", []string{"start"}) - - case "exit_group": - ev.Fields.Put("event.action", []string{"end"}) - ev.Fields.Put("event.type", []string{"end"}) - ev.Fields.Put("process.end", time.Now()) - } - } - return nil -} - func tryToMapStr(v interface{}) (mapstr.M, bool) { switch m := v.(type) { case mapstr.M: diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go index 1ef06c92b2ed..602f80f58367 100644 --- a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -32,8 +32,7 @@ var ( { testName: "enrich process", config: config{ - ReplaceFields: false, - PIDField: "process.pid", + PIDField: "process.pid", }, mockProcesses: []types.ProcessExecEvent{ { @@ -94,8 +93,7 @@ var ( { testName: "no PID field in event", config: config{ - ReplaceFields: false, - PIDField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -113,8 +111,7 @@ var ( { testName: "PID not number", config: config{ - ReplaceFields: false, - PIDField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -133,8 +130,7 @@ var ( { testName: "PID not in DB", config: config{ - ReplaceFields: false, - PIDField: "process.pid", + PIDField: "process.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -154,8 +150,7 @@ var ( testName: "process field not in event", // This event, without a "process" field, is not supported by enrich, it should be handled gracefully config: config{ - ReplaceFields: false, - PIDField: "action.pid", + PIDField: "action.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -170,8 +165,7 @@ var ( testName: "process field not mapstr", // Unsupported process field type should be handled gracefully config: config{ - ReplaceFields: false, - PIDField: "action.pid", + PIDField: "action.pid", }, input: beat.Event{ Fields: mapstr.M{ @@ -189,8 +183,7 @@ var ( { testName: "enrich event with map[string]any process field", config: config{ - ReplaceFields: false, - PIDField: "process.pid", + PIDField: "process.pid", }, mockProcesses: []types.ProcessExecEvent{ { diff --git a/x-pack/auditbeat/processors/sessionmd/config.go b/x-pack/auditbeat/processors/sessionmd/config.go index 31c07c9065f1..1aaa354c97d4 100644 --- a/x-pack/auditbeat/processors/sessionmd/config.go +++ b/x-pack/auditbeat/processors/sessionmd/config.go @@ -8,15 +8,13 @@ package sessionmd // Config for add_session_metadata processor. type config struct { - Backend string `config:"backend"` - ReplaceFields bool `config:"replace_fields"` - PIDField string `config:"pid_field"` + Backend string `config:"backend"` + PIDField string `config:"pid_field"` } func defaultConfig() config { return config{ - Backend: "auto", - ReplaceFields: false, - PIDField: "process.pid", + Backend: "auto", + PIDField: "process.pid", } } From 3e373f4478678fe163f82a810f8f1c3956e545ce Mon Sep 17 00:00:00 2001 From: Craig MacKenzie Date: Mon, 22 Apr 2024 16:06:57 -0400 Subject: [PATCH 10/12] Update test versions to 8.13.2 (#39136) --- metricbeat/docker-compose.yml | 12 ++++++------ x-pack/metricbeat/docker-compose.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index ab1ee57979f2..4eec70d9bb0b 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -17,11 +17,11 @@ services: # Used by base tests elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.12.1}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.13.2}-1 build: context: ./module/elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.12.1} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.13.2} environment: - "ES_JAVA_OPTS=-Xms256m -Xmx256m" - "transport.host=127.0.0.1" @@ -38,11 +38,11 @@ services: # Used by base tests kibana: - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.12.1}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.13.2}-1 build: context: ./module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.12.1} + KIBANA_VERSION: ${KIBANA_VERSION:-8.13.2} healthcheck: test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 @@ -53,11 +53,11 @@ services: # Used by base tests metricbeat: - image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.12.1}-1 + image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.13.2}-1 build: context: ./module/beat/_meta args: - BEAT_VERSION: ${BEAT_VERSION:-8.12.1} + BEAT_VERSION: ${BEAT_VERSION:-8.13.2} command: '-e' ports: - 5066:5066 diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index 42b946e42214..0bd47337aa10 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -24,11 +24,11 @@ services: kibana: # Copied configuration from OSS metricbeat because services with depends_on # cannot be extended with extends - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.12.1}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.13.2}-1 build: context: ../../metricbeat/module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.12.1} + KIBANA_VERSION: ${KIBANA_VERSION:-8.13.2} depends_on: - elasticsearch ports: From ddddf8022c8e6ebdb72d314f0c5502baa0318077 Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 22 Apr 2024 17:44:52 -0400 Subject: [PATCH 11/12] Osquerybeat: Strip osqueryd binary for linux (#38952) * Osquerybeat: Strip osqueryd binary for linux * Improve code comment, fix a typo * Address code review request to make it compatible with agentbeat build * Remove unused function --- dev-tools/mage/crossbuild.go | 13 +++++ x-pack/osquerybeat/magefile.go | 93 ++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 94b972ea25e0..ddeabbfdb79b 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -333,6 +333,19 @@ func (b GolangCrossBuilder) Build() error { "--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot), "-v", repoInfo.RootDir+":"+mountPoint, "-w", workDir, + ) + + // Ensure the proper platform is passed + // This fixes an issue where during arm64 linux build for the currently used docker image + // docker.elastic.co/beats-dev/golang-crossbuild:1.21.9-arm the image for amd64 arch is pulled + // and causes problems when using native arch tools on the binaries that are built for arm64 arch. + if strings.HasPrefix(b.Platform, "linux/") { + args = append(args, + "--platform", b.Platform, + ) + } + + args = append(args, image, // Arguments for docker crossbuild entrypoint. For details see diff --git a/x-pack/osquerybeat/magefile.go b/x-pack/osquerybeat/magefile.go index ae5c4c9874be..d8a9dee310bb 100644 --- a/x-pack/osquerybeat/magefile.go +++ b/x-pack/osquerybeat/magefile.go @@ -7,16 +7,23 @@ package main import ( + "context" "fmt" "os" "path/filepath" "runtime" + "strings" "time" "github.com/magefile/mage/mg" devtools "github.com/elastic/beats/v7/dev-tools/mage" "github.com/elastic/beats/v7/dev-tools/mage/target/build" + + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command" + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/distro" + "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fileutil" + osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage" // mage:import @@ -87,9 +94,95 @@ func Clean() error { return devtools.Clean(paths) } +func execCommand(ctx context.Context, name string, args ...string) error { + ps := strings.Join(append([]string{name}, args...), " ") + fmt.Println(ps) + output, err := command.Execute(ctx, name, args...) + if err != nil { + fmt.Println(ps, ", failed: ", err) + return err + } + fmt.Print(output) + return err +} + +// stripLinuxOsqueryd Strips osqueryd binary, that is not stripped in linux tar.gz distro +func stripLinuxOsqueryd() error { + if os.Getenv("GOOS") != "linux" { + return nil + } + + // Check that this step is called during x-pack/osquerybeat/ext/osquery-extension build + cwd, err := os.Getwd() + if err != nil { + return err + } + + // Strip osqueryd only once when osquery-extension is built + // There are two build paths at the moment both through GolangCrossBuild + // 1. Standlone osquerybeat package (this function is called twice: for osquerybeat and osquery-extension) + // 2. Agentbeat package, this function is only called once for osquery-extension + if !strings.HasSuffix(cwd, "/osquery-extension") { + return nil + } + + ctx := context.Background() + + osArchs := osquerybeat.OSArchs(devtools.Platforms) + + strip := func(oquerydPath string) error { + ok, err := fileutil.FileExists(oquerydPath) + if err != nil { + return err + } + if ok { + if err := execCommand(ctx, "strip", oquerydPath); err != nil { + return err + } + } + return nil + } + + for _, osarch := range osArchs { + // Skip everything but matching linux arch + if osarch.OS != os.Getenv("GOOS") || osarch.Arch != os.Getenv("GOARCH") { + continue + } + + // Strip osqueryd + // There are two scenarios where the build path is created depending on the type of build + // 1. Standlone osquerybeat build: the osqueryd binaries are downloaded into osquerybeat/build/data/install/[GOOS]/[GOARCH] + // 2. Agentbeat build: the osqueryd binaries are downloaded agentbeat/build/data/install/[GOOS]/[GOARCH] + + // This returns something like build/data/install/linux/amd64/osqueryd + querydRelativePath := distro.OsquerydPath(distro.GetDataInstallDir(osarch)) + + // Checking and stripping osqueryd binary and both paths osquerybeat/build and agentbeat/build + // because at the moment it's unclear if this step was initiated from osquerybeat or agentbeat build + osquerybeatPath := filepath.Clean(filepath.Join(cwd, "../..", querydRelativePath)) + err = strip(osquerybeatPath) + if err != nil { + return err + } + + agentbeatPath := filepath.Clean(filepath.Join(cwd, "../../../agentbeat", querydRelativePath)) + err = strip(agentbeatPath) + if err != nil { + return err + } + } + + return nil +} + // GolangCrossBuild build the Beat binary inside of the golang-builder. // Do not use directly, use crossBuild instead. func GolangCrossBuild() error { + // Strip linux osqueryd binary + if err := stripLinuxOsqueryd(); err != nil { + return err + } + return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs()) } From 27b4bae916d7cd9fe13b6246424c717901bd3ac1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz <87968844+sharbuz@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:36:12 +0300 Subject: [PATCH 12/12] move x-pack/packetbeat to static pipeline (#39072) * move x-pack/packetbeat to static pipeline * Update pull-requests.json * change pipeline.yml * update pipeline.xpack.packetbeat.yml * fix typo * fix dependency * fix remarks * Update common.sh * Update common.sh * Update common.sh * Update common.sh * fix remarks * fix remarks * fix remarks --- .buildkite/hooks/pre-command | 1 - .buildkite/pipeline.yml | 27 ++ .buildkite/pull-requests.json | 16 - .buildkite/scripts/common.sh | 10 - .../generate_xpack_packetbeat_pipeline.sh | 285 --------------- .../x-pack/pipeline.xpack.packetbeat.yml | 327 +++++++++++++++--- 6 files changed, 307 insertions(+), 359 deletions(-) delete mode 100644 .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index a014542c9c38..7144331fc284 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -61,7 +61,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=( "beats-metricbeat" "beats-winlogbeat" "beats-winlogbeat" - "beats-xpack-packetbeat" "beats-xpack-winlogbeat" "beats-xpack-dockerlogbeat" "beats-xpack-auditbeat" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 7deb787b6afd..00ce04b71d21 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -367,6 +367,33 @@ steps: - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger x-pack/packetbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - x-pack/packetbeat/ + - .buildkite/x-pack/pipeline.xpack.packetbeat.yml + - .buildkite/scripts/ + - .buildkite/hooks/ + # x-pack + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + - x-pack/libbeat/ + config: + trigger: "beats-xpack-packetbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger Winlogbeat" plugins: - monorepo-diff#v1.0.1: diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index c98c1f07578d..b79646acd1bd 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -47,22 +47,6 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^winlogbeat/.*", ".buildkite/winlogbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] - }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-packetbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/packetbeat$", - "always_trigger_comment_regex": "^/test x-pack/packetbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/packetbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] } ] } \ No newline at end of file diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2aebe4b5638c..9bd5cf93540c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,15 +14,12 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" -[ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" -[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" -[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" @@ -35,10 +32,6 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) -xpack_packetbeat_changeset=( - "^x-pack/packetbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) @@ -73,9 +66,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; - "beats-xpack-packetbeat") - BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} - ;; *) echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh deleted file mode 100644 index 97f37ca734bf..000000000000 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.xpack-packetbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":linux: Ubuntu System Tests" - key: "mandatory-linux-system-test" - command: "cd $BEATS_PROJECT_NAME && mage systemTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu System Tests" - - - label: ":rhel: RHEL9 Unit Tests" - key: "mandatory-rhel9-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - - - label: ":windows: Windows 2016 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - - label: ":windows: Windows 2022 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "mandatory-win-2022-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - - - label: ":windows: Windows 2022 System Tests" - skip: "see https://github.com/elastic/beats/issues/38142" - key: "mandatory-win-2022-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-10-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-11-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - - label: ":windows: Windows 2019 Unit Tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - key: "extended-win-2019-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - - - label: ":windows: Windows 10 System Tests" - skip: "see https://github.com/elastic/beats/issues/38142" - key: "extended-win-10-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 5e3ce87e2bf4..77fdf2af8483 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -3,7 +3,6 @@ name: "beats-xpack-packetbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" @@ -18,54 +17,288 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "x-pack/packetbeat Mandatory Tests" + key: "x-pack-packetbeat-mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu Unit Tests" + + - label: ":linux: Ubuntu System Tests" + key: "mandatory-linux-system-test" + command: | + cd x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu System Tests" + + - label: ":rhel: RHEL9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: RHEL9 Unit Tests" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 Unit Tests" + + - label: ":windows: Windows 2022 System Tests" + key: "mandatory-win-2022-system-tests" + skip: "skipping due to elastic/beats#38142" + command: | + Set-Location -Path x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 System Tests" + + - group: "x-pack/packetbeat Extended Windows Tests" + key: "x-pack-packetbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/packetbeat + mage build unitTest + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 System Tests" + key: "extended-win-10-system-tests" + skip: "skipping due to elastic/beats#38142" + command: | + Set-Location -Path x-pack/packetbeat + mage systemTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Windows 2022 System Tests" + + - group: "x-pack/packetbeat Linux arm Extended Tests" + key: "x-pack-packetbeat-extended-linux-arm-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + steps: + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: | + cd x-pack/packetbeat + mage build unitTest + if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Ubuntu ARM Unit Tests" + + - group: "x-pack/packetbeat MacOS Extended Tests" + key: "x-pack-packetbeat-extended-macos-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: MacOS Unit Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/packetbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "x-pack/packetbeat/build/*.xml" + - "x-pack/packetbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/packetbeat: MacOS arm64 Unit Tests" + + - group: "x-pack/packetbeat Packaging" + key: "x-pack-packetbeat-packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - step: "x-pack-packetbeat-mandatory-tests" + allow_failure: false + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd x-pack/packetbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Packaging Linux" - - input: "Input Parameters" - key: "force-run-stages" - fields: - - select: "Packetbeat - run_xpack_packetbeat" - key: "run_xpack_packetbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_xpack_packetbeat_macos_tests" - key: "run_xpack_packetbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Packetbeat - run_xpack_packetbeat_arm_tests" - key: "run_xpack_packetbeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - if: "build.source == 'ui'" - - - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack packetbeat pipeline" - key: "packetbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd x-pack/packetbeat + mage package + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "x-pack/packetbeat: Packaging Linux ARM"