From e28eeaafe7ed69b3df4992404e51c5ee468f4beb Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Fri, 19 Apr 2024 10:04:53 +0300 Subject: [PATCH 1/3] Moved winlogbeat pipeline to central pipeline (#38926) * updated common.sh * updated cnetral pipleine with winlogbeat * fixed typo (cherry picked from commit 2250273c77116f409d3d2243dc7a441dc2fb6d4f) # Conflicts: # .buildkite/winlogbeat/pipeline.winlogbeat.yml --- .buildkite/pipeline.yml | 26 +++ .buildkite/scripts/common.sh | 1 - .buildkite/winlogbeat/pipeline.winlogbeat.yml | 163 ++++++++++++++++-- 3 files changed, 175 insertions(+), 15 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 919347b8750d..1f131194df2b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -340,3 +340,29 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Winlogbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - winlogbeat/ + - .buildkite/winlogbeat/pipeline.winlogbeat.yml + - .buildkite/scripts + - .buildkite/hooks + #OSS + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + config: + trigger: "beats-winlogbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index f81363eadfd1..090ebc225c42 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,7 +14,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" -[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 7c5ee4d16090..f30785d623db 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,6 +2,7 @@ name: "beats-winlogbeat" env: +<<<<<<< HEAD IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_WIN_10: "family/general-windows-10" IMAGE_WIN_11: "family/general-windows-11" @@ -12,29 +13,163 @@ env: GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" BEATS_PROJECT_NAME: "winlogbeat" +======= + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + # Other deps + ASDF_MAGE_VERSION: 1.15.0 +>>>>>>> 2250273c77 (Moved winlogbeat pipeline to central pipeline (#38926)) steps: + - group: "Winlogbeat Mandatory Tests" + key: "winlogbeat-mandatory-tests" + + steps: + - label: ":ubuntu: Winlogbeat Crossccompile" + key: "mandatory-cross-compile" + command: "make -C winlogbeat crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Crosscompile" + + - label: ":windows: Winlogbeat Win-2016 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2016 Unit Tests" + + - label: ":windows: Winlogbeat Win-2019 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2019 Unit Tests" + + - label: ":windows: Winlogbeat Win-2022 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2022 Unit Tests" + + - group: "Winlogbeat Extended Windows Tests" + key: "winlogbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + + steps: + - label: ":windows: Winlogbeat Win-10 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-10 Unit Tests" - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Winlogbeat - run_winlogbeat" - key: "run_winlogbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" + - label: ":windows: Winlogbeat Win-11 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-11 Unit Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "winlogbeat-mandatory-tests" +<<<<<<< HEAD - label: ":linux: Load dynamic winlogbeat pipeline" key: "winlogbeat-pipeline" command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" +======= + - group: "Winlogbeat Packaging" + key: "winlogbeat-packaging" + steps: + - label: ":ubuntu: Winlogbeat Packaging Ubuntu x86_64" + key: "packaging-linux" + command: | + cd winlogbeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "Winlogbeat: Packaging Ubuntu x86_64" +>>>>>>> 2250273c77 (Moved winlogbeat pipeline to central pipeline (#38926)) From 2fcb2ed1744472c630c7a8a983efeb0b770b950b Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Fri, 19 Apr 2024 12:41:11 +0300 Subject: [PATCH 2/3] resolved conflicts --- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 24 +------------------ 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index f30785d623db..ead370d3fdeb 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,18 +2,6 @@ name: "beats-winlogbeat" env: -<<<<<<< HEAD - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "winlogbeat" -======= GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" @@ -27,7 +15,6 @@ env: # Other deps ASDF_MAGE_VERSION: 1.15.0 ->>>>>>> 2250273c77 (Moved winlogbeat pipeline to central pipeline (#38926)) steps: - group: "Winlogbeat Mandatory Tests" @@ -147,14 +134,6 @@ steps: if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "winlogbeat-mandatory-tests" -<<<<<<< HEAD - - label: ":linux: Load dynamic winlogbeat pipeline" - key: "winlogbeat-pipeline" - command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" -======= - group: "Winlogbeat Packaging" key: "winlogbeat-packaging" steps: @@ -168,8 +147,7 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: context: "Winlogbeat: Packaging Ubuntu x86_64" ->>>>>>> 2250273c77 (Moved winlogbeat pipeline to central pipeline (#38926)) From 8326634aa1ad305c0ec380cc65da6db0de6e06dd Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 22 Apr 2024 12:27:34 +0300 Subject: [PATCH 3/3] moved win-2016 tests to extended group --- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index ead370d3fdeb..88e43129c85b 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -35,24 +35,6 @@ steps: - github_commit_status: context: "Winlogbeat: Crosscompile" - - label: ":windows: Winlogbeat Win-2016 Unit Tests" - command: | - Set-Location -Path winlogbeat - mage build unitTest - key: "mandatory-win-2016-unit-tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "winlogbeat/build/*.xml" - - "winlogbeat/build/*.json" - notify: - - github_commit_status: - context: "Winlogbeat: Win-2016 Unit Tests" - - label: ":windows: Winlogbeat Win-2019 Unit Tests" command: | Set-Location -Path winlogbeat @@ -130,6 +112,24 @@ steps: - github_commit_status: context: "Winlogbeat: Win-11 Unit Tests" + - label: ":windows: Winlogbeat Win-2016 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2016 Unit Tests" + - wait: ~ if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: "winlogbeat-mandatory-tests"