From 14aef3f09b261f6bfa488faa599c1bc81c3ee30c Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 15 Apr 2024 10:23:25 +0300 Subject: [PATCH 1/3] updated common.sh --- .buildkite/scripts/common.sh | 5 - .buildkite/winlogbeat/pipeline.winlogbeat.yml | 154 +++++++++++++++--- 2 files changed, 129 insertions(+), 30 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index abc30997f08b..c665dc72b280 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -14,7 +14,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat [ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" -[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" @@ -55,10 +54,6 @@ xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) -xpack_winlogbeat_changeset=( - "^x-pack/winlogbeat/.*" - ) - ci_changeset=( "^.buildkite/.*" ) diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 254370db030b..c9e0ab4b21fd 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,7 +2,6 @@ name: "beats-winlogbeat" env: - BEATS_PROJECT_NAME: "winlogbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" @@ -17,33 +16,138 @@ env: PACKAGING_ARM_PLATFORMS: "linux/arm64" PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps + # Other deps ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "Winlogbeat Mandatory Tests" + key: "winlogbeat-mandatory-tests" - - input: "Input Parameters" - key: "input-run-all-stages" - fields: - - select: "Winlogbeat - run_winlogbeat" - key: "run_winlogbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" + steps: + - label: ":ubuntu: Crossccompile" + key: "mandatory-cross-compile" + command: "make -C winlogbeat crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Crosscompile" + + - label: ":windows: Winlogbeat Win-2016 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2016-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2016 Unit Tests" + + - label: ":windows: Winlogbeat Win-2019 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2019 Unit Tests" + + - label: ":windows: Winlogbeat Win-2022 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "mandatory-win-2022-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-2022 Unit Tests" + + - group: "Extended Windows Tests" + key: "winlogbeat-extended-win-tests" + + steps: + - label: ":windows: Winlogbeat Wind-10 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-10 Unit Tests" + + - label: ":windows: Winlogbeat Win-11 Unit Tests" + command: | + Set-Location -Path winlogbeat + mage build unitTest + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "winlogbeat/build/*.xml" + - "winlogbeat/build/*.json" + notify: + - github_commit_status: + context: "Winlogbeat: Win-11 Unit Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic winlogbeat pipeline" - key: "winlogbeat-pipeline" - command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + depends_on: + - step: "mandatory-tests" + + - group: "Winlogbeat Packaging" + key: "winlogbeat-packaging" + steps: + - label: ":ubuntu: Winlogbeat Packaging Ubuntu x86_64" + key: "packaging-linux" + command: "cd winlogbeat && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "Winlogbeat: Packaging Ubuntu x86_64" From 04806b39837145c8ab2100468c83a1bb36537399 Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Mon, 15 Apr 2024 12:05:59 +0300 Subject: [PATCH 2/3] updated cnetral pipleine with winlogbeat --- .buildkite/pipeline.yml | 26 +++++++++++++++++++ .buildkite/winlogbeat/pipeline.winlogbeat.yml | 11 +++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 63acda710aa6..01f62733cb1d 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -339,3 +339,29 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger Winlogbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - winlogbeat/ + - .buildkite/winlogbeat/pipeline.winlogbeat.yml + - .buildkite/scripts + - .buildkite/hooks + #OSS + - go.mod + - pytest.ini + - dev-tools/** + - libbeat/** + - testing/** + config: + trigger: "beats-winlogbeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index c9e0ab4b21fd..b8064a2a82b7 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -5,6 +5,7 @@ env: GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -12,10 +13,6 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - # Other deps ASDF_MAGE_VERSION: 1.15.0 @@ -24,7 +21,7 @@ steps: key: "winlogbeat-mandatory-tests" steps: - - label: ":ubuntu: Crossccompile" + - label: ":ubuntu: Winlogbeat Crossccompile" key: "mandatory-cross-compile" command: "make -C winlogbeat crosscompile" agents: @@ -134,7 +131,7 @@ steps: - wait: ~ depends_on: - - step: "mandatory-tests" + - step: "winlogbeat-mandatory-tests" - group: "Winlogbeat Packaging" key: "winlogbeat-packaging" @@ -147,7 +144,7 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "${PACKAGING_PLATFORMS}" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: context: "Winlogbeat: Packaging Ubuntu x86_64" From f7d1dc7b2e683d28432e8a4aa3b79d43521b886d Mon Sep 17 00:00:00 2001 From: Olga Naidjonoka Date: Wed, 17 Apr 2024 11:59:20 +0300 Subject: [PATCH 3/3] fixed typo --- .buildkite/pipeline.yml | 6 +++--- .buildkite/scripts/common.sh | 4 ++++ .buildkite/winlogbeat/pipeline.winlogbeat.yml | 13 ++++++++----- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 01f62733cb1d..eea60dcca412 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -353,9 +353,9 @@ steps: #OSS - go.mod - pytest.ini - - dev-tools/** - - libbeat/** - - testing/** + - dev-tools/ + - libbeat/ + - testing/ config: trigger: "beats-winlogbeat" build: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index c665dc72b280..faee83486d3b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -54,6 +54,10 @@ xpack_packetbeat_changeset=( "^x-pack/packetbeat/.*" ) +xpack_winlogbeat_changeset=( + "^x-pack/winlogbeat/.*" + ) + ci_changeset=( "^.buildkite/.*" ) diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index b8064a2a82b7..c71858b45b0a 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -89,11 +89,12 @@ steps: - github_commit_status: context: "Winlogbeat: Win-2022 Unit Tests" - - group: "Extended Windows Tests" + - group: "Winlogbeat Extended Windows Tests" key: "winlogbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Winlogbeat Wind-10 Unit Tests" + - label: ":windows: Winlogbeat Win-10 Unit Tests" command: | Set-Location -Path winlogbeat mage build unitTest @@ -130,15 +131,17 @@ steps: context: "Winlogbeat: Win-11 Unit Tests" - wait: ~ - depends_on: - - step: "winlogbeat-mandatory-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "winlogbeat-mandatory-tests" - group: "Winlogbeat Packaging" key: "winlogbeat-packaging" steps: - label: ":ubuntu: Winlogbeat Packaging Ubuntu x86_64" key: "packaging-linux" - command: "cd winlogbeat && mage package" + command: | + cd winlogbeat + mage package agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}"