From 436e78d7c9cb66636a12cb00a06a8882a41cc123 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 16 Apr 2024 10:29:47 +0300 Subject: [PATCH] Buildkite pipeline unification fixes (#38871) (#38937) In scope of elastic/ingest-dev#3072: added notifications for pipeline steps removed top level notifications for pipelines added BK hooks script to changeset checks removed BEATS_PROJECT_NAME env var from deploy/k8s and heartbeat (cherry picked from commit fe8ebd00df0a897d6d14e7aba0061c6a61a40ac9) Co-authored-by: Olga Naydyonock --- .../deploy/kubernetes/deploy-k8s-pipeline.yml | 36 +++++----- .buildkite/heartbeat/heartbeat-pipeline.yml | 65 ++++++++----------- .buildkite/pipeline.yml | 7 +- 3 files changed, 50 insertions(+), 58 deletions(-) diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml index 82959ec70675..d8a5354b1ef7 100644 --- a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -4,24 +4,20 @@ env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" -notify: - - github_commit_status: - context: "Deploy/K8S" - steps: - - group: "Depoy/K8S" + - group: "Deploy/K8S" if: build.env("BUILDKITE_PULL_REQUEST") != "false" && build.env("GITHUB_PR_LABELS") =~ /.*kubernetes.*/ steps: - label: "Checks" command: ".buildkite/deploy/kubernetes/scripts/make.sh" - notify: - - github_commit_status: - context: "Deploy/k8s-checks" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Deploy/k8s-checks" - label: "K8S Test/K8S version: v1.29.0" key: "k8s-test-129" @@ -30,13 +26,13 @@ steps: commands: - "MODULE=kubernetes make -C metricbeat integration-tests" - "make -C deploy/kubernetes test" - notify: - - github_commit_status: - context: "Deploy/k8s-test v1.29.0" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Deploy/k8s-test v1.29.0" - label: "K8S Test/K8S version: v1.28.0" key: "k8s-test-128" @@ -45,13 +41,13 @@ steps: commands: - "MODULE=kubernetes make -C metricbeat integration-tests" - "make -C deploy/kubernetes test" - notify: - - github_commit_status: - context: "Deploy/k8s-test v1.28.0" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Deploy/k8s-test v1.28.0" - label: "K8S Test/K8S version: v1.27.3" key: "k8s-test-1273" @@ -60,13 +56,13 @@ steps: commands: - "MODULE=kubernetes make -C metricbeat integration-tests" - "make -C deploy/kubernetes test" - notify: - - github_commit_status: - context: "Deploy/k8s-test v1.27.3" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Deploy/k8s-test v1.27.3" - label: "K8S Test/K8S version: v1.26.6" key: "k8s-test-1266" @@ -75,10 +71,10 @@ steps: commands: - "MODULE=kubernetes make -C metricbeat integration-tests" - "make -C deploy/kubernetes test" - notify: - - github_commit_status: - context: "Deploy/k8s-test v1.26.6" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + notify: + - github_commit_status: + context: "Deploy/k8s-test v1.26.6" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index ee3f1d649cee..0ab91594c253 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,10 +1,6 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json env: - BEATS_PROJECT_NAME: "heartbeat" - - ASDF_MAGE_VERSION: 1.15.0 - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" @@ -22,20 +18,15 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - -notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME" + # Other deps + ASDF_MAGE_VERSION: 1.15.0 steps: - group: "Heartbeat Mandatory Testing" key: "heartbeat-mandatory-tests" steps: - label: ":ubuntu: Unit Tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd heartbeat && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -44,8 +35,8 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":rhel:-9 Unit Tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + - label: ":rhel: Heartbeat Rhel9 Unit Tests" + command: "cd heartbeat && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -54,10 +45,10 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":windows:-2016 Unit Test" + - label: ":windows: Heartbeat Win-2016 Unit Test" key: "windows-2016" command: | - Set-Location -Path $BEATS_PROJECT_NAME + Set-Location -Path heartbeat mage build unitTest agents: provider: "gcp" @@ -68,10 +59,10 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":windows:-2022 Unit Test" + - label: ":windows: Heartbeat Win-2022 Unit Test" key: "windows-2022" command: | - Set-Location -Path $BEATS_PROJECT_NAME + Set-Location -Path heartbeat mage build unitTest agents: provider: "gcp" @@ -82,8 +73,8 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":ubuntu: Go Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" + - label: ":ubuntu: Heartbeat Go Integration Tests" + command: "cd heartbeat && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -92,8 +83,8 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":ubuntu: Python Integration Tests" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + - label: ":ubuntu: Heartbeat Python Integration Tests" + command: "cd heartbeat && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -106,10 +97,10 @@ steps: key: "heartbeat-extended-tests-arm" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ steps: - - label: ":linux: ARM64 Unit Tests" + - label: ":linux: Heartbeat ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd heartbeat && mage build unitTest" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" @@ -120,7 +111,7 @@ steps: key: "heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - - label: ":mac: MacOS Unit Tests" + - label: ":mac: Heartbeat MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" @@ -131,10 +122,10 @@ steps: - group: "Heartbeat Windows Extended Testing" key: "heartbeat-extended-tests-win" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Win 2019 Unit Tests" + - label: ":windows: Heartbeat Win-2019 Unit Tests" key: "heartbeat-win-extended-2019" command: "mage -d ${BEATS_PROJECT_NAME} unitTest" agents: @@ -146,10 +137,10 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":windows:-11 Unit Tests" + - label: ":windows: Heartbeat Win-11 Unit Tests" key: "heartbeat-windows-extended-11" command: | - Set-Location -Path $BEATS_PROJECT_NAME + Set-Location -Path heartbeat mage build unitTest agents: provider: "gcp" @@ -160,10 +151,10 @@ steps: - "heartbeat/build/*.xml" - "heartbeat/build/*.json" - - label: ":windows:-10 Unit Tests" + - label: ":windows: Heartbeat Win-10 Unit Tests" key: "heartbeat-windows-extended-10" command: | - Set-Location -Path $BEATS_PROJECT_NAME + Set-Location -Path heartbeat mage build unitTest agents: provider: "gcp" @@ -180,24 +171,24 @@ steps: depends_on: - "heartbeat-mandatory-tests" steps: - - label: ":ubuntu: Packaging Linux X86" + - label: ":ubuntu: Heartbeat Packaging Linux X86" key: "heartbeat-package-linux-x86" env: - PLATFORMS: "${PACKAGING_PLATFORMS}" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" SNAPSHOT: true - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "cd heartbeat && mage package" agents: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - - label: ":linux: Packaging Linux ARM" + - label: ":linux: Heartbeat Packaging Linux ARM" key: "heartbeat-package-linux-arm" env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PLATFORMS: "linux/arm64" PACKAGES: "docker" SNAPSHOT: true - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "cd heartbeat && mage package" agents: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1b4abe72e1c9..35c6e327523b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -36,6 +36,7 @@ steps: - heartbeat/ - .buildkite/heartbeat/ - .buildkite/scripts + - .buildkite/hooks/ #OSS - go.mod - pytest.ini @@ -87,6 +88,7 @@ steps: - path: - x-pack/dockerlogbeat/ - .buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml + - .buildkite/hooks/ - .buildkite/scripts #OSS - go.mod @@ -113,6 +115,7 @@ steps: - metricbeat/ - .buildkite/metricbeat/ - .buildkite/scripts + - .buildkite/hooks/ #OSS - go.mod - pytest.ini @@ -165,6 +168,7 @@ steps: - x-pack/winlogbeat/ - .buildkite/x-pack/pipeline.xpack.winlogbeat.yml - .buildkite/scripts + - .buildkite/hooks/ #OSS - go.mod - pytest.ini @@ -188,9 +192,10 @@ steps: watch: - path: - .buildkite/deploy/kubernetes/** + - .buildkite/hooks/ + - .buildkite/scripts - deploy/kubernetes/** - libbeat/docs/version.asciidoc - - .buildkite/scripts config: trigger: "deploy-k8s" build: