From edba3bc80a74f77b928c35db37446174e490c895 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Thu, 8 Feb 2024 19:59:25 +0200 Subject: [PATCH] Heartbeat pipeline migration (#37911) * added heartbeat scripts and updated pipeline * made scripts executable * removed gcp agent for skipped packaging step * added agent name from matrix to GH notifications * testing dynamic step * fixed typo * updated command * added agent * testing annotate step * code cleanup * restored dynamic step * updated GH notifictions * updated GH notifications * added comments --- .buildkite/auditbeat/auditbeat-pipeline.yml | 14 +- .buildkite/auditbeat/scripts/package-step.sh | 7 +- .buildkite/env-scripts/linux-env.sh | 2 - .buildkite/env-scripts/util.sh | 1 - .buildkite/env-scripts/win-env.sh | 1 - .buildkite/filebeat/filebeat-pipeline.yml | 6 +- .buildkite/filebeat/scripts/package-step.sh | 7 +- .buildkite/heartbeat/heartbeat-pipeline.yml | 141 +++++++++++++++++- .../heartbeat/scripts/integration-gotests.sh | 15 ++ .../heartbeat/scripts/integration-pytests.sh | 15 ++ .buildkite/heartbeat/scripts/package-step.sh | 49 ++++++ .buildkite/heartbeat/scripts/package.sh | 13 ++ .../heartbeat/scripts/unit-tests-win.ps1 | 51 +++++++ .buildkite/heartbeat/scripts/unit-tests.sh | 15 ++ .buildkite/hooks/pre-command | 2 +- .buildkite/pull-requests.json | 4 +- 16 files changed, 320 insertions(+), 23 deletions(-) create mode 100755 .buildkite/heartbeat/scripts/integration-gotests.sh create mode 100755 .buildkite/heartbeat/scripts/integration-pytests.sh create mode 100755 .buildkite/heartbeat/scripts/package-step.sh create mode 100755 .buildkite/heartbeat/scripts/package.sh create mode 100644 .buildkite/heartbeat/scripts/unit-tests-win.ps1 create mode 100755 .buildkite/heartbeat/scripts/unit-tests.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 147ca45ced16..62c0d86e259a 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -20,7 +20,7 @@ steps: - ".buildkite/auditbeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "auditbeat: Unit Tests" + context: "Auditbeat: linux/Unit Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -33,7 +33,7 @@ steps: - ".buildkite/auditbeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "auditbeat: Unit Tests" + context: "Auditbeat: rhel/Unit Tests" agents: provider: "gcp" image: "${IMAGE_RHEL9}" @@ -45,7 +45,7 @@ steps: command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" notify: - github_commit_status: - context: "auditbeat: Unit Tests" + context: "Auditbeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" @@ -68,7 +68,7 @@ steps: GOX_FLAGS: "-arch amd64" notify: - github_commit_status: - context: "auditbeat: Unit Tests" + context: "Auditbeat: Crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -85,7 +85,7 @@ steps: - ".buildkite/auditbeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "auditbeat/Extended: Unit Tests ARM" + context: "Auditbeat/Extended: Unit Tests ARM" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -99,7 +99,7 @@ steps: - ".buildkite/auditbeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "auditbeat/Extended: MacOS Unit Tests" + context: "Auditbeat/Extended: MacOS Unit Tests" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -115,7 +115,7 @@ steps: command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1" notify: - github_commit_status: - context: "auditbeat/Extended: Win-2019 Unit Tests" + context: "Auditbeat/Extended: Win-2019 Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" diff --git a/.buildkite/auditbeat/scripts/package-step.sh b/.buildkite/auditbeat/scripts/package-step.sh index 021240589923..6c113f4e3cae 100755 --- a/.buildkite/auditbeat/scripts/package-step.sh +++ b/.buildkite/auditbeat/scripts/package-step.sh @@ -13,7 +13,7 @@ changeset="^auditbeat/ ^\.buildkite/auditbeat/" if are_files_changed "$changeset"; then - cat <<-EOF + cat <<-YAML steps: - label: ":ubuntu: Packaging Linux X86" key: "package-linux-x86" @@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.large" -EOF +YAML +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 fi diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh index 5e6e5f7cbf05..1365aaace4a9 100644 --- a/.buildkite/env-scripts/linux-env.sh +++ b/.buildkite/env-scripts/linux-env.sh @@ -9,8 +9,6 @@ DEBIAN_FRONTEND="noninteractive" sudo mkdir -p /etc/needrestart echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null -echo "--- PLATFORM TYPE $PLATFORM_TYPE" - if [[ $PLATFORM_TYPE == "Linux" ]]; then # Remove this code once beats specific agent is set up if grep -q 'Ubuntu' /etc/*release; then diff --git a/.buildkite/env-scripts/util.sh b/.buildkite/env-scripts/util.sh index 7aef69cff389..6a5c36bcd047 100644 --- a/.buildkite/env-scripts/util.sh +++ b/.buildkite/env-scripts/util.sh @@ -85,7 +85,6 @@ are_files_changed() { if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then return 0; else - echo "WARN! No files changed in $changeset" return 1; fi } diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh index ccf5479b46e1..931051d550af 100644 --- a/.buildkite/env-scripts/win-env.sh +++ b/.buildkite/env-scripts/win-env.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash -echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}" if [[ ${PLATFORM_TYPE} = MINGW* ]]; then echo "--- Installing Python on Win" choco install mingw -y diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index eda9fb93a669..f86ad7da11d4 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -19,7 +19,7 @@ steps: - ".buildkite/filebeat/scripts/unit-tests.sh" notify: - github_commit_status: - context: "Filebeat: Unit Tests" + context: "Filebeat: linux/Unit Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -33,7 +33,7 @@ steps: - ".buildkite/filebeat/scripts/integration-gotests.sh" notify: - github_commit_status: - context: "Filebeat: Integration Tests" + context: "Filebeat: Go Integration Tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -60,7 +60,7 @@ steps: command: ".buildkite/filebeat/scripts/unit-tests-win.ps1" notify: - github_commit_status: - context: "Filebeat: Unit Tests" + context: "Filebeat: windows/Unit Tests" agents: provider: "gcp" image: "{{matrix.image}}" diff --git a/.buildkite/filebeat/scripts/package-step.sh b/.buildkite/filebeat/scripts/package-step.sh index 985125433cec..c2a1bf41a01d 100755 --- a/.buildkite/filebeat/scripts/package-step.sh +++ b/.buildkite/filebeat/scripts/package-step.sh @@ -13,7 +13,7 @@ changeset="^filebeat/ ^\.buildkite/filebeat/" if are_files_changed "$changeset"; then - cat <<-EOF + cat <<-YAML steps: - label: ":ubuntu: Packaging Linux X86" key: "package-linux-x86" @@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "t4g.large" -EOF +YAML +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 fi diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index 34321b61161b..93cbfc32008f 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -1,5 +1,142 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_RHEL9: "family/core-rhel-9" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + steps: - - label: "Example test" - command: echo "Hello!" + - group: "Heartbeat Mandatory Testing" + key: "mandatory-tests" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false" + + steps: + - label: ":linux: Unit Tests / {{matrix.image}}" + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat: linux/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + matrix: + setup: + image: + - "${IMAGE_UBUNTU_X86_64}" + - "${IMAGE_RHEL9}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":windows: Unit Tests / {{matrix.image}}" + command: + - ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Heartbeat: windows/Unit Tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":ubuntu: Go Integration Tests" + command: + - ".buildkite/heartbeat/scripts/integration-gotests.sh" + notify: + - github_commit_status: + context: "Heartbeat: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - label: ":ubuntu: Python Integration Tests" + command: + - ".buildkite/heartbeat/scripts/integration-pytests.sh" + notify: + - github_commit_status: + context: "Heartbeat: Python Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - group: "Extended Testing" + key: "extended-tests" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support" + + steps: + - label: ":linux: ARM64 Unit Tests" + key: "arm-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat/Extended: Unit Tests ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" + artifact_paths: "heartbeat/build/*.xml" + + - label: ":mac: MacOS Unit Tests" + key: "macos-extended" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: + - ".buildkite/heartbeat/scripts/unit-tests.sh" + notify: + - github_commit_status: + context: "Heartbeat/Extended: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "heartbeat/build/*.xml" + + - group: "Windows Extended Testing" + key: "extended-tests-win" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "win-extended-2019" + command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1" + notify: + - github_commit_status: + context: "Heartbeat/Extended: Win-2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_type: "pd-ssd" + artifact_paths: + - "heartbeat/build/*.xml" + - "heartbeat/build/*.json" + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: + - "mandatory-tests" + + steps: + - label: Package pipeline + commands: ".buildkite/heartbeat/scripts/package-step.sh | buildkite-agent pipeline upload" diff --git a/.buildkite/heartbeat/scripts/integration-gotests.sh b/.buildkite/heartbeat/scripts/integration-gotests.sh new file mode 100755 index 000000000000..8eab0e8b5d88 --- /dev/null +++ b/.buildkite/heartbeat/scripts/integration-gotests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage goIntegTest diff --git a/.buildkite/heartbeat/scripts/integration-pytests.sh b/.buildkite/heartbeat/scripts/integration-pytests.sh new file mode 100755 index 000000000000..729df5ae6f61 --- /dev/null +++ b/.buildkite/heartbeat/scripts/integration-pytests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Executing Integration Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage pythonIntegTest diff --git a/.buildkite/heartbeat/scripts/package-step.sh b/.buildkite/heartbeat/scripts/package-step.sh new file mode 100755 index 000000000000..05ef69b131ad --- /dev/null +++ b/.buildkite/heartbeat/scripts/package-step.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/util.sh + +changeset="^heartbeat/ +^go.mod +^pytest.ini +^dev-tools/ +^libbeat/ +^testing/ +^\.buildkite/heartbeat/" + +if are_files_changed "$changeset"; then + cat <<-YAML + steps: + - label: ":ubuntu: Packaging Linux X86" + key: "package-linux-x86" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + command: + - ".buildkite/heartbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "heartbeat/Packaging: Linux X86" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + + - label: ":linux: Packaging Linux ARM" + key: "package-linux-arm" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + command: + - ".buildkite/heartbeat/scripts/package.sh" + notify: + - github_commit_status: + context: "heartbeat/Packaging: ARM" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.large" +YAML +else + buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning' + exit 0 +fi diff --git a/.buildkite/heartbeat/scripts/package.sh b/.buildkite/heartbeat/scripts/package.sh new file mode 100755 index 000000000000..7f51a6b5ca19 --- /dev/null +++ b/.buildkite/heartbeat/scripts/package.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/env-scripts/linux-env.sh + +echo "--- Docker Version: $(docker --version)" + +echo "--- Start Packaging" +cd heartbeat +umask 0022 +mage package + diff --git a/.buildkite/heartbeat/scripts/unit-tests-win.ps1 b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 new file mode 100644 index 000000000000..17282813e13c --- /dev/null +++ b/.buildkite/heartbeat/scripts/unit-tests-win.ps1 @@ -0,0 +1,51 @@ +$ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually + +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF() { + Write-Host "--- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} + +function withGolang() { + Write-Host "--- Install golang $GoVersion --" + choco install golang -y --version $GoVersion + + $choco = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$choco\helpers\chocolateyProfile.psm1" + refreshenv + go version + go env +} + +function installGoDependencies() { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "github.com/tebeka/go2xunit" + ) + foreach ($pkg in $installPackages) { + go install "$pkg" + } +} + +fixCRLF + +$ErrorActionPreference = "Continue" # set +e + +Set-Location -Path heartbeat +New-Item -ItemType Directory -Force -Path "build" +withGolang +installGoDependencies + +mage build unitTest + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE diff --git a/.buildkite/heartbeat/scripts/unit-tests.sh b/.buildkite/heartbeat/scripts/unit-tests.sh new file mode 100755 index 000000000000..4b746da2d57c --- /dev/null +++ b/.buildkite/heartbeat/scripts/unit-tests.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Remove when custom image is set up +source .buildkite/env-scripts/linux-env.sh + +echo "--- Running Unit Tests" +# Remove when custom image is set up +sudo chmod -R go-w heartbeat/ + +cd heartbeat +# Remove when custom image is set up +umask 0022 +mage build unitTest diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 21e4f04eb13f..cf66e4edf6e0 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,7 +2,7 @@ set -euo pipefail -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh source .buildkite/env-scripts/win-env.sh diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 66c508e252c9..5d1310f101ed 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -73,8 +73,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^/test heartbeat$", - "always_trigger_comment_regex": "^/test heartbeat$", + "trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", + "always_trigger_comment_regex": "^/test heartbeat(for (arm|macos|windows|extended support))?$|^/package heartbeat$", "skip_ci_labels": [ ], "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ],