From e54f671d9ba29b400d8db15408eef08eb5fddd8f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 1 Feb 2024 18:12:31 +0200 Subject: [PATCH 01/34] migrate packetbeat pipeline --- .buildkite/hooks/post-checkout | 53 ++++ .buildkite/hooks/pre-command | 5 + .buildkite/packetbeat/pipeline.packetbeat.yml | 42 ++- .buildkite/scripts/common.sh | 242 ++++++++++++++++++ .../scripts/generate_packetbeat_pipeline.sh | 135 ++++++++++ .buildkite/scripts/install_tools.sh | 48 ++++ .buildkite/scripts/packaging.sh | 12 + .buildkite/scripts/setenv.sh | 15 ++ .buildkite/scripts/unit_tests.sh | 17 ++ .buildkite/scripts/win_unit_tests.ps1 | 70 +++++ 10 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 .buildkite/hooks/post-checkout create mode 100644 .buildkite/hooks/pre-command create mode 100755 .buildkite/scripts/common.sh create mode 100755 .buildkite/scripts/generate_packetbeat_pipeline.sh create mode 100755 .buildkite/scripts/install_tools.sh create mode 100755 .buildkite/scripts/packaging.sh create mode 100755 .buildkite/scripts/setenv.sh create mode 100755 .buildkite/scripts/unit_tests.sh create mode 100644 .buildkite/scripts/win_unit_tests.ps1 diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout new file mode 100644 index 000000000000..e10f15de7b65 --- /dev/null +++ b/.buildkite/hooks/post-checkout @@ -0,0 +1,53 @@ +#!/bin/bash + +set -euo pipefail + +checkout_merge() { + local target_branch=$1 + local pr_commit=$2 + local merge_branch=$3 + + if [[ -z "${target_branch}" ]]; then + echo "No pull request target branch" + exit 1 + fi + + git fetch -v origin "${target_branch}" + git checkout FETCH_HEAD + echo "Current branch: $(git rev-parse --abbrev-ref HEAD)" + + # create temporal branch to merge the PR with the target branch + git checkout -b ${merge_branch} + echo "New branch created: $(git rev-parse --abbrev-ref HEAD)" + + # set author identity so it can be run git merge + git config user.name "github-merged-pr-post-checkout" + git config user.email "auto-merge@buildkite" + + git merge --no-edit "${BUILDKITE_COMMIT}" || { + local merge_result=$? + echo "Merge failed: ${merge_result}" + git merge --abort + exit ${merge_result} + } +} + +pull_request="${BUILDKITE_PULL_REQUEST:-false}" + +if [[ "${pull_request}" == "false" ]]; then + echo "Not a pull request, skipping" + exit 0 +fi + +TARGET_BRANCH="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-master}" +PR_COMMIT="${BUILDKITE_COMMIT}" +PR_ID=${BUILDKITE_PULL_REQUEST} +MERGE_BRANCH="pr_merge_${PR_ID}" + +checkout_merge "${TARGET_BRANCH}" "${PR_COMMIT}" "${MERGE_BRANCH}" + +echo "Commit information" +git --no-pager log --format=%B -n 1 + +# Ensure buildkite groups are rendered +echo "" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 000000000000..80198244cc02 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +source .buildkite/scripts/setenv.sh diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 34321b61161b..61e68fa4e1dd 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -1,5 +1,43 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GO_AGENT_IMAGE: "golang:${GO_VERSION}" + BEATS_PROJECT_NAME: "packetbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - runAllStages" + key: "runAllStages" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + # - select: "Packetbeat - runMacOsTests" + # key: "UI_MACOS_TESTS" + # options: + # - label: "True" + # value: "true" + # - label: "False" + # value: "false" + # default: "false" + # if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + agents: + image: "${GO_AGENT_IMAGE}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh new file mode 100755 index 000000000000..9ae43b97bc9d --- /dev/null +++ b/.buildkite/scripts/common.sh @@ -0,0 +1,242 @@ +#!/bin/bash +set -euo pipefail + +WORKSPACE=${WORKSPACE:-"$(pwd)"} +BIN="${WORKSPACE}/bin" +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" +GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} +ONLY_DOCS=${ONLY_DOCS:-"true"} +UI_MACOS_TESTS="$(buildkite-agent meta-data get UI_MACOS_TESTS --default ${UI_MACOS_TESTS:-"false"})" +runAllStages="$(buildkite-agent meta-data get runAllStages --default ${runAllStages:-"false"})" +metricbeat_changeset=( + "^metricbeat/.*" + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" + ) +oss_changeset=( + "^go.mod" + "^pytest.ini" + "^dev-tools/.*" + "^libbeat/.*" + "^testing/.*" +) +ci_changeset=( + "^.buildkite/.*" +) +go_mod_changeset=( + "^go.mod" + ) +docs_changeset=( + ".*\\.(asciidoc|md)" + "deploy/kubernetes/.*-kubernetes\\.yaml" + ) +packaging_changeset=( + "^dev-tools/packaging/.*" + ".go-version" + ) + +with_docker_compose() { + local version=$1 + echo "Setting up the Docker-compose environment..." + create_workspace + retry 3 curl -sSL -o ${BIN}/docker-compose "https://github.com/docker/compose/releases/download/${version}/docker-compose-${platform_type_lowercase}-${arch_type}" + chmod +x ${BIN}/docker-compose + export PATH="${BIN}:${PATH}" + docker-compose version +} + +create_workspace() { + if [[ ! -d "${BIN}" ]]; then + mkdir -p "${BIN}" + fi +} + +add_bin_path() { + echo "Adding PATH to the environment variables..." + create_workspace + export PATH="${BIN}:${PATH}" +} + +check_platform_architeture() { + case "${arch_type}" in + "x86_64") + go_arch_type="amd64" + ;; + "aarch64") + go_arch_type="arm64" + ;; + "arm64") + go_arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} + +with_mage() { + local install_packages=( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report" + "gotest.tools/gotestsum" + ) + create_workspace + for pkg in "${install_packages[@]}"; do + go install "${pkg}@latest" + done +} + +with_go() { + echo "Setting up the Go environment..." + create_workspace + check_platform_architeture + retry 5 curl -sL -o "${BIN}/gvm" "https://github.com/andrewkroh/gvm/releases/download/${SETUP_GVM_VERSION}/gvm-${platform_type_lowercase}-${go_arch_type}" + chmod +x "${BIN}/gvm" + eval "$(gvm $GO_VERSION)" + go version + which go + local go_path="$(go env GOPATH):$(go env GOPATH)/bin" + export PATH="${go_path}:${PATH}" +} + +with_python() { + if [ "${platform_type}" == "Linux" ]; then + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv libsystemd-dev + elif [ "${platform_type}" == "Darwin" ]; then + brew update + pip3 install virtualenv + ulimit -Sn 10000 + fi +} + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +are_paths_changed() { + local patterns=("${@}") + local changelist=() + + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + else + return 1 + fi +} + +are_conditions_met_mandatory_tests() { + if [[ "${BUILDKITE_PULL_REQUEST}" == "" ]] || [[ "${runAllStages}" == "true" ]] || [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L107-L137 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_extended_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + return 0 + else + return 1 + fi +} + +are_conditions_met_macos_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "${UI_MACOS_TESTS}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_extended_windows_tests() { + if [[ "${ONLY_DOCS}" == "false" && "${BUILDKITE_PULL_REQUEST}" != "" ]] || [[ "${runAllStages}" == "true" ]]; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" ]] || [[ "${GITHUB_PR_LABELS}" =~ Metricbeat ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +are_conditions_met_packaging() { + if are_conditions_met_extended_windows_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 + return 0 + else + return 1 + fi + else + return 1 + fi +} + +if ! are_changed_only_paths "${docs_changeset[@]}" ; then + ONLY_DOCS="false" + echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." +else + echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." +fi + +if are_paths_changed "${go_mod_changeset[@]}" ; then + GO_MOD_CHANGES="true" +fi + +if are_paths_changed "${packaging_changeset[@]}" ; then + PACKAGING_CHANGES="true" +fi diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh new file mode 100755 index 000000000000..7e0c8c8d3db1 --- /dev/null +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.metricbeat-dynamic.yml" + +cat > $pipelineName <<- YAML + +steps: + +# YAML + +# if are_conditions_met_mandatory_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_ARM_64}" + machineType: "c2-standard-16" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_windows_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Extended Windowds Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_windows_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2-standard-16" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + + depends_on: + - step: "mandatory-tests" + allow_failure: false + - step: "extended-tests" + allow_failure: true + - step: "extended-win-tests" + allow_failure: true + +YAML +# fi + +cat $pipelineName #remove after tests + +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh new file mode 100755 index 000000000000..ff7b3ae19ab3 --- /dev/null +++ b/.buildkite/scripts/install_tools.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +echo "--- Env preparation" + +# Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE +# It could be removed when we use our own image for the BK agent. +if [ "${platform_type}" == "Linux" ]; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null +fi + +add_bin_path + +if command -v docker-compose &> /dev/null +then + echo "Found docker-compose. Checking version.." + FOUND_DOCKER_COMPOSE_VERSION=$(docker-compose --version | awk '{print $4}'|sed s/\,//) + if [ $FOUND_DOCKER_COMPOSE_VERSION == $DOCKER_COMPOSE_VERSION ]; then + echo "Versions match. No need to install docker-compose. Exiting." + elif [ "${platform_type}" == "Linux" && "${arch_type}" == "aarch64" ]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION_AARCH64}" + elif [ "${platform_type}" == "Linux" && "${arch_type}" == "x86_64" ]; then + with_docker_compose "${DOCKER_COMPOSE_VERSION}" + fi +else + with_docker_compose "${DOCKER_COMPOSE_VERSION}" +fi + +with_go "${GO_VERSION}" +with_mage +with_python + +#sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) +#sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #fix the fulesystem permissions issue like this:https://buildkite.com/elastic/beats-metricbeat/builds/1154#018d12db-dc0c-4bcd-b9b4-d5dece0b42c6/272-1267 + +sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #fix the fulesystem permissions issue like this:https://buildkite.com/elastic/beats-metricbeat/builds/1154#018d12db-dc0c-4bcd-b9b4-d5dece0b42c6/272-1267 + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +umask 0022 # fix the filesystem permissions issue like this: https://buildkite.com/elastic/beats-metricbeat/builds/1329#018d3179-25a9-475b-a2c8-64329dfe092b/320-1696 + +popd > /dev/null diff --git a/.buildkite/scripts/packaging.sh b/.buildkite/scripts/packaging.sh new file mode 100755 index 000000000000..1539d3ab430c --- /dev/null +++ b/.buildkite/scripts/packaging.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Packaging for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +mage package + +popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh new file mode 100755 index 000000000000..509b858f7b25 --- /dev/null +++ b/.buildkite/scripts/setenv.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +SETUP_GVM_VERSION="v0.5.1" +DOCKER_COMPOSE_VERSION="1.21.0" +DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +SETUP_WIN_PYTHON_VERSION="3.11.0" +GO_VERSION=$(cat .go-version) + +export SETUP_GVM_VERSION +export DOCKER_COMPOSE_VERSION +export DOCKER_COMPOSE_VERSION_AARCH64 +export SETUP_WIN_PYTHON_VERSION +export GO_VERSION diff --git a/.buildkite/scripts/unit_tests.sh b/.buildkite/scripts/unit_tests.sh new file mode 100755 index 000000000000..5d197877e65a --- /dev/null +++ b/.buildkite/scripts/unit_tests.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +source .buildkite/scripts/install_tools.sh + +set -euo pipefail + +echo "--- Run Unit Tests" + +#sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) +sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #fix the fulesystem permissions issue like this:https://buildkite.com/elastic/beats-metricbeat/builds/1154#018d12db-dc0c-4bcd-b9b4-d5dece0b42c6/272-1267 + +pushd "${BEATS_PROJECT_NAME}" > /dev/null + +umask 0022 # fix the filesystem permissions issue like this: https://buildkite.com/elastic/beats-metricbeat/builds/1329#018d3179-25a9-475b-a2c8-64329dfe092b/320-1696 +mage build unitTest + +popd > /dev/null diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 new file mode 100644 index 000000000000..34833d183ffa --- /dev/null +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -0,0 +1,70 @@ +$ErrorActionPreference = "Stop" # set -e +$WorkFolder = "metricbeat" +# Forcing to checkout again all the files with a correct autocrlf. +# Doing this here because we cannot set git clone options before. +function fixCRLF { + Write-Host "-- Fixing CRLF in git checkout --" + git config core.autocrlf false + git rm --quiet --cached -r . + git reset --quiet --hard +} +function withChoco { + Write-Host "-- Configure Choco --" + $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." + Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" +} +function withGolang($version) { + Write-Host "-- Install golang $version --" + choco install -y golang --version=$version + refreshenv + go version +} +function withPython($version) { + Write-Host "-- Install Python $version --" + choco install python --version=$version + refreshenv + python --version +} +function withMinGW { + Write-Host "-- Install MinGW --" + choco install mingw -y + refreshenv +} +function installGoDependencies { + $installPackages = @( + "github.com/magefile/mage" + "github.com/elastic/go-licenser" + "golang.org/x/tools/cmd/goimports" + "github.com/jstemmer/go-junit-report/v2" + "gotest.tools/gotestsum" + ) + foreach ($pkg in $installPackages) { + go install "$pkg@latest" + } +} + +fixCRLF + +withChoco + +withGolang $env:GO_VERSION + +installGoDependencies + +withPython $env:SETUP_WIN_PYTHON_VERSION + +withMinGW + +$ErrorActionPreference = "Continue" # set +e + +Push-Location $WorkFolder + +New-Item -ItemType Directory -Force -Path "build" +mage build unitTest + +Pop-Location + +$EXITCODE=$LASTEXITCODE +$ErrorActionPreference = "Stop" + +Exit $EXITCODE From d1ad060fe29fa64ce17b6c336e2974c999e22a91 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 1 Feb 2024 18:52:32 +0200 Subject: [PATCH 02/34] fix typo --- .buildkite/packetbeat/pipeline.packetbeat.yml | 2 +- .buildkite/scripts/common.sh | 12 +++++++++++- .buildkite/scripts/generate_packetbeat_pipeline.sh | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 61e68fa4e1dd..0784cda19233 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -30,7 +30,7 @@ steps: # - label: "False" # value: "false" # default: "false" - # if: "build.source == 'ui'" + if: "build.source == 'ui'" - wait: ~ if: "build.source == 'ui'" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9ae43b97bc9d..a27fa820a7ab 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -110,7 +110,7 @@ with_python() { if [ "${platform_type}" == "Linux" ]; then #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo apt-get update - sudo apt-get install -y python3-pip python3-venv libsystemd-dev + sudo apt-get install -y python3-pip python3-venv elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv @@ -118,6 +118,16 @@ with_python() { fi } +with_dependencies() { + if [ "${platform_type}" == "Linux" ]; then + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo apt-get update + sudo apt-get install -y libsystemd-dev libpcap-dev + elif [ "${platform_type}" == "Darwin" ]; then + pip3 install libpcap + fi +} + retry() { local retries=$1 shift diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 7e0c8c8d3db1..096a98ebde30 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -64,9 +64,9 @@ steps: key: "extended-arm64-unit-test" command: ".buildkite/scripts/unit_tests.sh" agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_ARM_64}" - machineType: "c2-standard-16" + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" # YAML From 0174a2a5eb708fcc778c6b214c777b5e465eb594 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 1 Feb 2024 18:58:40 +0200 Subject: [PATCH 03/34] add with_dependencies --- .buildkite/scripts/install_tools.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index ff7b3ae19ab3..60ceef3acdd4 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -35,6 +35,7 @@ fi with_go "${GO_VERSION}" with_mage with_python +with_dependencies #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) #sudo chmod -R go-w "${BEATS_PROJECT_NAME}/" #fix the fulesystem permissions issue like this:https://buildkite.com/elastic/beats-metricbeat/builds/1154#018d12db-dc0c-4bcd-b9b4-d5dece0b42c6/272-1267 From 9e88642609cb729c7e4ee11b3bfd435d2a68f164 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 18:07:56 +0200 Subject: [PATCH 04/34] run all pipeline's steps --- .buildkite/packetbeat/pipeline.packetbeat.yml | 7 ++- .buildkite/scripts/common.sh | 7 +++ .../scripts/generate_packetbeat_pipeline.sh | 54 ++++++++++++++----- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 0784cda19233..255e3c2c00f5 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -2,11 +2,13 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_RHEL9_X86_64: "family/core-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - GO_AGENT_IMAGE: "golang:${GO_VERSION}" BEATS_PROJECT_NAME: "packetbeat" steps: @@ -40,4 +42,5 @@ steps: key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" agents: - image: "${GO_AGENT_IMAGE}" + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a27fa820a7ab..fea468c496e5 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -128,6 +128,13 @@ with_dependencies() { fi } +config_git() { + if [ -z "$(git config --get user.email)" ]; then + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" + fi +} + retry() { local retries=$1 shift diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 096a98ebde30..cb5f55f48581 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -4,7 +4,7 @@ source .buildkite/scripts/common.sh set -euo pipefail -pipelineName="pipeline.metricbeat-dynamic.yml" +pipelineName="pipeline.packetbeat-dynamic.yml" cat > $pipelineName <<- YAML @@ -24,9 +24,17 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" @@ -89,12 +97,40 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # Temporary disabled https://github.com/elastic/beats/issues/37841 + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # YAML # fi # if are_conditions_met_extended_windows_tests; then # cat >> $pipelineName <<- YAML + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 key: "packaging" steps: @@ -104,7 +140,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + machineType: "c2d-highcpu-16" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -119,17 +155,11 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" - depends_on: - - step: "mandatory-tests" - allow_failure: false - - step: "extended-tests" - allow_failure: true - - step: "extended-win-tests" - allow_failure: true - YAML # fi -cat $pipelineName #remove after tests +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName +echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName From 073b241577c158d3df1ca13c08ff2d7ba2d04032 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 18:48:37 +0200 Subject: [PATCH 05/34] disable the failed Windows test and change pre-command --- .buildkite/hooks/pre-command | 2 +- .../scripts/generate_packetbeat_pipeline.sh | 22 +++++++++---------- metricbeat/tests/system/test_reload.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ef38478a4327..4f20a0e47803 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -16,7 +16,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index cb5f55f48581..5f063ee702d4 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -97,17 +97,17 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # Temporary disabled https://github.com/elastic/beats/issues/37841 - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "n2-standard-8" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # # Temporary disabled https://github.com/elastic/beats/issues/37841 + # - label: ":windows: Windows 10 Unit Tests" + # key: "extended-win-10-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machine_type: "n2-standard-8" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" diff --git a/metricbeat/tests/system/test_reload.py b/metricbeat/tests/system/test_reload.py index 29d82bbf82b2..aa0b8e8a5373 100644 --- a/metricbeat/tests/system/test_reload.py +++ b/metricbeat/tests/system/test_reload.py @@ -42,7 +42,7 @@ def test_reload(self): self.wait_until(lambda: self.output_lines() > 0) proc.check_kill_and_wait() - @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") + @unittest.skipUnless(re.match("(?i)linux|darwin|freebsd|openbsd", sys.platform), "os") def test_start_stop(self): """ Test if module is properly started and stopped From 3e2554783d5488dafabafe4aa8f2f93ba7cf0de7 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 18:52:38 +0200 Subject: [PATCH 06/34] uncomment Windows 10 tests --- .../scripts/generate_packetbeat_pipeline.sh | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 5f063ee702d4..cb5f55f48581 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -97,17 +97,17 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # # Temporary disabled https://github.com/elastic/beats/issues/37841 - # - label: ":windows: Windows 10 Unit Tests" - # key: "extended-win-10-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machine_type: "n2-standard-8" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # Temporary disabled https://github.com/elastic/beats/issues/37841 + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" From 572dc150fe6154e8e680ba4b776efb9bcaf5bdc0 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 5 Feb 2024 21:24:51 +0200 Subject: [PATCH 07/34] fir RHEL9 step --- .buildkite/scripts/common.sh | 11 ++++++++--- .buildkite/scripts/install_tools.sh | 10 ++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index fea468c496e5..4c3e6fdf137c 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -108,9 +108,14 @@ with_go() { with_python() { if [ "${platform_type}" == "Linux" ]; then - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo apt-get update - sudo apt-get install -y python3-pip python3-venv + if grep -q "ID=ubuntu" /etc/*release; then + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo apt-get update + sudo apt-get install -y python3-pip python3-venv + elif grep grep -qE "ID=centos|ID=rhel" /etc/*release; then + sudo dnf update + sudo dnf install -y python3-pip python3-venv + fi elif [ "${platform_type}" == "Darwin" ]; then brew update pip3 install virtualenv diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index 1a1e3a29f54e..a92bbc4acc48 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -9,10 +9,12 @@ echo "--- Env preparation" # Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE # It could be removed when we use our own image for the BK agent. if [ "${platform_type}" == "Linux" ]; then - DEBIAN_FRONTEND="noninteractive" - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo mkdir -p /etc/needrestart - echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + if grep -q "ID=ubuntu" /etc/*release; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + fi fi add_bin_path From 64c2fe2d897e0cb28adb3610943328c8273c24ef Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 10:55:48 +0200 Subject: [PATCH 08/34] fir RHEL9 step --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 4c3e6fdf137c..07c55bf46727 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -112,7 +112,7 @@ with_python() { #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) sudo apt-get update sudo apt-get install -y python3-pip python3-venv - elif grep grep -qE "ID=centos|ID=rhel" /etc/*release; then + elif grep grep -q "ID=rhel" /etc/*release; then sudo dnf update sudo dnf install -y python3-pip python3-venv fi From d35edb8b79fd5a9e01885c08c9595d63782c9382 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 14:33:33 +0200 Subject: [PATCH 09/34] fix RHEL9 --- .buildkite/scripts/common.sh | 33 +++++++++++++++++++++++------ .buildkite/scripts/install_tools.sh | 12 ++++++----- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 07c55bf46727..81582cc126bb 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -106,13 +106,29 @@ with_go() { export PATH="${go_path}:${PATH}" } -with_python() { +checkLinuxType() { if [ "${platform_type}" == "Linux" ]; then if grep -q "ID=ubuntu" /etc/*release; then - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + echo "ubuntu" + elif grep grep -q "ID=rhel" /etc/*release; then + echo "rhel" + elif grep grep -q "ID=centos" /etc/*release; then + echo "centos" + else + echo "Unsupported Linux" + fi + else + echo "This is not a Linux" + fi +} + +with_python() { + local linuxType="$(checkLinuxType)" + if [ "${platform_type}" == "Linux" ]; then + if [ "${linuxType}" = "ubuntu" ]; then sudo apt-get update sudo apt-get install -y python3-pip python3-venv - elif grep grep -q "ID=rhel" /etc/*release; then + elif [ "${linuxType}" = "rhel" ]; then sudo dnf update sudo dnf install -y python3-pip python3-venv fi @@ -124,10 +140,15 @@ with_python() { } with_dependencies() { + local linuxType="$(checkLinuxType)" if [ "${platform_type}" == "Linux" ]; then - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo apt-get update - sudo apt-get install -y libsystemd-dev libpcap-dev + if [ "${linuxType}" = "ubuntu" ]; then + sudo apt-get update + sudo apt-get install -y libsystemd-dev libpcap-dev + elif [ "${linuxType}" = "rhel" ]; then + sudo dnf update + sudo dnf install -y libsystemd-dev libpcap-dev + fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap fi diff --git a/.buildkite/scripts/install_tools.sh b/.buildkite/scripts/install_tools.sh index a92bbc4acc48..80e70ae96c53 100755 --- a/.buildkite/scripts/install_tools.sh +++ b/.buildkite/scripts/install_tools.sh @@ -9,11 +9,13 @@ echo "--- Env preparation" # Temporary solution to fix the issues with "sudo apt get...." https://elastic.slack.com/archives/C0522G6FBNE/p1706003603442859?thread_ts=1706003209.424539&cid=C0522G6FBNE # It could be removed when we use our own image for the BK agent. if [ "${platform_type}" == "Linux" ]; then - if grep -q "ID=ubuntu" /etc/*release; then - DEBIAN_FRONTEND="noninteractive" - #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) - sudo mkdir -p /etc/needrestart - echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + if [ "${platform_type}" == "Linux" ]; then + if [ $(checkLinuxType) = "ubuntu" ]; then + DEBIAN_FRONTEND="noninteractive" + #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) + sudo mkdir -p /etc/needrestart + echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null + fi fi fi From 9ebe4c6b8b8e9e3026dfccd50dad1dfc5235179d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 15:00:41 +0200 Subject: [PATCH 10/34] fix RHEL9 --- .buildkite/scripts/common.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 81582cc126bb..1ddd11bddb23 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -124,13 +124,14 @@ checkLinuxType() { with_python() { local linuxType="$(checkLinuxType)" + echo "${linuxType}" if [ "${platform_type}" == "Linux" ]; then if [ "${linuxType}" = "ubuntu" ]; then sudo apt-get update sudo apt-get install -y python3-pip python3-venv elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update - sudo dnf install -y python3-pip python3-venv + sudo yum update -y + sudo yum install -y python3-pip python3-venv fi elif [ "${platform_type}" == "Darwin" ]; then brew update @@ -141,13 +142,14 @@ with_python() { with_dependencies() { local linuxType="$(checkLinuxType)" + echo "${linuxType}" if [ "${platform_type}" == "Linux" ]; then if [ "${linuxType}" = "ubuntu" ]; then sudo apt-get update sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update - sudo dnf install -y libsystemd-dev libpcap-dev + sudo yum update -y + sudo yum install -y libsystemd-dev libpcap-dev fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From dda216e6c7cc5ae5cf74d58bd728e6693e61c793 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 15:33:28 +0200 Subject: [PATCH 11/34] fix RHEL9 --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 1ddd11bddb23..fcd5c71305b6 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -110,9 +110,9 @@ checkLinuxType() { if [ "${platform_type}" == "Linux" ]; then if grep -q "ID=ubuntu" /etc/*release; then echo "ubuntu" - elif grep grep -q "ID=rhel" /etc/*release; then + elif grep -q "ID=rhel" /etc/*release; then echo "rhel" - elif grep grep -q "ID=centos" /etc/*release; then + elif grep -q "ID=centos" /etc/*release; then echo "centos" else echo "Unsupported Linux" From 8809bb7b0635d7c633b004d2ddcbd9172a26de1c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 15:46:54 +0200 Subject: [PATCH 12/34] fix RHEL9 --- .buildkite/scripts/common.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index fcd5c71305b6..d4db129cec15 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -108,12 +108,10 @@ with_go() { checkLinuxType() { if [ "${platform_type}" == "Linux" ]; then - if grep -q "ID=ubuntu" /etc/*release; then + if grep -q 'ubuntu' /etc/*release; then echo "ubuntu" - elif grep -q "ID=rhel" /etc/*release; then + elif grep -q 'rhel' /etc/*release; then echo "rhel" - elif grep -q "ID=centos" /etc/*release; then - echo "centos" else echo "Unsupported Linux" fi From 386f384c662e94609e3869ee4500d0b92940f633 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 16:26:38 +0200 Subject: [PATCH 13/34] fix RHEL9 --- .buildkite/scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d4db129cec15..f4ba3bf6940d 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -129,7 +129,8 @@ with_python() { sudo apt-get install -y python3-pip python3-venv elif [ "${linuxType}" = "rhel" ]; then sudo yum update -y - sudo yum install -y python3-pip python3-venv + sudo yum install -y python3 python3-pip + pip3 install virtualenv fi elif [ "${platform_type}" == "Darwin" ]; then brew update From a6b450463ec108b3d9c44bb16d2c8e45f2f62449 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 16:58:28 +0200 Subject: [PATCH 14/34] fix RHEL9 --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index f4ba3bf6940d..a9dda70cc948 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -148,7 +148,7 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo yum update -y - sudo yum install -y libsystemd-dev libpcap-dev + sudo yum install -y systemd-devel libpcap-dev fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From b4933f33b9fb892f6af52e00c95ad57c4f8b8564 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 17:53:36 +0200 Subject: [PATCH 15/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a9dda70cc948..c9a426ad3d01 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -148,7 +148,7 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo yum update -y - sudo yum install -y systemd-devel libpcap-dev + sudo yum install -y systemd-devel fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From 6f4b03536fd00f067ec39f7d1f791d1ec4671573 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 18:10:04 +0200 Subject: [PATCH 16/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index c9a426ad3d01..d84fe97fccba 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -148,7 +148,7 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo yum update -y - sudo yum install -y systemd-devel + sudo yum install -y systemd-devel libpcap-devel fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From d1e0d4d17cf319413aaf07de352d46f29f17ba38 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 18:22:47 +0200 Subject: [PATCH 17/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 1 + .../scripts/generate_packetbeat_pipeline.sh | 252 +++++++++--------- 2 files changed, 127 insertions(+), 126 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index d84fe97fccba..a7c1ebcfc533 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -148,6 +148,7 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo yum update -y + sudo yum install -y epel-release sudo yum install -y systemd-devel libpcap-devel fi elif [ "${platform_type}" == "Darwin" ]; then diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index cb5f55f48581..a962787892e7 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -18,13 +18,13 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":linux: Ubuntu Unit Tests" + # key: "mandatory-linux-unit-test" + # command: ".buildkite/scripts/unit_tests.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":rhel: RHEL-9 Unit Tests" key: "mandatory-rhel9-unit-test" @@ -35,125 +35,125 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "n2-standard-8" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then -# cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# if are_conditions_met_extended_windows_tests; then -# cat >> $pipelineName <<- YAML - - - group: "Extended Windowds Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # Temporary disabled https://github.com/elastic/beats/issues/37841 - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "n2-standard-8" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "n2-standard-8" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# if are_conditions_met_extended_windows_tests; then -# cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" - env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" +# - label: ":windows: Windows Unit Tests - {{matrix.image}}" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# key: "mandatory-win-unit-tests" +# agents: +# provider: "gcp" +# image: "{{matrix.image}}" +# machine_type: "n2-standard-8" +# disk_size: 100 +# disk_type: "pd-ssd" +# matrix: +# setup: +# image: +# - "${IMAGE_WIN_2016}" +# - "${IMAGE_WIN_2022}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# # YAML +# # fi + +# # if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then +# # cat >> $pipelineName <<- YAML + +# - group: "Extended Tests" +# key: "extended-tests" +# steps: +# - label: ":mac: MacOS Unit Tests" +# key: "extended-macos-unit-tests" +# command: ".buildkite/scripts/unit_tests.sh" +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# - label: ":linux: ARM Ubuntu Unit Tests" +# key: "extended-arm64-unit-test" +# command: ".buildkite/scripts/unit_tests.sh" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "t4g.xlarge" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# # YAML +# # fi + +# # if are_conditions_met_extended_windows_tests; then +# # cat >> $pipelineName <<- YAML + +# - group: "Extended Windowds Tests" +# key: "extended-win-tests" +# steps: +# - label: ":windows: Win 2019 Unit Tests" +# key: "extended-win-2019-unit-tests" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_2019}" +# machine_type: "n2-standard-8" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# # Temporary disabled https://github.com/elastic/beats/issues/37841 +# - label: ":windows: Windows 10 Unit Tests" +# key: "extended-win-10-unit-tests" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_10}" +# machine_type: "n2-standard-8" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# - label: ":windows: Windows 11 Unit Tests" +# key: "extended-win-11-unit-tests" +# command: ".buildkite/scripts/win_unit_tests.ps1" +# agents: +# provider: "gcp" +# image: "${IMAGE_WIN_11}" +# machine_type: "n2-standard-8" +# disk_size: 100 +# disk_type: "pd-ssd" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# # YAML +# # fi + +# # if are_conditions_met_extended_windows_tests; then +# # cat >> $pipelineName <<- YAML + +# - wait: ~ +# depends_on: +# - step: "mandatory-tests" +# allow_failure: false + +# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 +# key: "packaging" +# steps: +# - label: ":linux: Packaging Linux" +# key: "packaging-linux" +# command: ".buildkite/scripts/packaging.sh" +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "c2d-highcpu-16" +# env: +# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + +# - label: ":linux: Packaging ARM" +# key: "packaging-arm" +# command: ".buildkite/scripts/packaging.sh" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "t4g.xlarge" +# env: +# PLATFORMS: "linux/arm64" +# PACKAGES: "docker" YAML # fi From c6fa36068b8a71ab058ddffbd3194184ac711b8f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 6 Feb 2024 18:39:04 +0200 Subject: [PATCH 18/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a7c1ebcfc533..7136b456c683 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -128,8 +128,8 @@ with_python() { sudo apt-get update sudo apt-get install -y python3-pip python3-venv elif [ "${linuxType}" = "rhel" ]; then - sudo yum update -y - sudo yum install -y python3 python3-pip + sudo dnf update -y + sudo dnf install -y python3 python3-pip pip3 install virtualenv fi elif [ "${platform_type}" == "Darwin" ]; then @@ -147,9 +147,9 @@ with_dependencies() { sudo apt-get update sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then - sudo yum update -y - sudo yum install -y epel-release - sudo yum install -y systemd-devel libpcap-devel + sudo dnf update -y + sudo dnf --enablerepo=crb install -y libpcap-devel + sudo dnf install -y systemd-devel fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From ded6036bec4740a274c8a743ac913e831426b3e5 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 08:42:54 +0200 Subject: [PATCH 19/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7136b456c683..a0647bd7bd41 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -148,8 +148,8 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo dnf update -y - sudo dnf --enablerepo=crb install -y libpcap-devel - sudo dnf install -y systemd-devel + sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + sudo dnf install -y systemd-devel libpcap-devel fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From f267e6c603c50e323f72dcaee0e5d5a952301e25 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 09:14:25 +0200 Subject: [PATCH 20/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a0647bd7bd41..f8f46403e238 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -139,6 +139,13 @@ with_python() { fi } +with_rhel_epel() { + if [ "${linuxType}" = "rhel" ]; then + sudo subscription-manager repos --enable codeready-builder-for-rhel-9-${arch_type}-rpms + sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + fi +} + with_dependencies() { local linuxType="$(checkLinuxType)" echo "${linuxType}" @@ -148,7 +155,7 @@ with_dependencies() { sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then sudo dnf update -y - sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + with_rhel_epel sudo dnf install -y systemd-devel libpcap-devel fi elif [ "${platform_type}" == "Darwin" ]; then From 11c3dade474399e16543d1dd15563eed02d847da Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 09:29:38 +0200 Subject: [PATCH 21/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index f8f46403e238..afc37623e558 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -141,8 +141,8 @@ with_python() { with_rhel_epel() { if [ "${linuxType}" = "rhel" ]; then - sudo subscription-manager repos --enable codeready-builder-for-rhel-9-${arch_type}-rpms - sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + sudo dnf update -y + sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm fi } @@ -154,8 +154,8 @@ with_dependencies() { sudo apt-get update sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y with_rhel_epel + sudo dnf update -y sudo dnf install -y systemd-devel libpcap-devel fi elif [ "${platform_type}" == "Darwin" ]; then @@ -192,7 +192,6 @@ retry() { are_paths_changed() { local patterns=("${@}") local changelist=() - for pattern in "${patterns[@]}"; do changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) if [ "${#changed_files[@]}" -gt 0 ]; then From f0079cdae782dbdf8f9cc5140c64e2a9307b5027 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 09:45:45 +0200 Subject: [PATCH 22/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index afc37623e558..479eb8409210 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -139,13 +139,6 @@ with_python() { fi } -with_rhel_epel() { - if [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y - sudo dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm - fi -} - with_dependencies() { local linuxType="$(checkLinuxType)" echo "${linuxType}" @@ -154,9 +147,10 @@ with_dependencies() { sudo apt-get update sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then - with_rhel_epel sudo dnf update -y - sudo dnf install -y systemd-devel libpcap-devel + sudo dnf install -y systemd-devel + wget http://mirror.centos.org/centos/9/BaseOS/${arch_type}/os/Packages/libpcap-devel-1.10.1-10.el9.${arch_type}.rpm + sudo dnf install -y libpcap-devel-1.10.1-10.el9.x86_64.rpm fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From fdff8397918e8a89825fb88c9dc43da3c588446d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 10:06:37 +0200 Subject: [PATCH 23/34] fix RHEL9 dependencies --- .buildkite/scripts/common.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 479eb8409210..7135814c056a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -147,10 +147,10 @@ with_dependencies() { sudo apt-get update sudo apt-get install -y libsystemd-dev libpcap-dev elif [ "${linuxType}" = "rhel" ]; then - sudo dnf update -y + # sudo dnf update -y sudo dnf install -y systemd-devel - wget http://mirror.centos.org/centos/9/BaseOS/${arch_type}/os/Packages/libpcap-devel-1.10.1-10.el9.${arch_type}.rpm - sudo dnf install -y libpcap-devel-1.10.1-10.el9.x86_64.rpm + wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm + sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap From 64238cb4ae28a6aaa40fa5bd45b72e52517a2c3b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 10:32:08 +0200 Subject: [PATCH 24/34] full test --- .buildkite/scripts/common.sh | 4 +- .../scripts/generate_packetbeat_pipeline.sh | 252 +++++++++--------- 2 files changed, 128 insertions(+), 128 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7135814c056a..883538b3796a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -149,8 +149,8 @@ with_dependencies() { elif [ "${linuxType}" = "rhel" ]; then # sudo dnf update -y sudo dnf install -y systemd-devel - wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm - sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm + wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image + sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image fi elif [ "${platform_type}" == "Darwin" ]; then pip3 install libpcap diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index a962787892e7..cb5f55f48581 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -18,13 +18,13 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - # - label: ":linux: Ubuntu Unit Tests" - # key: "mandatory-linux-unit-test" - # command: ".buildkite/scripts/unit_tests.sh" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":rhel: RHEL-9 Unit Tests" key: "mandatory-rhel9-unit-test" @@ -35,125 +35,125 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -# - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" -# key: "mandatory-win-unit-tests" -# agents: -# provider: "gcp" -# image: "{{matrix.image}}" -# machine_type: "n2-standard-8" -# disk_size: 100 -# disk_type: "pd-ssd" -# matrix: -# setup: -# image: -# - "${IMAGE_WIN_2016}" -# - "${IMAGE_WIN_2022}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# # YAML -# # fi - -# # if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then -# # cat >> $pipelineName <<- YAML - -# - group: "Extended Tests" -# key: "extended-tests" -# steps: -# - label: ":mac: MacOS Unit Tests" -# key: "extended-macos-unit-tests" -# command: ".buildkite/scripts/unit_tests.sh" -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_X86_64}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# - label: ":linux: ARM Ubuntu Unit Tests" -# key: "extended-arm64-unit-test" -# command: ".buildkite/scripts/unit_tests.sh" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "t4g.xlarge" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# # YAML -# # fi - -# # if are_conditions_met_extended_windows_tests; then -# # cat >> $pipelineName <<- YAML - -# - group: "Extended Windowds Tests" -# key: "extended-win-tests" -# steps: -# - label: ":windows: Win 2019 Unit Tests" -# key: "extended-win-2019-unit-tests" -# command: ".buildkite/scripts/win_unit_tests.ps1" -# agents: -# provider: "gcp" -# image: "${IMAGE_WIN_2019}" -# machine_type: "n2-standard-8" -# disk_size: 100 -# disk_type: "pd-ssd" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# # Temporary disabled https://github.com/elastic/beats/issues/37841 -# - label: ":windows: Windows 10 Unit Tests" -# key: "extended-win-10-unit-tests" -# command: ".buildkite/scripts/win_unit_tests.ps1" -# agents: -# provider: "gcp" -# image: "${IMAGE_WIN_10}" -# machine_type: "n2-standard-8" -# disk_size: 100 -# disk_type: "pd-ssd" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# - label: ":windows: Windows 11 Unit Tests" -# key: "extended-win-11-unit-tests" -# command: ".buildkite/scripts/win_unit_tests.ps1" -# agents: -# provider: "gcp" -# image: "${IMAGE_WIN_11}" -# machine_type: "n2-standard-8" -# disk_size: 100 -# disk_type: "pd-ssd" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# # YAML -# # fi - -# # if are_conditions_met_extended_windows_tests; then -# # cat >> $pipelineName <<- YAML - -# - wait: ~ -# depends_on: -# - step: "mandatory-tests" -# allow_failure: false - -# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 -# key: "packaging" -# steps: -# - label: ":linux: Packaging Linux" -# key: "packaging-linux" -# command: ".buildkite/scripts/packaging.sh" -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "c2d-highcpu-16" -# env: -# PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - -# - label: ":linux: Packaging ARM" -# key: "packaging-arm" -# command: ".buildkite/scripts/packaging.sh" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "t4g.xlarge" -# env: -# PLATFORMS: "linux/arm64" -# PACKAGES: "docker" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_windows_tests; then +# cat >> $pipelineName <<- YAML + + - group: "Extended Windowds Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # Temporary disabled https://github.com/elastic/beats/issues/37841 + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "n2-standard-8" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_extended_windows_tests; then +# cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "c2d-highcpu-16" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: ".buildkite/scripts/packaging.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" YAML # fi From 1adfd990a7a17a04e237f10138522f5a1a94397f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 16:46:37 +0200 Subject: [PATCH 25/34] add/change conditions for dynamic pipelines --- .buildkite/packetbeat/pipeline.packetbeat.yml | 28 +++--- .buildkite/scripts/common.sh | 63 ++++++++++--- .../scripts/generate_libbeat_pipeline.sh | 2 +- .../scripts/generate_metricbeat_pipeline.sh | 4 +- .../scripts/generate_packetbeat_pipeline.sh | 88 ++++++++++--------- 5 files changed, 117 insertions(+), 68 deletions(-) diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 255e3c2c00f5..7cc39ffc0ea5 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -16,22 +16,30 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - runAllStages" - key: "runAllStages" + - select: "Packetbeat - runPacketbeat" + key: "runPacketbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - runPacketbeatArmTest" + key: "runPacketbeatArmTest" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - runPacketbeatMacOsTests" + key: "runPacketbeatMacOsTests" options: - label: "True" value: "true" - label: "False" value: "false" default: "false" - # - select: "Packetbeat - runMacOsTests" - # key: "UI_MACOS_TESTS" - # options: - # - label: "True" - # value: "true" - # - label: "False" - # value: "false" - # default: "false" if: "build.source == 'ui'" - wait: ~ diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index c6f9445022f0..7d4b9c48960b 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -10,17 +10,24 @@ GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" +runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" +runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" metricbeat_changeset=( "^metricbeat/.*" - "^go.mod" - "^pytest.ini" - "^dev-tools/.*" + ) + +libbeat_changeset=( "^libbeat/.*" - "^testing/.*" ) + +packetbeat_changeset=( + "^packetbeat/.*" + ) + oss_changeset=( "^go.mod" "^pytest.ini" @@ -28,16 +35,20 @@ oss_changeset=( "^libbeat/.*" "^testing/.*" ) + ci_changeset=( "^.buildkite/.*" ) + go_mod_changeset=( "^go.mod" ) + docs_changeset=( ".*\\.(asciidoc|md)" "deploy/kubernetes/.*-kubernetes\\.yaml" ) + packaging_changeset=( "^dev-tools/packaging/.*" ".go-version" @@ -77,7 +88,7 @@ check_platform_architeture() { go_arch_type="arm64" ;; *) - echo "The current platform/OS type is unsupported yet" + echo "The current platform or OS type is unsupported yet" ;; esac } @@ -218,13 +229,17 @@ are_changed_only_paths() { } are_conditions_met_mandatory_tests() { - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then + if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then return 0 fi elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ packetbeat || "${runPacketbeat}" == "true" ]]; then return 0 fi fi @@ -232,27 +247,47 @@ are_conditions_met_mandatory_tests() { return 1 } -are_conditions_met_libbeat_arm_tests() { +are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then return 0 fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then + return 0 + fi fi fi return 1 } -are_conditions_met_metricbeat_macos_tests() { - if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 +are_conditions_met_macos_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then + return 0 + fi + fi fi return 1 } are_conditions_met_packaging() { - if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 - return 0 + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 + return 0 + fi + elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then + return 0 + fi + fi fi return 1 } diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 6da1bef711dc..edadbf8cb6a9 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -64,7 +64,7 @@ YAML fi echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_libbeat_arm_tests; then +if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 0ea19734c4fd..8876f095675b 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -110,7 +110,7 @@ YAML fi echo "Check and add the Extended Tests into the pipeline" -if are_conditions_met_metricbeat_macos_tests; then +if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -128,7 +128,7 @@ YAML fi echo "Check and add the Packaging into the pipeline" -if are_conditions_met_mandatory_tests && are_conditions_met_packaging; then +if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - wait: ~ diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index cb5f55f48581..4fa1b78e1a78 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -10,10 +10,10 @@ cat > $pipelineName <<- YAML steps: -# YAML +YAML -# if are_conditions_met_mandatory_tests; then -# cat >> $pipelineName <<- YAML +if are_conditions_met_mandatory_tests; then + cat >> $pipelineName <<- YAML - group: "Mandatory Tests" key: "mandatory-tests" @@ -51,38 +51,6 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -# YAML -# fi - -# if are_conditions_met_extended_tests && are_conditions_met_macos_tests; then -# cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# if are_conditions_met_extended_windows_tests; then -# cat >> $pipelineName <<- YAML - - group: "Extended Windowds Tests" key: "extended-win-tests" steps: @@ -97,7 +65,6 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # Temporary disabled https://github.com/elastic/beats/issues/37841 - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" command: ".buildkite/scripts/win_unit_tests.ps1" @@ -120,11 +87,50 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -# YAML -# fi +YAML +fi + +if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "t4g.xlarge" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + -# if are_conditions_met_extended_windows_tests; then -# cat >> $pipelineName <<- YAML +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML - wait: ~ depends_on: @@ -156,7 +162,7 @@ steps: PACKAGES: "docker" YAML -# fi +fi echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public cat $pipelineName From 6e037942f457ad814e6f3fabbdc0f997964c4ed1 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 16:59:43 +0200 Subject: [PATCH 26/34] update conditions for dynamic pipelines --- .buildkite/scripts/common.sh | 2 +- .buildkite/scripts/generate_libbeat_pipeline.sh | 3 +++ .buildkite/scripts/generate_metricbeat_pipeline.sh | 3 +++ .buildkite/scripts/generate_packetbeat_pipeline.sh | 10 +++++----- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7d4b9c48960b..9d859c2cd588 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -239,7 +239,7 @@ are_conditions_met_mandatory_tests() { return 0 fi elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ packetbeat || "${runPacketbeat}" == "true" ]]; then + if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then return 0 fi fi diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index edadbf8cb6a9..221a6df3f412 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -61,6 +61,9 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 1 fi echo "Check and add the Extended Tests into the pipeline" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 8876f095675b..ab09de70f26a 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -107,6 +107,9 @@ steps: disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 1 fi echo "Check and add the Extended Tests into the pipeline" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 4fa1b78e1a78..ab38568bd4a0 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -6,15 +6,12 @@ set -euo pipefail pipelineName="pipeline.packetbeat-dynamic.yml" +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then cat > $pipelineName <<- YAML steps: -YAML - -if are_conditions_met_mandatory_tests; then - cat >> $pipelineName <<- YAML - - group: "Mandatory Tests" key: "mandatory-tests" steps: @@ -88,6 +85,9 @@ if are_conditions_met_mandatory_tests; then artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 1 fi if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then From 444771b8fe349d9ea08c8b211bc9b8f1ee8d443c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 18:04:08 +0200 Subject: [PATCH 27/34] update setenv.sh --- .buildkite/scripts/common.sh | 3 -- .../scripts/generate_packetbeat_pipeline.sh | 2 +- .buildkite/scripts/setenv.sh | 34 +++++++++++++++++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9d859c2cd588..358326625a1f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -3,9 +3,6 @@ set -euo pipefail WORKSPACE=${WORKSPACE:-"$(pwd)"} BIN="${WORKSPACE}/bin" -platform_type="$(uname)" -platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') -arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index ab38568bd4a0..11bb151d6829 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -8,7 +8,7 @@ pipelineName="pipeline.packetbeat-dynamic.yml" echo "Add the mandatory and extended tests without additional conditions into the pipeline" if are_conditions_met_mandatory_tests; then -cat > $pipelineName <<- YAML + cat > $pipelineName <<- YAML steps: diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 901ba9891c20..a17a355199e1 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -7,9 +7,43 @@ DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" GO_VERSION=$(cat .go-version) +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export GO_VERSION + +exportVars() { + if [ "${arch_type}" == "x86_64" ]; then + case "${platform_type}" in + Linux|Darwin) + export GOX_FLAGS="-arch amd64" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + ;; + MINGW*) + export GOX_FLAGS="-arch 386" + export testResults="**\\build\\TEST*.xml" + export artifacts="**\\build\\TEST*.out" + ;; + esac + elif [[ "${arch_type}" == "aarch64" || "${arch_type}" == "arm64" ]]; then + export GOX_FLAGS="-arch arm" + export testResults="**/build/TEST*.xml" + export artifacts="**/build/TEST*.out" + else + echo "Unsupported OS" + fi +} + + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then + exportVars + export RACE_DETECTOR="true" + export TEST_COVERAGE="true" + export DOCKER_PULL="0" +fi \ No newline at end of file From fbc73d58bbe53780ac811fbcc1f506f175a60386 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 7 Feb 2024 18:11:23 +0200 Subject: [PATCH 28/34] update setenv.sh --- .buildkite/scripts/common.sh | 3 +++ .buildkite/scripts/setenv.sh | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 358326625a1f..9d859c2cd588 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -3,6 +3,9 @@ set -euo pipefail WORKSPACE=${WORKSPACE:-"$(pwd)"} BIN="${WORKSPACE}/bin" +platform_type="$(uname)" +platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') +arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index a17a355199e1..dbf23c3e5ef9 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -7,9 +7,7 @@ DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" GO_VERSION=$(cat .go-version) -platform_type="$(uname)" -platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') -arch_type="$(uname -m)" + export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -18,6 +16,8 @@ export SETUP_WIN_PYTHON_VERSION export GO_VERSION exportVars() { + local platform_type="$(uname)" + local arch_type="$(uname -m)" if [ "${arch_type}" == "x86_64" ]; then case "${platform_type}" in Linux|Darwin) @@ -46,4 +46,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then export RACE_DETECTOR="true" export TEST_COVERAGE="true" export DOCKER_PULL="0" -fi \ No newline at end of file +fi From 3ddb056df3b2a4d68c32678114b065ff9ec13196 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 11:25:45 +0200 Subject: [PATCH 29/34] change exit 1 to exit 0 for the dynamic pipeline --- .buildkite/scripts/generate_libbeat_pipeline.sh | 2 +- .buildkite/scripts/generate_metricbeat_pipeline.sh | 2 +- .buildkite/scripts/generate_packetbeat_pipeline.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 221a6df3f412..47bdb8997cf6 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -63,7 +63,7 @@ steps: YAML else echo "The conditions don't match to requirements for generating pipeline steps." - exit 1 + exit 0 fi echo "Check and add the Extended Tests into the pipeline" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index ab09de70f26a..7ef89dd01163 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -109,7 +109,7 @@ steps: YAML else echo "The conditions don't match to requirements for generating pipeline steps." - exit 1 + exit 0 fi echo "Check and add the Extended Tests into the pipeline" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 11bb151d6829..59b3f70c9c9b 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -87,7 +87,7 @@ steps: YAML else echo "The conditions don't match to requirements for generating pipeline steps." - exit 1 + exit 0 fi if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then From c120fab563831a03c60959430635cc7c682199d8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 12:10:56 +0200 Subject: [PATCH 30/34] update dynamic pipeline --- .buildkite/libbeat/pipeline.libbeat.yml | 2 +- .buildkite/metricbeat/pipeline.yml | 2 +- .buildkite/packetbeat/pipeline.packetbeat.yml | 4 ++++ .../scripts/generate_packetbeat_pipeline.sh | 16 +++++++++------- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 1fb185b59d19..cfaf78183609 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -4,7 +4,7 @@ env: IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "libbeat" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index c42f17d2a363..d882edaaa0a1 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -11,7 +11,7 @@ env: GO_AGENT_IMAGE: "golang:${GO_VERSION}" BEATS_PROJECT_NAME: "metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 7cc39ffc0ea5..0dbb87fc1d10 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -9,6 +9,10 @@ env: IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "packetbeat" steps: diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 59b3f70c9c9b..89ea7a33e20a 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -21,6 +21,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":rhel: RHEL-9 Unit Tests" @@ -29,6 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -38,7 +40,7 @@ steps: agents: provider: "gcp" image: "{{matrix.image}}" - machine_type: "n2-standard-8" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" matrix: @@ -57,7 +59,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -68,7 +70,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_WIN_10}" - machine_type: "n2-standard-8" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -79,7 +81,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_WIN_11}" - machine_type: "n2-standard-8" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" @@ -122,7 +124,7 @@ if are_conditions_met_arm_tests; then agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML @@ -146,7 +148,7 @@ if are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2d-highcpu-16" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" @@ -156,7 +158,7 @@ if are_conditions_met_packaging; then agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" From d6a798b9cc0f7c8f216c52ee62c808c55f7e015d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 12:28:33 +0200 Subject: [PATCH 31/34] fix VAR name --- .buildkite/scripts/generate_libbeat_pipeline.sh | 6 +++--- .buildkite/scripts/generate_metricbeat_pipeline.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 47bdb8997cf6..0674d0b186b5 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -30,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":negative_squared_cross_mark: Cross compile" @@ -48,7 +48,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" - label: ":testengine: Stress Tests" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 7ef89dd01163..850ef14f9bdf 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -30,7 +30,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":python: Python Integration Tests" @@ -39,7 +39,7 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":negative_squared_cross_mark: Cross compile" @@ -148,7 +148,7 @@ if are_conditions_met_packaging; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MASHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" From 9b3d894ec59dfd1a1c1eb157fc434a623fb83198 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 15:13:49 +0200 Subject: [PATCH 32/34] change checkLinuxType --- .buildkite/scripts/common.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 9d859c2cd588..8fab6b4c4d1f 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -122,9 +122,9 @@ with_go() { checkLinuxType() { if [ "${platform_type}" == "Linux" ]; then - if grep -q 'ubuntu' /etc/*release; then + if grep -q 'ubuntu' /etc/os-release; then echo "ubuntu" - elif grep -q 'rhel' /etc/*release; then + elif grep -q 'rhel' /etc/os-release; then echo "rhel" else echo "Unsupported Linux" From a6f7a12df6e36c3df332ff1bf5e1e49ef6d05d06 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 19:04:19 +0200 Subject: [PATCH 33/34] fix BK VARs --- .buildkite/scripts/common.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 8fab6b4c4d1f..5acd7c2fb250 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -8,13 +8,14 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" -runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" -runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" -runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" -runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" + +[ -z "$runLibbeat" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "$runMetricbeat" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" +[ -z "$runPacketbeat" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" +[ -z "$runLibBeatArmTest" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "$runPacketbeatArmTest" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" +[ -z "$runMetricbeatMacOsTests" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" +[ -z "$runPacketbeatMacOsTests" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" metricbeat_changeset=( "^metricbeat/.*" From 4019ce6d36f10146e9f448e23d9d0defc3f5c759 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 8 Feb 2024 19:14:21 +0200 Subject: [PATCH 34/34] fix BK VARs --- .buildkite/scripts/common.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 5acd7c2fb250..f51097091f4a 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,13 +9,13 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -[ -z "$runLibbeat" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -[ -z "$runMetricbeat" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" -[ -z "$runPacketbeat" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" -[ -z "$runLibBeatArmTest" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" -[ -z "$runPacketbeatArmTest" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" -[ -z "$runMetricbeatMacOsTests" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" -[ -z "$runPacketbeatMacOsTests" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" +[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" +[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" +[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" +[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" +[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" +[ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" metricbeat_changeset=( "^metricbeat/.*"