From 2b5dd7bf28cd79e8fcd30cf13cfcbc679b9e046f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 12:55:18 +0200 Subject: [PATCH 01/59] migrate xpack-metricbeat --- .buildkite/pipeline.yml | 55 +++++- .buildkite/scripts/common.sh | 18 ++ .../scripts/generate_packetbeat_pipeline.sh | 2 +- .../generate_xpack_metricbeat_pipeline.sh | 182 ++++++++++++++++++ .buildkite/x-pack/pipeline.xpack.libbeat.yml | 8 +- .../x-pack/pipeline.xpack.metricbeat.yml | 58 +++++- 6 files changed, 314 insertions(+), 9 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 34321b61161b..e07cfa87fe0f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,5 +1,56 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" - command: echo "Hello!" + - trigger: "beats-packetbeat" + label: ":esbuild: Downstream - beats-packetbeat" + key: "downstream-beats-packetbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-libbeat" + label: ":esbuild: Downstream - beats-libbeat" + key: "downstream-beats-libbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-metricbeat" + label: ":esbuild: Downstream - beats-metricbeat" + key: "downstream-beats-metricbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-winlogbeat" + label: ":esbuild: Downstream - beats-winlogbeat" + key: "downstream-beats-winlogbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-xpack-libbeat" + label: ":esbuild: Downstream - beats-xpack-libbeat" + key: "downstream-beats-xpack-libbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + + - trigger: "beats-xpack-metricbeat" + label: ":esbuild: Downstream - beats-xpack-metricbeat" + key: "downstream-beats-xpack-metricbeat" + async: true + build: + env: + BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b797ec718aa7..846738662175 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -17,6 +17,9 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" +[ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" +[ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" +[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" metricbeat_changeset=( "^metricbeat/.*" @@ -89,6 +92,7 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" @@ -101,6 +105,7 @@ check_and_set_beat_vars() { BEATS_GH_LABEL=${BEATS_PROJECT_NAME} TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_PROJECT_NAME" @@ -113,8 +118,10 @@ check_and_set_beat_vars() { BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" + BEATS_GH_AWS_COMMENT="${BEATS_GH_COMMENT} for aws cloud" BAETS_GH_MACOS_LABEL="macOS" BAETS_GH_ARM_LABEL="arm" + BAETS_GH_AWS_LABEL="aws" } with_docker_compose() { @@ -320,6 +327,17 @@ are_conditions_met_macos_tests() { return 1 } +re_conditions_met_aws_tests() { + if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_AWS_LABEL}" || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + return 0 + fi + fi + fi + return 1 +} + are_conditions_met_packaging() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 89ea7a33e20a..b35554ce6695 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -92,7 +92,7 @@ else exit 0 fi -if are_conditions_met_arm_tests && are_conditions_met_macos_tests; then +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh new file mode 100755 index 000000000000..fe5b5a8ca49f --- /dev/null +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-metricbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_cloud_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Cloud Tests" + key: "extended-cloud-test" + command: "cd $BEATS_PROJECT_NAME && mage build test" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 01695fa4fb65..5da3b6a3ac0c 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -20,7 +20,7 @@ steps: - input: "Input Parameters" key: "input-run-all-stages" fields: - - select: "Packetbeat - run_xpack_libbeat" + - select: "Libbeat - run_xpack_libbeat" key: "run_xpack_libbeat" options: - label: "True" @@ -28,7 +28,7 @@ steps: - label: "False" value: "false" default: "false" - - select: "Packetbeat - run_xpack_libbeat_arm_tests" + - select: "Libbeat - run_xpack_libbeat_arm_tests" key: "run_xpack_libbeat_arm_tests" options: - label: "True" @@ -42,8 +42,8 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic packetbeat pipeline" - key: "packetbeat-pipeline" + - label: ":linux: Load dynamic libbeat pipeline" + key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: - github_commit_status: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 34321b61161b..8f7b19ec3aaa 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -1,5 +1,59 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-metricbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch6" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Metricbeat - run_xpack_metricbeat" + key: "run_xpack_metricbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_macos_tests" + key: "run_xpack_metricbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Metricbeat - run_xpack_metricbeat_aws_tests" + key: "run_xpack_metricbeat_aws_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic metricbeat pipeline" + key: "metricbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From cf44101119051a59f2e7810dd1dde4debab472ac Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 13:29:47 +0200 Subject: [PATCH 02/59] change pre-command --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0ac7c51099c0..1096f7c34333 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" From e432525e19726ab2466beda5a05285eb9477a04a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:41:10 +0200 Subject: [PATCH 03/59] add mage_version --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/scripts/setenv.sh | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fe5b5a8ca49f..bb3d1ab91a93 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage build test" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 25121de212fd..921e8ee5f980 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -8,6 +8,7 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) +MAGE_VERSION="1.15.0" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From 0783eee4bfb56a43b35336a81af6a24ac1c5ec73 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:45:58 +0200 Subject: [PATCH 04/59] add mage_version --- .buildkite/scripts/setenv.sh | 1 + .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 921e8ee5f980..7464e30d86d3 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -16,6 +16,7 @@ export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION +export MAGE_VERSION exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 5da3b6a3ac0c..cf0494ffef88 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -42,7 +42,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic libbeat pipeline" + - label: ":linux: Load dynamic **x-pack libbeat** pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 8f7b19ec3aaa..88695cb176cb 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -51,7 +51,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic metricbeat pipeline" + - label: ":linux: Load dynamic **x-pack metricbeat** pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: From 2a1de268b0a7cadefdadef6431c16bcadc213995 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:46:29 +0200 Subject: [PATCH 05/59] disable some trigger steps --- .buildkite/pipeline.yml | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e07cfa87fe0f..c28e8d09bf30 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,41 +1,41 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - trigger: "beats-packetbeat" - label: ":esbuild: Downstream - beats-packetbeat" - key: "downstream-beats-packetbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-packetbeat" + # label: ":esbuild: Downstream - beats-packetbeat" + # key: "downstream-beats-packetbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-libbeat" - label: ":esbuild: Downstream - beats-libbeat" - key: "downstream-beats-libbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-libbeat" + # label: ":esbuild: Downstream - beats-libbeat" + # key: "downstream-beats-libbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-metricbeat" - label: ":esbuild: Downstream - beats-metricbeat" - key: "downstream-beats-metricbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-metricbeat" + # label: ":esbuild: Downstream - beats-metricbeat" + # key: "downstream-beats-metricbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-winlogbeat" - label: ":esbuild: Downstream - beats-winlogbeat" - key: "downstream-beats-winlogbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-winlogbeat" + # label: ":esbuild: Downstream - beats-winlogbeat" + # key: "downstream-beats-winlogbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - trigger: "beats-xpack-libbeat" label: ":esbuild: Downstream - beats-xpack-libbeat" From 7e55863013e3096dad24337d4b63d1960cab5080 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 14:59:06 +0200 Subject: [PATCH 06/59] change command with mage --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 2 +- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bb3d1ab91a93..7396d7e88c16 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage build test" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "mage $MAGE_VERSION && cd $BEATS_PROJECT_NAME && mage package" + command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index cf0494ffef88..0c745b1a09d6 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -42,7 +42,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic **x-pack libbeat** pipeline" + - label: ":linux: Load dynamic x-pack libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" notify: diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 88695cb176cb..a2ca41a620f9 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -51,7 +51,7 @@ steps: if: "build.source == 'ui'" allow_dependency_failure: false - - label: ":linux: Load dynamic **x-pack metricbeat** pipeline" + - label: ":linux: Load dynamic x-pack metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" notify: From cf55e5bf32e2836127e1e2f2da936ddae8a5df29 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 15:21:07 +0200 Subject: [PATCH 07/59] add ASDF_MAGE_VERSION --- .../scripts/generate_xpack_metricbeat_pipeline.sh | 14 +++++++------- .buildkite/scripts/setenv.sh | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 7396d7e88c16..d607572b9462 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,7 +17,7 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -26,7 +26,7 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -35,7 +35,7 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" @@ -129,7 +129,7 @@ if are_conditions_met_cloud_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION build test" + command: "cd $BEATS_PROJECT_NAME && mage build test" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -153,7 +153,7 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -163,7 +163,7 @@ if are_conditions_met_packaging; then - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage@$MAGE_VERSION package" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7464e30d86d3..eb374e71c875 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -8,7 +8,7 @@ DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) -MAGE_VERSION="1.15.0" +ASDF_MAGE_VERSION="1.15.0" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -16,7 +16,7 @@ export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION -export MAGE_VERSION +export ASDF_MAGE_VERSION exportVars() { local platform_type="$(uname)" From def5673254040db0e8194900b8722dc231da853d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 16:21:29 +0200 Subject: [PATCH 08/59] fix arm image name --- .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index a2ca41a620f9..eca0442fb11c 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -3,7 +3,7 @@ name: "beats-xpack-metricbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch6" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/general-windows-10" IMAGE_WIN_11: "family/general-windows-11" IMAGE_WIN_2016: "family/core-windows-2016" From 85bfec853512915c6a04dae6f8b68efe71ede6ef Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 21 Feb 2024 17:11:32 +0200 Subject: [PATCH 09/59] fix macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index d607572b9462..fe5b5a8ca49f 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -116,7 +116,7 @@ if are_conditions_met_macos_tests; then - label: ":mac: MacOS Unit Tests" key: "extended-macos-unit-tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: ".buildkite/scripts/unit_tests.sh" agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" From 5af9510b6a9eda5b4133f20e5edee0544519059a Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 10:58:22 +0200 Subject: [PATCH 10/59] test arm packaging --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fe5b5a8ca49f..790e73c3bb34 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -159,7 +159,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - label: ":linux: Packaging ARM" key: "packaging-arm" From 90bec63d1cf8d527b184e5ebac30230f7d7b4e84 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 12:22:16 +0200 Subject: [PATCH 11/59] test aws/macos --- .buildkite/scripts/common.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 846738662175..44cfab1a37bc 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -308,7 +308,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -318,8 +318,8 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -330,7 +330,7 @@ are_conditions_met_macos_tests() { re_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_AWS_LABEL}" || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi From bd038c2144983f46a22c3a31453814eb2fca51db Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 14:50:59 +0200 Subject: [PATCH 12/59] test macos --- .buildkite/scripts/generate_metricbeat_pipeline.sh | 8 ++++++-- .buildkite/scripts/generate_packetbeat_pipeline.sh | 4 ++-- .buildkite/scripts/generate_winlogbeat_pipeline.sh | 2 +- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 7 +++++-- .buildkite/scripts/setenv.sh | 4 ++++ .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index e91896eb70c5..15d0a737523d 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -111,6 +111,10 @@ else exit 0 fi +#---------------debug--------------- +echo "${GITHUB_PR_LABELS}" +#------------end of debug----------- + echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML @@ -149,7 +153,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -159,7 +163,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index b35554ce6695..97bdb531a6b0 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -150,7 +150,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -160,7 +160,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index 1eb1b459c921..ce812016e795 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -96,7 +96,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" YAML diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 790e73c3bb34..e1cf6d3f2774 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -110,6 +110,9 @@ if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then YAML fi +#---------------debug--------------- +echo "${GITHUB_PR_LABELS}" +#------------end of debug----------- if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML @@ -159,7 +162,7 @@ if are_conditions_met_packaging; then image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: "${PACKAGING_PLATFORMS}" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -169,7 +172,7 @@ if are_conditions_met_packaging; then imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "linux/arm64" + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" YAML diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index eb374e71c875..7bc0cfd16949 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -9,6 +9,8 @@ SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" +PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" +PACKAGING_ARM_PLATFORMS="linux/arm64" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -17,6 +19,8 @@ export SETUP_WIN_PYTHON_VERSION export NMAP_WIN_VERSION export GO_VERSION export ASDF_MAGE_VERSION +export PACKAGING_PLATFORMS +export PACKAGING_ARM_PLATFORMS exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index eca0442fb11c..17529a338589 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -18,7 +18,7 @@ env: steps: - input: "Input Parameters" - key: "input-run-all-stages" + key: "force-run-stages" fields: - select: "Metricbeat - run_xpack_metricbeat" key: "run_xpack_metricbeat" From 77b29b26983e23b68ac26db40af10af142e4b5c9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 15:23:48 +0200 Subject: [PATCH 13/59] test macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index e1cf6d3f2774..c501c9eecad6 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -101,7 +101,7 @@ else exit 0 fi -if are_conditions_met_cloud_tests || are_conditions_met_macos_tests; then +if are_conditions_met_aws_tests || are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML - group: "Extended Tests" From 5b41d3b3e99be1b333cd429b5daac87c60b75ebf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 15:29:11 +0200 Subject: [PATCH 14/59] test macos --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c501c9eecad6..bc1bde53a4e2 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -128,7 +128,7 @@ if are_conditions_met_macos_tests; then YAML fi -if are_conditions_met_cloud_tests; then +if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - label: ":linux: Cloud Tests" key: "extended-cloud-test" From 2751e38c5a3ce2556c34930dd019b754621ae9d6 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:03:36 +0200 Subject: [PATCH 15/59] test macos --- .buildkite/pipeline.yml | 32 +++++++++---------- .buildkite/scripts/common.sh | 2 +- .../generate_xpack_metricbeat_pipeline.sh | 2 +- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c28e8d09bf30..1bd21ef6acb5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -37,20 +37,20 @@ steps: # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-xpack-libbeat" - label: ":esbuild: Downstream - beats-xpack-libbeat" - key: "downstream-beats-xpack-libbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-xpack-libbeat" + # label: ":esbuild: Downstream - beats-xpack-libbeat" + # key: "downstream-beats-xpack-libbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - trigger: "beats-xpack-metricbeat" - label: ":esbuild: Downstream - beats-xpack-metricbeat" - key: "downstream-beats-xpack-metricbeat" - async: true - build: - env: - BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" + # - trigger: "beats-xpack-metricbeat" + # label: ":esbuild: Downstream - beats-xpack-metricbeat" + # key: "downstream-beats-xpack-metricbeat" + # async: true + # build: + # env: + # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" + # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 44cfab1a37bc..ee8cef670172 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -327,7 +327,7 @@ are_conditions_met_macos_tests() { return 1 } -re_conditions_met_aws_tests() { +are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_AWS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_AWS_LABEL} || "${!TRIGGER_SPECIFIC_AWS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index bc1bde53a4e2..a640349ae388 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -101,7 +101,7 @@ else exit 0 fi -if are_conditions_met_aws_tests || are_conditions_met_macos_tests; then +if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then cat >> $pipelineName <<- YAML - group: "Extended Tests" From 88800c82d858220f95fb87f5916cb75847d2a92b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:09:16 +0200 Subject: [PATCH 16/59] test macos --- .buildkite/pipeline.yml | 4 ++++ .buildkite/x-pack/pipeline.xpack.metricbeat.yml | 1 + 2 files changed, 5 insertions(+) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1bd21ef6acb5..c5dc4a978a21 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,10 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: + + - label: "Example test" + command: echo "Hello!" + # - trigger: "beats-packetbeat" # label: ":esbuild: Downstream - beats-packetbeat" # key: "downstream-beats-packetbeat" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 17529a338589..83a7fe886c8b 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -9,6 +9,7 @@ env: IMAGE_WIN_2016: "family/core-windows-2016" IMAGE_WIN_2019: "family/core-windows-2019" IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" From 4805ccb6a20ae680b0643381dea3345d2ed26e67 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Thu, 22 Feb 2024 16:29:48 +0200 Subject: [PATCH 17/59] fix macos declare --- .buildkite/scripts/common.sh | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index ee8cef670172..b33368f184c1 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -84,6 +84,32 @@ packaging_changeset=( ".go-version" ) +case "${BUILDKITE_PIPELINE_SLUG}" in + "beats-metricbeat") + BEAT_CHANGESET_REFERENCE=${metricbeat_changeset[@]} + ;; + "beats-libbeat") + BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} + ;; + "beats-packetbeat") + BEAT_CHANGESET_REFERENCE=${packetbeat_changeset[@]} + ;; + "beats-winlogbeat") + BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]} + ;; + "beats-xpack-libbeat") + BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} + ;; + "beats-xpack-metricbeat") + BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} + ;; + *) + echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + ;; +esac + + + check_and_set_beat_vars() { if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - @@ -94,7 +120,6 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" @@ -107,7 +132,6 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" - declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" echo "Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" From e2425682f71c6cc5606358b3c22c2646cf5bb9ab Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 09:34:42 +0200 Subject: [PATCH 18/59] add export MODULE --- .buildkite/scripts/common.sh | 18 +++++++++++++++--- .../scripts/generate_metricbeat_pipeline.sh | 4 ---- .../generate_xpack_metricbeat_pipeline.sh | 3 --- .buildkite/scripts/setenv.sh | 10 +++++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b33368f184c1..870677f07c20 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -379,18 +379,30 @@ config_git() { } if ! are_changed_only_paths "${docs_changeset[@]}" ; then - ONLY_DOCS="false" + export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." else echo "All changes are related to DOCS. ONLY_DOCS=$ONLY_DOCS." fi if are_paths_changed "${go_mod_changeset[@]}" ; then - GO_MOD_CHANGES="true" + export GO_MOD_CHANGES="true" fi if are_paths_changed "${packaging_changeset[@]}" ; then - PACKAGING_CHANGES="true" + export PACKAGING_CHANGES="true" +fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") + AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then + export MODULE="true" + else + export MODULE="false" + fi + + fi check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 15d0a737523d..477f8fb25a34 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -111,10 +111,6 @@ else exit 0 fi -#---------------debug--------------- -echo "${GITHUB_PR_LABELS}" -#------------end of debug----------- - echo "Check and add the Extended Tests into the pipeline" if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index a640349ae388..06ce06e03d04 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -110,9 +110,6 @@ if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then YAML fi -#---------------debug--------------- -echo "${GITHUB_PR_LABELS}" -#------------end of debug----------- if are_conditions_met_macos_tests; then cat >> $pipelineName <<- YAML diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7bc0cfd16949..7f455758b641 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,6 +11,8 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -47,8 +49,14 @@ exportVars() { fi } +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + exportVars + export RACE_DETECTOR="true" + export TEST_COVERAGE="true" + export DOCKER_PULL="0" +fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" From d41b68d929e5bc245664cb86937fd9b92918945c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 12:42:06 +0200 Subject: [PATCH 19/59] test aws --- .buildkite/scripts/common.sh | 9 +- .../generate_xpack_metricbeat_pipeline.sh | 126 +++++++++--------- 2 files changed, 68 insertions(+), 67 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 870677f07c20..e7ef3dae2d02 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -395,13 +395,14 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + AWS_MODULE_PATH_TRNSFORMED=$(echo "$directory" | sed 's/\//\\\//g') + AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then - export MODULE="true" + MODULE="aws" else - export MODULE="false" + MODULE="" fi - + export MODULE fi diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 06ce06e03d04..2bcdf8cebfd5 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -59,41 +59,41 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - group: "Extended Windows Tests" + # key: "extended-win-tests" + # steps: + # - label: ":windows: Windows 10 Unit Tests" + # key: "extended-win-10-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Windows 11 Unit Tests" + # key: "extended-win-11-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_11}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Win 2019 Unit Tests" + # key: "extended-win-2019-unit-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2019}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -143,34 +143,34 @@ echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" + # - wait: ~ + # depends_on: + # - step: "mandatory-tests" + # allow_failure: false + + # - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + # key: "packaging" + # steps: + # - label: ":linux: Packaging Linux" + # key: "packaging-linux" + # command: "cd $BEATS_PROJECT_NAME && mage package" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + # env: + # PLATFORMS: "${PACKAGING_PLATFORMS}" + + # - label: ":linux: Packaging ARM" + # key: "packaging-arm" + # command: "cd $BEATS_PROJECT_NAME && mage package" + # agents: + # provider: "aws" + # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + # instanceType: "${AWS_ARM_INSTANCE_TYPE}" + # env: + # PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + # PACKAGES: "docker" YAML fi From 531975317242253a1bf39a017c6ea3db38a9d76e Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 12:46:14 +0200 Subject: [PATCH 20/59] test aws --- .buildkite/scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index e7ef3dae2d02..21c48dbb2950 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -395,7 +395,7 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_TRNSFORMED=$(echo "$directory" | sed 's/\//\\\//g') + AWS_MODULE_PATH_TRNSFORMED=$(echo "$AWS_MODULE_PATH" | sed 's/\//\\\//g') AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then MODULE="aws" From 32229ce7bd8e2a64388790cc861ffb8561f1b56b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 13:59:34 +0200 Subject: [PATCH 21/59] add withModule --- .buildkite/scripts/common.sh | 33 +++++++++++++------ .../generate_xpack_metricbeat_pipeline.sh | 30 ++++++++--------- .buildkite/scripts/setenv.sh | 2 -- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 21c48dbb2950..da3fea4f89e3 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,6 +9,8 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -378,6 +380,26 @@ config_git() { fi } +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -394,16 +416,7 @@ if are_paths_changed "${packaging_changeset[@]}" ; then fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - AWS_MODULE_PATH=("x-pack/metricbeat/module/aws") - AWS_MODULE_PATH_TRNSFORMED=$(echo "$AWS_MODULE_PATH" | sed 's/\//\\\//g') - AWS_MODULE_PATH_EXCLUDE=("^(${AWS_MODULE_PATH_TRNSFORMED}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${AWS_MODULE_PATH[@]}" && ! are_changed_only_paths "${AWS_MODULE_PATH_EXCLUDE[@]}"; then - MODULE="aws" - else - MODULE="" - fi - export MODULE - + withModule "x-pack/metricbeat/module/aws" fi check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 2bcdf8cebfd5..19bd87fbb8eb 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -42,21 +42,21 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1" + # key: "mandatory-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_2016}" + # - "${IMAGE_WIN_2022}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h # - group: "Extended Windows Tests" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 7f455758b641..d85c0e9c0eaf 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,8 +11,6 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From a09803e99651ea6ed4e7ce7a6874e7cef219fac4 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:13:15 +0200 Subject: [PATCH 22/59] move withModule to pre-command --- .buildkite/scripts/common.sh | 55 ------------------------------------ .buildkite/scripts/setenv.sh | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 55 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index da3fea4f89e3..a3e2c7aa5572 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -293,37 +293,6 @@ retry() { return 0 } -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then - return 0 - fi - return 1 -} - are_conditions_met_mandatory_tests() { if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 @@ -380,26 +349,6 @@ config_git() { fi } -withModule() { - local module_path=$1 - if [[ "$module_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local module_name="${module_path#*module/}" - local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') - local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" - local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE="${module_name}" - else - MODULE="" - fi - echo "MODULE=$MODULE" - export MODULE -} - if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -415,8 +364,4 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "x-pack/metricbeat/module/aws" -fi - check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index d85c0e9c0eaf..dc3eb4c26e8d 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -47,6 +47,57 @@ exportVars() { fi } +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + fi + return 1 +} + +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" @@ -60,3 +111,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export TEST_COVERAGE="true" export DOCKER_PULL="0" fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + withModule "x-pack/metricbeat/module/aws" +fi From 271c788e0a165afae1174179c620f0a55887ac78 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:16:50 +0200 Subject: [PATCH 23/59] move withModule to pre-command --- .buildkite/scripts/common.sh | 2 -- .buildkite/scripts/setenv.sh | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index a3e2c7aa5572..309886d973f3 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,8 +9,6 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dc3eb4c26e8d..119728b18093 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,6 +11,8 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION From cd2282de13c1022803a29d9386a413f2def05304 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:24:12 +0200 Subject: [PATCH 24/59] move withModule to command --- .buildkite/scripts/common.sh | 57 ++++++++++++++++++++++++++++++++++++ .buildkite/scripts/setenv.sh | 53 --------------------------------- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 309886d973f3..da3fea4f89e3 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,6 +9,8 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} +OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" +XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -291,6 +293,37 @@ retry() { return 0 } +are_paths_changed() { + local patterns=("${@}") + local changelist=() + for pattern in "${patterns[@]}"; do + changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) + if [ "${#changed_files[@]}" -gt 0 ]; then + changelist+=("${changed_files[@]}") + fi + done + + if [ "${#changelist[@]}" -gt 0 ]; then + echo "Files changed:" + echo "${changelist[*]}" + return 0 + else + echo "No files changed within specified changeset:" + echo "${patterns[*]}" + return 1 + fi +} + +are_changed_only_paths() { + local patterns=("${@}") + local changelist=() + local changed_files=$(git diff --name-only HEAD@{1} HEAD) + if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + return 0 + fi + return 1 +} + are_conditions_met_mandatory_tests() { if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 @@ -347,6 +380,26 @@ config_git() { fi } +withModule() { + local module_path=$1 + if [[ "$module_path" == *"x-pack/"* ]]; then + local pattern=("$XPACK_MODULE_PATTERN") + else + local pattern=("$OSS_MODULE_PATTERN") + fi + local module_name="${module_path#*module/}" + local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') + local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" + local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") + if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then + MODULE="${module_name}" + else + MODULE="" + fi + echo "MODULE=$MODULE" + export MODULE +} + if ! are_changed_only_paths "${docs_changeset[@]}" ; then export ONLY_DOCS="false" echo "Changes include files outside the docs_changeset vairiabe. ONLY_DOCS=$ONLY_DOCS." @@ -362,4 +415,8 @@ if are_paths_changed "${packaging_changeset[@]}" ; then export PACKAGING_CHANGES="true" fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + withModule "x-pack/metricbeat/module/aws" +fi + check_and_set_beat_vars diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 119728b18093..97ca2102c845 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,8 +11,6 @@ GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" -OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" -XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -49,57 +47,6 @@ exportVars() { fi } -are_paths_changed() { - local patterns=("${@}") - local changelist=() - for pattern in "${patterns[@]}"; do - changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) - if [ "${#changed_files[@]}" -gt 0 ]; then - changelist+=("${changed_files[@]}") - fi - done - - if [ "${#changelist[@]}" -gt 0 ]; then - echo "Files changed:" - echo "${changelist[*]}" - return 0 - else - echo "No files changed within specified changeset:" - echo "${patterns[*]}" - return 1 - fi -} - -are_changed_only_paths() { - local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then - return 0 - fi - return 1 -} - -withModule() { - local module_path=$1 - if [[ "$module_path" == *"x-pack/"* ]]; then - local pattern=("$XPACK_MODULE_PATTERN") - else - local pattern=("$OSS_MODULE_PATTERN") - fi - local module_name="${module_path#*module/}" - local module_path_transformed=$(echo "$module_path" | sed 's/\//\\\//g') - local module_path_exclussion="((?!^${module_path_transformed}\\/).)*\$" - local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") - if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then - MODULE="${module_name}" - else - MODULE="" - fi - echo "MODULE=$MODULE" - export MODULE -} - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then exportVars export RACE_DETECTOR="true" From 8eba9dce5b1f2def25250aeb48a67800fb19e07b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 14:31:46 +0200 Subject: [PATCH 25/59] fix withModule --- .buildkite/scripts/common.sh | 1 - .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 6 ++++++ .buildkite/scripts/setenv.sh | 4 ---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index da3fea4f89e3..2d938dbc6a83 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -397,7 +397,6 @@ withModule() { MODULE="" fi echo "MODULE=$MODULE" - export MODULE } if ! are_changed_only_paths "${docs_changeset[@]}" ; then diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 19bd87fbb8eb..40723f63c91e 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -27,6 +27,8 @@ steps: - label: ":go: Go Integration Tests" key: "mandatory-int-test" command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -36,6 +38,8 @@ steps: - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -130,6 +134,8 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: "cd $BEATS_PROJECT_NAME && mage build test" + env: + MODULE: "$MODULE" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 97ca2102c845..d85c0e9c0eaf 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -60,7 +60,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then export TEST_COVERAGE="true" export DOCKER_PULL="0" fi - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - withModule "x-pack/metricbeat/module/aws" -fi From 1781fc004b0fa4ee03b5cfba826794e2f4fc281b Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 19:39:33 +0200 Subject: [PATCH 26/59] test aws changes --- .buildkite/scripts/common.sh | 2 +- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 2d938dbc6a83..7855a84a9917 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -393,7 +393,7 @@ withModule() { local exclude=("^(${module_path_transformed}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)") if are_paths_changed "${pattern[@]}" && ! are_changed_only_paths "${exclude[@]}"; then MODULE="${module_name}" - else + elif [ -d "${module_path}" ]; then MODULE="" fi echo "MODULE=$MODULE" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 40723f63c91e..fc506b69d8af 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -139,7 +139,7 @@ if are_conditions_met_aws_tests; then agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml index 5401ba81f419..262bf7bf87b9 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml @@ -9,6 +9,7 @@ input: name: - BucketSizeBytes - NumberOfObjects + - TestName processors: - rename: From c12fe5ab52774bec32fb73597f8ef3b13a51a755 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Fri, 23 Feb 2024 19:44:13 +0200 Subject: [PATCH 27/59] test aws changes --- x-pack/metricbeat/module/aws/variables.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index e4b95d471b2c..ea5ecaea96b7 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,3 +20,7 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } + +variable "TESTVAR" { + default = "unknown-repo-name" +} From dd1e93ebe8bc6eb2d9056dfbf4c99813c066ac43 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 24 Feb 2024 12:06:05 +0200 Subject: [PATCH 28/59] change the are_changed_only_paths --- .buildkite/scripts/common.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 7855a84a9917..b5e57e1b5483 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -316,9 +316,15 @@ are_paths_changed() { are_changed_only_paths() { local patterns=("${@}") - local changelist=() - local changed_files=$(git diff --name-only HEAD@{1} HEAD) - if [ -z "$changed_files" ] || grep -qE "$(IFS=\|; echo "${patterns[*]}")" <<< "$changed_files"; then + local changed_files=($(git diff --name-only HEAD@{1} HEAD)) + local matched_files=() + for pattern in "${patterns[@]}"; do + local matched=($(grep -E "${pattern}" <<< "${changed_files[@]}")) + if [ "${#matched[@]}" -gt 0 ]; then + matched_files+=("${matched[@]}") + fi + done + if [ "${#matched_files[@]}" -eq "${#changed_files[@]}" ] || [ "${#changed_files[@]}" -eq 0 ]; then return 0 fi return 1 From d0f433b9708261bcf6599e94310a3f2ee5949eed Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Sat, 24 Feb 2024 12:35:07 +0200 Subject: [PATCH 29/59] revert aws changes and test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml | 1 - x-pack/metricbeat/module/aws/variables.tf | 4 ---- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index fc506b69d8af..76352b206b64 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -131,7 +131,7 @@ fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" + - label: ":linux: ${MODULE^^} Cloud Tests" key: "extended-cloud-test" command: "cd $BEATS_PROJECT_NAME && mage build test" env: diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml index 262bf7bf87b9..5401ba81f419 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/manifest.yml @@ -9,7 +9,6 @@ input: name: - BucketSizeBytes - NumberOfObjects - - TestName processors: - rename: diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf index ea5ecaea96b7..e4b95d471b2c 100644 --- a/x-pack/metricbeat/module/aws/variables.tf +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -20,7 +20,3 @@ variable "ENVIRONMENT" { variable "REPO" { default = "unknown-repo-name" } - -variable "TESTVAR" { - default = "unknown-repo-name" -} From 5696ea284baff2351a03678b05b4cec45dfd9ebf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 13:42:27 +0200 Subject: [PATCH 30/59] test with 240min timeout --- .buildkite/scripts/common.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b5e57e1b5483..073997ab3f04 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -110,8 +110,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in ;; esac - - check_and_set_beat_vars() { if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - From c3f04b34c39310364082ea4fad4a99cd1a370eeb Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 15:36:14 +0200 Subject: [PATCH 31/59] migrate-xpack-packetbeat --- .buildkite/hooks/pre-command | 11 +- .buildkite/scripts/common.sh | 7 +- .../generate_xpack_packetbeat_pipeline.sh | 191 ++++++++++++++++++ .buildkite/scripts/win_unit_tests.ps1 | 20 +- .../x-pack/pipeline.xpack.packetbeat.yml | 60 +++++- 5 files changed, 280 insertions(+), 9 deletions(-) create mode 100644 .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1096f7c34333..3baee823592b 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,6 +2,8 @@ set -euo pipefail +CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -12,10 +14,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" exit 0 fi fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then + # Set GCP credentials + GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) + echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json + export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) +fi diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 073997ab3f04..722ef9e35172 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -22,6 +22,9 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")" [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" +[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" +[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" + metricbeat_changeset=( "^metricbeat/.*" @@ -337,7 +340,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -348,7 +351,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BAETS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh new file mode 100644 index 000000000000..0ebb18329577 --- /dev/null +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.xpack-metricbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":linux: Ubuntu System Tests" + key: "mandatory-linux-system-test" + command: "cd $BEATS_PROJECT_NAME && mage systemTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2022 System Tests" + key: "mandatory-win-2022-system-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 System Tests" + key: "extended-win-10-system-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index b3c5c58fac0a..32167ba98284 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -1,3 +1,7 @@ +param( + [string]$testType +) + $ErrorActionPreference = "Stop" # set -e $WorkFolder = $env:BEATS_PROJECT_NAME $WORKSPACE = Get-Location @@ -142,10 +146,18 @@ $env:MAGEFILE_CACHE = $magefile New-Item -ItemType Directory -Force -Path "build" -if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { - mage -w reader/etw build goUnitTest -} else { - mage build unitTest +if ($testType -eq "unittests") { + if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { + mage -w reader/etw build goUnitTest + } else { + mage build unitTest + } +} +elseif ($testType -eq "systemtests") { + mage systemTest +} +else { + Write-Host "Unknown test type. Please specify 'unittests' or 'systemtests'." } $EXITCODE=$LASTEXITCODE diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 34321b61161b..750b59e716d8 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -1,5 +1,61 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-packetbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/packetbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "force-run-stages" + fields: + - select: "Packetbeat - run_xpack_packetbeat" + key: "run_xpack_packetbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_packetbeat_macos_tests" + key: "run_xpack_packetbeat_macos_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_packetbeat_arm_tests" + key: "run_xpack_packetbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic x-pack packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" From 420cd1bd80d35a3af2c25e6b87526fddd7ecd373 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 15:44:43 +0200 Subject: [PATCH 32/59] add retry --- .buildkite/hooks/pre-command | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 3baee823592b..ec1f8f32d069 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -4,6 +4,25 @@ set -euo pipefail CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh From 2711e89e5bfec7cea745b0cdf4421389adf77173 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 18:22:36 +0200 Subject: [PATCH 33/59] comment out GCP auth and update dynamic pipelines --- .buildkite/hooks/pre-command | 13 +- .../scripts/generate_metricbeat_pipeline.sh | 8 +- .../scripts/generate_packetbeat_pipeline.sh | 8 +- .../scripts/generate_winlogbeat_pipeline.sh | 6 +- .../generate_xpack_libbeat_pipeline.sh | 8 +- .../generate_xpack_metricbeat_pipeline.sh | 190 +++++++++--------- 6 files changed, 119 insertions(+), 114 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ec1f8f32d069..aae64af02699 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -41,9 +41,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then - # Set GCP credentials - GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) - echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json - export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) -fi +# uncomement when PR is merged https://github.com/elastic/ci/pull/2704 +# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then +# # Set GCP credentials +# GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) +# echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json +# export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) +# fi diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 477f8fb25a34..c3ddd9efb0d3 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -52,7 +52,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -75,7 +75,7 @@ steps: steps: - label: ":windows: Windows 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -86,7 +86,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -97,7 +97,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 97bdb531a6b0..91484b190a31 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -35,7 +35,7 @@ steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -55,7 +55,7 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -66,7 +66,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -77,7 +77,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index ce812016e795..ec72e75adfe3 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -26,7 +26,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -51,7 +51,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -62,7 +62,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh index 66f0750ab6fa..903acc0bbe91 100755 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -43,7 +43,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -64,7 +64,7 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -75,7 +75,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -86,7 +86,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 76352b206b64..c84fb09bbc21 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -46,58 +46,58 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # key: "mandatory-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_2016}" - # - "${IMAGE_WIN_2022}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - # - group: "Extended Windows Tests" - # key: "extended-win-tests" - # steps: - # - label: ":windows: Windows 10 Unit Tests" - # key: "extended-win-10-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Windows 11 Unit Tests" - # key: "extended-win-11-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_11}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Win 2019 Unit Tests" - # key: "extended-win-2019-unit-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2019}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -105,7 +105,9 @@ else exit 0 fi -if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +#TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved +# if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +if [[ are_conditions_met_macos_tests ]]; then cat >> $pipelineName <<- YAML - group: "Extended Tests" @@ -129,54 +131,56 @@ if are_conditions_met_macos_tests; then YAML fi -if are_conditions_met_aws_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ${MODULE^^} Cloud Tests" - key: "extended-cloud-test" - command: "cd $BEATS_PROJECT_NAME && mage build test" - env: - MODULE: "$MODULE" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" +#TODO: uncomment the commented-out below step when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -YAML -fi +# if are_conditions_met_aws_tests; then +# cat >> $pipelineName <<- YAML +# - label: ":linux: ${MODULE^^} Cloud Tests" +# key: "extended-cloud-test" +# command: "cd $BEATS_PROJECT_NAME && mage build test" +# env: +# MODULE: "$MODULE" +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi echo "Check and add the Packaging into the pipeline" if are_conditions_met_packaging; then cat >> $pipelineName <<- YAML - # - wait: ~ - # depends_on: - # - step: "mandatory-tests" - # allow_failure: false - - # - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - # key: "packaging" - # steps: - # - label: ":linux: Packaging Linux" - # key: "packaging-linux" - # command: "cd $BEATS_PROJECT_NAME && mage package" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - # env: - # PLATFORMS: "${PACKAGING_PLATFORMS}" - - # - label: ":linux: Packaging ARM" - # key: "packaging-arm" - # command: "cd $BEATS_PROJECT_NAME && mage package" - # agents: - # provider: "aws" - # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - # instanceType: "${AWS_ARM_INSTANCE_TYPE}" - # env: - # PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - # PACKAGES: "docker" + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" YAML fi From dfe71d021ac7026d8c73edae91f872f38d632e4d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Mon, 26 Feb 2024 18:47:15 +0200 Subject: [PATCH 34/59] add reference fot xpack_packetbeat_changeset --- .buildkite/scripts/common.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 722ef9e35172..112781927e0d 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -108,6 +108,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-metricbeat") BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]} ;; + "beats-xpack-packetbeat") + BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]} + ;; *) echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; From 91e7176c696dea2189901727f8e0f0ec076fdf98 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 13:03:15 +0200 Subject: [PATCH 35/59] test --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index c84fb09bbc21..0c013889ecd3 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -62,7 +62,7 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h +## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: From d48fd2093ec8c1f77b9f62902ce6073ca922986d Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 14:24:15 +0200 Subject: [PATCH 36/59] change pre-command --- .buildkite/hooks/pre-command | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index aae64af02699..ec1f8f32d069 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -41,10 +41,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -# uncomement when PR is merged https://github.com/elastic/ci/pull/2704 -# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then -# # Set GCP credentials -# GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) -# echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json -# export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) -# fi +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then + # Set GCP credentials + GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) + echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json + export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) +fi From ebf072a2b07107252b256a8b75ae6689b104eaca Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 16:36:49 +0200 Subject: [PATCH 37/59] update magefile.go and GCP credentials --- .buildkite/hooks/pre-command | 15 ++++++++++++--- x-pack/packetbeat/magefile.go | 10 +++++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ec1f8f32d069..bc0792bdfbf8 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,7 +2,8 @@ set -euo pipefail -CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" +# CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" retry() { local retries=$1 @@ -41,9 +42,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi +# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then +# # Set GCP credentials +# GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) +# echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json +# export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) +# fi + + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then # Set GCP credentials - GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) - echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ./gcp.json export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) fi diff --git a/x-pack/packetbeat/magefile.go b/x-pack/packetbeat/magefile.go index 03104ab9157e..357e5e235855 100644 --- a/x-pack/packetbeat/magefile.go +++ b/x-pack/packetbeat/magefile.go @@ -172,6 +172,13 @@ func SystemTest(ctx context.Context) error { return devtools.GoTest(ctx, args) } +func getBucketName() string { + if os.Getenv("BUILDKITE") == "true" { + return "ingest-buildkite-ci" + } + return "obs-ci-cache" +} + // getNpcapInstaller gets the installer from the Google Cloud Storage service. // // On Windows platforms, if getNpcapInstaller is invoked with the environment variables @@ -198,7 +205,8 @@ func getNpcapInstaller() error { return err } } + ciBucketName := getBucketName() fmt.Printf("getting %s from private cache\n", installer) - return sh.RunV("gsutil", "cp", "gs://obs-ci-cache/private/"+installer, dstPath) + return sh.RunV("gsutil", "cp", "gs://"+ciBucketName+"/private/"+installer, dstPath) } From 577a435ffdf6841a1cec67e05f048e63cf789a66 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 17:07:35 +0200 Subject: [PATCH 38/59] update pre-command --- .buildkite/hooks/pre-command | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index bc0792bdfbf8..7886723777d9 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -4,6 +4,16 @@ set -euo pipefail # CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" +TMP_FOLDER_TEMPLATE_BASE="tmp.beats" +WORKSPACE="$(pwd)" + +google_cloud_auth() { + local secretFileLocation + secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json + echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} + gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null + export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation} +} retry() { local retries=$1 @@ -53,6 +63,5 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then # Set GCP credentials PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) - echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ./gcp.json - export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) + google_cloud_auth fi From 7c9fcd8337b0ece31cce5b8dec30852c99371ecf Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 18:19:38 +0200 Subject: [PATCH 39/59] test --- .buildkite/hooks/pre-command | 7 +- .../generate_xpack_packetbeat_pipeline.sh | 286 +++++++++--------- 2 files changed, 147 insertions(+), 146 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7886723777d9..306ead06a6a0 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -4,14 +4,13 @@ set -euo pipefail # CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -TMP_FOLDER_TEMPLATE_BASE="tmp.beats" WORKSPACE="$(pwd)" google_cloud_auth() { local secretFileLocation - secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json + secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "tmp.beats.XXXXXXXXX")/google-cloud-credentials.json echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} - gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null + #gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation} } @@ -61,7 +60,9 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == "kibana-discover" || "$BUILDKITE_STEP_KEY" == "refresh-k8s-tmpls" || "$BUILDKITE_STEP_KEY" =~ ^sync-bundled-packages ]]; then # Set GCP credentials PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + export PRIVATE_CI_GCS_CREDENTIALS_SECRET google_cloud_auth fi diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 0ebb18329577..ff25d859ba87 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -15,49 +15,49 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":linux: Ubuntu System Tests" - key: "mandatory-linux-system-test" - command: "cd $BEATS_PROJECT_NAME && mage systemTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" - key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":linux: Ubuntu Unit Tests" + # key: "mandatory-linux-unit-test" + # command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + # - label: ":linux: Ubuntu System Tests" + # key: "mandatory-linux-system-test" + # command: "cd $BEATS_PROJECT_NAME && mage systemTest" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + # - label: ":rhel: RHEL-9 Unit Tests" + # key: "mandatory-rhel9-unit-test" + # command: ".buildkite/scripts/unit_tests.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_RHEL9_X86_64}" + # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + # key: "mandatory-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_2016}" + # - "${IMAGE_WIN_2022}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2022 System Tests" key: "mandatory-win-2022-system-tests" @@ -76,33 +76,33 @@ steps: key: "extended-win-tests" steps: - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" - key: "extended-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_10}" - - "${IMAGE_WIN_11}" - - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 System Tests" - key: "extended-win-10-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + # key: "extended-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_10}" + # - "${IMAGE_WIN_11}" + # - "${IMAGE_WIN_2019}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Windows 10 System Tests" + # key: "extended-win-10-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -110,79 +110,79 @@ else exit 0 fi -if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - -YAML -fi +# if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then +# cat >> $pipelineName <<- YAML + +# - group: "Extended Tests" +# key: "extended-tests" +# steps: + +# YAML +# fi + +# if are_conditions_met_macos_tests; then +# cat >> $pipelineName <<- YAML + +# - label: ":mac: MacOS Unit Tests" +# key: "extended-macos-unit-tests" +# command: ".buildkite/scripts/unit_tests.sh" +# agents: +# provider: "orka" +# imagePrefix: "${IMAGE_MACOS_X86_64}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# if are_conditions_met_arm_tests; then +# cat >> $pipelineName <<- YAML +# - label: ":linux: ARM Ubuntu Unit Tests" +# key: "extended-arm64-unit-test" +# command: "cd $BEATS_PROJECT_NAME && mage build unitTest" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +# YAML +# fi + +# echo "Check and add the Packaging into the pipeline" +# if are_conditions_met_packaging; then +# cat >> $pipelineName <<- YAML + +# - wait: ~ +# depends_on: +# - step: "mandatory-tests" +# allow_failure: false + +# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 +# key: "packaging" +# steps: +# - label: ":linux: Packaging Linux" +# key: "packaging-linux" +# command: "cd $BEATS_PROJECT_NAME && mage package" +# agents: +# provider: "gcp" +# image: "${IMAGE_UBUNTU_X86_64}" +# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +# env: +# PLATFORMS: "${PACKAGING_PLATFORMS}" + +# - label: ":linux: Packaging ARM" +# key: "packaging-arm" +# command: "cd $BEATS_PROJECT_NAME && mage package" +# agents: +# provider: "aws" +# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" +# instanceType: "${AWS_ARM_INSTANCE_TYPE}" +# env: +# PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" +# PACKAGES: "docker" + +# YAML +# fi echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public cat $pipelineName From 3de4774b2faf5d026924663c179a9ffe347e60a8 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 18:24:07 +0200 Subject: [PATCH 40/59] test --- .buildkite/hooks/pre-command | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 306ead06a6a0..fff46d0a6c80 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -60,7 +60,6 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then - if [[ "$BUILDKITE_STEP_KEY" == "kibana-discover" || "$BUILDKITE_STEP_KEY" == "refresh-k8s-tmpls" || "$BUILDKITE_STEP_KEY" =~ ^sync-bundled-packages ]]; then # Set GCP credentials PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET From fc5d80a5d64ae4860c09317f7916095a4da9c06f Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 27 Feb 2024 18:30:25 +0200 Subject: [PATCH 41/59] test --- .../generate_xpack_packetbeat_pipeline.sh | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index ff25d859ba87..00756ad08601 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -72,37 +72,37 @@ steps: ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - - group: "Extended Windows Tests" - key: "extended-win-tests" - steps: - - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" - # key: "extended-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_10}" - # - "${IMAGE_WIN_11}" - # - "${IMAGE_WIN_2019}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Windows 10 System Tests" - # key: "extended-win-10-system-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # - group: "Extended Windows Tests" + # key: "extended-win-tests" + # steps: + + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + # key: "extended-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_10}" + # - "${IMAGE_WIN_11}" + # - "${IMAGE_WIN_2019}" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + # - label: ":windows: Windows 10 System Tests" + # key: "extended-win-10-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else From 86bf1742916170e4e1c9100f7c4eea2dbd07c7bb Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 08:38:26 +0200 Subject: [PATCH 42/59] update windows script --- .buildkite/hooks/pre-command | 11 ----------- .buildkite/scripts/win_unit_tests.ps1 | 8 ++++++++ 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index fff46d0a6c80..280d430c95b9 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,17 +2,7 @@ set -euo pipefail -# CI_GCP_OBS_PATH="kv/ci-shared/observability-ingest/cloud/gcp" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" -WORKSPACE="$(pwd)" - -google_cloud_auth() { - local secretFileLocation - secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "tmp.beats.XXXXXXXXX")/google-cloud-credentials.json - echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation} - #gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null - export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation} -} retry() { local retries=$1 @@ -63,5 +53,4 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then # Set GCP credentials PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET - google_cloud_auth fi diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index 32167ba98284..7134fe3cdfc8 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -125,6 +125,13 @@ function withNmap($version) { Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait } +function google_cloud_auth { + $secretFileLocation = New-TemporaryFile -Path $env:WORKSPACE -Name "google-cloud-credentials.json" + Set-Content -Path $secretFileLocation.FullName -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET + gcloud auth activate-service-account --key-file $secretFileLocation.FullName > $null 2>&1 + $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation.FullName +} + fixCRLF withGolang $env:GO_VERSION @@ -154,6 +161,7 @@ if ($testType -eq "unittests") { } } elseif ($testType -eq "systemtests") { + google_cloud_auth mage systemTest } else { From fbc07039f2bce1a4317909b46c5cacea3239e24c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 10:08:51 +0200 Subject: [PATCH 43/59] update windows script --- .buildkite/scripts/win_unit_tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index 7134fe3cdfc8..df5155026d05 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -126,7 +126,7 @@ function withNmap($version) { } function google_cloud_auth { - $secretFileLocation = New-TemporaryFile -Path $env:WORKSPACE -Name "google-cloud-credentials.json" + $secretFileLocation = New-TemporaryFile -Name "google-cloud-credentials.json" -Directory $env:WORKSPACE Set-Content -Path $secretFileLocation.FullName -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET gcloud auth activate-service-account --key-file $secretFileLocation.FullName > $null 2>&1 $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation.FullName From a1588868ee44078d490002eb44757ba8700f8484 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 10:44:59 +0200 Subject: [PATCH 44/59] update windows script --- .buildkite/scripts/win_unit_tests.ps1 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index df5155026d05..1a39e90f9213 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -124,12 +124,13 @@ function withNmap($version) { } Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait } - function google_cloud_auth { - $secretFileLocation = New-TemporaryFile -Name "google-cloud-credentials.json" -Directory $env:WORKSPACE - Set-Content -Path $secretFileLocation.FullName -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET - gcloud auth activate-service-account --key-file $secretFileLocation.FullName > $null 2>&1 - $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation.FullName + $tempFileName = "google-cloud-credentials.json" + $secretFileLocation = Join-Path $env:TEMP $tempFileName + $null = New-Item -ItemType File -Path $secretFileLocation + Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET + gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1 + $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation } fixCRLF From 781181ba91853f78e7419e3dbf711ede881ea2be Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 11:56:26 +0200 Subject: [PATCH 45/59] update windows script --- .../generate_xpack_packetbeat_pipeline.sh | 294 +++++++++--------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 00756ad08601..0ebb18329577 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -15,49 +15,49 @@ steps: - group: "Mandatory Tests" key: "mandatory-tests" steps: - # - label: ":linux: Ubuntu Unit Tests" - # key: "mandatory-linux-unit-test" - # command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - # - label: ":linux: Ubuntu System Tests" - # key: "mandatory-linux-system-test" - # command: "cd $BEATS_PROJECT_NAME && mage systemTest" - # agents: - # provider: "gcp" - # image: "${IMAGE_UBUNTU_X86_64}" - # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - # - label: ":rhel: RHEL-9 Unit Tests" - # key: "mandatory-rhel9-unit-test" - # command: ".buildkite/scripts/unit_tests.sh" - # agents: - # provider: "gcp" - # image: "${IMAGE_RHEL9_X86_64}" - # machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" - # key: "mandatory-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_2016}" - # - "${IMAGE_WIN_2022}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":linux: Ubuntu System Tests" + key: "mandatory-linux-system-test" + command: "cd $BEATS_PROJECT_NAME && mage systemTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":rhel: RHEL-9 Unit Tests" + key: "mandatory-rhel9-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2022 System Tests" key: "mandatory-win-2022-system-tests" @@ -72,37 +72,37 @@ steps: ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - # - group: "Extended Windows Tests" - # key: "extended-win-tests" - # steps: - - # - label: ":windows: Windows Unit Tests - {{matrix.image}}" - # command: ".buildkite/scripts/win_unit_tests.ps1 unittests" - # key: "extended-win-unit-tests" - # agents: - # provider: "gcp" - # image: "{{matrix.image}}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # matrix: - # setup: - # image: - # - "${IMAGE_WIN_10}" - # - "${IMAGE_WIN_11}" - # - "${IMAGE_WIN_2019}" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - # - label: ":windows: Windows 10 System Tests" - # key: "extended-win-10-system-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + key: "extended-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 System Tests" + key: "extended-win-10-system-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else @@ -110,79 +110,79 @@ else exit 0 fi -# if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then -# cat >> $pipelineName <<- YAML - -# - group: "Extended Tests" -# key: "extended-tests" -# steps: - -# YAML -# fi - -# if are_conditions_met_macos_tests; then -# cat >> $pipelineName <<- YAML - -# - label: ":mac: MacOS Unit Tests" -# key: "extended-macos-unit-tests" -# command: ".buildkite/scripts/unit_tests.sh" -# agents: -# provider: "orka" -# imagePrefix: "${IMAGE_MACOS_X86_64}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# if are_conditions_met_arm_tests; then -# cat >> $pipelineName <<- YAML -# - label: ":linux: ARM Ubuntu Unit Tests" -# key: "extended-arm64-unit-test" -# command: "cd $BEATS_PROJECT_NAME && mage build unitTest" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "${AWS_ARM_INSTANCE_TYPE}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -# YAML -# fi - -# echo "Check and add the Packaging into the pipeline" -# if are_conditions_met_packaging; then -# cat >> $pipelineName <<- YAML - -# - wait: ~ -# depends_on: -# - step: "mandatory-tests" -# allow_failure: false - -# - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 -# key: "packaging" -# steps: -# - label: ":linux: Packaging Linux" -# key: "packaging-linux" -# command: "cd $BEATS_PROJECT_NAME && mage package" -# agents: -# provider: "gcp" -# image: "${IMAGE_UBUNTU_X86_64}" -# machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -# env: -# PLATFORMS: "${PACKAGING_PLATFORMS}" - -# - label: ":linux: Packaging ARM" -# key: "packaging-arm" -# command: "cd $BEATS_PROJECT_NAME && mage package" -# agents: -# provider: "aws" -# imagePrefix: "${IMAGE_UBUNTU_ARM_64}" -# instanceType: "${AWS_ARM_INSTANCE_TYPE}" -# env: -# PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" -# PACKAGES: "docker" - -# YAML -# fi +if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: ARM Ubuntu Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then + cat >> $pipelineName <<- YAML + + - wait: ~ + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + +YAML +fi echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public cat $pipelineName From e202bb73b108106817bf7151bdd7d2fa5678cefd Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 15:24:35 +0200 Subject: [PATCH 46/59] full test --- .../generate_xpack_packetbeat_pipeline.sh | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 0ebb18329577..d9f305b26a8f 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -59,16 +59,17 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - label: ":windows: Windows 2022 System Tests" - key: "mandatory-win-2022-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + # - label: ":windows: Windows 2022 System Tests" + # key: "mandatory-win-2022-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2022}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -93,16 +94,17 @@ steps: - "${IMAGE_WIN_2019}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - label: ":windows: Windows 10 System Tests" - key: "extended-win-10-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + # - label: ":windows: Windows 10 System Tests" + # key: "extended-win-10-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else From 67ff7996049f5d630d6778cc8f11ea15744f7c04 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 15:39:08 +0200 Subject: [PATCH 47/59] update windows script and dynamic pipelines --- .buildkite/hooks/pre-command | 8 +++++--- .buildkite/scripts/generate_packetbeat_pipeline.sh | 8 ++++---- .buildkite/scripts/generate_winlogbeat_pipeline.sh | 6 +++--- .buildkite/scripts/generate_xpack_libbeat_pipeline.sh | 8 ++++---- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 8 ++++---- .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh | 8 ++++---- .buildkite/scripts/win_unit_tests.ps1 | 8 ++++---- 7 files changed, 28 insertions(+), 26 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 280d430c95b9..8fc84481dffc 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -50,7 +50,9 @@ fi if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then - # Set GCP credentials - PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) - export PRIVATE_CI_GCS_CREDENTIALS_SECRET + if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then + # Set GCP credentials + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + export PRIVATE_CI_GCS_CREDENTIALS_SECRET + fi fi diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 91484b190a31..97bdb531a6b0 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -35,7 +35,7 @@ steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -55,7 +55,7 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -66,7 +66,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -77,7 +77,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index ec72e75adfe3..ce812016e795 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -26,7 +26,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -51,7 +51,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -62,7 +62,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh index 903acc0bbe91..66f0750ab6fa 100755 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -43,7 +43,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -64,7 +64,7 @@ steps: steps: - label: ":windows: Win 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -75,7 +75,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -86,7 +86,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_11}" diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 0c013889ecd3..61313067e015 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -47,7 +47,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -68,7 +68,7 @@ steps: steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -79,7 +79,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -90,7 +90,7 @@ steps: - label: ":windows: Win 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index d9f305b26a8f..e197be7ed6f5 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -44,7 +44,7 @@ steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -62,7 +62,7 @@ steps: ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 2022 System Tests" # key: "mandatory-win-2022-system-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" # agents: # provider: "gcp" # image: "${IMAGE_WIN_2022}" @@ -78,7 +78,7 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "extended-win-unit-tests" agents: provider: "gcp" @@ -97,7 +97,7 @@ steps: ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 10 System Tests" # key: "extended-win-10-system-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1 systemtests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" # agents: # provider: "gcp" # image: "${IMAGE_WIN_10}" diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index 1a39e90f9213..def9df01d2f7 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -1,5 +1,5 @@ param( - [string]$testType + [string]$testType = "unittest" ) $ErrorActionPreference = "Stop" # set -e @@ -154,19 +154,19 @@ $env:MAGEFILE_CACHE = $magefile New-Item -ItemType Directory -Force -Path "build" -if ($testType -eq "unittests") { +if ($testType -eq "unittest") { if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { mage -w reader/etw build goUnitTest } else { mage build unitTest } } -elseif ($testType -eq "systemtests") { +elseif ($testType -eq "systemtest") { google_cloud_auth mage systemTest } else { - Write-Host "Unknown test type. Please specify 'unittests' or 'systemtests'." + Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." } $EXITCODE=$LASTEXITCODE From 2a1eb07181e0a94ad22c25ca373bbedd97b0c077 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 28 Feb 2024 16:34:52 +0200 Subject: [PATCH 48/59] cleanup --- .buildkite/hooks/pre-command | 9 ---- .buildkite/pipeline.yml | 54 ------------------- .../scripts/generate_metricbeat_pipeline.sh | 8 +-- 3 files changed, 4 insertions(+), 67 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8fc84481dffc..e5e6651c0454 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -41,17 +41,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S fi fi -# if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then -# # Set GCP credentials -# GOOGLE_APPLICATION_GCP_SECRET=$(retry 5 vault kv get -format=json -field=data ${CI_GCP_OBS_PATH}) -# echo "${GOOGLE_APPLICATION_GCP_SECRET}" > ./gcp.json -# export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ./gcp.json) -# fi - - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then - # Set GCP credentials PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) export PRIVATE_CI_GCS_CREDENTIALS_SECRET fi diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c5dc4a978a21..c9153448be23 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -4,57 +4,3 @@ steps: - label: "Example test" command: echo "Hello!" - - # - trigger: "beats-packetbeat" - # label: ":esbuild: Downstream - beats-packetbeat" - # key: "downstream-beats-packetbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-libbeat" - # label: ":esbuild: Downstream - beats-libbeat" - # key: "downstream-beats-libbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-metricbeat" - # label: ":esbuild: Downstream - beats-metricbeat" - # key: "downstream-beats-metricbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-winlogbeat" - # label: ":esbuild: Downstream - beats-winlogbeat" - # key: "downstream-beats-winlogbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-xpack-libbeat" - # label: ":esbuild: Downstream - beats-xpack-libbeat" - # key: "downstream-beats-xpack-libbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" - - # - trigger: "beats-xpack-metricbeat" - # label: ":esbuild: Downstream - beats-xpack-metricbeat" - # key: "downstream-beats-xpack-metricbeat" - # async: true - # build: - # env: - # BUILDKITE_REFSPEC: "refs/pull/${BUILDKITE_PULL_REQUEST}/head" - # BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}" diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index c3ddd9efb0d3..477f8fb25a34 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -52,7 +52,7 @@ steps: artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -75,7 +75,7 @@ steps: steps: - label: ":windows: Windows 2019 Unit Tests" key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -86,7 +86,7 @@ steps: - label: ":windows: Windows 10 Unit Tests" key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_10}" @@ -97,7 +97,7 @@ steps: - label: ":windows: Windows 11 Unit Tests" key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 unittests" + command: ".buildkite/scripts/win_unit_tests.ps1" agents: provider: "gcp" image: "${IMAGE_WIN_11}" From 74a4d1f4d91c571429a51fb458c82983d0249eeb Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 18:32:46 +0200 Subject: [PATCH 49/59] update setenv.sh --- .buildkite/scripts/setenv.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 30dbbb3532af..dd3ff9125f8a 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -11,15 +11,12 @@ SETUP_WIN_PYTHON_VERSION="3.11.0" NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) ASDF_MAGE_VERSION="1.15.0" -<<<<<<< HEAD PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" -======= ASDF_TERRAFORM_VERSION="1.0.2" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" AWS_REGION="eu-central-1" ->>>>>>> 843010c1cbddd201bb4c08a79fd649a0af3c04ce export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION @@ -30,14 +27,11 @@ export GO_VERSION export ASDF_MAGE_VERSION export PACKAGING_PLATFORMS export PACKAGING_ARM_PLATFORMS -<<<<<<< HEAD -======= export REPO export TMP_FOLDER export DOCKER_REGISTRY export ASDF_TERRAFORM_VERSION export AWS_REGION ->>>>>>> 843010c1cbddd201bb4c08a79fd649a0af3c04ce exportVars() { local platform_type="$(uname)" @@ -65,16 +59,7 @@ exportVars() { } if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then -<<<<<<< HEAD - exportVars - export RACE_DETECTOR="true" - export TEST_COVERAGE="true" - export DOCKER_PULL="0" -fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then -======= ->>>>>>> 843010c1cbddd201bb4c08a79fd649a0af3c04ce exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" From 70bf4de04f6284e5e3f926ccfbeaa046125830c9 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 18:33:12 +0200 Subject: [PATCH 50/59] solving merge conflicts --- .buildkite/scripts/setenv.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dd3ff9125f8a..e0929bc9f71c 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -59,7 +59,6 @@ exportVars() { } if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then - exportVars export RACE_DETECTOR="true" export TEST_COVERAGE="true" From 426014fac0fef98a61c7955737f78902509d189c Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 18:38:46 +0200 Subject: [PATCH 51/59] conflict solving --- .buildkite/hooks/pre-command | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 0ac7c51099c0..7d90b6142227 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,6 +2,27 @@ set -euo pipefail +AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" + +retry() { + local retries=$1 + shift + local count=0 + until "$@"; do + exit=$? + wait=$((2 ** count)) + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + >&2 echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + >&2 echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh source .buildkite/env-scripts/util.sh @@ -12,10 +33,17 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" exit 0 fi fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP_KEY" == "extended-cloud-test" ]]; then + BEATS_AWS_SECRET_KEY=$(retry 5 vault kv get -field secret_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_SECRET_KEY + BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) + export BEATS_AWS_ACCESS_KEY +fi From 1a839aaad91c4254854c9f2e9388dc8f58e8a295 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 12 Mar 2024 18:39:52 +0200 Subject: [PATCH 52/59] conflict solving --- .buildkite/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c9153448be23..34321b61161b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json steps: - - label: "Example test" command: echo "Hello!" From 5543e57b64fbb14facca10495fe0dadd48ec1a70 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 07:46:23 +0200 Subject: [PATCH 53/59] fix dinamic pipeline's condition --- .buildkite/scripts/generate_xpack_metricbeat_pipeline.sh | 2 +- .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 13f8b2fb7256..8fc074aee4ca 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -106,7 +106,7 @@ else fi #TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved -if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then +if are_conditions_met_aws_tests || are_conditions_met_macos_tests ; then cat >> $pipelineName <<- YAML - group: "Extended Tests" diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index e197be7ed6f5..e7190ed67260 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -112,7 +112,7 @@ else exit 0 fi -if [[ are_conditions_met_arm_tests || are_conditions_met_macos_tests ]]; then +if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then cat >> $pipelineName <<- YAML - group: "Extended Tests" From 554c3a5f7455f066456ed8892d2ecb7ba4783793 Mon Sep 17 00:00:00 2001 From: sharbuz <87968844+sharbuz@users.noreply.github.com> Date: Wed, 13 Mar 2024 08:10:58 +0200 Subject: [PATCH 54/59] Update .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh Co-authored-by: Victor Martinez --- .buildkite/scripts/generate_xpack_packetbeat_pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index e7190ed67260..a076bd00f996 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -4,7 +4,7 @@ source .buildkite/scripts/common.sh set -euo pipefail -pipelineName="pipeline.xpack-metricbeat-dynamic.yml" +pipelineName="pipeline.xpack-packetbeat-dynamic.yml" echo "Add the mandatory and extended tests without additional conditions into the pipeline" if are_conditions_met_mandatory_tests; then From d02580b612334c9b2875a1d3c3a8bda71216ba83 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 12:17:43 +0200 Subject: [PATCH 55/59] add gcp_credentials_cleanup --- .../generate_xpack_metricbeat_pipeline.sh | 8 ++------ .../generate_xpack_packetbeat_pipeline.sh | 2 ++ .buildkite/scripts/setenv.sh | 2 -- .buildkite/scripts/win_unit_tests.ps1 | 19 +++++++++++++++++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index 8fc074aee4ca..af116b17209d 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -31,8 +31,6 @@ steps: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":python: Python Integration Tests" @@ -42,8 +40,6 @@ steps: provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - label: ":windows: Windows Unit Tests - {{matrix.image}}" @@ -139,8 +135,6 @@ if are_conditions_met_aws_tests; then provider: "gcp" image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML @@ -165,6 +159,8 @@ if are_conditions_met_packaging; then provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index e7190ed67260..ec510915c20f 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -169,6 +169,8 @@ if are_conditions_met_packaging; then provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index e0929bc9f71c..29a8a05446ec 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -14,8 +14,6 @@ ASDF_MAGE_VERSION="1.15.0" PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" PACKAGING_ARM_PLATFORMS="linux/arm64" ASDF_TERRAFORM_VERSION="1.0.2" -PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -PACKAGING_ARM_PLATFORMS="linux/arm64" AWS_REGION="eu-central-1" export SETUP_GVM_VERSION diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index def9df01d2f7..205398db7e74 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -133,6 +133,17 @@ function google_cloud_auth { $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation } +function google_cloud_auth_cleanup { + if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { + Write-Host "$env:GOOGLE_APPLICATION_CREDENTIALS" # debug + Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force + Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS + Write-Host "$env:GOOGLE_APPLICATION_CREDENTIALS" # debug + } else { + Write-Host "No GCP credentials were added" + } +} + fixCRLF withGolang $env:GO_VERSION @@ -162,8 +173,12 @@ if ($testType -eq "unittest") { } } elseif ($testType -eq "systemtest") { - google_cloud_auth - mage systemTest + try { + google_cloud_auth + mage systemTest + } finally { + google_cloud_auth_cleanup + } } else { Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." From b23146a27a25647aa823d4326973aa3745290651 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 12:42:20 +0200 Subject: [PATCH 56/59] test removing gcp credentials --- .../generate_xpack_packetbeat_pipeline.sh | 22 +++++++++---------- .buildkite/scripts/win_unit_tests.ps1 | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 4eb2a1c3e049..efa7d873ff80 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -59,17 +59,17 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved - # - label: ":windows: Windows 2022 System Tests" - # key: "mandatory-win-2022-system-tests" - # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2022}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + - label: ":windows: Windows 2022 System Tests" + key: "mandatory-win-2022-system-tests" + command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index 205398db7e74..5be8a7604dab 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -135,10 +135,10 @@ function google_cloud_auth { function google_cloud_auth_cleanup { if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { - Write-Host "$env:GOOGLE_APPLICATION_CREDENTIALS" # debug + Write-Host $env:GOOGLE_APPLICATION_CREDENTIALS # debug Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS - Write-Host "$env:GOOGLE_APPLICATION_CREDENTIALS" # debug + Write-Host $env:GOOGLE_APPLICATION_CREDENTIALS # debug } else { Write-Host "No GCP credentials were added" } From 29546185a75ac52170279d7dcb032d55bcefa266 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 13:11:07 +0200 Subject: [PATCH 57/59] add gcp credentials --- .buildkite/hooks/pre-command | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 7d90b6142227..21062e416455 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -47,3 +47,10 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" && "$BUILDKITE_STEP BEATS_AWS_ACCESS_KEY=$(retry 5 vault kv get -field access_key ${AWS_SERVICE_ACCOUNT_SECRET_PATH}) export BEATS_AWS_ACCESS_KEY fi + +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then + if [[ "$BUILDKITE_STEP_KEY" == "extended-win-10-system-tests" || "$BUILDKITE_STEP_KEY" == "mandatory-win-2022-system-tests" ]]; then + PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) + export PRIVATE_CI_GCS_CREDENTIALS_SECRET + fi +fi From e2e12d13396757d5908068bf37b85b2eb726a884 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 13:17:18 +0200 Subject: [PATCH 58/59] add gcp credentials --- .buildkite/hooks/pre-command | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 21062e416455..c23f5c3af645 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,6 +3,7 @@ set -euo pipefail AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" +PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" retry() { local retries=$1 From 37b956a18ca5f0aeb0330d120689a97587914c80 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Wed, 13 Mar 2024 14:21:26 +0200 Subject: [PATCH 59/59] remove debug and cleanup --- .../generate_xpack_packetbeat_pipeline.sh | 22 +++++++++---------- .buildkite/scripts/win_unit_tests.ps1 | 2 -- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index efa7d873ff80..4eb2a1c3e049 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -59,17 +59,17 @@ steps: - "${IMAGE_WIN_2022}" artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - # TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved - - label: ":windows: Windows 2022 System Tests" - key: "mandatory-win-2022-system-tests" - command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved + # - label: ":windows: Windows 2022 System Tests" + # key: "mandatory-win-2022-system-tests" + # command: ".buildkite/scripts/win_unit_tests.ps1 systemtest" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2022}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index 5be8a7604dab..b61e4107c3cb 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -135,10 +135,8 @@ function google_cloud_auth { function google_cloud_auth_cleanup { if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { - Write-Host $env:GOOGLE_APPLICATION_CREDENTIALS # debug Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS - Write-Host $env:GOOGLE_APPLICATION_CREDENTIALS # debug } else { Write-Host "No GCP credentials were added" }