From 21ebdae0b950000b9e80de065daad3b2ca279eba Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 11 Apr 2024 00:22:42 +0200 Subject: [PATCH 01/11] [CI] x-pack/heartbeat monorepo --- .buildkite/pipeline.yml | 25 ++ .buildkite/pull-requests.json | 16 -- .buildkite/scripts/common.sh | 11 +- .../generate_xpack_heartbeat_pipeline.sh | 213 ------------------ .../x-pack/pipeline.xpack.heartbeat.yml | 210 ++++++++++++++--- 5 files changed, 204 insertions(+), 271 deletions(-) delete mode 100755 .buildkite/scripts/generate_xpack_heartbeat_pipeline.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 8515ebc76527..abfaaeaefb0a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -46,3 +46,28 @@ steps: env: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + + - label: "Trigger x-pack/Heartbeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only HEAD~1" + watch: + - path: + - x-pack/heartbeat/ + - .buildkite/x-pack/pipeline.xpack.heartbeat.yml + #XPACK + - go.mod + - pytest.ini + - dev-tools/ + - libbeat/ + - testing/ + - x-pack/libbeat/ + config: + trigger: "beats-xpack-heartbeat" + label: ":package: x-pack/heartbeat pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 1cc18f0f0b1d..639120750a4a 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -224,22 +224,6 @@ "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": ["^x-pack/filebeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] }, - { - "enabled": true, - "pipelineSlug": "beats-xpack-heartbeat", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": [ ], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^/test x-pack/heartbeat$", - "always_trigger_comment_regex": "^/test x-pack/heartbeat$", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ ], - "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"] - }, { "enabled": true, "pipelineSlug": "beats-xpack-osquerybeat", diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 07de21a2dbca..419c58ca6382 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -24,7 +24,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" [ -z "${run_xpack_dockerlogbeat+x}" ] && run_xpack_dockerlogbeat="$(buildkite-agent meta-data get run_xpack_dockerlogbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" -[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat @@ -44,7 +43,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" [ -z "${run_xpack_osquerybeat_macos_tests+x}" ] && run_xpack_osquerybeat_macos_tests="$(buildkite-agent meta-data get run_xpack_osquerybeat_macos_tests --default "false")" # define if needed run Windows platform-specific tests for the particular beat @@ -83,10 +81,6 @@ xpack_dockerlogbeat_changeset=( "^x-pack/dockerlogbeat/.*" ) -xpack_heartbeat_changeset=( - "^x-pack/heartbeat/.*" - ) - xpack_filebeat_changeset=( "^x-pack/filebeat/.*" ) @@ -167,9 +161,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in "beats-xpack-filebeat") BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]} ;; - "beats-xpack-heartbeat") - BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]} - ;; "beats-xpack-libbeat") BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]} ;; @@ -435,7 +426,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh deleted file mode 100755 index 94eed4f7c6a7..000000000000 --- a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.xpack-heartbeat-dynamic.yml" - -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML -steps: - - group: "Mandatory Tests" - key: "mandatory-tests" - steps: - - label: ":linux: Ubuntu Unit Tests" - key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - - label: ":go: Go Integration Tests" - key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - - ####### skip: "see elastic/beats#23957 and elastic/beats#23958" - # - label: ":windows: Windows 2016 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "mandatory-win-2016-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2016}" - # machine_type: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - # - label: ":windows: Windows 2022 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "mandatory-win-2022-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2022}" - # machine_type: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - - # - group: "Extended Windows Tests" ## TODO: this condition will be changed in the Phase 3 - # key: "extended-win-tests" - # steps: - # - label: ":windows: Windows 10 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-10-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - # - label: ":windows: Windows 11 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-11-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_11}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - # - label: ":windows: Windows 2019 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-2019-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2019}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - ####### skip: "see elastic/beats#23957 and elastic/beats#23958" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - group: "Extended Tests" - key: "extended-tests" - steps: - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then - cat >> $pipelineName <<- YAML - - wait: ~ - depends_on: - - step: "mandatory-tests" - allow_failure: false - - - group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589 - key: "packaging" - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -yq . -P -e $pipelineName || (echo "Yaml formatting error, below is the YAML"; cat $pipelineName; exit 1) - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 1b04fcf99225..f2a64227c7da 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -26,38 +26,184 @@ env: ASDF_MAGE_VERSION: 1.15.0 steps: + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - - input: "Input Parameters" - key: "force-run-stages" - fields: - - select: "Heartbeat - run_xpack_heartbeat" - key: "run_xpack_heartbeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "Heartbeat - run_xpack_heartbeat_macos_tests" - key: "run_xpack_heartbeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - if: "build.source == 'ui'" + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" + + ####### skip: "see elastic/beats#23957 and elastic/beats#23958" + # - label: ":windows: Windows 2016 Unit Tests" + # command: | + # Set-Location -Path $BEATS_PROJECT_NAME + # mage build unitTest + # key: "mandatory-win-2016-unit-tests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2016}" + # machine_type: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" + # notify: + # - github_commit_status: + # context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + + # - label: ":windows: Windows 2022 Unit Tests" + # command: | + # Set-Location -Path $BEATS_PROJECT_NAME + # mage build unitTest + # key: "mandatory-win-2022-unit-tests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2022}" + # machine_type: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" + # notify: + # - github_commit_status: + # context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + + # - group: "Extended Windows Tests" ## TODO: this condition will be changed in the Phase 3 + # key: "extended-win-tests" + # if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ + # steps: + # - label: ":windows: Windows 10 Unit Tests" + # command: | + # Set-Location -Path $BEATS_PROJECT_NAME + # mage build unitTest + # key: "extended-win-10-unit-tests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_10}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" + # notify: + # - github_commit_status: + # context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + + # - label: ":windows: Windows 11 Unit Tests" + # command: | + # Set-Location -Path $BEATS_PROJECT_NAME + # mage build unitTest + # key: "extended-win-11-unit-tests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_11}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" + # notify: + # - github_commit_status: + # context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" + + # - label: ":windows: Windows 2019 Unit Tests" + # command: | + # Set-Location -Path $BEATS_PROJECT_NAME + # mage build unitTest + # key: "extended-win-2019-unit-tests" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2019}" + # machineType: "${GCP_WIN_MACHINE_TYPE}" + # disk_size: 100 + # disk_type: "pd-ssd" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" + # notify: + # - github_commit_status: + # context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + ####### skip: "see elastic/beats#23957 and elastic/beats#23958" + + - group: "MacOs Extended Tests" + key: "extended-tests-macos" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: + - label: ":mac: MacOS Unit Tests" + key: "extended-macos-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic x-pack heartbeat pipeline" - key: "xpack-heartbeat-pipeline" - command: ".buildkite/scripts/generate_xpack_heartbeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + depends_on: + - step: "mandatory-tests" + allow_failure: false + + - group: "Packaging" + key: "packaging" + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" From 22aeeed501ce179dadf660ccb36053bcbd7683ae Mon Sep 17 00:00:00 2001 From: Pavel Zorin Date: Thu, 11 Apr 2024 00:35:22 +0200 Subject: [PATCH 02/11] notification --- .buildkite/x-pack/pipeline.xpack.heartbeat.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index f2a64227c7da..68db552d87f0 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -25,6 +25,10 @@ env: #Deps ASDF_MAGE_VERSION: 1.15.0 +notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME" + steps: - group: "Mandatory Tests" key: "mandatory-tests" From ebdcb4db196eb75234ca3686e4436e78f381b5cd Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:02:59 +0300 Subject: [PATCH 03/11] fix trigger --- .buildkite/pipeline.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e2ed71136134..72ff9f846e2a 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -286,15 +286,17 @@ steps: - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} - - label: "Trigger x-pack/Heartbeat" + - label: "Trigger x-pack/heartbeat" plugins: - monorepo-diff#v1.0.1: - diff: "git diff --name-only HEAD~1" + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" watch: - path: - x-pack/heartbeat/ - .buildkite/x-pack/pipeline.xpack.heartbeat.yml - #XPACK + - .buildkite/scripts/ + - .buildkite/hooks/ + # x-pack - go.mod - pytest.ini - dev-tools/ @@ -303,10 +305,10 @@ steps: - x-pack/libbeat/ config: trigger: "beats-xpack-heartbeat" - label: ":package: x-pack/heartbeat pipeline" build: commit: "${BUILDKITE_COMMIT}" branch: "${BUILDKITE_BRANCH}" env: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} From bbb1c924e3fa8f3cb246e7329941fdde28db2c16 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:27:12 +0300 Subject: [PATCH 04/11] pipeline and script fixes --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/common.sh | 4 +- .buildkite/scripts/install_macos_tools.sh | 4 + .buildkite/scripts/setenv.sh | 13 +- .../x-pack/pipeline.xpack.heartbeat.yml | 298 ++++++++++-------- 5 files changed, 174 insertions(+), 147 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index d8862d9cc949..41c695d032c6 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat"]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index b2a7b720d5a9..9ce1071c8088 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -19,7 +19,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" -[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" @@ -33,7 +32,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")" [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")" -[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" @@ -375,7 +373,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index 124025b4a5ad..ba376fe75423 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -133,3 +133,7 @@ with_go "${GO_VERSION}" with_mage with_python config_git + +# prevent "OSError: [Errno 24] Too many open files" on macOS +ulimit -Sn 150000 +echo "~~~ Setting ulimit: $(ulimit)" diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f6b03f53f9bd..9de97285c3db 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -49,18 +49,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle" fi -if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then +if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then source .buildkite/scripts/common.sh - if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then - export ELASTIC_SYNTHETICS_CAPABLE=true - else - # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" - fi -fi - -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. - source .buildkite/scripts/common.sh defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" + fi fi diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 68db552d87f0..1186ca3061fb 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -3,10 +3,11 @@ name: "beats-xpack-heartbeat" env: AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/heartbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" @@ -18,166 +19,195 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + # pipeline specific + ELASTIC_SYNTHETICS_CAPABLE: "true" - #Deps + # Other deps ASDF_MAGE_VERSION: 1.15.0 - -notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME" + ASDF_NODEJS_VERSION: 18.17.1 steps: - - group: "Mandatory Tests" - key: "mandatory-tests" + - group: "x-pack/heartbeat Mandatory Tests" + key: "x-pack-heartbeat-mandatory-tests" steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: | + set -euo pipefail + cd x-pack/heartbeat + echo "~~~ Installing @elastic/synthetics" + npm i -g @elastic/synthetics + echo "~~~ Running tests" + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" notify: - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + context: "x-pack/heartbeat: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + command: | + set -euo pipefail + cd x-pack/heartbeat + echo "~~~ Installing @elastic/synthetics" + npm i -g @elastic/synthetics + echo "~~~ Running tests" + cd x-pack/heartbeat + mage goIntegTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: Go Integration Tests" + + - label: ":windows: Windows 2016 Unit Tests" + key: "mandatory-win-2016-unit-tests" + skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" + command: | + Set-Location -Path x-pack/heartbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + key: "mandatory-win-2022-unit-tests" + skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" + command: | + Set-Location -Path x-pack/heartbeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" notify: - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - - ####### skip: "see elastic/beats#23957 and elastic/beats#23958" - # - label: ":windows: Windows 2016 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "mandatory-win-2016-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2016}" - # machine_type: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - - # - label: ":windows: Windows 2022 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "mandatory-win-2022-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2022}" - # machine_type: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - - # - group: "Extended Windows Tests" ## TODO: this condition will be changed in the Phase 3 - # key: "extended-win-tests" - # if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ - # steps: - # - label: ":windows: Windows 10 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-10-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_10}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - - # - label: ":windows: Windows 11 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-11-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_11}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - - # - label: ":windows: Windows 2019 Unit Tests" - # command: | - # Set-Location -Path $BEATS_PROJECT_NAME - # mage build unitTest - # key: "extended-win-2019-unit-tests" - # agents: - # provider: "gcp" - # image: "${IMAGE_WIN_2019}" - # machineType: "${GCP_WIN_MACHINE_TYPE}" - # disk_size: 100 - # disk_type: "pd-ssd" - # artifact_paths: - # - "$BEATS_PROJECT_NAME/build/*.xml" - # - "$BEATS_PROJECT_NAME/build/*.json" - # notify: - # - github_commit_status: - # context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - ####### skip: "see elastic/beats#23957 and elastic/beats#23958" - - - group: "MacOs Extended Tests" - key: "extended-tests-macos" + context: "x-pack/heartbeat: Windows 2022 Unit Tests" + + - group: "x-pack/heartbeat Extended Windows Tests" + key: "x-pack-heartbeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" + steps: + - label: ":windows: Windows 10 Unit Tests" + command: | + Set-Location -Path x-pack/heartbeat + mage build test + key: "extended-win-10-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + command: | + Set-Location -Path x-pack/heartbeat + mage build test + key: "extended-win-11-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: Windows 11 Unit Tests" + + - label: ":windows: Windows 2019 Unit Tests" + command: | + Set-Location -Path x-pack/heartbeat + mage build test + key: "extended-win-2019-unit-tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: Windows 2019 Unit Tests" + + - group: "x-pack/heartbeat MacOS Extended Tests" + key: "x-pack-heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ steps: - - label: ":mac: MacOS Unit Tests" - key: "extended-macos-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + - label: ":mac: MacOS x86_64 Unit Tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/heartbeat + mage build unitTest agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" + notify: + - github_commit_status: + context: "x-pack/heartbeat: MacOS x86_64 Extended Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd x-pack/heartbeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "x-pack/heartbeat/build/*.xml" + - "x-pack/heartbeat/build/*.json" notify: - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + context: "x-pack/heartbeat: MacOS arm64 Extended Tests" - wait: ~ + if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - - step: "mandatory-tests" - allow_failure: false + - step: "x-pack-heartbeat-mandatory-tests" - group: "Packaging" key: "packaging" @@ -185,7 +215,9 @@ steps: steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: | + cd x-pack/heartbeat + mage package agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" @@ -193,21 +225,23 @@ steps: disk_size: 100 disk_type: "pd-ssd" env: - PLATFORMS: "${PACKAGING_PLATFORMS}" + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" notify: - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" + context: "x-pack/heartbeat: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" + command: | + cd x-pack/heartbeat + mage package agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + context: "x-pack/heartbeat: Packaging Linux ARM" From 54a660f2484e557caf692a329e92cac218a71ebb Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:33:47 +0300 Subject: [PATCH 05/11] typo --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 41c695d032c6..ab608b9ddde2 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat"]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") From 86b1d375c9050c654a2b1c0a7530d98fd9957241 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:37:06 +0300 Subject: [PATCH 06/11] fix script bug in setenv --- .buildkite/scripts/setenv.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index 9de97285c3db..e68308b58573 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -53,5 +53,4 @@ if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_K source .buildkite/scripts/common.sh # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines. defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}" - fi fi From 7bcbfb1270a601a4296f92c5bfb64784b4f0ad53 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:41:50 +0300 Subject: [PATCH 07/11] fix paths --- .buildkite/x-pack/pipeline.xpack.heartbeat.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 1186ca3061fb..9b714bd52f3d 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -34,9 +34,8 @@ steps: key: "mandatory-linux-unit-test" command: | set -euo pipefail - cd x-pack/heartbeat echo "~~~ Installing @elastic/synthetics" - npm i -g @elastic/synthetics + npm install -g @elastic/synthetics echo "~~~ Running tests" mage build unitTest agents: @@ -54,9 +53,8 @@ steps: key: "mandatory-int-test" command: | set -euo pipefail - cd x-pack/heartbeat echo "~~~ Installing @elastic/synthetics" - npm i -g @elastic/synthetics + npm install -g @elastic/synthetics echo "~~~ Running tests" cd x-pack/heartbeat mage goIntegTest From 377f407468eb6c82a305aa4a3c600ca76667fd70 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 18:53:30 +0300 Subject: [PATCH 08/11] fix cd --- .buildkite/x-pack/pipeline.xpack.heartbeat.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 9b714bd52f3d..e2762f5d524d 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -37,6 +37,7 @@ steps: echo "~~~ Installing @elastic/synthetics" npm install -g @elastic/synthetics echo "~~~ Running tests" + cd x-pack/heartbeat mage build unitTest agents: provider: "gcp" From ab4fd311088714f52725a868ddd2aae1985b93b5 Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Wed, 17 Apr 2024 19:08:30 +0300 Subject: [PATCH 09/11] restore heartbeat in pre-command --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index ab608b9ddde2..d8862d9cc949 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then source .buildkite/env-scripts/env.sh if [[ -z "${GO_VERSION-""}" ]]; then export GO_VERSION=$(cat "${WORKSPACE}/.go-version") From 8fdec96bf6ffff3c0c0af7571e2d8071634efbfe Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 18 Apr 2024 10:57:23 +0300 Subject: [PATCH 10/11] add nodejs on macos --- .buildkite/x-pack/pipeline.xpack.heartbeat.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index e2762f5d524d..107dfa65f1b2 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -175,6 +175,8 @@ steps: command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh + withNodeJSEnv $ASDF_NODEJS_VERSION + installNodeJsDependencies cd x-pack/heartbeat mage build unitTest agents: @@ -191,6 +193,8 @@ steps: command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh + withNodeJSEnv $ASDF_NODEJS_VERSION + installNodeJsDependencies cd x-pack/heartbeat mage build unitTest agents: From c18fa184a0e2c5ec7f87a017b82fefe0bf7fc56a Mon Sep 17 00:00:00 2001 From: Dimitrios Liappis Date: Thu, 18 Apr 2024 11:24:36 +0300 Subject: [PATCH 11/11] unfold ulimit in bk output --- .buildkite/scripts/install_macos_tools.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh index ba376fe75423..4572afb4cdc5 100755 --- a/.buildkite/scripts/install_macos_tools.sh +++ b/.buildkite/scripts/install_macos_tools.sh @@ -137,3 +137,4 @@ config_git # prevent "OSError: [Errno 24] Too many open files" on macOS ulimit -Sn 150000 echo "~~~ Setting ulimit: $(ulimit)" +echo "~~~ Resuming commands"