From 10a74e25060dfe441f5a4d8dc6916644f46644c6 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 3 May 2024 12:36:50 +0300 Subject: [PATCH 1/7] Auto trigger DRA for all 8.x branches (#39386) (#39388) Now that DRA support via Buildkite has also been backported to 8.13[^1] this commit widens the whitelisted branches that can trigger Buildkite DRA builds to include all 8.x branches. Relates: https://github.com/elastic/ingest-dev/issues/3095 [^1]: https://github.com/elastic/beats/pull/39198 (cherry picked from commit 7e203ad7fd945be9b05e22b991cdf385935d36b7) Co-authored-by: Dimitrios Liappis --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index 34d9e397ca3e..0e79d2ad8484 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -1040,7 +1040,7 @@ spec: spec: repository: elastic/beats pipeline_file: ".buildkite/packaging.pipeline.yml" - branch_configuration: "main 8.14" + branch_configuration: "main 8.*" # TODO enable after packaging backports for release branches # branch_configuration: "main 8.* 7.17" cancel_intermediate_builds: false From e313c8057079e1702d18ca3925f1ab28dd6b0709 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 3 May 2024 12:42:15 -0400 Subject: [PATCH 2/7] chore: Update snapshot.yml (#39399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index c31c98d11c4f..7ff4b505dfa5 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21-86bc1e1e-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21-4f451c48-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -22,7 +22,7 @@ services: - "ingest.geoip.downloader.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.17.21-86bc1e1e-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.17.21-4f451c48-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -32,7 +32,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.17.21-86bc1e1e-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.17.21-4f451c48-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 From daa9253fee1528518fdf3df94c89f0b6ea13ab5b Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Mon, 6 May 2024 07:30:08 +0100 Subject: [PATCH 3/7] [Release] update test environment (#39315) Co-authored-by: Pierre HILBERT --- metricbeat/docker-compose.yml | 12 ++++++------ metricbeat/module/logstash/docker-compose.yml | 8 ++++---- testing/environments/latest.yml | 6 +++--- x-pack/metricbeat/docker-compose.yml | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 618cf962841d..0fea379dd65f 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -15,11 +15,11 @@ services: # Used by base tests elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.17.21}-1 build: context: ./module/elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.17.20} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.17.21} environment: - "ES_JAVA_OPTS=-Xms256m -Xmx256m" - "network.host=" @@ -37,11 +37,11 @@ services: # Used by base tests kibana: - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.17.21}-1 build: context: ./module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-7.17.20} + KIBANA_VERSION: ${KIBANA_VERSION:-7.17.21} depends_on: - elasticsearch ports: @@ -49,11 +49,11 @@ services: # Used by base tests metricbeat: - image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-7.17.21}-1 build: context: ./module/beat/_meta args: - BEAT_VERSION: ${BEAT_VERSION:-7.17.20} + BEAT_VERSION: ${BEAT_VERSION:-7.17.21} command: '-e' ports: - 5066 diff --git a/metricbeat/module/logstash/docker-compose.yml b/metricbeat/module/logstash/docker-compose.yml index 25d75e29b877..d82b10cc23a3 100644 --- a/metricbeat/module/logstash/docker-compose.yml +++ b/metricbeat/module/logstash/docker-compose.yml @@ -2,22 +2,22 @@ version: '2.3' services: logstash: - image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-logstash:${LOGSTASH_VERSION:-7.17.21}-1 build: context: ./_meta args: - LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.17.20} + LOGSTASH_VERSION: ${LOGSTASH_VERSION:-7.17.21} ports: - 9600 depends_on: - elasticsearch elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-7.17.21}-1 build: context: ../elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.17.20} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-7.17.21} environment: - "network.host=" - "transport.host=127.0.0.1" diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 197f199dfdba..cad5db51e8c3 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.20 + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21 healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.17.20 + image: docker.elastic.co/logstash/logstash:7.17.21 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.17.20 + image: docker.elastic.co/kibana/kibana:7.17.21 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300 diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index 8344b0468753..896418833622 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -24,11 +24,11 @@ services: kibana: # Copied configuration from OSS metricbeat because services with depends_on # cannot be extended with extends - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.17.20}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-7.17.21}-1 build: context: ../../metricbeat/module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-7.17.20} + KIBANA_VERSION: ${KIBANA_VERSION:-7.17.21} depends_on: - elasticsearch ports: From 315724b01e26f12de8e698a3f63275531f59bad7 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 6 May 2024 12:42:13 -0400 Subject: [PATCH 4/7] chore: Update snapshot.yml (#39415) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 7ff4b505dfa5..e7c84bfe7ed9 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.21-4f451c48-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.22-3712b6ed-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -22,7 +22,7 @@ services: - "ingest.geoip.downloader.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:7.17.21-4f451c48-SNAPSHOT + image: docker.elastic.co/logstash/logstash:7.17.22-3712b6ed-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -32,7 +32,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:7.17.21-4f451c48-SNAPSHOT + image: docker.elastic.co/kibana/kibana:7.17.22-3712b6ed-SNAPSHOT healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 From 94fd141043600c3de0e7d010573cbe4ab8a83891 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:24:41 +0300 Subject: [PATCH 5/7] [7.17](backport #39126) Beats DRA pipeline (#39197) Add DRA Buildkite packaging pipelines for 7.17 --------- Signed-off-by: Alexandros Sapranidis Co-authored-by: Pavel Zorin Co-authored-by: Dimitrios Liappis Co-authored-by: Alexandros Sapranidis --- .buildkite/packaging.pipeline.md | 23 ++ .buildkite/packaging.pipeline.yml | 248 ++++++++++++++++++ .buildkite/scripts/dra.sh | 92 +++++++ .buildkite/scripts/packaging/package-dra.sh | 21 ++ .../packaging/prepare-release-manager.sh | 41 +++ 5 files changed, 425 insertions(+) create mode 100644 .buildkite/packaging.pipeline.md create mode 100644 .buildkite/packaging.pipeline.yml create mode 100755 .buildkite/scripts/dra.sh create mode 100755 .buildkite/scripts/packaging/package-dra.sh create mode 100755 .buildkite/scripts/packaging/prepare-release-manager.sh diff --git a/.buildkite/packaging.pipeline.md b/.buildkite/packaging.pipeline.md new file mode 100644 index 000000000000..577310d0104e --- /dev/null +++ b/.buildkite/packaging.pipeline.md @@ -0,0 +1,23 @@ +### Beats Packaging pipeline +[Buildkite packaging pipeline](https://buildkite.com/elastic/beats-packaging-pipeline) is used to build and publish the packages for the Beats. The pipeline is triggered by a commit to the `main` or release branches. +The pipeline definition is located in the `.buildkite/packaging.pipeline.yml` + +### Triggers +Staging packaging DRA is triggered for release branches only. +Snapshot is triggered for `main` and release branches. + +### Pipeline steps + +#### Beats dashboard +Generates `build/distributions/dependencies.csv` and `tar.gz` and adds them to the `beats-dashboards` artifact. This is required by the release-manager configuration. + +#### Packaging snapshot/staging + +- Builds the Beats packages for all supported platforms and architectures (`mage package, mage ironbank`) +- Copies artifacts `build/distributions//` directory and adds it as an artifact, where `` is the corresponding beat name. +- x-pack artifacts are also copied to `build/distributions//` directory, where `` is the name of the beat. For example, `auditbeat`, not `x-pack/auditbeat`. It's required for the DRA publish step by [release-manager configuration](https://github.com/elastic/infra/blob/master/cd/release/release-manager/project-configs/master/beats.gradle). + +#### DRA publish +Downloads the artifacts from the `packaging snapshot/staging` step and publishes them to the Elastic DRA registry. + + diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml new file mode 100644 index 000000000000..fc5197c4314d --- /dev/null +++ b/.buildkite/packaging.pipeline.yml @@ -0,0 +1,248 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +# TODO: Pre-cache beats-dev/golang-crossbuild container image + +env: + ASDF_MAGE_VERSION: 1.15.0 + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + PLATFORMS: "+all linux/amd64 linux/386 linux/arm64 windows/amd64 windows/386 darwin/amd64" + PLATFORMS_ARM: "linux/arm64" + +steps: + # we use concurrency gates (https://buildkite.com/blog/concurrency-gates) + # to implement two FIFO queues for DRA-snapshot and DRA-staging + # this prevents parallel builds and possibility of publishing out of order DRA artifacts if the first job takes longer than the second + + - name: Start of concurrency group for DRA Snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + command: echo "--> Start of concurrency gate dra-snapshot" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" + concurrency: 1 + key: start-gate-snapshot + + - name: Start of concurrency group for DRA Staging + if: build.branch =~ /^\d+\.\d+$$/ + command: echo "--> Start of concurrency gate dra-staging" + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" + concurrency: 1 + key: start-gate-staging + + - wait + + - group: Beats dashboards + key: dashboards + steps: + - label: Snapshot dashboards + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + depends_on: start-gate-snapshot + key: dashboards-snapshot + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: true + DEV: true + artifact_paths: + - build/distributions/**/* + + - label: Staging dashboards + if: build.branch =~ /^\d+\.\d+$$/ + depends_on: start-gate-staging + key: dashboards-staging + # TODO: container with go and make + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + commands: + - make build/distributions/dependencies.csv + - make beats-dashboards + env: + SNAPSHOT: false + DEV: false + artifact_paths: + - build/distributions/**/* + + - group: Packaging snapshot + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + key: packaging-snapshot + depends_on: start-gate-snapshot + steps: + - label: "SNAPSHOT: {{matrix}}" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: true + DEV: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + - label: "SNAPSHOT: {{matrix}} docker Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: true + DEV: true + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + - group: Packaging staging + key: packaging-staging + depends_on: start-gate-staging + ## Only for release + if: build.branch =~ /^\d+\.\d+$$/ + steps: + - label: "STAGING: {{matrix}}" + env: + PLATFORMS: "${PLATFORMS}" + SNAPSHOT: false + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - winlogbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/functionbeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/osquerybeat + - x-pack/packetbeat + - x-pack/winlogbeat + + - label: "STAGING: {{matrix}} docker Linux/arm64" + env: + PLATFORMS: "${PLATFORMS_ARM}" + PACKAGES: "docker" + SNAPSHOT: false + DEV: false + command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - build/distributions/**/* + matrix: + - auditbeat + - filebeat + - heartbeat + - metricbeat + - packetbeat + - x-pack/auditbeat + - x-pack/dockerlogbeat + - x-pack/filebeat + - x-pack/heartbeat + - x-pack/metricbeat + - x-pack/packetbeat + + - group: DRA publish + key: dra + steps: + - label: DRA Snapshot + ## Only for release branches and main + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + key: dra-snapshot + env: + DRA_WORKFLOW: snapshot + depends_on: + - start-gate-snapshot + - packaging-snapshot + - dashboards-snapshot + command: | + buildkite-agent artifact download "build/**/*" . + .buildkite/scripts/packaging/prepare-release-manager.sh snapshot + .buildkite/scripts/dra.sh + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - label: DRA Staging + ## Only for release branches + if: build.branch =~ /^\d+\.\d+$$/ + key: dra-staging + env: + DRA_WORKFLOW: staging + depends_on: + - start-gate-staging + - packaging-staging + - dashboards-staging + command: | + buildkite-agent artifact download "build/**" . + .buildkite/scripts/packaging/prepare-release-manager.sh staging + .buildkite/scripts/dra.sh + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + + - wait + + - command: echo "End of concurrency gate dra-snapshot <--" + if: build.branch =~ /^\d+\.\d+$$/ || build.branch == 'main' || build.env('RUN_SNAPSHOT') == "true" + concurrency_group: "dra-gate-snapshot-$BUILDKITE_BRANCH" + concurrency: 1 + key: end-gate-snapshot + + - command: echo "End of concurrency gate dra-staging <--" + if: build.branch =~ /^\d+\.\d+$$/ + concurrency_group: "dra-gate-staging-$BUILDKITE_BRANCH" + concurrency: 1 + key: end-gate-staging diff --git a/.buildkite/scripts/dra.sh b/.buildkite/scripts/dra.sh new file mode 100755 index 000000000000..aeee80908a7e --- /dev/null +++ b/.buildkite/scripts/dra.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# TODO: uncomment out below when Jenkins packaging has been stopped +# if [[ "$DRY_RUN" == "true" ]]; then +# echo "~~~ Running in dry-run mode -- will NOT publish artifacts" +# DRY_RUN="--dry-run" +# else +# echo "~~~ Running in publish mode" +# DRY_RUN="" +# fi + +# TODO: delete the conditional below (and replace it with the above, uncommented out, section) after Jenkins packaging has been stopped +if [[ "$DRY_RUN" == "false" ]]; then + echo "~~~ Running in publish mode" + DRY_RUN="" +else + echo "~~~ Running in dry-run mode -- will NOT publish artifacts" + DRY_RUN="--dry-run" +fi + +set -euo pipefail + +# DRA_BRANCH can be used for manually testing packaging with PRs +# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build +BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}" + +BEAT_VERSION=$(make get-version) + +CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role" + +function release_manager_login { + DRA_CREDS_SECRET=$(retry -t 5 -- vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH}) + VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr') + VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id') + VAULT_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id') + export VAULT_ADDR_SECRET VAULT_ROLE_ID_SECRET VAULT_SECRET +} + +set +x +release_manager_login + +# required by the release-manager docker image, otherwise we hit: +# > java.io.FileNotFoundException: /artifacts/build/distributions/agentbeat/agentbeat-8.15.0-SNAPSHOT-darwin-x86_64.tar.gz.sha512 (Permission denied) +chmod -R a+r build/* +chmod -R a+w build + +echo "+++ :clipboard: Listing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH] and workflow [$DRA_WORKFLOW]" +set +x +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli list \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" + +echo "+++ :hammer_and_pick: Publishing DRA artifacts for version [$BEAT_VERSION], branch [$BRANCH], workflow [$DRA_WORKFLOW] and DRY_RUN: [$DRY_RUN]" + +set +x +docker run --rm \ + --name release-manager \ + -e VAULT_ADDR="${VAULT_ADDR_SECRET}" \ + -e VAULT_ROLE_ID="${VAULT_ROLE_ID_SECRET}" \ + -e VAULT_SECRET_ID="${VAULT_SECRET}" \ + --mount type=bind,readonly=false,src="${PWD}",target=/artifacts \ + docker.elastic.co/infra/release-manager:latest \ + cli collect \ + --project "beats" \ + --branch "${BRANCH}" \ + --commit "${BUILDKITE_COMMIT}" \ + --workflow "${DRA_WORKFLOW}" \ + --version "${BEAT_VERSION}" \ + --artifact-set "main" \ + ${DRY_RUN} | tee rm-output.txt + + +if [[ "$DRY_RUN" != "--dry-run" ]]; then + # extract the summary URL from a release manager output line like: + # Report summary-18.22.0.html can be found at https://artifacts-staging.elastic.co/beats/18.22.0-ABCDEFGH/summary-18.22.0.html + SUMMARY_URL=$(grep -E '^Report summary-.* can be found at ' rm-output.txt | grep -oP 'https://\S+' | awk '{print $1}') + rm rm-output.txt + + # and make it easily clickable as a Builkite annotation + printf "**${DRA_WORKFLOW} summary link:** [${SUMMARY_URL}](${SUMMARY_URL})\n" | buildkite-agent annotate --style=success --append +fi diff --git a/.buildkite/scripts/packaging/package-dra.sh b/.buildkite/scripts/packaging/package-dra.sh new file mode 100755 index 000000000000..10d5425da366 --- /dev/null +++ b/.buildkite/scripts/packaging/package-dra.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -ueo pipefail +BEAT_DIR=${1:-""} + +if [ -z "$BEAT_DIR" ]; then + echo "Error: Beat directory must be specified." + exit 1 +fi + +echo "~~~ Packaging : $BEAT_DIR" + +WORKSPACE=$(pwd) +BEAT_NAME_SLUG=$(echo "$BEAT_DIR" | sed 's/x-pack\///g') + +cd $BEAT_DIR +mage package +mage ironbank + +mkdir -p $WORKSPACE/build/distributions/$BEAT_NAME_SLUG +cp build/distributions/* $WORKSPACE/build/distributions/$BEAT_NAME_SLUG/ +cd $WORKSPACE \ No newline at end of file diff --git a/.buildkite/scripts/packaging/prepare-release-manager.sh b/.buildkite/scripts/packaging/prepare-release-manager.sh new file mode 100755 index 000000000000..2727fd986cc0 --- /dev/null +++ b/.buildkite/scripts/packaging/prepare-release-manager.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# +# This script is executed by the DRA stage. +# It prepares the required files to be consumed by the release-manager +# It can be published as snapshot or staging, for such you use +# the paramater $0 "snapshot" or $0 "staging" +# +set -ueo pipefail + +readonly TYPE=${1:-snapshot} + +# rename dependencies.csv to the name expected by release-manager. +VERSION=$(make get-version) +FINAL_VERSION=$VERSION-SNAPSHOT +if [ "$TYPE" != "snapshot" ] ; then + FINAL_VERSION=$VERSION +fi +echo "Rename dependencies to $FINAL_VERSION" +mv build/distributions/dependencies.csv \ + build/distributions/dependencies-"$FINAL_VERSION".csv + +# rename docker files to support the unified release format. +# TODO: this could be supported by the package system itself +# or the unified release process the one to do the transformation +# See https://github.com/elastic/beats/pull/30895 +find build/distributions -name '*linux-arm64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-arm64.docker.tar.gz/docker-image-linux-arm64.tar.gz}" + done + +find build/distributions -name '*linux-amd64.docker.tar.gz*' -print0 | + while IFS= read -r -d '' file + do + echo "Rename file $file" + mv "$file" "${file/linux-amd64.docker.tar.gz/docker-image-linux-amd64.tar.gz}" + done + +echo 'List all the files' +find build/distributions -type f -ls || true From 10141582753159561a6410b3091ab8fe29824f36 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 11:44:32 +0300 Subject: [PATCH 6/7] [7.17](backport #39332) More resilient DRA packaging (#39339) Occasionally packaging steps from the DRA pipeline may get stuck[^1]. This causes a breach of the global pipeline timeout (currently 1hr) and cancels the job. This commit increases the global timeout to 90min, adds one retry per step and limits the runtime per step to 40min (so that a single stuck step doesn't exhaust the entire global timeout). Finally, we shush slack notifications if the retry recovered the step. In a future PR we will consider also adding a daily DRA build to cover for cases where the retries didn't help and there were no subsequent commits to trigger a new build. [^1]: https://buildkite.com/elastic/beats-packaging-pipeline/builds/114 (cherry picked from commit 726f6e9bdec715f958ba47500e77feb5655b0a48) --------- Co-authored-by: Dimitrios Liappis --- .buildkite/packaging.pipeline.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml index fc5197c4314d..1acab32aa9df 100644 --- a/.buildkite/packaging.pipeline.yml +++ b/.buildkite/packaging.pipeline.yml @@ -44,6 +44,10 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -62,6 +66,10 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 commands: - make build/distributions/dependencies.csv - make beats-dashboards @@ -86,6 +94,10 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 artifact_paths: - build/distributions/**/* matrix: @@ -116,6 +128,10 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 artifact_paths: - build/distributions/**/* matrix: @@ -147,6 +163,10 @@ steps: provider: gcp image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 artifact_paths: - build/distributions/**/* matrix: @@ -177,6 +197,10 @@ steps: provider: "aws" imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" + timeout_in_minutes: 40 + retry: + automatic: + - limit: 1 artifact_paths: - build/distributions/**/* matrix: From f487125b1bee65957d338102938472a3e10bc39b Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 7 May 2024 13:53:50 +0300 Subject: [PATCH 7/7] [7.17](backport #39239) Update x-pack/heartbeat pipeline to match Jenkins (#39421) * Update x-pack/heartbeat pipeline to match Jenkins (#39239) * updated win-2016 test cmd * updated macos namimg * pr fixes * removed pipeline masking where irrelevant (cherry picked from commit 4f5b23a89f7f1b8a3b31c999409a001b414acafc) # Conflicts: # .buildkite/x-pack/pipeline.xpack.heartbeat.yml * fixed conflicts --------- Co-authored-by: Olga Naydyonock --- .../x-pack/pipeline.xpack.heartbeat.yml | 123 +++++------------- 1 file changed, 33 insertions(+), 90 deletions(-) diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index f2befa0e3b0b..5a76b42c8dae 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -2,7 +2,8 @@ name: "beats-xpack-heartbeat" env: - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" @@ -11,7 +12,6 @@ env: IMAGE_MACOS_ARM: "generic-13-ventura-arm" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" @@ -30,7 +30,7 @@ steps: - group: "x-pack/heartbeat Mandatory Tests" key: "x-pack-heartbeat-mandatory-tests" steps: - - label: ":linux: Ubuntu Unit Tests" + - label: ":ubuntu: x-pack/heartbeat Ubuntu Unit Tests" key: "mandatory-linux-unit-test" command: | set -euo pipefail @@ -53,7 +53,7 @@ steps: - github_commit_status: context: "x-pack/heartbeat: Ubuntu Unit Tests" - - label: ":go: Go Integration Tests" + - label: ":ubuntu: x-pack/heartbeat Go Integration Tests" key: "mandatory-int-test" command: | set -euo pipefail @@ -76,15 +76,15 @@ steps: - github_commit_status: context: "x-pack/heartbeat: Go Integration Tests" - - label: ":windows: Windows 2019 Unit Tests" + - label: ":windows: x-pack/heartbeat Windows 2019 Unit Tests" skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" command: | Set-Location -Path x-pack/heartbeat mage build test - key: "extended-win-2019-unit-tests" retry: automatic: - - limit: 3 + - limit: 3 + key: "extended-win-2019-unit-tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -102,57 +102,17 @@ steps: key: "x-pack-heartbeat-extended-win-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" - steps: - - label: ":windows: Windows 10 Unit Tests" - command: | - Set-Location -Path x-pack/heartbeat - mage build test - key: "extended-win-10-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/heartbeat/build/*.xml" - - "x-pack/heartbeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/heartbeat: Windows 10 Unit Tests" - - - label: ":windows: Windows 11 Unit Tests" - command: | - Set-Location -Path x-pack/heartbeat - mage build test - key: "extended-win-11-unit-tests" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "x-pack/heartbeat/build/*.xml" - - "x-pack/heartbeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/heartbeat: Windows 11 Unit Tests" - - label: ":windows: Windows 2016 Unit Tests" + steps: + - label: ":windows: x-pack/heartbeat Windows 2016 Unit Tests" key: "mandatory-win-2016-unit-tests" + skip: "skipping due to elastic/beats#23957 and elastic/beats#23958" command: | Set-Location -Path x-pack/heartbeat - mage build unitTest + mage build test retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -166,18 +126,18 @@ steps: - github_commit_status: context: "x-pack/heartbeat: Windows 2016 Unit Tests" - - label: ":windows: Windows 2022 Unit Tests" - key: "mandatory-win-2022-unit-tests" + - label: ":windows: x-pack/heartbeat Windows 10 Unit Tests" command: | Set-Location -Path x-pack/heartbeat - mage build unitTest + mage build test + key: "extended-win-10-unit-tests" retry: automatic: - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" artifact_paths: @@ -185,13 +145,14 @@ steps: - "x-pack/heartbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/heartbeat: Windows 2022 Unit Tests" - - - group: "x-pack/heartbeat MacOS Extended Tests" + context: "x-pack/heartbeat: Windows 10 Unit Tests" + + - group: "x-pack/heartbeat macOS Extended Tests" key: "x-pack-heartbeat-extended-tests-macos" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + steps: - - label: ":mac: MacOS x86_64 Unit Tests" + - label: ":mac: x-pack/heartbeat macOS x86_64 Unit Tests" command: | set -euo pipefail source .buildkite/scripts/install_macos_tools.sh @@ -210,39 +171,21 @@ steps: - "x-pack/heartbeat/build/*.json" notify: - github_commit_status: - context: "x-pack/heartbeat: MacOS x86_64 Extended Tests" - - - label: ":mac: MacOS arm64 Unit Tests" - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - withNodeJSEnv $ASDF_NODEJS_VERSION - installNodeJsDependencies - cd x-pack/heartbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "x-pack/heartbeat/build/*.xml" - - "x-pack/heartbeat/build/*.json" - notify: - - github_commit_status: - context: "x-pack/heartbeat: MacOS arm64 Extended Tests" + context: "x-pack/heartbeat: macOS x86_64 Extended Tests" - wait: ~ + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests if: build.env("BUILDKITE_PULL_REQUEST") != "false" depends_on: - step: "x-pack-heartbeat-mandatory-tests" - - group: "Packaging" - key: "packaging" + - group: "x-pack/heartbeat Packaging" + key: "x-pack-heartbeat-packaging" if: build.env("BUILDKITE_PULL_REQUEST") != "false" steps: - - label: ":linux: Packaging Linux" + - label: ":ubuntu: x-pack/heartbeat Packaging Linux x86_64" key: "packaging-linux" command: | cd x-pack/heartbeat @@ -257,20 +200,20 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" notify: - github_commit_status: - context: "x-pack/heartbeat: Packaging Linux" + context: "x-pack/heartbeat: Packaging Linux x86_64" - - label: ":linux: Packaging ARM" + - label: ":ubuntu: x-pack/heartbeat Packaging Linux arm64" key: "packaging-arm" command: | cd x-pack/heartbeat mage package agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "linux/arm64" PACKAGES: "docker" notify: - github_commit_status: - context: "x-pack/heartbeat: Packaging Linux ARM" + context: "x-pack/heartbeat: Packaging Linux arm64"