Skip to content

Commit

Permalink
Merge branch 'main' into hybrid-agent-test
Browse files Browse the repository at this point in the history
  • Loading branch information
mauri870 committed Feb 7, 2025
2 parents 1238a43 + b59d51a commit 541729d
Show file tree
Hide file tree
Showing 43 changed files with 866 additions and 177 deletions.
16 changes: 9 additions & 7 deletions .buildkite/bk.integration.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ steps:
steps:
- label: "Win2022:sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/integration-tests.ps1 {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -45,7 +45,7 @@ steps:

- label: "Win2022:non-sudo:{{matrix}}"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/integration-tests.ps1 {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -63,8 +63,9 @@ steps:
- integration-ess
steps:
- label: "x86_64:non-sudo: {{matrix}}"
# only packaging-ubuntu-x86-64 artifact dependency is required
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -77,8 +78,9 @@ steps:
- default

- label: "x86_64:sudo: {{matrix}}"
# due to deb group present in matrix tar.gz and deb packages artifacts are required
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -102,7 +104,7 @@ steps:
- label: "arm:sudo: {{matrix}}"
skip: true
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
artifact_paths:
- build/**
Expand All @@ -126,7 +128,7 @@ steps:
- label: "arm:non-sudo: {{matrix}}"
skip: true
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
artifact_paths:
- build/**
Expand All @@ -145,7 +147,7 @@ steps:
steps:
- label: "x86_64:sudo:rpm"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
.buildkite/scripts/steps/integration_tests_tf.sh rpm true
artifact_paths:
- build/**
Expand Down
87 changes: 69 additions & 18 deletions .buildkite/integration.pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,80 +5,131 @@ env:
VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"

steps:
- label: "Integration tests: packaging"
key: "package-it"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n1-standard-8"
- group: "Integration tests: packaging"
key: "int-packaging"
steps:
- label: "Packaging: Ubuntu x86_64"
key: "packaging-ubuntu-x86-64"
env:
PACKAGES: "tar.gz"
PLATFORMS: "linux/amd64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n2-standard-8"

- label: "Packaging: Ubuntu arm64"
key: "packaging-ubuntu-arm64"
env:
PACKAGES: "tar.gz"
PLATFORMS: "linux/arm64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n2-standard-8"

- label: "Packaging: Windows"
key: "packaging-windows"
env:
PACKAGES: "zip"
PLATFORMS: "windows/amd64"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n2-standard-8"

- label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
key: "packaging-containers"
env:
PACKAGES: "{{matrix.ext}}"
PLATFORMS: "{{matrix.arch}}"
command: ".buildkite/scripts/steps/integration-package.sh"
artifact_paths:
- build/distributions/**
agents:
provider: "gcp"
machineType: "n2-standard-8"
matrix:
setup:
arch:
- linux/amd64
- linux/arm64
ext:
- rpm
- deb

- label: "Serverless integration test"
key: "serverless-integration-tests"
depends_on:
- package-it
- int-packaging
concurrency_group: elastic-agent-extended-testing/serverless-integration
concurrency: 8
env:
# we run each step in a different data center to spread the load
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
.buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
agents:
provider: "gcp"
machineType: "n1-standard-8"
machineType: "n2-standard-8"
notify:
- github_commit_status:
context: "buildkite/elastic-agent-extended-testing - Serverless integration test"

- label: "Extended runtime leak tests"
key: "extended-integration-tests"
depends_on:
- package-it
- int-packaging
concurrency_group: elastic-agent-extended-testing/leak-tests
concurrency: 8
env:
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
.buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
artifact_paths:
- "build/TEST-**"
- "build/diagnostics/*"
agents:
provider: "gcp"
machineType: "n1-standard-8"
machineType: "n2-standard-8"
notify:
- github_commit_status:
context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"

- label: "Triggering Integration tests"
depends_on:
- package-it
- int-packaging
trigger: "elastic-agent-extended-testing-bk"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"

- label: "Serverless Beats Tests"
# To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
depends_on:
- package-it
- packaging-ubuntu-x86-64
key: "serverless-beats-integration-tests"
concurrency_group: elastic-agent-extended-testing/beats-integration
concurrency: 8
env:
TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
command: |
buildkite-agent artifact download build/distributions/** . --step 'package-it'
buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
.buildkite/scripts/steps/beats_tests.sh
agents:
provider: "gcp"
machineType: "n1-standard-8"
machineType: "n2-standard-8"
notify:
- github_commit_status:
context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/pull-requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
"always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
"skip_ci_labels": [ "skip-ci" ],
"skip_target_branches": [ ],
"skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
"skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.mergify.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
"always_require_ci_on_changed": [ ]
},
{
Expand Down
8 changes: 7 additions & 1 deletion .buildkite/scripts/steps/integration-package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,11 @@
set -euo pipefail

source .buildkite/scripts/common.sh

# Remove AGENT_PACKAGE_VERSION pinning as soon as 9.0.0 is released
AGENT_PACKAGE_VERSION=9.0.0 PACKAGES=tar.gz,zip,rpm,deb PLATFORMS=linux/amd64,linux/arm64,windows/amd64 SNAPSHOT=true EXTERNAL=true mage package
export AGENT_PACKAGE_VERSION=9.0.0

export SNAPSHOT="true"
export EXTERNAL="true"

mage package
13 changes: 13 additions & 0 deletions .mergify.yml
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,19 @@ pull_request_rules:
labels:
- "backport"
title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
- name: backport patches to 9.0 branch
conditions:
- merged
- label=backport-9.0
actions:
backport:
assignees:
- "{{ author }}"
branches:
- "9.0"
labels:
- "backport"
title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

- name: backport patches to all active minor branches for the 8 major.
conditions:
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Kind can be one of:
# - breaking-change: a change to previously-documented behavior
# - deprecation: functionality that is being removed in a later release
# - bug-fix: fixes a problem in a previous version
# - enhancement: extends functionality but does not break or fix existing behavior
# - feature: new functionality
# - known-issue: problems that we are aware of in a given version
# - security: impacts on the security of a product or a user’s deployment.
# - upgrade: important information for someone upgrading from a prior version
# - other: does not fit into any of the other categories
kind: feature

# Change summary; a 80ish characters long description of the change.
summary: Add context variable support to outputs

# Long description; in case the summary is not enough to describe the change
# this field accommodate a description without length limits.
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
description: |
Adds support for using context variable providers in the outputs section of a policy. Includes fallback support
to reference env provider when no provider prefix is provided in the variable.
# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
component: elastic-agent

# PR URL; optional; the PR number that added the changeset.
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
# Please provide it if you are adding a fragment for a different PR.
pr: https://github.com/elastic/elastic-agent/pull/6602

# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
# If not present is automatically filled by the tooling with the issue linked to the PR number.
issue: https://github.com/elastic/elastic-agent/issues/6376
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Kind can be one of:
# - breaking-change: a change to previously-documented behavior
# - deprecation: functionality that is being removed in a later release
# - bug-fix: fixes a problem in a previous version
# - enhancement: extends functionality but does not break or fix existing behavior
# - feature: new functionality
# - known-issue: problems that we are aware of in a given version
# - security: impacts on the security of a product or a user’s deployment.
# - upgrade: important information for someone upgrading from a prior version
# - other: does not fit into any of the other categories
kind: bug-fix

# Change summary; a 80ish characters long description of the change.
summary: Fix secret_paths redaction along complex paths

# Long description; in case the summary is not enough to describe the change
# this field accommodate a description without length limits.
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
#description:

# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
component: elastic-agent

# PR URL; optional; the PR number that added the changeset.
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
# Please provide it if you are adding a fragment for a different PR.
pr: https://github.com/elastic/elastic-agent/pull/6710

# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
# If not present is automatically filled by the tooling with the issue linked to the PR number.
#issue: https://github.com/owner/repo/1234
Loading

0 comments on commit 541729d

Please sign in to comment.