Merge branch 'main' into dependabot/github_actions/updatecli/updatecl…

…i-action-2.77.0

.buildkite/bk.integration.pipeline.yml

@@ -28,7 +28,7 @@ steps:
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -40,12 +40,16 @@ steps:
         matrix:
           - default
           - fleet
+          - fleet-endpoint-security
           - fleet-privileged
+          - standalone-upgrade
           - upgrade
+          - upgrade-flavor
+          - install-uninstall
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -63,8 +67,9 @@ steps:
       - integration-ess
     steps:
       - label: "x86_64:non-sudo: {{matrix}}"
+        # only packaging-ubuntu-x86-64 artifact dependency is required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -77,8 +82,9 @@ steps:
           - default
 
       - label: "x86_64:sudo: {{matrix}}"
+        # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -89,20 +95,24 @@ steps:
           image: "family/platform-ingest-elastic-agent-ubuntu-2404"
         matrix:
           - default
-          - container
-          - fleet-upgrade-to-pr-build
           - upgrade
+          - upgrade-flavor
+          - standalone-upgrade
           - fleet
-          - fqdn
-          - deb
+          - fleet-endpoint-security
           - fleet-airgapped
-          - fleet-privileged
           - fleet-airgapped-privileged
+          - fleet-privileged
+          - fleet-upgrade-to-pr-build
+          - install-uninstall
+          - fqdn
+          - deb
+          - container
 
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -113,20 +123,24 @@ steps:
           instanceType: "m6g.2xlarge"
         matrix:
           - default
-          - container
-          - fleet-upgrade-to-pr-build
           - upgrade
+          - upgrade-flavor
+          - standalone-upgrade
           - fleet
-          - fqdn
-          - deb
+          - fleet-endpoint-security
           - fleet-airgapped
-          - fleet-privileged
           - fleet-airgapped-privileged
+          - fleet-privileged
+          - fleet-upgrade-to-pr-build
+          - install-uninstall
+          - fqdn
+          - deb
+          - container
 
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -145,7 +159,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'package-it' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -5,80 +5,131 @@ env:
   VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
 
 steps:
-  - label: "Integration tests: packaging"
-    key: "package-it"
-    command: ".buildkite/scripts/steps/integration-package.sh"
-    artifact_paths:
-      - build/distributions/**
-    agents:
-      provider: "gcp"
-      machineType: "n1-standard-8"
+  - group: "Integration tests: packaging"
+    key: "int-packaging"
+    steps:
+      - label: "Packaging: Ubuntu x86_64"
+        key: "packaging-ubuntu-x86-64"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/amd64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+        - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Ubuntu arm64"
+        key: "packaging-ubuntu-arm64"
+        env:
+          PACKAGES: "tar.gz"
+          PLATFORMS: "linux/arm64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Windows"
+        key: "packaging-windows"
+        env:
+          PACKAGES: "zip"
+          PLATFORMS: "windows/amd64"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+
+      - label: "Packaging: Containers {{matrix.ext}} {{matrix.arch}}"
+        key: "packaging-containers"
+        env:
+          PACKAGES: "{{matrix.ext}}"
+          PLATFORMS: "{{matrix.arch}}"
+        command: ".buildkite/scripts/steps/integration-package.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+        matrix:
+          setup:
+            arch:
+              - linux/amd64
+              - linux/arm64
+            ext:
+              - rpm
+              - deb
 
   - label: "Serverless integration test"
     key: "serverless-integration-tests"
     depends_on:
-      - package-it
+      - int-packaging
     concurrency_group: elastic-agent-extended-testing/serverless-integration
     concurrency: 8
     env:
       # we run each step in a different data center to spread the load
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/integration_tests.sh serverless integration:single TestLogIngestionFleetManaged #right now, run a single test in serverless mode as a sort of smoke test, instead of re-running the entire suite
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless integration test"
 
   - label: "Extended runtime leak tests"
     key: "extended-integration-tests"
     depends_on:
-      - package-it
+      - int-packaging
     concurrency_group: elastic-agent-extended-testing/leak-tests
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-b"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/integration_tests.sh stateful integration:TestForResourceLeaks
     artifact_paths:
       - "build/TEST-**"
       - "build/diagnostics/*"
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Extended runtime leak tests"
 
   - label: "Triggering Integration tests"
     depends_on:
-      - package-it
+      - int-packaging
     trigger: "elastic-agent-extended-testing-bk"
     build:
       commit: "${BUILDKITE_COMMIT}"
       branch: "${BUILDKITE_BRANCH}"
 
   - label: "Serverless Beats Tests"
+    # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required
     depends_on:
-      - package-it
+      - packaging-ubuntu-x86-64
     key: "serverless-beats-integration-tests"
     concurrency_group: elastic-agent-extended-testing/beats-integration
     concurrency: 8
     env:
       TEST_INTEG_AUTH_GCP_DATACENTER: "us-central1-a"
     command: |
-      buildkite-agent artifact download build/distributions/** . --step 'package-it'
+      buildkite-agent artifact download "build/distributions/**" . $BUILDKITE_BUILD_ID
       .buildkite/scripts/steps/beats_tests.sh
     agents:
       provider: "gcp"
-      machineType: "n1-standard-8"
+      machineType: "n2-standard-8"
     notify:
       - github_commit_status:
           context: "buildkite/elastic-agent-extended-testing - Serverless Beats Tests"

.buildkite/pipeline.elastic-agent-binary-dra.yml

@@ -47,8 +47,9 @@ steps:
     if: build.branch =~ /^[0-9]+\.[0-9x]+\$/ || build.env("RUN_STAGING") == "true" || build.env('VERSION_QUALIFIER') != null
     steps:
     - label: ":package: Build Elastic-Agent Core staging"
-      commands:
-        - .buildkite/scripts/steps/build-agent-core.sh
+      commands: |
+        source .buildkite/scripts/version_qualifier.sh
+        .buildkite/scripts/steps/build-agent-core.sh
       key: "build-dra-staging"
       artifact_paths:
         - "build/distributions/**/*"
@@ -62,6 +63,7 @@ steps:
 
     - label: ":hammer: DRA Publish Elastic-Agent Core staging"
       command: |
+        source .buildkite/scripts/version_qualifier.sh
         echo "+++ Restoring Artifacts"
         buildkite-agent artifact download "build/**/*" .
         echo "+++ Changing permissions for the release manager"

.buildkite/pipeline.yml

@@ -275,6 +275,7 @@ steps:
                 - dev-tools/
                 - pkg/
                 - deploy/
+                - test_infra/
                 - testing/
                 - version/
                 - specs/

.buildkite/pull-requests.json

@@ -13,7 +13,7 @@
             "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it|extended))|^/test\\W*(?:extended|)",
             "skip_ci_labels": [ "skip-ci" ],
             "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
+            "skip_ci_on_only_changed": [ "^.ci/", "^.github/", "^updatecli-compose.yaml", "^changelog", "^docs/", "\\.md$", "^docker-compose.yml", "^.mergify.yml", "^.pre-commit-config.yaml", "skaffold.yaml", "^Dockerfile.skaffold", "^Dockerfile"],
             "always_require_ci_on_changed": [ ]
         },
         {

.buildkite/scripts/buildkite-integration-tests.sh

@@ -26,15 +26,18 @@ fi
 asdf install
 
 echo "~~~ Running integration tests as $USER"
-echo "~~~ Integration tests: ${GROUP_NAME}"
 
 go install gotest.tools/gotestsum
 gotestsum --version
 
-PACKAGE_VERSION="$(cat .package-version)"
-if [[ -n "$PACKAGE_VERSION" ]]; then
-    PACKAGE_VERSION=${PACKAGE_VERSION}"-SNAPSHOT"
-fi
+# Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
+#AGENT_VERSION=$(grep "const defaultBeatVersion =" version/version.go | cut -d\" -f2)
+#AGENT_VERSION="${AGENT_VERSION}-SNAPSHOT"
+
+# Remove agent pinning once 9.0.0 is released
+AGENT_VERSION=9.0.0-SNAPSHOT
+export AGENT_VERSION
+echo "~~~ Agent version: ${AGENT_VERSION}"
 
 os_data=$(uname -spr | tr ' ' '_')
 root_suffix=""
@@ -44,8 +47,11 @@ fi
 fully_qualified_group_name="${GROUP_NAME}${root_suffix}_${os_data}"
 outputXML="build/${fully_qualified_group_name}.integration.xml"
 outputJSON="build/${fully_qualified_group_name}.integration.out.json"
+
+echo "~~~ Integration tests: ${GROUP_NAME}"
+
 set +e
-TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${PACKAGE_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
+TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/scripts/integration-tests.ps1

@@ -18,11 +18,22 @@ if ($PACKAGE_VERSION) {
     $PACKAGE_VERSION = "${PACKAGE_VERSION}-SNAPSHOT"
 }
 $env:TEST_BINARY_NAME = "elastic-agent"
-$env:AGENT_VERSION = $PACKAGE_VERSION
+# Parsing version.go. Will be simplified here: https://github.com/elastic/ingest-dev/issues/4925
+# $AGENT_VERSION = (Get-Content version/version.go | Select-String -Pattern 'const defaultBeatVersion =' | ForEach-Object { $_ -replace '.*?"(.*?)".*', '$1' })
+# $env:AGENT_VERSION = $AGENT_VERSION + "-SNAPSHOT"
+
+# Remove agent pinning once 9.0.0 is released
+$env:AGENT_VERSION = "9.0.0-SNAPSHOT"
+
+echo "~~~ Agent version: $env:AGENT_VERSION"
 $env:SNAPSHOT = $true
 
 echo "~~~ Building test binaries"
-mage build:testBinaries
+& mage build:testBinaries
+if ($LASTEXITCODE -ne 0) {    
+    Write-Error "Failed to build test binaries"
+    exit 1
+}
 $osInfo = (Get-CimInstance Win32_OperatingSystem).Caption + " " + (Get-CimInstance Win32_OperatingSystem).OSArchitecture -replace " ", "_"
 $root_suffix=""
 if ($TEST_SUDO -eq "true") {
@@ -31,10 +42,12 @@ if ($TEST_SUDO -eq "true") {
 $fully_qualified_group_name="${GROUP_NAME}${root_suffix}_${osInfo}"
 $outputXML = "build/${fully_qualified_group_name}.integration.xml"
 $outputJSON = "build/${fully_qualified_group_name}.integration.out.json"
+$TestsExitCode = 0
 try {
-    Get-Ess-Stack -StackVersion $PACKAGE_VERSION    
+    Get-Ess-Stack -StackVersion $PACKAGE_VERSION
     Write-Output "~~~ Running integration test group: $GROUP_NAME as user: $env:USERNAME"
-    gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=$TEST_SUDO" 
+    & gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags=integration -shuffle=on -timeout=2h0m0s "github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=$GROUP_NAME" "-integration.sudo=$TEST_SUDO"
+    $TestsExitCode = $LASTEXITCODE    
 } finally {
     ess_down
 
@@ -46,3 +59,7 @@ try {
         Write-Output "Cannot generate HTML test report: $outputXML not found"
     }
 }
+
+if ($TestsExitCode -ne 0) {    
+    exit 1
+}