Fold triggered pipelines under elastic-agent (#6978)

In order for the Buildkite PR bot to be able to publish meaningful comments for failures, the pipelines need to be collapsed / uploaded rather than triggered, otherwise we lose the context and the ability to have the comment include specific failures for the failed steps in triggered pipelines.

This commit switches away from triggering to pipeline upload commands to enable this functionality.

(cherry picked from commit 33d0137)

# Conflicts:
#	.buildkite/bk.integration.pipeline.yml

.buildkite/bk.integration.pipeline.yml

@@ -28,7 +28,7 @@ steps:
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -48,7 +48,7 @@ steps:
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -60,6 +60,44 @@ steps:
         matrix:
           - default
 
+<<<<<<< HEAD
+=======
+      - label: "Win2025:sudo:{{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-windows-2025"
+        matrix:
+          - default
+          - fleet
+          - fleet-endpoint-security
+          - fleet-privileged
+          - standalone-upgrade
+          - upgrade
+          - upgrade-flavor
+          - install-uninstall
+
+      - label: "Win2025:non-sudo:{{matrix}}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/integration-tests.ps1 {{matrix}} false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "family/platform-ingest-elastic-agent-windows-2025"
+        matrix:
+          - default
+
+>>>>>>> 33d01371e (Fold triggered pipelines under elastic-agent (#6978))
   - group: "Stateful:Ubuntu"
     key: integration-tests-ubuntu
     depends_on:
@@ -68,7 +106,7 @@ steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         # only packaging-ubuntu-x86-64 artifact dependency is required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -83,7 +121,7 @@ steps:
       - label: "x86_64:sudo: {{matrix}}"
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** .
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -110,7 +148,7 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -137,7 +175,7 @@ steps:
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -156,7 +194,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** .
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

.buildkite/integration.pipeline.yml

@@ -110,10 +110,7 @@ steps:
   - label: "Triggering Integration tests"
     depends_on:
       - int-packaging
-    trigger: "elastic-agent-extended-testing-bk"
-    build:
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
+    command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
 
   - label: "Serverless Beats Tests"
     # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required

.buildkite/pipeline.yml

@@ -245,7 +245,7 @@ steps:
   - label: "Trigger k8s sync"
     branches: main
     plugins:
-      - monebag/monorepo-diff#v2.5.9:
+      - monorepo-diff#v1.2.0:
           diff: "git diff --name-only HEAD~1"
           watch:
             - path:
@@ -267,7 +267,7 @@ steps:
       build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
 
     plugins:
-      - monorepo-diff#v1.0.1:
+      - monorepo-diff#v1.2.0:
           diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
           watch:
             - path:
@@ -293,14 +293,12 @@ steps:
                 - .buildkite/hooks/
 
               config:
-                trigger: "elastic-agent-extended-testing"
-                build:
-                  commit: "${BUILDKITE_COMMIT}"
-                  branch: "${BUILDKITE_BRANCH}"
-                  env:
-                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
-                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
-                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+                label: ":pipeline: Upload extended testing Pipeline"
+                command: "buildkite-agent pipeline upload .buildkite/integration.pipeline.yml"
+                env:
+                  - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                  - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                  - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
   # Trigger for branches
   - label: "Triggering Extended tests for branches"