Fold triggered pipelines under elastic-agent (#6978)

In order for the Buildkite PR bot to be able to publish meaningful comments for failures, the pipelines need to be collapsed / uploaded rather than triggered, otherwise we lose the context and the ability to have the comment include specific failures for the failed steps in triggered pipelines. This commit switches away from triggering to pipeline upload commands to enable this functionality.
elastic · Feb 26, 2025 · 33d0137 · 33d0137
1 parent 2614cb5
commit 33d0137
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 23 deletions.
diff --git a/.buildkite/bk.integration.pipeline.yml b/.buildkite/bk.integration.pipeline.yml
@@ -28,7 +28,7 @@ steps:
     steps:
       - label: "Win2022:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -49,7 +49,7 @@ steps:
 
       - label: "Win2022:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -63,7 +63,7 @@ steps:
 
       - label: "Win2025:sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} true
         artifact_paths:
           - build/**
@@ -84,7 +84,7 @@ steps:
 
       - label: "Win2025:non-sudo:{{matrix}}"
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
           .buildkite/scripts/integration-tests.ps1 {{matrix}} false
         artifact_paths:
           - build/**
@@ -104,7 +104,7 @@ steps:
       - label: "x86_64:non-sudo: {{matrix}}"
         # only packaging-ubuntu-x86-64 artifact dependency is required
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -119,7 +119,7 @@ steps:
       - label: "x86_64:sudo: {{matrix}}"
         # due to deb group present in matrix tar.gz and deb packages artifacts are required
         command: |
-          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** .
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -147,7 +147,7 @@ steps:
       - label: "arm:sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
         artifact_paths:
           - build/**
@@ -175,7 +175,7 @@ steps:
       - label: "arm:non-sudo: {{matrix}}"
         skip: true
         command: |
-          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64' --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64'
           .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
         artifact_paths:
           - build/**
@@ -194,7 +194,7 @@ steps:
     steps:
       - label: "x86_64:sudo:rpm"
         command: |
-          buildkite-agent artifact download build/distributions/** . --build ${BUILDKITE_TRIGGERED_FROM_BUILD_ID}
+          buildkite-agent artifact download build/distributions/** .
           .buildkite/scripts/steps/integration_tests_tf.sh rpm true
         artifact_paths:
           - build/**

diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -110,10 +110,7 @@ steps:
   - label: "Triggering Integration tests"
     depends_on:
       - int-packaging
-    trigger: "elastic-agent-extended-testing-bk"
-    build:
-      commit: "${BUILDKITE_COMMIT}"
-      branch: "${BUILDKITE_BRANCH}"
+    command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
 
   - label: "Serverless Beats Tests"
     # To speedup the build process only packaging-ubuntu-x86-64 artifact dependency is required

diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -245,7 +245,7 @@ steps:
   - label: "Trigger k8s sync"
     branches: main
     plugins:
-      - monebag/monorepo-diff#v2.5.9:
+      - monorepo-diff#v1.2.0:
           diff: "git diff --name-only HEAD~1"
           watch:
             - path:
@@ -267,7 +267,7 @@ steps:
       build.env("GITHUB_PR_TRIGGER_COMMENT") =~ /.*extended.*/
 
     plugins:
-      - monorepo-diff#v1.0.1:
+      - monorepo-diff#v1.2.0:
           diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
           watch:
             - path:
@@ -294,14 +294,12 @@ steps:
                 - .buildkite/hooks/
 
               config:
-                trigger: "elastic-agent-extended-testing"
-                build:
-                  commit: "${BUILDKITE_COMMIT}"
-                  branch: "${BUILDKITE_BRANCH}"
-                  env:
-                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
-                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
-                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+                label: ":pipeline: Upload extended testing Pipeline"
+                command: "buildkite-agent pipeline upload .buildkite/integration.pipeline.yml"
+                env:
+                  - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                  - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                  - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
   # Trigger for branches
   - label: "Triggering Extended tests for branches"