Merge branch 'main' into salesforce_remove_dashbaords

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -24,28 +24,39 @@ env:
 steps:
   - group: "Heartbeat Mandatory Testing"
     key: "heartbeat-mandatory-tests"
+
     steps:
-      - label: ":ubuntu: Unit Tests"
-        command: "cd heartbeat && mage build unitTest"
+      - label: ":ubuntu: Heartbeat Unit Tests"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Ubuntu Unit Tests"
 
       - label: ":rhel: Heartbeat Rhel9 Unit Tests"
-        command: "cd heartbeat && mage build unitTest"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
           machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Rhel9 Unit Tests"
 
-      - label: ":windows: Heartbeat Win-2016 Unit Test"
+      - label: ":windows: Heartbeat Win-2016 Unit Tests"
         key: "windows-2016"
         command: |
           Set-Location -Path heartbeat
@@ -58,6 +69,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2016 Unit Tests"
 
       - label: ":windows: Heartbeat Win-2022 Unit Test"
         key: "windows-2022"
@@ -72,53 +86,97 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2022 Unit Tests"
 
       - label: ":ubuntu: Heartbeat Go Integration Tests"
-        command: "cd heartbeat && mage goIntegTest"
+        command:  |
+          cd heartbeat
+          mage goIntegTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Go Integration Tests"
 
       - label: ":ubuntu: Heartbeat Python Integration Tests"
-        command: "cd heartbeat && mage pythonIntegTest"
+        command: |
+          cd heartbeat
+          mage pythonIntegTest
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Python Integration Tests"
 
   - group: "Heartbeat ARM Tests"
     key: "heartbeat-extended-tests-arm"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+
     steps:
       - label: ":linux: Heartbeat ARM64 Unit Tests"
         key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
-        command: "cd heartbeat && mage build unitTest"
+        command: |
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths: "heartbeat/build/*.xml"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Ubuntu ARM64 Unit Tests"
 
   - group: "Heartbeat Extended Testing MacOS"
     key: "heartbeat-extended-tests-macos"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
       - label: ":mac: Heartbeat MacOS Unit Tests"
         key: "macos-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-        command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd heartbeat
+          mage build unitTest
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
-        artifact_paths: "heartbeat/build/*.xml"
+        artifact_paths:
+          - "metricbeat/build/*.xml"
+          - "metricbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS Unit Tests"
+
+      - label: ":mac: Heartbeat MacOS ARM Unit Tests"
+        key: "macos-arm-extended"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd heartbeat
+          mage build unitTest
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        artifact_paths:
+          - "metricbeat/build/*.xml"
+          - "metricbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: MacOS ARM Unit Tests"
 
   - group: "Heartbeat Windows Extended Testing"
     key: "heartbeat-extended-tests-win"
@@ -127,7 +185,9 @@ steps:
     steps:
       - label: ":windows: Heartbeat Win-2019 Unit Tests"
         key: "heartbeat-win-extended-2019"
-        command: "mage -d ${BEATS_PROJECT_NAME} unitTest"
+        command:  |
+          Set-Location -Path heartbeat
+          mage build unitTest
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"
@@ -136,6 +196,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-2019 Unit Tests"
 
       - label: ":windows: Heartbeat Win-11 Unit Tests"
         key: "heartbeat-windows-extended-11"
@@ -150,6 +213,9 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-11 Unit Tests"
 
       - label: ":windows: Heartbeat Win-10 Unit Tests"
         key: "heartbeat-windows-extended-10"
@@ -164,32 +230,46 @@ steps:
         artifact_paths:
           - "heartbeat/build/*.xml"
           - "heartbeat/build/*.json"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Win-10 Unit Tests"
+
+  - wait: ~
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on: "heartbeat-mandatory-tests"
 
   - group: "Heartbeat Packaging"
     key: "heartbeat-packaging"
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on:
-      - "heartbeat-mandatory-tests"
     steps:
       - label: ":ubuntu: Heartbeat Packaging Linux X86"
         key: "heartbeat-package-linux-x86"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
-        command: "cd heartbeat && mage package"
+        command: |
+          cd heartbeat
+          mage package
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Packaging Ubuntu x86_64"
 
       - label: ":linux: Heartbeat Packaging Linux ARM"
         key: "heartbeat-package-linux-arm"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: true
-        command: "cd heartbeat && mage package"
+        command: |
+          cd heartbeat
+          mage package
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Packaging Ubuntu ARM"

.buildkite/hooks/pre-command

@@ -11,7 +11,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-inges
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token"
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" ]]; then
   source .buildkite/env-scripts/env.sh
   if [[ -z "${GO_VERSION-""}" ]]; then
     export GO_VERSION=$(cat "${WORKSPACE}/.go-version")
@@ -56,6 +56,7 @@ fi
 
 ENABLED_BEATS_PIPELINES_SLUGS=(
   "auditbeat"
+  "heartbeat"
   "filebeat"
   "beats-metricbeat"
   "beats-packetbeat"
@@ -66,7 +67,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
   "beats-xpack-winlogbeat"
   "beats-xpack-dockerlogbeat"
   "beats-xpack-auditbeat"
-  "beats-xpack-filebeat"
   "beats-xpack-metricbeat"
   "beats-xpack-heartbeat"
   "deploy-k8s"

.buildkite/hooks/scripts/util.sh

@@ -25,10 +25,11 @@ google_cloud_logout_active_account() {
 }
 
 cleanup() {
-  # TODO: disable for upload pipeline
+  if [[ "$BUILDKITE_COMMAND" != *"buildkite-agent pipeline upload"* ]]; then
   echo "Deleting temporary files..."
-  if [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
-    rm -rf "${BIN}/${TMP_FOLDER}.*"
-  fi
+    if [[ -n "${BIN:-}" ]] && [[ -e "${BIN}/${TMP_FOLDER}" ]]; then
+      rm -rf "${BIN}/${TMP_FOLDER}.*"
+    fi
   echo "Done."
+  fi
 }

.buildkite/pipeline.yml

@@ -80,6 +80,33 @@ steps:
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
 
+  - label: "Trigger x-pack/filebeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/filebeat/
+                - x-pack/libbeat/
+                - .buildkite/x-pack/pipeline.xpack.filebeat.yml
+                - .buildkite/scripts
+                - .buildkite/hooks/
+                #OSS
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/**
+                - testing/**
+              config:
+                trigger: "beats-xpack-filebeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
   - label: "Trigger Xpack/Dockerlogbeat"
     plugins:
       - monorepo-diff#v1.0.1:
@@ -258,3 +285,57 @@ steps:
                   env:
                     - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - label: "Trigger x-pack/auditbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/auditbeat/
+                - .buildkite/x-pack/pipeline.xpack.auditbeat.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+                # x-pack
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/
+                - testing/
+                - x-pack/libbeat/
+              config:
+                trigger: "beats-xpack-auditbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - label: "Trigger x-pack/heartbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/heartbeat/
+                - .buildkite/x-pack/pipeline.xpack.heartbeat.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+                # x-pack
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/
+                - testing/
+                - x-pack/libbeat/
+              config:
+                trigger: "beats-xpack-heartbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

.buildkite/pull-requests.json

@@ -95,22 +95,6 @@
             "skip_target_branches": [ ],
             "skip_ci_on_only_changed": [ ],
             "always_require_ci_on_changed": ["^x-pack/libbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
-        },
-        {
-            "enabled": true,
-            "pipelineSlug": "beats-xpack-heartbeat",
-            "allow_org_users": true,
-            "allowed_repo_permissions": ["admin", "write"],
-            "allowed_list": [ ],
-            "set_commit_status": true,
-            "build_on_commit": true,
-            "build_on_comment": true,
-            "trigger_comment_regex": "^/test x-pack/heartbeat$",
-            "always_trigger_comment_regex": "^/test x-pack/heartbeat$",
-            "skip_ci_labels": [  ],
-            "skip_target_branches": [ ],
-            "skip_ci_on_only_changed": [ ],
-            "always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
         }
     ]
 }