Heartbeat pipeline migration

.buildkite/auditbeat/auditbeat-pipeline.yml

@@ -20,7 +20,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: linux/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -33,7 +33,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: rhel/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_RHEL9}"
@@ -45,7 +45,7 @@ steps:
         command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: windows/Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"
@@ -68,7 +68,7 @@ steps:
           GOX_FLAGS: "-arch amd64"
         notify:
           - github_commit_status:
-              context: "auditbeat: Unit Tests"
+              context: "Auditbeat: Crosscompile"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -85,7 +85,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: Unit Tests ARM"
+              context: "Auditbeat/Extended: Unit Tests ARM"
         agents:
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
@@ -99,7 +99,7 @@ steps:
           - ".buildkite/auditbeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: MacOS Unit Tests"
+              context: "Auditbeat/Extended: MacOS Unit Tests"
         agents:
           provider: "orka"
           imagePrefix: "${IMAGE_MACOS_X86_64}"
@@ -115,7 +115,7 @@ steps:
         command: ".buildkite/auditbeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "auditbeat/Extended: Win-2019 Unit Tests"
+              context: "Auditbeat/Extended: Win-2019 Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_WIN_2019}"

.buildkite/auditbeat/scripts/package-step.sh

@@ -13,7 +13,7 @@ changeset="^auditbeat/
 ^\.buildkite/auditbeat/"
 
 if are_files_changed "$changeset"; then
-  cat <<-EOF
+  cat <<-YAML
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
@@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
-EOF
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
 fi

.buildkite/env-scripts/linux-env.sh

@@ -9,8 +9,6 @@ DEBIAN_FRONTEND="noninteractive"
 sudo mkdir -p /etc/needrestart
 echo "\$nrconf{restart} = 'a';" | sudo tee -a /etc/needrestart/needrestart.conf > /dev/null
 
-echo "--- PLATFORM TYPE $PLATFORM_TYPE"
-
 if [[ $PLATFORM_TYPE == "Linux" ]]; then
   # Remove this code once beats specific agent is set up
   if grep -q 'Ubuntu' /etc/*release; then

.buildkite/env-scripts/util.sh

@@ -85,7 +85,6 @@ are_files_changed() {
   if git diff --name-only HEAD@{1} HEAD | grep -qE "$changeset"; then
     return 0;
   else
-    echo "WARN! No files changed in $changeset"
     return 1;
   fi
 }

.buildkite/env-scripts/win-env.sh

@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
 
-echo "--- PLATFORM TYPE: ${PLATFORM_TYPE}"
 if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
   echo "--- Installing Python on Win"
   choco install mingw -y

.buildkite/filebeat/filebeat-pipeline.yml

@@ -19,7 +19,7 @@ steps:
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat: Unit Tests"
+              context: "Filebeat: linux/Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -33,7 +33,7 @@ steps:
           - ".buildkite/filebeat/scripts/integration-gotests.sh"
         notify:
           - github_commit_status:
-              context: "Filebeat: Integration Tests"
+              context: "Filebeat: Go Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -60,7 +60,7 @@ steps:
         command: ".buildkite/filebeat/scripts/unit-tests-win.ps1"
         notify:
           - github_commit_status:
-              context: "Filebeat: Unit Tests"
+              context: "Filebeat: windows/Unit Tests"
         agents:
           provider: "gcp"
           image: "{{matrix.image}}"

.buildkite/filebeat/scripts/package-step.sh

@@ -13,7 +13,7 @@ changeset="^filebeat/
 ^\.buildkite/filebeat/"
 
 if are_files_changed "$changeset"; then
-  cat <<-EOF
+  cat <<-YAML
     steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
@@ -42,5 +42,8 @@ if are_files_changed "$changeset"; then
           provider: "aws"
           imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
           instanceType: "t4g.large"
-EOF
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
 fi

.buildkite/heartbeat/heartbeat-pipeline.yml

@@ -1,5 +1,142 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_RHEL9: "family/core-rhel-9"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+  - group: "Heartbeat Mandatory Testing"
+    key: "mandatory-tests"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat" || build.env("BUILDKITE_PULL_REQUEST") != "false"
+
+    steps:
+      - label: ":linux: Unit Tests / {{matrix.image}}"
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: linux/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_UBUNTU_X86_64}"
+              - "${IMAGE_RHEL9}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":windows: Unit Tests / {{matrix.image}}"
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: windows/Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machine_type: "n2-standard-8"
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":ubuntu: Go Integration Tests"
+        command:
+          - ".buildkite/heartbeat/scripts/integration-gotests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Go Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+      - label: ":ubuntu: Python Integration Tests"
+        command:
+          - ".buildkite/heartbeat/scripts/integration-pytests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat: Python Integration Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+  - group: "Extended Testing"
+    key: "extended-tests"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false" || build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for extended support"
+
+    steps:
+      - label: ":linux: ARM64 Unit Tests"
+        key: "arm-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: Unit Tests ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+        artifact_paths: "heartbeat/build/*.xml"
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "macos-extended"
+        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+        command:
+          - ".buildkite/heartbeat/scripts/unit-tests.sh"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: MacOS Unit Tests"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "heartbeat/build/*.xml"
+
+  - group: "Windows Extended Testing"
+    key: "extended-tests-win"
+    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+
+    steps:
+      - label: ":windows: Win 2019 Unit Tests"
+        key: "win-extended-2019"
+        command: ".buildkite/heartbeat/scripts/unit-tests-win.ps1"
+        notify:
+          - github_commit_status:
+              context: "Heartbeat/Extended: Win-2019 Unit Tests"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2019}"
+          machine_type: "n2-standard-8"
+          disk_type: "pd-ssd"
+        artifact_paths:
+          - "heartbeat/build/*.xml"
+          - "heartbeat/build/*.json"
+
+  - group: "Packaging"
+    key: "packaging"
+    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+    depends_on:
+      - "mandatory-tests"
+
+    steps:
+      - label: Package pipeline
+        commands: ".buildkite/heartbeat/scripts/package-step.sh | buildkite-agent pipeline upload"

.buildkite/heartbeat/scripts/integration-gotests.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove when custom image is set up
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Integration Tests"
+# Remove when custom image is set up
+sudo chmod -R go-w heartbeat/
+
+cd heartbeat
+# Remove when custom image is set up
+umask 0022
+mage goIntegTest

.buildkite/heartbeat/scripts/integration-pytests.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Remove when custom image is set up
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Executing Integration Tests"
+# Remove when custom image is set up
+sudo chmod -R go-w heartbeat/
+
+cd heartbeat
+# Remove when custom image is set up
+umask 0022
+mage pythonIntegTest

.buildkite/heartbeat/scripts/package-step.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/util.sh
+
+changeset="^heartbeat/
+^go.mod
+^pytest.ini
+^dev-tools/
+^libbeat/
+^testing/
+^\.buildkite/heartbeat/"
+
+if are_files_changed "$changeset"; then
+  cat <<-YAML
+    steps:
+      - label: ":ubuntu: Packaging Linux X86"
+        key: "package-linux-x86"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+        command:
+          - ".buildkite/heartbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "heartbeat/Packaging: Linux X86"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+
+      - label: ":linux: Packaging Linux ARM"
+        key: "package-linux-arm"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+        command:
+          - ".buildkite/heartbeat/scripts/package.sh"
+        notify:
+          - github_commit_status:
+              context: "heartbeat/Packaging: ARM"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.large"
+YAML
+else
+  buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
+  exit 0
+fi

.buildkite/heartbeat/scripts/package.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/env-scripts/linux-env.sh
+
+echo "--- Docker Version: $(docker --version)"
+
+echo "--- Start Packaging"
+cd heartbeat
+umask 0022
+mage package
+