[CI] x-pack/heartbeat monorepo (#38845)

Use mono repo plugin for xpack/heartbeat Relates: https://github.com/elastic/ingest-dev/issues/3072 --------- Co-authored-by: Dimitrios Liappis <[email protected]> (cherry picked from commit 0d3a8b6) # Conflicts: # .buildkite/scripts/common.sh # .buildkite/scripts/setenv.sh # .buildkite/x-pack/pipeline.xpack.heartbeat.yml
elastic · Apr 18, 2024 · 3d83a1e · 3d83a1e
1 parent 6872842
commit 3d83a1e
Show file tree

Hide file tree

Showing 6 changed files with 308 additions and 247 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -286,3 +286,30 @@ steps:
                     - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
                     - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
                     - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
+
+  - label: "Trigger x-pack/heartbeat"
+    plugins:
+      - monorepo-diff#v1.0.1:
+          diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
+          watch:
+            - path:
+                - x-pack/heartbeat/
+                - .buildkite/x-pack/pipeline.xpack.heartbeat.yml
+                - .buildkite/scripts/
+                - .buildkite/hooks/
+                # x-pack
+                - go.mod
+                - pytest.ini
+                - dev-tools/
+                - libbeat/
+                - testing/
+                - x-pack/libbeat/
+              config:
+                trigger: "beats-xpack-heartbeat"
+                build:
+                  commit: "${BUILDKITE_COMMIT}"
+                  branch: "${BUILDKITE_BRANCH}"
+                  env:
+                    - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
+                    - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
+                    - GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh
@@ -18,9 +18,18 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")"
 [ -z "${run_xpack_metricbeat+x}" ] && run_xpack_metricbeat="$(buildkite-agent meta-data get run_xpack_metricbeat --default "false")"
 [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")"
+<<<<<<< HEAD
 [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")"
 [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")"
 
+=======
+<<<<<<< HEAD
+[ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")"
+
+=======
+[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")"
+>>>>>>> main
+>>>>>>> 0d3a8b68d9 ([CI] x-pack/heartbeat monorepo (#38845))
 # define if needed run ARM platform-specific tests for the particular beat
 [ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")"
 [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
@@ -33,7 +42,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")"
 [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
 [ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
-[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")"
 
 # define if needed run cloud-specific tests for the particular beat
 [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
@@ -55,8 +63,13 @@ xpack_dockerlogbeat_changeset=(
   "^x-pack/dockerlogbeat/.*"
   )
 
+<<<<<<< HEAD
+xpack_filebeat_changeset=(
+  "^x-pack/filebeat/.*"
+=======
 xpack_heartbeat_changeset=(
   "^x-pack/heartbeat/.*"
+>>>>>>> main
   )
 
 xpack_filebeat_changeset=(
@@ -116,11 +129,19 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
   "beats-winlogbeat")
     BEAT_CHANGESET_REFERENCE=${winlogbeat_changeset[@]}
     ;;
+<<<<<<< HEAD
   "beats-xpack-filebeat")
     BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]}
     ;;
+=======
+<<<<<<< HEAD
+  "beats-xpack-filebeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_filebeat_changeset[@]}
+=======
+>>>>>>> 0d3a8b68d9 ([CI] x-pack/heartbeat monorepo (#38845))
   "beats-xpack-heartbeat")
     BEAT_CHANGESET_REFERENCE=${xpack_heartbeat_changeset[@]}
+>>>>>>> main
     ;;
   "beats-xpack-libbeat")
     BEAT_CHANGESET_REFERENCE=${xpack_libbeat_changeset[@]}
@@ -393,7 +414,11 @@ are_conditions_met_arm_tests() {
 
 are_conditions_met_macos_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+<<<<<<< HEAD
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then
+=======
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+>>>>>>> 0d3a8b68d9 ([CI] x-pack/heartbeat monorepo (#38845))
       if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
         return 0
       fi

diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh
diff --git a/.buildkite/scripts/install_macos_tools.sh b/.buildkite/scripts/install_macos_tools.sh
@@ -133,3 +133,8 @@ with_go "${GO_VERSION}"
 with_mage
 with_python
 config_git
+
+# prevent "OSError: [Errno 24] Too many open files" on macOS
+ulimit -Sn 150000
+echo "~~~ Setting ulimit: $(ulimit)"
+echo "~~~ Resuming commands"
diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh
@@ -67,18 +67,12 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_S
   export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}oracle"
 fi
 
+<<<<<<< HEAD
 if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-filebeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then
+=======
+if [[ "$BUILDKITE_STEP_KEY" == "xpack-winlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-metricbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "xpack-dockerlogbeat-pipeline" || "$BUILDKITE_STEP_KEY" == "metricbeat-pipeline" ]]; then
+>>>>>>> 0d3a8b68d9 ([CI] x-pack/heartbeat monorepo (#38845))
   source .buildkite/scripts/common.sh
-  if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then
-    export ELASTIC_SYNTHETICS_CAPABLE=true
-  else
-    # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines.
-    defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
-  fi
-fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" ]]; then
   # Set the MODULE env variable if possible, it should be defined before generating pipeline's steps. It is used in multiple pipelines.
-  source .buildkite/scripts/common.sh
   defineModuleFromTheChangeSet "${BEATS_PROJECT_NAME}"
 fi