moved env setup to separate script

elastic · Jan 17, 2024 · d1ca7df · d1ca7df
1 parent f3e74b9
commit d1ca7df
Show file tree

Hide file tree

Showing 7 changed files with 74 additions and 46 deletions.
diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -17,35 +17,35 @@ steps:
     steps:
       - label: ":ubuntu: Unit Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt install -y python3-pip"
-          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Mandatory: Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-unit.xml"
+          - "filebeat/build/TEST-go-unit.out.json"
 
       - label: ":ubuntu: Go Integration Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
           - ".buildkite/filebeat/scripts/integration-gotests.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Mandatory: Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-integration.xml"
+          - "filebeat/build/TEST-go-integration.out.json"
 
       - label: ":ubuntu: Python Integration Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y libsystemd-dev"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/integration-pytests.sh"
         notify:
           - github_commit_status:
@@ -70,6 +70,9 @@ steps:
             image:
               - "${IMAGE_WIN_2016}"
               - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-unit.xml"
+          - "filebeat/build/TEST-go-unit.out.json"
 
   - group: "Extended Testing"
     key: "extended-tests"
@@ -80,10 +83,10 @@ steps:
         key: "arm-extended"
         if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt install -y python3-pip"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y libsystemd-dev"
+#          - "sudo apt install -y python3-pip"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
@@ -97,7 +100,6 @@ steps:
         key: "macos-extended"
 #        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command:
-#          - "ulimit -Sn 10000"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
@@ -137,8 +139,8 @@ steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:
@@ -153,8 +155,8 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:

diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh
@@ -3,13 +3,14 @@
 set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+source .buildkite/util-scripts/linux-env.sh
+#
+## ToDo - remove after Beats agent is created"
+#echo ":: Setup Env ::"
+#add_bin_path
+#with_go
+#with_mage
+## ToDo - end
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh
@@ -3,13 +3,14 @@
 set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+source .buildkite/util-scripts/linux-env.sh
+#
+## ToDo - remove after Beats agent is created"
+#echo ":: Setup Env ::"
+#add_bin_path
+#with_go
+#with_mage
+## ToDo - end
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh
@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
+source .buildkite/util-scripts/linux-env.sh
 
 echo ":: Evaluate Filebeat Changes ::"
 
@@ -22,12 +23,12 @@ if ! are_files_changed "$changeset" ; then
   exit 0
 fi
 
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+## ToDo - remove after Beats agent is created"
+#echo ":: Setup Env ::"
+#add_bin_path
+#with_go
+#with_mage
+## ToDo - end
 
 echo ":: Start Packaging ::"
 cd filebeat

diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh
@@ -2,14 +2,11 @@
 
 set -euo pipefail
 
-source .buildkite/filebeat/scripts/common.sh
+source .buildkite/util-scripts/linux-env.sh
 
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi
 
 echo ":: Execute Unit Tests ::"
 sudo chmod -R go-w filebeat/

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -4,6 +4,10 @@ set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
 
+SETUP_GVM_VERSION="v0.5.1"
+
+export SETUP_GVM_VERSION
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   export WORKSPACE="$(pwd)"
 
@@ -14,6 +18,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
   fi
 
   if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    ulimit -Sn 30000
+    # To bypass file descriptor errors like "Too many open files error" on MacOS
+    ulimit -Sn 70000
   fi
 fi
diff --git a/.buildkite/util-scripts/linux-env.sh b/.buildkite/util-scripts/linux-env.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source .buildkite/filebeat/scripts/common.sh
+
+DEBIAN_FRONTEND="noninteractive"
+
+export DEBIAN_FRONTEND
+
+# Remove this code once beats specific agent is set up
+if [[ $PLATFORM_TYPE == "Linux" ]]; then
+  echo ":: Installing libs ::"
+  sudo apt-get update
+  sudo apt-get install -y libsystemd-dev
+  sudo apt install -y python3-pip
+  sudo apt-get install -y python3-venv
+fi
+
+echo ":: Setting up environment ::"
+add_bin_path
+with_go
+with_mage