moved env setup to separate script

.buildkite/env-scripts/env.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+SETUP_GVM_VERSION="v0.5.1"
+WORKSPACE="$(pwd)"
+BIN="${WORKSPACE}/bin"
+HW_TYPE="$(uname -m)"
+PLATFORM_TYPE="$(uname)"
+
+export SETUP_GVM_VERSION
+export WORKSPACE
+export BIN
+export HW_TYPE
+export PLATFORM_TYPE

.buildkite/env-scripts/linux-env.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+source .buildkite/filebeat/scripts/common.sh
+
+DEBIAN_FRONTEND="noninteractive"
+
+export DEBIAN_FRONTEND
+
+# Remove this code once beats specific agent is set up
+if [[ $PLATFORM_TYPE == "Linux" ]]; then
+  echo ":: Installing libs ::"
+  sudo apt-get update
+  sudo apt-get install -y libsystemd-dev
+  sudo apt install -y python3-pip
+  sudo apt-get install -y python3-venv
+fi
+
+echo ":: Setting up environment ::"
+add_bin_path
+with_go
+with_mage

.buildkite/env-scripts/macos-env.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
+  echo ":: Setting larger ulimit on MacOS ::"
+  # To bypass file descriptor errors like "Too many open files error" on MacOS
+  ulimit -Sn 10000
+fi

.buildkite/env-scripts/win-env.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+install_python_win() {
+  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+    choco install mingw -y
+    choco install python --version=3.11.0 -y
+  fi
+}

.buildkite/filebeat/filebeat-pipeline.yml

@@ -17,35 +17,35 @@ steps:
     steps:
       - label: ":ubuntu: Unit Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt install -y python3-pip"
-          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Mandatory: Unit Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-unit.xml"
+          - "filebeat/build/TEST-go-unit.out.json"
 
       - label: ":ubuntu: Go Integration Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
           - ".buildkite/filebeat/scripts/integration-gotests.sh"
         notify:
           - github_commit_status:
               context: "Filebeat/Mandatory: Integration Tests"
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-integration.xml"
+          - "filebeat/build/TEST-go-integration.out.json"
 
       - label: ":ubuntu: Python Integration Tests"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y libsystemd-dev"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/integration-pytests.sh"
         notify:
           - github_commit_status:
@@ -70,6 +70,9 @@ steps:
             image:
               - "${IMAGE_WIN_2016}"
               - "${IMAGE_WIN_2022}"
+        artifact_paths:
+          - "filebeat/build/TEST-go-unit.xml"
+          - "filebeat/build/TEST-go-unit.out.json"
 
   - group: "Extended Testing"
     key: "extended-tests"
@@ -78,12 +81,12 @@ steps:
     steps:
       - label: ":linux: ARM64 Unit Tests"
         key: "arm-extended"
-        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+#        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y libsystemd-dev"
-          - "sudo apt install -y python3-pip"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y libsystemd-dev"
+#          - "sudo apt install -y python3-pip"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
@@ -97,7 +100,6 @@ steps:
         key: "macos-extended"
 #        if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command:
-#          - "ulimit -Sn 10000"
           - ".buildkite/filebeat/scripts/unit-tests.sh"
         notify:
           - github_commit_status:
@@ -109,7 +111,7 @@ steps:
 
   - group: "Windows Extended Testing"
     key: "extended-tests-win"
-    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
+#    if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/
 
     steps:
       - label: ":windows: Win 2019 Unit Tests"
@@ -137,8 +139,8 @@ steps:
       - label: ":ubuntu: Packaging Linux X86"
         key: "package-linux-x86"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:
@@ -153,8 +155,8 @@ steps:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         command:
-          - "sudo apt-get update"
-          - "sudo apt-get install -y python3-venv"
+#          - "sudo apt-get update"
+#          - "sudo apt-get install -y python3-venv"
           - ".buildkite/filebeat/scripts/package.sh"
         notify:
           - github_commit_status:

.buildkite/filebeat/scripts/common.sh

@@ -2,10 +2,10 @@
 
 set -euo pipefail
 
-WORKSPACE="$(pwd)"
-BIN="${WORKSPACE}/bin"
-HW_TYPE="$(uname -m)"
-PLATFORM_TYPE="$(uname)"
+#WORKSPACE="$(pwd)"
+#BIN="${WORKSPACE}/bin"
+#HW_TYPE="$(uname -m)"
+#PLATFORM_TYPE="$(uname)"
 
 add_bin_path() {
     echo "Adding PATH to the environment variables..."
@@ -95,11 +95,11 @@ are_files_changed() {
   fi
 }
 
-prepare_win() {
-  local os
-  os="$(uname)"
-  if [[ $os = MINGW* ]]; then
-    choco install mingw -y
-    choco install python --version=3.11.0 -y
-  fi
-}
+#prepare_win() {
+##  local os
+##  os="$(uname)"
+#  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+#    choco install mingw -y
+#    choco install python --version=3.11.0 -y
+#  fi
+#}

.buildkite/filebeat/scripts/integration-gotests.sh

@@ -3,13 +3,14 @@
 set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+source .buildkite/env-scripts/linux-env.sh
+#
+## ToDo - remove after Beats agent is created"
+#echo ":: Setup Env ::"
+#add_bin_path
+#with_go
+#with_mage
+## ToDo - end
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/filebeat/scripts/integration-pytests.sh

@@ -2,14 +2,7 @@
 
 set -euo pipefail
 
-source .buildkite/filebeat/scripts/common.sh
-
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+source .buildkite/env-scripts/linux-env.sh
 
 echo ":: Execute Integration Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/filebeat/scripts/package.sh

@@ -3,6 +3,7 @@
 set -euo pipefail
 
 source .buildkite/filebeat/scripts/common.sh
+source .buildkite/env-scripts/linux-env.sh
 
 echo ":: Evaluate Filebeat Changes ::"
 
@@ -22,12 +23,12 @@ if ! are_files_changed "$changeset" ; then
   exit 0
 fi
 
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+## ToDo - remove after Beats agent is created"
+#echo ":: Setup Env ::"
+#add_bin_path
+#with_go
+#with_mage
+## ToDo - end
 
 echo ":: Start Packaging ::"
 cd filebeat

.buildkite/filebeat/scripts/unit-tests-win.ps1

@@ -1,4 +1,5 @@
 $ErrorActionPreference = "Stop" # set -e
+$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually
 
 # Forcing to checkout again all the files with a correct autocrlf.
 # Doing this here because we cannot set git clone options before.
@@ -10,11 +11,8 @@ function fixCRLF() {
 }
 
 function withGolang() {
-    #    Write-Host "-- Install golang $env:GOLANG_VERSION --"
-    #    choco install golang -y --version $env:GOLANG_VERSION
-
-    Write-Host "-- Install golang 1.20.7 --"
-    choco install golang -y --version 1.20.7
+    Write-Host "-- Install golang $GoVersion --"
+    choco install golang -y --version $GoVersion
 
     $choco = Convert-Path "$((Get-Command choco).Path)\..\.."
     Import-Module "$choco\helpers\chocolateyProfile.psm1"

.buildkite/filebeat/scripts/unit-tests.sh

@@ -2,14 +2,12 @@
 
 set -euo pipefail
 
-source .buildkite/filebeat/scripts/common.sh
+source .buildkite/env-scripts/linux-env.sh
+source .buildkite/env-scripts/macos-env.sh
 
-# ToDo - remove after Beats agent is created"
-echo ":: Setup Env ::"
-add_bin_path
-with_go
-with_mage
-# ToDo - end
+if [[ $PLATFORM_TYPE == Darwin* ]]; then
+  echo ":: ULIMIT :: $(ulimit -n)"
+fi
 
 echo ":: Execute Unit Tests ::"
 sudo chmod -R go-w filebeat/

.buildkite/hooks/pre-command

@@ -2,18 +2,23 @@
 
 set -euo pipefail
 
+source .buildkite/env-scripts/env.sh
 source .buildkite/filebeat/scripts/common.sh
+source .buildkite/env-scripts/win-env.sh
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
-  export WORKSPACE="$(pwd)"
+#SETUP_GVM_VERSION="v0.5.1"
+#WORKSPACE="$(pwd)"
+#
+#export SETUP_GVM_VERSION
+#export WORKSPACE
 
-  prepare_win
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then
+  if [[ ${PLATFORM_TYPE} = MINGW* ]]; then
+    install_python_win
+  fi
+  #  prepare_win
 
   if [[ -z "${GOLANG_VERSION-""}" ]]; then
     export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version")
   fi
-
-  if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then
-    ulimit -Sn 30000
-  fi
 fi