diff --git a/.buildkite/env-scripts/env.sh b/.buildkite/env-scripts/env.sh new file mode 100644 index 000000000000..d94d03aad53b --- /dev/null +++ b/.buildkite/env-scripts/env.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SETUP_GVM_VERSION="v0.5.1" +WORKSPACE="$(pwd)" +BIN="${WORKSPACE}/bin" +HW_TYPE="$(uname -m)" +PLATFORM_TYPE="$(uname)" + +export SETUP_GVM_VERSION +export WORKSPACE +export BIN +export HW_TYPE +export PLATFORM_TYPE diff --git a/.buildkite/env-scripts/linux-env.sh b/.buildkite/env-scripts/linux-env.sh new file mode 100644 index 000000000000..1453807a073d --- /dev/null +++ b/.buildkite/env-scripts/linux-env.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source .buildkite/filebeat/scripts/common.sh + +DEBIAN_FRONTEND="noninteractive" + +export DEBIAN_FRONTEND + +# Remove this code once beats specific agent is set up +if [[ $PLATFORM_TYPE == "Linux" ]]; then + echo ":: Installing libs ::" + sudo apt-get update + sudo apt-get install -y libsystemd-dev + sudo apt install -y python3-pip + sudo apt-get install -y python3-venv +fi + +echo ":: Setting up environment ::" +add_bin_path +with_go +with_mage diff --git a/.buildkite/env-scripts/macos-env.sh b/.buildkite/env-scripts/macos-env.sh new file mode 100644 index 000000000000..c247c5a03a73 --- /dev/null +++ b/.buildkite/env-scripts/macos-env.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then + echo ":: Setting larger ulimit on MacOS ::" + # To bypass file descriptor errors like "Too many open files error" on MacOS + ulimit -Sn 10000 +fi diff --git a/.buildkite/env-scripts/win-env.sh b/.buildkite/env-scripts/win-env.sh new file mode 100644 index 000000000000..aa5f67ca4cee --- /dev/null +++ b/.buildkite/env-scripts/win-env.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +install_python_win() { + if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + choco install mingw -y + choco install python --version=3.11.0 -y + fi +} diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 38e56ed0a154..8bcfa72f69f2 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -17,10 +17,6 @@ steps: steps: - label: ":ubuntu: Unit Tests" command: - - "sudo apt-get update" - - "sudo apt-get install -y libsystemd-dev" - - "sudo apt install -y python3-pip" - - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/unit-tests.sh" notify: - github_commit_status: @@ -28,11 +24,12 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "filebeat/build/TEST-go-unit.xml" + - "filebeat/build/TEST-go-unit.out.json" - label: ":ubuntu: Go Integration Tests" command: - - "sudo apt-get update" - - "sudo apt-get install -y libsystemd-dev" - ".buildkite/filebeat/scripts/integration-gotests.sh" notify: - github_commit_status: @@ -40,12 +37,15 @@ steps: agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" + artifact_paths: + - "filebeat/build/TEST-go-integration.xml" + - "filebeat/build/TEST-go-integration.out.json" - label: ":ubuntu: Python Integration Tests" command: - - "sudo apt-get update" - - "sudo apt-get install -y libsystemd-dev" - - "sudo apt-get install -y python3-venv" +# - "sudo apt-get update" +# - "sudo apt-get install -y libsystemd-dev" +# - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/integration-pytests.sh" notify: - github_commit_status: @@ -70,6 +70,9 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" + artifact_paths: + - "filebeat/build/TEST-go-unit.xml" + - "filebeat/build/TEST-go-unit.out.json" - group: "Extended Testing" key: "extended-tests" @@ -78,12 +81,12 @@ steps: steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ command: - - "sudo apt-get update" - - "sudo apt-get install -y libsystemd-dev" - - "sudo apt install -y python3-pip" - - "sudo apt-get install -y python3-venv" +# - "sudo apt-get update" +# - "sudo apt-get install -y libsystemd-dev" +# - "sudo apt install -y python3-pip" +# - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/unit-tests.sh" notify: - github_commit_status: @@ -97,7 +100,6 @@ steps: key: "macos-extended" # if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: -# - "ulimit -Sn 10000" - ".buildkite/filebeat/scripts/unit-tests.sh" notify: - github_commit_status: @@ -109,7 +111,7 @@ steps: - group: "Windows Extended Testing" key: "extended-tests-win" - if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ +# if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "filebeat for windows" || build.env("GITHUB_PR_LABELS") =~ /.*windows.*/ steps: - label: ":windows: Win 2019 Unit Tests" @@ -137,8 +139,8 @@ steps: - label: ":ubuntu: Packaging Linux X86" key: "package-linux-x86" command: - - "sudo apt-get update" - - "sudo apt-get install -y python3-venv" +# - "sudo apt-get update" +# - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/package.sh" notify: - github_commit_status: @@ -153,8 +155,8 @@ steps: PLATFORMS: "linux/arm64" PACKAGES: "docker" command: - - "sudo apt-get update" - - "sudo apt-get install -y python3-venv" +# - "sudo apt-get update" +# - "sudo apt-get install -y python3-venv" - ".buildkite/filebeat/scripts/package.sh" notify: - github_commit_status: diff --git a/.buildkite/filebeat/scripts/common.sh b/.buildkite/filebeat/scripts/common.sh index 943d3b98a6f0..e0cfd8460441 100644 --- a/.buildkite/filebeat/scripts/common.sh +++ b/.buildkite/filebeat/scripts/common.sh @@ -2,10 +2,10 @@ set -euo pipefail -WORKSPACE="$(pwd)" -BIN="${WORKSPACE}/bin" -HW_TYPE="$(uname -m)" -PLATFORM_TYPE="$(uname)" +#WORKSPACE="$(pwd)" +#BIN="${WORKSPACE}/bin" +#HW_TYPE="$(uname -m)" +#PLATFORM_TYPE="$(uname)" add_bin_path() { echo "Adding PATH to the environment variables..." @@ -95,11 +95,11 @@ are_files_changed() { fi } -prepare_win() { - local os - os="$(uname)" - if [[ $os = MINGW* ]]; then - choco install mingw -y - choco install python --version=3.11.0 -y - fi -} +#prepare_win() { +## local os +## os="$(uname)" +# if [[ ${PLATFORM_TYPE} = MINGW* ]]; then +# choco install mingw -y +# choco install python --version=3.11.0 -y +# fi +#} diff --git a/.buildkite/filebeat/scripts/integration-gotests.sh b/.buildkite/filebeat/scripts/integration-gotests.sh index 5060a18047f5..2a3d19e5ead8 100755 --- a/.buildkite/filebeat/scripts/integration-gotests.sh +++ b/.buildkite/filebeat/scripts/integration-gotests.sh @@ -3,13 +3,14 @@ set -euo pipefail source .buildkite/filebeat/scripts/common.sh - -# ToDo - remove after Beats agent is created" -echo ":: Setup Env ::" -add_bin_path -with_go -with_mage -# ToDo - end +source .buildkite/env-scripts/linux-env.sh +# +## ToDo - remove after Beats agent is created" +#echo ":: Setup Env ::" +#add_bin_path +#with_go +#with_mage +## ToDo - end echo ":: Execute Integration Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/integration-pytests.sh b/.buildkite/filebeat/scripts/integration-pytests.sh index 4e5057a7efd0..5e2e403dda87 100755 --- a/.buildkite/filebeat/scripts/integration-pytests.sh +++ b/.buildkite/filebeat/scripts/integration-pytests.sh @@ -2,14 +2,7 @@ set -euo pipefail -source .buildkite/filebeat/scripts/common.sh - -# ToDo - remove after Beats agent is created" -echo ":: Setup Env ::" -add_bin_path -with_go -with_mage -# ToDo - end +source .buildkite/env-scripts/linux-env.sh echo ":: Execute Integration Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/filebeat/scripts/package.sh b/.buildkite/filebeat/scripts/package.sh index 050bbda53fe2..748de8a7e7bc 100755 --- a/.buildkite/filebeat/scripts/package.sh +++ b/.buildkite/filebeat/scripts/package.sh @@ -3,6 +3,7 @@ set -euo pipefail source .buildkite/filebeat/scripts/common.sh +source .buildkite/env-scripts/linux-env.sh echo ":: Evaluate Filebeat Changes ::" @@ -22,12 +23,12 @@ if ! are_files_changed "$changeset" ; then exit 0 fi -# ToDo - remove after Beats agent is created" -echo ":: Setup Env ::" -add_bin_path -with_go -with_mage -# ToDo - end +## ToDo - remove after Beats agent is created" +#echo ":: Setup Env ::" +#add_bin_path +#with_go +#with_mage +## ToDo - end echo ":: Start Packaging ::" cd filebeat diff --git a/.buildkite/filebeat/scripts/unit-tests-win.ps1 b/.buildkite/filebeat/scripts/unit-tests-win.ps1 index 48571b95036f..8990eb30a093 100644 --- a/.buildkite/filebeat/scripts/unit-tests-win.ps1 +++ b/.buildkite/filebeat/scripts/unit-tests-win.ps1 @@ -1,4 +1,5 @@ $ErrorActionPreference = "Stop" # set -e +$GoVersion = $env:GOLANG_VERSION # If Choco doesn't have the version specified in .go-version file, should be changed manually # Forcing to checkout again all the files with a correct autocrlf. # Doing this here because we cannot set git clone options before. @@ -10,11 +11,8 @@ function fixCRLF() { } function withGolang() { - # Write-Host "-- Install golang $env:GOLANG_VERSION --" - # choco install golang -y --version $env:GOLANG_VERSION - - Write-Host "-- Install golang 1.20.7 --" - choco install golang -y --version 1.20.7 + Write-Host "-- Install golang $GoVersion --" + choco install golang -y --version $GoVersion $choco = Convert-Path "$((Get-Command choco).Path)\..\.." Import-Module "$choco\helpers\chocolateyProfile.psm1" diff --git a/.buildkite/filebeat/scripts/unit-tests.sh b/.buildkite/filebeat/scripts/unit-tests.sh index 25f7f751bd7b..583a9a30231a 100755 --- a/.buildkite/filebeat/scripts/unit-tests.sh +++ b/.buildkite/filebeat/scripts/unit-tests.sh @@ -2,14 +2,12 @@ set -euo pipefail -source .buildkite/filebeat/scripts/common.sh +source .buildkite/env-scripts/linux-env.sh +source .buildkite/env-scripts/macos-env.sh -# ToDo - remove after Beats agent is created" -echo ":: Setup Env ::" -add_bin_path -with_go -with_mage -# ToDo - end +if [[ $PLATFORM_TYPE == Darwin* ]]; then + echo ":: ULIMIT :: $(ulimit -n)" +fi echo ":: Execute Unit Tests ::" sudo chmod -R go-w filebeat/ diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 88697d41d1a3..4338c4016fbc 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,18 +2,23 @@ set -euo pipefail +source .buildkite/env-scripts/env.sh source .buildkite/filebeat/scripts/common.sh +source .buildkite/env-scripts/win-env.sh -if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then - export WORKSPACE="$(pwd)" +#SETUP_GVM_VERSION="v0.5.1" +#WORKSPACE="$(pwd)" +# +#export SETUP_GVM_VERSION +#export WORKSPACE - prepare_win +if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then + if [[ ${PLATFORM_TYPE} = MINGW* ]]; then + install_python_win + fi + # prepare_win if [[ -z "${GOLANG_VERSION-""}" ]]; then export GOLANG_VERSION=$(cat "${WORKSPACE}/.go-version") fi - - if [[ "$BUILDKITE_STEP_KEY" == macos* ]]; then - ulimit -Sn 30000 - fi fi