From 44ddf8030826858907e7f6be5a79ebaec7010b65 Mon Sep 17 00:00:00 2001 From: Siarhei Harbuz Date: Tue, 16 Jan 2024 18:24:24 +0200 Subject: [PATCH] debug MacOS and Ubuntu unit tests --- .buildkite/metricbeat/pipeline.yml | 154 ++++++++++---------- .buildkite/metricbeat/scripts/common.sh | 49 ++++--- .buildkite/metricbeat/scripts/packaging.sh | 2 - .buildkite/metricbeat/scripts/unit_tests.sh | 8 +- 4 files changed, 113 insertions(+), 100 deletions(-) diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index a4444efecefa..3b642b6c6424 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -24,45 +24,45 @@ steps: image: "${IMAGE_UBUNTU_X86_64}" machineType: "c2-standard-16" - - label: ":go: Go Intergration Tests" - key: "mandatory-int-test" - command: ".buildkite/metricbeat/scripts/go_int_tests.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # - label: ":go: Go Intergration Tests" + # key: "mandatory-int-test" + # command: ".buildkite/metricbeat/scripts/go_int_tests.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" - - label: ":python: Python Integration Tests" - key: "mandatory-python-int-test" - command: ".buildkite/metricbeat/scripts/py_int_tests.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # - label: ":python: Python Integration Tests" + # key: "mandatory-python-int-test" + # command: ".buildkite/metricbeat/scripts/py_int_tests.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" - - label: ":negative_squared_cross_mark: Cross compile" - key: "mandatory-cross-compile" - command: ".buildkite/metricbeat/scripts/crosscompile.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # - label: ":negative_squared_cross_mark: Cross compile" + # key: "mandatory-cross-compile" + # command: ".buildkite/metricbeat/scripts/crosscompile.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" - - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/metricbeat/scripts/win_unit_tests.ps1" - key: "mandatory-win-unit-tests" - agents: - provider: "gcp" - image: "{{matrix.image}}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - matrix: - setup: - image: - - "${IMAGE_WIN_2016}" - - "${IMAGE_WIN_2022}" - artifact_paths: "metricbeat/build/*.*" + # - label: ":windows: Windows Unit Tests - {{matrix.image}}" + # command: ".buildkite/metricbeat/scripts/win_unit_tests.ps1" + # key: "mandatory-win-unit-tests" + # agents: + # provider: "gcp" + # image: "{{matrix.image}}" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # matrix: + # setup: + # image: + # - "${IMAGE_WIN_2016}" + # - "${IMAGE_WIN_2022}" + # artifact_paths: "metricbeat/build/*.*" - group: "Extended Tests" key: "extended-tests" @@ -77,46 +77,46 @@ steps: provider: "orka" imagePrefix: "generic-13-ventura-x64" - - group: "Extended Windowds Tests" - key: "extended-win-tests" - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/metricbeat/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "n2-standard-8" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "metricbeat/build/*.*" + # - group: "Extended Windowds Tests" + # key: "extended-win-tests" + # steps: + # - label: ":windows: Win 2019 Unit Tests" + # key: "extended-win-2019-unit-tests" + # command: ".buildkite/metricbeat/scripts/win_unit_tests.ps1" + # agents: + # provider: "gcp" + # image: "${IMAGE_WIN_2019}" + # machine_type: "n2-standard-8" + # disk_size: 200 + # disk_type: "pd-ssd" + # artifact_paths: "metricbeat/build/*.*" - - group: "Packaging" - key: "packaging" - if: build.env("BUILDKITE_TAG") == "" && build.env("BUILDKITE_PULL_REQUEST") != "" - depends_on: - - step: "mandatory-tests" - allow_failure: false - - step: "extended-tests" - allow_failure: false - - step: "extended-win-tests" - allow_failure: false - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: ".buildkite/metricbeat/scripts/packaging.sh" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "c2-standard-16" + # - group: "Packaging" + # key: "packaging" + # if: build.env("BUILDKITE_TAG") == "" && build.env("BUILDKITE_PULL_REQUEST") != "" + # depends_on: + # - step: "mandatory-tests" + # allow_failure: false + # - step: "extended-tests" + # allow_failure: false + # - step: "extended-win-tests" + # allow_failure: false + # steps: + # - label: ":linux: Packaging Linux" + # key: "packaging-linux" + # command: ".buildkite/metricbeat/scripts/packaging.sh" + # agents: + # provider: "gcp" + # image: "${IMAGE_UBUNTU_X86_64}" + # machineType: "c2-standard-16" - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: ".buildkite/metricbeat/scripts/packaging.sh" - agents: - provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" - instanceType: "t4g.xlarge" - env: - PLATFORMS: "linux/arm64" - PACKAGES: "docker" + # - label: ":linux: Packaging ARM" + # key: "packaging-arm" + # command: ".buildkite/metricbeat/scripts/packaging.sh" + # agents: + # provider: "aws" + # imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + # instanceType: "t4g.xlarge" + # env: + # PLATFORMS: "linux/arm64" + # PACKAGES: "docker" diff --git a/.buildkite/metricbeat/scripts/common.sh b/.buildkite/metricbeat/scripts/common.sh index 0ddf949167e6..a16229615dbc 100755 --- a/.buildkite/metricbeat/scripts/common.sh +++ b/.buildkite/metricbeat/scripts/common.sh @@ -2,6 +2,8 @@ set -euo pipefail WORKSPACE=${WORKSPACE:-"$(pwd)"} +platform_type="$(uname)" +arch_type="$(uname -m)" create_workspace() { if [[ ! -d "${WORKSPACE}/bin" ]]; then @@ -15,6 +17,22 @@ add_bin_path() { export PATH="${PATH}:${WORKSPACE}/bin" } +check_platform_architeture() { + case "${hw_type}" in + "x86_64") + arch_type="amd64" + ;; + "aarch64") + arch_type="arm64" + ;; + "arm64") + arch_type="arm64" + ;; + *) + echo "The current platform/OS type is unsupported yet" + ;; + esac +} # with_yq() { # pip install yq @@ -36,34 +54,31 @@ with_mage() { with_go() { go_version=$1 - url=$(get_gvm_link "${SETUP_GVM_VERSION}") + echo "Setting up the Go environment..." create_workspace - retry 5 curl -sL -o "${WORKSPACE}/bin/gvm" "${url}" + check_platform_architeture + local platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') + retry 5 curl -sL -o "${WORKSPACE}/bin/gvm" "https://github.com/andrewkroh/gvm/releases/download/${go_version}/gvm-${platform_type_lowercase}-${arch_type}" chmod +x "${WORKSPACE}/bin/gvm" - ls -l ${WORKSPACE}/bin/ - eval "$(gvm $go_version)" + eval "$(gvm $(cat .go-version))" + go version + which go go_path="$(go env GOPATH):$(go env GOPATH)/bin" export PATH="${PATH}:${go_path}" - go version } with_python() { - if [ "$(uname)" == "Linux" ]; then + if [ "${platform_type}" == "Linux" ]; then sudo apt-get update - sudo apt-get install -y python3 python3-venv python3-pip libsystemd-dev + sudo apt-get install -y python3-venv python3-pip libsystemd-dev + elif [ "${platform_type}" == "Darwin" ]; then + brew update + brew install python3 libsystemd + pip3 install --upgrade pip + pip3 install virtualenv fi } -get_gvm_link() { - gvm_version=$1 - platform_type="$(uname)" - platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') - arch_type="$(uname -m)" - [[ ${arch_type} == "aarch64" ]] && arch_type="arm64" - [[ ${arch_type} == "x86_64" ]] && arch_type="amd64" - echo "https://github.com/andrewkroh/gvm/releases/download/${gvm_version}/gvm-${platform_type_lowercase}-${arch_type}" -} - retry() { local retries=$1 shift diff --git a/.buildkite/metricbeat/scripts/packaging.sh b/.buildkite/metricbeat/scripts/packaging.sh index 59b5bafe37f4..977b2530af8d 100755 --- a/.buildkite/metricbeat/scripts/packaging.sh +++ b/.buildkite/metricbeat/scripts/packaging.sh @@ -24,7 +24,5 @@ with_python echo "--- Packaging" pushd "metricbeat" > /dev/null -# chmod -R go-w ./mb/testdata/ -# umask 0022 mage package popd > /dev/null diff --git a/.buildkite/metricbeat/scripts/unit_tests.sh b/.buildkite/metricbeat/scripts/unit_tests.sh index 85337f4588ce..48c3aca14ba9 100755 --- a/.buildkite/metricbeat/scripts/unit_tests.sh +++ b/.buildkite/metricbeat/scripts/unit_tests.sh @@ -4,16 +4,16 @@ source .buildkite/metricbeat/scripts/common.sh set -euo pipefail -echo "--- prepare env" +echo "--- Prepare env" ulimit -Sn 50000 add_bin_path with_go ${GO_VERSION} with_mage with_python -echo "--- run unit tests" +echo "--- Run unit tests" +sudo chmod -R go-w metricbeat/ pushd "metricbeat" > /dev/null -# chmod -R go-w ./mb/testdata/ -# umask 0022 +umask 0022 mage build unitTest popd > /dev/null