From 30c7cbebc960291b680d32a08199584f6f954a40 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 11:11:18 +0200 Subject: [PATCH] Migrate xpack libbeat pipeline (#38006) (#38073) Migrate x-pack libbeat pipeline Fix the Windows script remove the Choco --------- Co-authored-by: Victor Martinez (cherry picked from commit 1f38627297cadfd3f56b00623da0278c8e39a58e) Co-authored-by: sharbuz <87968844+sharbuz@users.noreply.github.com> --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/common.sh | 84 +++++++++--- .../generate_xpack_libbeat_pipeline.sh | 127 ++++++++++++++++++ .buildkite/scripts/setenv.sh | 3 +- .buildkite/scripts/win_unit_tests.ps1 | 127 ++++++++++++++---- .buildkite/x-pack/pipeline.xpack.libbeat.yml | 49 ++++++- 6 files changed, 348 insertions(+), 44 deletions(-) create mode 100755 .buildkite/scripts/generate_xpack_libbeat_pipeline.sh diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 4dda7a884dda..0ac7c51099c0 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -12,7 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == fi fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" ]]; then source .buildkite/scripts/setenv.sh if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then echo "Skipped pre-command when running the Upload pipeline" diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 6e7e19ea7d2d..b797ec718aa7 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -9,7 +9,6 @@ arch_type="$(uname -m)" GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""} ONLY_DOCS=${ONLY_DOCS:-"true"} - [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_metricbeat+x}" ] && run_metricbeat="$(buildkite-agent meta-data get run_metricbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" @@ -18,9 +17,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_metricbeat_macos_tests+x}" ] && run_metricbeat_macos_tests="$(buildkite-agent meta-data get run_metricbeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" -trigger_specific_beat="run_${BEATS_PROJECT_NAME}" -trigger_specific_arm_tests="run_${BEATS_PROJECT_NAME}_arm_tests" -trigger_specific_macos_tests="run_${BEATS_PROJECT_NAME}_macos_tests" metricbeat_changeset=( "^metricbeat/.*" @@ -38,6 +34,30 @@ winlogbeat_changeset=( "^winlogbeat/.*" ) +xpack_libbeat_changeset=( + "^x-pack/libbeat/.*" + ) + +xpack_metricbeat_changeset=( + "^x-pack/metricbeat/.*" + ) + +xpack_packetbeat_changeset=( + "^x-pack/packetbeat/.*" + ) + +xpack_winlogbeat_changeset=( + "^x-pack/winlogbeat/.*" + ) + +ci_changeset=( + "^.buildkite/.*" + ) + +go_mod_changeset=( + "^go.mod" + ) + oss_changeset=( "^go.mod" "^pytest.ini" @@ -46,14 +66,11 @@ oss_changeset=( "^testing/.*" ) -ci_changeset=( - "^.buildkite/.*" +xpack_changeset=( + "${xpack_libbeat_changeset[@]}" + "${oss_changeset[@]}" ) -go_mod_changeset=( - "^go.mod" - ) - docs_changeset=( ".*\\.(asciidoc|md)" "deploy/kubernetes/.*-kubernetes\\.yaml" @@ -64,6 +81,41 @@ packaging_changeset=( ".go-version" ) +check_and_set_beat_vars() { + if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then + BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - + BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ + BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels + BEATS_GH_LABEL=${BEATS_XPACK_LABEL_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_XPACK_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_XPACK_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${xpack_changeset[@]}" + "${ci_changeset[@]}" + ) + else + BEATS_GH_LABEL=${BEATS_PROJECT_NAME} + TRIGGER_SPECIFIC_BEAT="run_${BEATS_PROJECT_NAME}" + TRIGGER_SPECIFIC_ARM_TESTS="run_${BEATS_PROJECT_NAME}_arm_tests" + TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" + declare -n BEAT_CHANGESET_REFERENCE="${BEATS_PROJECT_NAME}_changeset" + echo "Beats project name is $BEATS_PROJECT_NAME" + mandatory_changeset=( + "${BEAT_CHANGESET_REFERENCE[@]}" + "${oss_changeset[@]}" + "${ci_changeset[@]}" + ) + fi + BEATS_GH_COMMENT="/test ${BEATS_PROJECT_NAME}" + BEATS_GH_MACOS_COMMENT="${BEATS_GH_COMMENT} for macos" + BEATS_GH_ARM_COMMENT="${BEATS_GH_COMMENT} for arm" + BAETS_GH_MACOS_LABEL="macOS" + BAETS_GH_ARM_LABEL="arm" +} with_docker_compose() { local version=$1 @@ -240,11 +292,7 @@ are_changed_only_paths() { } are_conditions_met_mandatory_tests() { - declare -n beat_changeset_reference="${BEATS_PROJECT_NAME}_changeset" - if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 - return 0 - fi - if are_paths_changed "${beat_changeset_reference[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_PROJECT_NAME}/ || "${!trigger_specific_beat}" == "true" ]]; then + if are_paths_changed "${mandatory_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_COMMENT}" || "${GITHUB_PR_LABELS}" =~ /(?i)${BEATS_GH_LABEL}/ || "${!TRIGGER_SPECIFIC_BEAT}" == "true" ]]; then return 0 fi return 1 @@ -253,7 +301,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${!trigger_specific_arm_tests}" == "true" ]]; then + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_ARM_LABEL}" || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi fi @@ -264,7 +312,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test ${BEATS_PROJECT_NAME} for macos" || "${GITHUB_PR_LABELS}" =~ macOS || "${!trigger_specific_macos_tests}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 + if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ "${BAETS_GH_MACOS_LABEL}" || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi fi @@ -302,3 +350,5 @@ fi if are_paths_changed "${packaging_changeset[@]}" ; then PACKAGING_CHANGES="true" fi + +check_and_set_beat_vars diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh new file mode 100755 index 000000000000..66f0750ab6fa --- /dev/null +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.libbeat-dynamic.yml" + +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + + - group: "Mandatory Tests" + key: "mandatory-tests" + steps: + - label: ":linux: Ubuntu Unit Tests" + key: "mandatory-linux-unit-test" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":go: Go Integration Tests" + key: "mandatory-int-test" + command: ".buildkite/scripts/go_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":python: Python Integration Tests" + key: "mandatory-python-int-test" + command: ".buildkite/scripts/py_int_tests.sh" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: ".buildkite/scripts/win_unit_tests.ps1" + key: "mandatory-win-unit-tests" + agents: + provider: "gcp" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + matrix: + setup: + image: + - "${IMAGE_WIN_2016}" + - "${IMAGE_WIN_2022}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h + - group: "Extended Windows Tests" + key: "extended-win-tests" + steps: + - label: ":windows: Win 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: ".buildkite/scripts/win_unit_tests.ps1" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machineType: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + - label: ":linux: Arm64 Unit Tests" + key: "extended-arm64-unit-tests" + command: ".buildkite/scripts/unit_tests.sh" + agents: + provider: "aws" + imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + +YAML +fi + +echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public +cat $pipelineName + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index dbf23c3e5ef9..25121de212fd 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -6,13 +6,14 @@ SETUP_GVM_VERSION="v0.5.1" DOCKER_COMPOSE_VERSION="1.21.0" DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" SETUP_WIN_PYTHON_VERSION="3.11.0" +NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) GO_VERSION=$(cat .go-version) - export SETUP_GVM_VERSION export DOCKER_COMPOSE_VERSION export DOCKER_COMPOSE_VERSION_AARCH64 export SETUP_WIN_PYTHON_VERSION +export NMAP_WIN_VERSION export GO_VERSION exportVars() { diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 index da0ffc105a51..b3c5c58fac0a 100644 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ b/.buildkite/scripts/win_unit_tests.ps1 @@ -1,5 +1,6 @@ $ErrorActionPreference = "Stop" # set -e -$WorkFolder = "metricbeat" +$WorkFolder = $env:BEATS_PROJECT_NAME +$WORKSPACE = Get-Location # Forcing to checkout again all the files with a correct autocrlf. # Doing this here because we cannot set git clone options before. function fixCRLF { @@ -8,30 +9,93 @@ function fixCRLF { git rm --quiet --cached -r . git reset --quiet --hard } -function withChoco { - Write-Host "-- Configure Choco --" - $env:ChocolateyInstall = Convert-Path "$((Get-Command choco).Path)\..\.." - Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +function retry { + param( + [int]$retries, + [ScriptBlock]$scriptBlock + ) + $count = 0 + while ($count -lt $retries) { + $count++ + try { + & $scriptBlock + return + } catch { + $exitCode = $_.Exception.ErrorCode + Write-Host "Retry $count/$retries exited $exitCode, retrying..." + Start-Sleep -Seconds ([Math]::Pow(2, $count)) + } + } + Write-Host "Retry $count/$retries exited, no more retries left." +} + +function verifyFileChecksum { + param ( + [string]$filePath, + [string]$checksumFilePath + ) + $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash + $checksumData = Get-Content -Path $checksumFilePath + $expectedHash = ($checksumData -split "\s+")[0] + if ($actualHash -eq $expectedHash) { + Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " + return $true + } else { + Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" + return $false + } } + function withGolang($version) { - $downloadPath = Join-Path $env:TEMP "go_installer.msi" + Write-Host "-- Installing Go $version --" + $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" - Invoke-WebRequest -Uri $goInstallerUrl -OutFile $downloadPath - Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $downloadPath /quiet" -Wait - $goBinPath = "${env:ProgramFiles}\Go\bin" - $env:Path += ";$goBinPath" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath + } + Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait + $env:GOPATH = "${env:ProgramFiles}\Go" + $env:GOBIN = "${env:GOPATH}\bin" + $env:Path += ";$env:GOPATH;$env:GOBIN" go version + installGoDependencies } + function withPython($version) { - Write-Host "-- Install Python $version --" - choco install python --version=$version - refreshenv + Write-Host "-- Installing Python $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" + $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath + } + Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait + $pyBinPath = "${env:ProgramFiles}\Python311" + $env:Path += ";$pyBinPath" python --version } + function withMinGW { - Write-Host "-- Install MinGW --" - choco install mingw -y - refreshenv + Write-Host "-- Installing MinGW --" + [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" + $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" + $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" + $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" + $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath + Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath + } + $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath + if ($comparingResult) { + Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" + $gwBinPath = "$env:TEMP\mingw64\bin" + $env:Path += ";$gwBinPath" + } else { + exit 1 + } + } function installGoDependencies { $installPackages = @( @@ -46,26 +110,43 @@ function installGoDependencies { } } -fixCRLF +function withNmap($version) { + Write-Host "-- Installing Nmap $version --" + [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" + $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" + $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" + retry -retries 5 -scriptBlock { + Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath + } + Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait +} -withChoco +fixCRLF withGolang $env:GO_VERSION -installGoDependencies - withPython $env:SETUP_WIN_PYTHON_VERSION withMinGW +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat") { + withNmap $env:NMAP_WIN_VERSION +} + $ErrorActionPreference = "Continue" # set +e -Push-Location $WorkFolder +Set-Location -Path $WorkFolder + +$magefile = "$WORKSPACE\$WorkFolder\.magefile" +$env:MAGEFILE_CACHE = $magefile New-Item -ItemType Directory -Force -Path "build" -mage build unitTest -Pop-Location +if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { + mage -w reader/etw build goUnitTest +} else { + mage build unitTest +} $EXITCODE=$LASTEXITCODE $ErrorActionPreference = "Stop" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 34321b61161b..01695fa4fb65 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -1,5 +1,50 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +name: "beats-xpack-libbeat" + +env: + IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" + IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" + IMAGE_WIN_10: "family/general-windows-10" + IMAGE_WIN_11: "family/general-windows-11" + IMAGE_WIN_2016: "family/core-windows-2016" + IMAGE_WIN_2019: "family/core-windows-2019" + IMAGE_WIN_2022: "family/core-windows-2022" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" steps: - - label: "Example test" - command: echo "Hello!" + + - input: "Input Parameters" + key: "input-run-all-stages" + fields: + - select: "Packetbeat - run_xpack_libbeat" + key: "run_xpack_libbeat" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + - select: "Packetbeat - run_xpack_libbeat_arm_tests" + key: "run_xpack_libbeat_arm_tests" + options: + - label: "True" + value: "true" + - label: "False" + value: "false" + default: "false" + if: "build.source == 'ui'" + + - wait: ~ + if: "build.source == 'ui'" + allow_dependency_failure: false + + - label: ":linux: Load dynamic packetbeat pipeline" + key: "packetbeat-pipeline" + command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + notify: + - github_commit_status: + context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"