-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
migrate packetbeat pipeline (#37825)
- Loading branch information
Showing
10 changed files
with
385 additions
and
43 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,58 @@ | ||
| # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json | ||
| env: | ||
| IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" | ||
| IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" | ||
| IMAGE_RHEL9_X86_64: "family/core-rhel-9" | ||
| IMAGE_WIN_10: "family/general-windows-10" | ||
| IMAGE_WIN_11: "family/general-windows-11" | ||
| IMAGE_WIN_2016: "family/core-windows-2016" | ||
| IMAGE_WIN_2019: "family/core-windows-2019" | ||
| IMAGE_WIN_2022: "family/core-windows-2022" | ||
| IMAGE_MACOS_X86_64: "generic-13-ventura-x64" | ||
| GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" | ||
| GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" | ||
| GCP_WIN_MACHINE_TYPE: "n2-standard-8" | ||
| AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" | ||
| BEATS_PROJECT_NAME: "packetbeat" | ||
|
|
||
| steps: | ||
| - label: "Example test" | ||
| command: echo "Hello!" | ||
|
|
||
| - input: "Input Parameters" | ||
| key: "input-run-all-stages" | ||
| fields: | ||
| - select: "Packetbeat - runPacketbeat" | ||
| key: "runPacketbeat" | ||
| options: | ||
| - label: "True" | ||
| value: "true" | ||
| - label: "False" | ||
| value: "false" | ||
| default: "false" | ||
| - select: "Packetbeat - runPacketbeatArmTest" | ||
| key: "runPacketbeatArmTest" | ||
| options: | ||
| - label: "True" | ||
| value: "true" | ||
| - label: "False" | ||
| value: "false" | ||
| default: "false" | ||
| - select: "Packetbeat - runPacketbeatMacOsTests" | ||
| key: "runPacketbeatMacOsTests" | ||
| options: | ||
| - label: "True" | ||
| value: "true" | ||
| - label: "False" | ||
| value: "false" | ||
| default: "false" | ||
| if: "build.source == 'ui'" | ||
|
|
||
| - wait: ~ | ||
| if: "build.source == 'ui'" | ||
| allow_dependency_failure: false | ||
|
|
||
| - label: ":linux: Load dynamic packetbeat pipeline" | ||
| key: "packetbeat-pipeline" | ||
| command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" | ||
| agents: | ||
| provider: "gcp" | ||
| image: "${IMAGE_UBUNTU_X86_64}" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,36 +8,48 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]') | |
| arch_type="$(uname -m)" | ||
| GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""} | ||
| ONLY_DOCS=${ONLY_DOCS:-"true"} | ||
| runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" | ||
| runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" | ||
| runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" | ||
| runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" | ||
|
|
||
| [ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" | ||
| [ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})" | ||
| [ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})" | ||
| [ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})" | ||
| [ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})" | ||
| [ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})" | ||
| [ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})" | ||
|
|
||
| metricbeat_changeset=( | ||
| "^metricbeat/.*" | ||
| "^go.mod" | ||
| "^pytest.ini" | ||
| "^dev-tools/.*" | ||
| ) | ||
|
|
||
| libbeat_changeset=( | ||
| "^libbeat/.*" | ||
| "^testing/.*" | ||
| ) | ||
|
|
||
| packetbeat_changeset=( | ||
| "^packetbeat/.*" | ||
| ) | ||
|
|
||
| oss_changeset=( | ||
| "^go.mod" | ||
| "^pytest.ini" | ||
| "^dev-tools/.*" | ||
| "^libbeat/.*" | ||
| "^testing/.*" | ||
| ) | ||
|
|
||
| ci_changeset=( | ||
| "^.buildkite/.*" | ||
| ) | ||
|
|
||
| go_mod_changeset=( | ||
| "^go.mod" | ||
| ) | ||
|
|
||
| docs_changeset=( | ||
| ".*\\.(asciidoc|md)" | ||
| "deploy/kubernetes/.*-kubernetes\\.yaml" | ||
| ) | ||
|
|
||
| packaging_changeset=( | ||
| "^dev-tools/packaging/.*" | ||
| ".go-version" | ||
|
|
@@ -77,7 +89,7 @@ check_platform_architeture() { | |
| go_arch_type="arm64" | ||
| ;; | ||
| *) | ||
| echo "The current platform/OS type is unsupported yet" | ||
| echo "The current platform or OS type is unsupported yet" | ||
| ;; | ||
| esac | ||
| } | ||
|
|
@@ -109,11 +121,32 @@ with_go() { | |
| export PATH="${go_path}:${PATH}" | ||
| } | ||
|
|
||
| checkLinuxType() { | ||
| if [ "${platform_type}" == "Linux" ]; then | ||
| if grep -q 'ubuntu' /etc/os-release; then | ||
| echo "ubuntu" | ||
| elif grep -q 'rhel' /etc/os-release; then | ||
| echo "rhel" | ||
| else | ||
| echo "Unsupported Linux" | ||
| fi | ||
| else | ||
| echo "This is not a Linux" | ||
| fi | ||
| } | ||
|
|
||
| with_python() { | ||
| local linuxType="$(checkLinuxType)" | ||
| echo "${linuxType}" | ||
| if [ "${platform_type}" == "Linux" ]; then | ||
| #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) | ||
| sudo apt-get update | ||
| sudo apt-get install -y python3-pip python3-venv | ||
| if [ "${linuxType}" = "ubuntu" ]; then | ||
| sudo apt-get update | ||
| sudo apt-get install -y python3-pip python3-venv | ||
| elif [ "${linuxType}" = "rhel" ]; then | ||
| sudo dnf update -y | ||
| sudo dnf install -y python3 python3-pip | ||
| pip3 install virtualenv | ||
| fi | ||
| elif [ "${platform_type}" == "Darwin" ]; then | ||
| brew update | ||
| pip3 install virtualenv | ||
|
|
@@ -122,15 +155,30 @@ with_python() { | |
| } | ||
|
|
||
| with_dependencies() { | ||
| local linuxType="$(checkLinuxType)" | ||
| echo "${linuxType}" | ||
| if [ "${platform_type}" == "Linux" ]; then | ||
| #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions) | ||
| sudo apt-get update | ||
| sudo apt-get install -y libsystemd-dev libpcap-dev | ||
| if [ "${linuxType}" = "ubuntu" ]; then | ||
| sudo apt-get update | ||
| sudo apt-get install -y libsystemd-dev libpcap-dev | ||
| elif [ "${linuxType}" = "rhel" ]; then | ||
| # sudo dnf update -y | ||
| sudo dnf install -y systemd-devel | ||
| wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image | ||
| sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm #TODO: move this step to our own image | ||
| fi | ||
| elif [ "${platform_type}" == "Darwin" ]; then | ||
| pip3 install libpcap | ||
| fi | ||
| } | ||
|
|
||
| config_git() { | ||
| if [ -z "$(git config --get user.email)" ]; then | ||
| git config --global user.email "[email protected]" | ||
| git config --global user.name "beatsmachine" | ||
| fi | ||
| } | ||
|
|
||
| retry() { | ||
| local retries=$1 | ||
| shift | ||
|
|
@@ -153,7 +201,6 @@ retry() { | |
| are_paths_changed() { | ||
| local patterns=("${@}") | ||
| local changelist=() | ||
|
|
||
| for pattern in "${patterns[@]}"; do | ||
| changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern")) | ||
| if [ "${#changed_files[@]}" -gt 0 ]; then | ||
|
|
@@ -183,41 +230,65 @@ are_changed_only_paths() { | |
| } | ||
|
|
||
| are_conditions_met_mandatory_tests() { | ||
| if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 | ||
| if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 | ||
| if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then | ||
| if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then | ||
| if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then | ||
| return 0 | ||
| fi | ||
| elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then | ||
| if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then | ||
| if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then | ||
| return 0 | ||
| fi | ||
| elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then | ||
| if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then | ||
| return 0 | ||
| fi | ||
| fi | ||
| fi | ||
| return 1 | ||
| } | ||
|
|
||
| are_conditions_met_libbeat_arm_tests() { | ||
| are_conditions_met_arm_tests() { | ||
| if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 | ||
| if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then | ||
| if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then | ||
| return 0 | ||
| fi | ||
| elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then | ||
| if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then | ||
| return 0 | ||
| fi | ||
| fi | ||
| fi | ||
| return 1 | ||
| } | ||
|
|
||
| are_conditions_met_metricbeat_macos_tests() { | ||
| if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 | ||
| return 0 | ||
| are_conditions_met_macos_tests() { | ||
| if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 | ||
| if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then | ||
| if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 | ||
| return 0 | ||
| fi | ||
| elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then | ||
| if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then | ||
| return 0 | ||
| fi | ||
| fi | ||
| fi | ||
| return 1 | ||
| } | ||
|
|
||
| are_conditions_met_packaging() { | ||
| if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 | ||
| return 0 | ||
| if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 | ||
| if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then | ||
| if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103 | ||
| return 0 | ||
| fi | ||
| elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then | ||
| if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then | ||
| return 0 | ||
| fi | ||
| fi | ||
| fi | ||
| return 1 | ||
| } | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.