migrate packetbeat pipeline

.buildkite/hooks/pre-command

@@ -12,7 +12,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"

.buildkite/libbeat/pipeline.libbeat.yml

@@ -4,7 +4,7 @@ env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
-  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
   BEATS_PROJECT_NAME: "libbeat"

.buildkite/metricbeat/pipeline.yml

@@ -11,7 +11,7 @@ env:
   GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   BEATS_PROJECT_NAME: "metricbeat"
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
-  GCP_HI_PERF_MASHINE_TYPE: "c2d-highcpu-16"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
   AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
 

.buildkite/packetbeat/pipeline.packetbeat.yml

@@ -1,5 +1,58 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
+  IMAGE_RHEL9_X86_64: "family/core-rhel-9"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  BEATS_PROJECT_NAME: "packetbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "input-run-all-stages"
+    fields:
+    - select: "Packetbeat - runPacketbeat"
+      key: "runPacketbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - runPacketbeatArmTest"
+      key: "runPacketbeatArmTest"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - runPacketbeatMacOsTests"
+      key: "runPacketbeatMacOsTests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic packetbeat pipeline"
+    key: "packetbeat-pipeline"
+    command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
+    agents:
+      provider: "gcp"
+      image: "${IMAGE_UBUNTU_X86_64}"

.buildkite/scripts/common.sh

@@ -8,36 +8,48 @@ platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
 arch_type="$(uname -m)"
 GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
 ONLY_DOCS=${ONLY_DOCS:-"true"}
-runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
-runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
-runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
-runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+
+[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
+[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})"
+[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
+[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})"
+[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
+[ -z "${runPacketbeatMacOsTests+x}" ] && runPacketbeatMacOsTests="$(buildkite-agent meta-data get runPacketbeatMacOsTests --default ${runPacketbeatMacOsTests:-"false"})"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
-  "^go.mod"
-  "^pytest.ini"
-  "^dev-tools/.*"
+  )
+
+libbeat_changeset=(
   "^libbeat/.*"
-  "^testing/.*"
   )
+
+packetbeat_changeset=(
+  "^packetbeat/.*"
+  )
+
 oss_changeset=(
   "^go.mod"
   "^pytest.ini"
   "^dev-tools/.*"
   "^libbeat/.*"
   "^testing/.*"
 )
+
 ci_changeset=(
   "^.buildkite/.*"
 )
+
 go_mod_changeset=(
   "^go.mod"
   )
+
 docs_changeset=(
   ".*\\.(asciidoc|md)"
   "deploy/kubernetes/.*-kubernetes\\.yaml"
   )
+
 packaging_changeset=(
   "^dev-tools/packaging/.*"
   ".go-version"
@@ -77,7 +89,7 @@ check_platform_architeture() {
       go_arch_type="arm64"
       ;;
     *)
-    echo "The current platform/OS type is unsupported yet"
+    echo "The current platform or OS type is unsupported yet"
     ;;
   esac
 }
@@ -109,11 +121,32 @@ with_go() {
   export PATH="${go_path}:${PATH}"
 }
 
+checkLinuxType() {
+  if [ "${platform_type}" == "Linux" ]; then
+    if grep -q 'ubuntu' /etc/os-release; then
+      echo "ubuntu"
+    elif grep -q 'rhel' /etc/os-release; then
+      echo "rhel"
+    else
+      echo "Unsupported Linux"
+    fi
+  else
+      echo "This is not a Linux"
+  fi
+}
+
 with_python() {
+  local linuxType="$(checkLinuxType)"
+  echo "${linuxType}"
   if [ "${platform_type}" == "Linux" ]; then
-    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
-    sudo apt-get update
-    sudo apt-get install -y python3-pip python3-venv
+    if [ "${linuxType}" = "ubuntu" ]; then
+      sudo apt-get update
+      sudo apt-get install -y python3-pip python3-venv
+    elif [ "${linuxType}" = "rhel" ]; then
+      sudo dnf update -y
+      sudo dnf install -y python3 python3-pip
+      pip3 install virtualenv
+    fi
   elif [ "${platform_type}" == "Darwin" ]; then
     brew update
     pip3 install virtualenv
@@ -122,15 +155,30 @@ with_python() {
 }
 
 with_dependencies() {
+  local linuxType="$(checkLinuxType)"
+  echo "${linuxType}"
   if [ "${platform_type}" == "Linux" ]; then
-    #sudo command doesn't work at the "pre-command" hook because of another user environment (root with strange permissions)
-    sudo apt-get update
-    sudo apt-get install -y libsystemd-dev libpcap-dev
+    if [ "${linuxType}" = "ubuntu" ]; then
+      sudo apt-get update
+      sudo apt-get install -y libsystemd-dev libpcap-dev
+    elif [ "${linuxType}" = "rhel" ]; then
+      # sudo dnf update -y
+      sudo dnf install -y systemd-devel
+      wget https://mirror.stream.centos.org/9-stream/CRB/${arch_type}/os/Packages/libpcap-devel-1.10.0-4.el9.${arch_type}.rpm     #TODO: move this step to our own image
+      sudo dnf install -y libpcap-devel-1.10.0-4.el9.${arch_type}.rpm     #TODO: move this step to our own image
+    fi
   elif [ "${platform_type}" == "Darwin" ]; then
     pip3 install libpcap
   fi
 }
 
+config_git() {
+  if [ -z "$(git config --get user.email)" ]; then
+    git config --global user.email "[email protected]"
+    git config --global user.name "beatsmachine"
+  fi
+}
+
 retry() {
   local retries=$1
   shift
@@ -153,7 +201,6 @@ retry() {
 are_paths_changed() {
   local patterns=("${@}")
   local changelist=()
-
   for pattern in "${patterns[@]}"; do
     changed_files=($(git diff --name-only HEAD@{1} HEAD | grep -E "$pattern"))
     if [ "${#changed_files[@]}" -gt 0 ]; then
@@ -183,41 +230,65 @@ are_changed_only_paths() {
 }
 
 are_conditions_met_mandatory_tests() {
-  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+  if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
+      if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
         return 0
       fi
     elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
-      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+      if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
         return 0
       fi
     fi
   fi
   return 1
 }
 
-are_conditions_met_libbeat_arm_tests() {
+are_conditions_met_arm_tests() {
   if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
     if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
       if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runLibBeatArmTest}" == "true" ]]; then
         return 0
       fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for arm" || "${GITHUB_PR_LABELS}" =~ arm || "${runPacketbeatArmTest}" == "true" ]]; then
+        return 0
+      fi
     fi
   fi
   return 1
 }
 
-are_conditions_met_metricbeat_macos_tests() {
-  if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
-    return 0
+are_conditions_met_macos_tests() {
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+      if [[ "${runMetricbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${runPacketbeatMacOsTests}" == true ]] || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat for macos" ]] || [[ "${GITHUB_PR_LABELS}" =~ macOS ]]; then
+        return 0
+      fi
+    fi
   fi
   return 1
 }
 
 are_conditions_met_packaging() {
-  if are_paths_changed "${metricbeat_changeset[@]}" || are_paths_changed "${oss_changeset[@]}" || [[ "${BUILDKITE_TAG}" == "" ]] || [[ "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
-    return 0
+  if are_conditions_met_mandatory_tests; then    #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
+    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
+      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then   # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
+        return 0
+      fi
+    elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
+      if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
+        return 0
+      fi
+    fi
   fi
   return 1
 }

.buildkite/scripts/generate_libbeat_pipeline.sh

@@ -30,7 +30,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":python: Python Integration Tests"
@@ -39,7 +39,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":negative_squared_cross_mark: Cross compile"
@@ -48,7 +48,7 @@ steps:
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MASHINE_TYPE}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml"
 
       - label: ":testengine: Stress Tests"
@@ -61,10 +61,13 @@ steps:
         artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml"
 
 YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
 fi
 
 echo "Check and add the Extended Tests into the pipeline"
-if are_conditions_met_libbeat_arm_tests; then
+if are_conditions_met_arm_tests; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"