Migrate xpack packetbeat

.buildkite/hooks/pre-command

@@ -33,7 +33,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" ]]; then
   source .buildkite/scripts/setenv.sh
   if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
     echo "Skipped pre-command when running the Upload pipeline"

.buildkite/scripts/common.sh

@@ -22,6 +22,8 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
 [ -z "${run_xpack_libbeat_arm_tests+x}" ] && run_xpack_libbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_libbeat_arm_tests --default "false")"
 [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")"
 [ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
+[ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")"
+[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
 
 metricbeat_changeset=(
   "^metricbeat/.*"
@@ -105,6 +107,9 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
   "beats-xpack-metricbeat")
     BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
     ;;
+  "beats-xpack-packetbeat")
+    BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]}
+    ;;
   *)
   echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet."
   ;;

.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh

@@ -106,8 +106,7 @@ else
 fi
 
 #TODO: replace by commented-out below condition when issues mentioned in the PR https://github.com/elastic/beats/pull/38081 are resolved
-if [[ are_conditions_met_aws_tests || are_conditions_met_macos_tests ]]; then
-# if [[ are_conditions_met_macos_tests ]]; then
+if are_conditions_met_aws_tests || are_conditions_met_macos_tests ; then
   cat >> $pipelineName <<- YAML
 
   - group: "Extended Tests"

.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh

@@ -0,0 +1,193 @@
+#!/usr/bin/env bash
+
+source .buildkite/scripts/common.sh
+
+set -euo pipefail
+
+pipelineName="pipeline.xpack-packetbeat-dynamic.yml"
+
+echo "Add the mandatory and extended tests without additional conditions into the pipeline"
+if are_conditions_met_mandatory_tests; then
+  cat > $pipelineName <<- YAML
+
+steps:
+
+  - group: "Mandatory Tests"
+    key: "mandatory-tests"
+    steps:
+      - label: ":linux: Ubuntu Unit Tests"
+        key: "mandatory-linux-unit-test"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":linux: Ubuntu System Tests"
+        key: "mandatory-linux-system-test"
+        command: "cd $BEATS_PROJECT_NAME && mage systemTest"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml"
+
+      - label: ":rhel: RHEL-9 Unit Tests"
+        key: "mandatory-rhel9-unit-test"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_RHEL9_X86_64}"
+          machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "mandatory-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_2016}"
+              - "${IMAGE_WIN_2022}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      ##  TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
+      # - label: ":windows: Windows 2022 System Tests"
+      #   key: "mandatory-win-2022-system-tests"
+      #   command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
+      #   agents:
+      #     provider: "gcp"
+      #     image: "${IMAGE_WIN_2022}"
+      #     machineType: "${GCP_WIN_MACHINE_TYPE}"
+      #     disk_size: 100
+      #     disk_type: "pd-ssd"
+      #   artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h
+
+  - group: "Extended Windows Tests"
+    key: "extended-win-tests"
+    steps:
+
+      - label: ":windows: Windows Unit Tests - {{matrix.image}}"
+        command: ".buildkite/scripts/win_unit_tests.ps1"
+        key: "extended-win-unit-tests"
+        agents:
+          provider: "gcp"
+          image: "{{matrix.image}}"
+          machineType: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 100
+          disk_type: "pd-ssd"
+        matrix:
+          setup:
+            image:
+              - "${IMAGE_WIN_10}"
+              - "${IMAGE_WIN_11}"
+              - "${IMAGE_WIN_2019}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+      ##  TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved
+      # - label: ":windows: Windows 10 System Tests"
+      #   key: "extended-win-10-system-tests"
+      #   command: ".buildkite/scripts/win_unit_tests.ps1 systemtest"
+      #   agents:
+      #     provider: "gcp"
+      #     image: "${IMAGE_WIN_10}"
+      #     machineType: "${GCP_WIN_MACHINE_TYPE}"
+      #     disk_size: 100
+      #     disk_type: "pd-ssd"
+      #   artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+else
+  echo "The conditions don't match to requirements for generating pipeline steps."
+  exit 0
+fi
+
+if are_conditions_met_arm_tests || are_conditions_met_macos_tests ; then
+  cat >> $pipelineName <<- YAML
+
+  - group: "Extended Tests"
+    key: "extended-tests"
+    steps:
+
+YAML
+fi
+
+if  are_conditions_met_macos_tests; then
+  cat >> $pipelineName <<- YAML
+
+      - label: ":mac: MacOS Unit Tests"
+        key: "extended-macos-unit-tests"
+        command: ".buildkite/scripts/unit_tests.sh"
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+if  are_conditions_met_arm_tests; then
+  cat >> $pipelineName <<- YAML
+      - label: ":linux: ARM Ubuntu Unit Tests"
+        key: "extended-arm64-unit-test"
+        command: "cd $BEATS_PROJECT_NAME && mage build unitTest"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
+
+YAML
+fi
+
+echo "Check and add the Packaging into the pipeline"
+if are_conditions_met_packaging; then
+  cat >> $pipelineName <<- YAML
+
+  - wait: ~
+    depends_on:
+      - step: "mandatory-tests"
+        allow_failure: false
+
+  - group: "Packaging"    # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
+    key: "packaging"
+    steps:
+      - label: ":linux: Packaging Linux"
+        key: "packaging-linux"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        env:
+          PLATFORMS: "${PACKAGING_PLATFORMS}"
+
+      - label: ":linux: Packaging ARM"
+        key: "packaging-arm"
+        command: "cd $BEATS_PROJECT_NAME && mage package"
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        env:
+          PLATFORMS: "${PACKAGING_ARM_PLATFORMS}"
+          PACKAGES: "docker"
+
+YAML
+fi
+
+echo "--- Printing dynamic steps"     #TODO: remove if the pipeline is public
+cat $pipelineName
+
+echo "--- Loading dynamic steps"
+buildkite-agent pipeline upload $pipelineName

.buildkite/scripts/setenv.sh

@@ -11,6 +11,8 @@ SETUP_WIN_PYTHON_VERSION="3.11.0"
 NMAP_WIN_VERSION="7.12"           # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI)
 GO_VERSION=$(cat .go-version)
 ASDF_MAGE_VERSION="1.15.0"
+PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+PACKAGING_ARM_PLATFORMS="linux/arm64"
 ASDF_TERRAFORM_VERSION="1.0.2"
 PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
 PACKAGING_ARM_PLATFORMS="linux/arm64"

.buildkite/scripts/win_unit_tests.ps1

@@ -1,3 +1,7 @@
+param(
+    [string]$testType = "unittest"
+)
+
 $ErrorActionPreference = "Stop" # set -e
 $WorkFolder = $env:BEATS_PROJECT_NAME
 $WORKSPACE = Get-Location
@@ -120,6 +124,14 @@ function withNmap($version) {
     }
     Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait
 }
+function google_cloud_auth {
+    $tempFileName = "google-cloud-credentials.json"
+    $secretFileLocation = Join-Path $env:TEMP $tempFileName
+    $null = New-Item -ItemType File -Path $secretFileLocation
+    Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET
+    gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1
+    $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation
+}
 
 fixCRLF
 
@@ -142,10 +154,19 @@ $env:MAGEFILE_CACHE = $magefile
 
 New-Item -ItemType Directory -Force -Path "build"
 
-if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") {
-    mage -w reader/etw build goUnitTest
-} else {
-    mage build unitTest
+if ($testType -eq "unittest") {
+    if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") {
+        mage -w reader/etw build goUnitTest
+    } else {
+        mage build unitTest
+    }
+}
+elseif ($testType -eq "systemtest") {
+    google_cloud_auth
+    mage systemTest
+}
+else {
+    Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'."
 }
 
 $EXITCODE=$LASTEXITCODE

.buildkite/x-pack/pipeline.xpack.packetbeat.yml

@@ -1,5 +1,61 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+name: "beats-xpack-packetbeat"
+
+env:
+  IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+  IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
+  IMAGE_WIN_10: "family/general-windows-10"
+  IMAGE_WIN_11: "family/general-windows-11"
+  IMAGE_WIN_2016: "family/core-windows-2016"
+  IMAGE_WIN_2019: "family/core-windows-2019"
+  IMAGE_WIN_2022: "family/core-windows-2022"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+  GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
+  GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
+  GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  BEATS_PROJECT_NAME: "x-pack/packetbeat"
 
 steps:
-  - label: "Example test"
-    command: echo "Hello!"
+
+  - input: "Input Parameters"
+    key: "force-run-stages"
+    fields:
+    - select: "Packetbeat - run_xpack_packetbeat"
+      key: "run_xpack_packetbeat"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - run_xpack_packetbeat_macos_tests"
+      key: "run_xpack_packetbeat_macos_tests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+    - select: "Packetbeat - run_xpack_packetbeat_arm_tests"
+      key: "run_xpack_packetbeat_arm_tests"
+      options:
+        - label: "True"
+          value: "true"
+        - label: "False"
+          value: "false"
+      default: "false"
+
+    if: "build.source == 'ui'"
+
+  - wait: ~
+    if: "build.source == 'ui'"
+    allow_dependency_failure: false
+
+  - label: ":linux: Load dynamic x-pack packetbeat pipeline"
+    key: "packetbeat-pipeline"
+    command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh"
+    notify:
+      - github_commit_status:
+          context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"

x-pack/packetbeat/magefile.go

@@ -172,6 +172,13 @@ func SystemTest(ctx context.Context) error {
 	return devtools.GoTest(ctx, args)
 }
 
+func getBucketName() string {
+	if os.Getenv("BUILDKITE") == "true" {
+		return "ingest-buildkite-ci"
+	}
+	return "obs-ci-cache"
+}
+
 // getNpcapInstaller gets the installer from the Google Cloud Storage service.
 //
 // On Windows platforms, if getNpcapInstaller is invoked with the environment variables
@@ -198,7 +205,8 @@ func getNpcapInstaller() error {
 			return err
 		}
 	}
+	ciBucketName := getBucketName()
 
 	fmt.Printf("getting %s from private cache\n", installer)
-	return sh.RunV("gsutil", "cp", "gs://obs-ci-cache/private/"+installer, dstPath)
+	return sh.RunV("gsutil", "cp", "gs://"+ciBucketName+"/private/"+installer, dstPath)
 }