diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index a22e96c974aa..1cc2f1585387 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -4,7 +4,6 @@ name: "beats-filebeat" env: AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - BEATS_PROJECT_NAME: "filebeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" @@ -17,60 +16,276 @@ env: IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - #Packaging - PACKAGING_ARM_PLATFORMS: "linux/arm64" - PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" - #Deps ASDF_MAGE_VERSION: 1.15.0 + # Integration Tests + K8S_VERSION: "v1.29.0" + ASDF_KIND_VERSION: "0.20.0" steps: - - input: "Input Parameters" - key: "run_filebeat" - fields: - - select: "filebeat - run_filebeat" - key: "run_filebeat" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "filebeat - run_filebeat_macos_tests" - key: "run_filebeat_macos_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "filebeat - run_filebeat_arm_tests" - key: "run_filebeat_arm_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - - select: "filebeat - run_filebeat_win_tests" - key: "run_filebeat_win_tests" - options: - - label: "True" - value: "true" - - label: "False" - value: "false" - default: "false" - if: "build.source == 'ui'" + - group: "Filebeat Mandatory Tests" + key: "filebeat-mandatory-tests" + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: | + cd filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Ubuntu Unit Tests" + + - label: ":ubuntu: Ubuntu Go Integration Tests" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet filebeat + echo "~~~ Will run tests with env var MODULE=$$MODULE" + # TODO move this section to base image / pre-command hook + echo "~~~ Installing kind" + asdf plugin add kind + asdf install kind $ASDF_KIND_VERSION + .buildkite/deploy/kubernetes/scripts/kind-setup.sh + + export KUBECONFIG="$$PWD/kubecfg" + echo "~~~ Running tests" + cd filebeat + mage goIntegTest + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Ubuntu Go Integration Tests" + + - label: ":ubuntu: Ubuntu Python Integration Tests" + command: | + set -euo pipefail + # defines the MODULE env var based on what's changed in a PR + source .buildkite/scripts/changesets.sh + defineModuleFromTheChangeSet filebeat + echo "~~~ Will run tests with env var MODULE=$$MODULE" + # TODO move this section to base image / pre-command hook + echo "~~~ Installing kind" + asdf plugin add kind + asdf install kind $ASDF_KIND_VERSION + .buildkite/deploy/kubernetes/scripts/kind-setup.sh + + export KUBECONFIG="$$PWD/kubecfg" + echo "~~~ Running tests" + cd filebeat + mage pythonIntegTest + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Python Integration Tests" + + - label: ":windows: Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" + command: | + Set-Location -Path filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Windows 2016 Unit Tests" + + - label: ":windows: Windows 2022 Unit Tests" + key: "windows-2022-unit-tests" + command: | + Set-Location -Path filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Windows 2022 Unit Tests" + + - group: "Filebeat Extended Tests" + key: "filebeat-extended-tests" + steps: + - label: ":mac: MacOS x64_64 Unit Tests" + key: "macos-unit-tests-extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd filebeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended MacOS Unit Tests" + + - label: ":mac: MacOS arm64 Unit Tests" + key: "macos-arm64-unit-tests-extended" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ + command: | + set -euo pipefail + source .buildkite/scripts/install_macos_tools.sh + cd filebeat + mage build unitTest + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended MacOS ARM Unit Tests" + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ + command: | + cd filebeat + mage build unitTest + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended Ubuntu ARM Unit Tests" + + - group: "Extended Windows Tests" + key: "filebeat-extended-win-tests" + if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "windows-extended-2019" + command: | + Set-Location -Path filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended Windows 2019 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + key: "windows-extended-11" + command: | + Set-Location -Path filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended Windows 11 Unit Tests" + + - label: ":windows: Windows 10 Unit Tests" + key: "windows-extended-10" + command: | + Set-Location -Path filebeat + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + notify: + - github_commit_status: + context: "filebeat: Extended Windows 10 Unit Tests" - wait: ~ - if: "build.source == 'ui'" - allow_dependency_failure: false - - - label: ":linux: Load dynamic filebeat pipeline" - key: "filebeat-pipeline" - command: ".buildkite/scripts/generate_filebeat_pipeline.sh" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" - notify: - - github_commit_status: - context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" + # with PRs, we want to run packaging only if mandatory tests succeed + # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in mandatory tests + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "filebeat-mandatory-tests" + + - group: "Filebeat Packaging" + key: "packaging" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: | + cd filebeat + mage package + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + notify: + - github_commit_status: + context: "filebeat: Packaging" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: | + cd filebeat + mage package + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "linux/arm64" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "filebeat: Packaging ARM" diff --git a/.buildkite/filebeat/generate_filebeat_pipeline.sh b/.buildkite/filebeat/generate_filebeat_pipeline.sh deleted file mode 100644 index aa0a03eadc1a..000000000000 --- a/.buildkite/filebeat/generate_filebeat_pipeline.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.filebeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Filebeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Filebeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":ubuntu: Go Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" - notify: - - github_commit_status: - context: "Filebeat: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":ubuntu: Python Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" - notify: - - github_commit_status: - context: "Filebeat: Python Integration Tests" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2016 Unit Tests" - key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Filebeat: windows-2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2022 Unit Tests" - key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Filebeat: windows-2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Filebeat: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Filebeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Filebeat: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Win 2019 Unit Tests" - key: "windows-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Filebeat: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-11 Unit Tests" - key: "windows-extended-11" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Filebeat: Win-11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-10 Unit Tests" - key: "windows-extended-10" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Filebeat: Win-10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Filebeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 978949b51d0f..1b4abe72e1c9 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -52,6 +52,33 @@ steps: - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger Filebeat" + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + watch: + - path: + - filebeat/ + - .buildkite/filebeat/ + # CI related scripts + - .buildkite/scripts + - .buildkite/hooks/ + #OSS + - go.mod + - pytest.ini + - dev-tools + - libbeat/** + - testing/** + config: + trigger: "filebeat" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - label: "Trigger Xpack/Dockerlogbeat" plugins: - monorepo-diff#v1.0.1: diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 386bc5aaf427..d69ae12db457 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -12,7 +12,6 @@ ONLY_DOCS=${ONLY_DOCS:-"true"} OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run the whole pipeline for the particular beat -[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")" [ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")" [ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")" [ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")" @@ -25,7 +24,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" # define if needed run ARM platform-specific tests for the particular beat -[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")" [ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")" [ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")" [ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")" @@ -34,7 +32,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat_arm_tests+x}" ] && run_xpack_packetbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_arm_tests --default "false")" # define if needed run MacOS platform-specific tests for the particular beat -[ -z "${run_filebeat_tests+x}" ] && run_filebeat_macos_tests="$(buildkite-agent meta-data get run_filebeat_macos_tests --default "false")" [ -z "${run_packetbeat_macos_tests+x}" ] && run_packetbeat_macos_tests="$(buildkite-agent meta-data get run_packetbeat_macos_tests --default "false")" [ -z "${run_xpack_auditbeat_macos_tests+x}" ] && run_xpack_auditbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_macos_tests --default "false")" [ -z "${run_xpack_filebeat_macos_tests+x}" ] && run_xpack_filebeat_macos_tests="$(buildkite-agent meta-data get run_xpack_filebeat_macos_tests --default "false")" @@ -45,15 +42,11 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" # define if needed run Windows platform-specific tests for the particular beat [ -z "${run_auditbeat_win_tests+x}" ] && run_auditbeat_win_tests="$(buildkite-agent meta-data get run_auditbeat_win_tests --default "false")" -[ -z "${run_filebeat_win_tests+x}" ] && run_filebeat_win_tests="$(buildkite-agent meta-data get run_filebeat_win_tests --default "false")" # define if needed run cloud-specific tests for the particular beat [ -z "${run_xpack_metricbeat_aws_tests+x}" ] && run_xpack_metricbeat_aws_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_aws_tests --default "false")" [ -z "${run_xpack_filebeat_aws_tests+x}" ] && run_xpack_filebeat_aws_tests="$(buildkite-agent meta-data get run_xpack_filebeat_aws_tests --default "false")" -filebeat_changeset=( - "^filebeat/.*" - ) libbeat_changeset=( "^libbeat/.*" @@ -131,9 +124,6 @@ packaging_changeset=( ) case "${BUILDKITE_PIPELINE_SLUG}" in - "filebeat") - BEAT_CHANGESET_REFERENCE=${filebeat_changeset[@]} - ;; "beats-libbeat") BEAT_CHANGESET_REFERENCE=${libbeat_changeset[@]} ;; @@ -403,7 +393,7 @@ are_conditions_met_mandatory_tests() { are_conditions_met_arm_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_ARM_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_ARM_LABEL} || "${!TRIGGER_SPECIFIC_ARM_TESTS}" == "true" ]]; then return 0 fi @@ -414,7 +404,7 @@ are_conditions_met_arm_tests() { are_conditions_met_macos_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12 return 0 fi @@ -423,17 +413,6 @@ are_conditions_met_macos_tests() { return 1 } -are_conditions_met_win_tests() { - if are_conditions_met_mandatory_tests; then - if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then - if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then - return 0 - fi - fi - fi - return 1 -} - are_conditions_met_aws_tests() { if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh deleted file mode 100644 index a3fe1108dc3b..000000000000 --- a/.buildkite/scripts/generate_filebeat_pipeline.sh +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.filebeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - - -steps: - - group: "Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Ubuntu Unit Tests" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Ubuntu Go Integration Tests" - command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Go Integration Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":ubuntu: Ubuntu Python Integration Tests" - command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Python Integration Tests" - agents: - provider: gcp - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: - - "filebeat/build/*.xml" - - "filebeat/build/*.json" - - - label: ":windows: Windows 2016 Unit Tests" - key: "windows-2016-unit-tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 2022 Unit Tests" - key: "windows-2022-unit-tests" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - - label: ":linux: Ubuntu ARM Unit Tests" - key: "extended-arm64-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - - group: "Windows Extended Testing" - key: "extended-tests-win" - steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "windows-extended-2019" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":windows: Windows 11 Unit Tests" - key: "windows-extended-11" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" - - - label: ":windows: Windows 10 Unit Tests" - key: "windows-extended-10" - command: | - Set-Location -Path $BEATS_PROJECT_NAME - mage build unitTest - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "$BEATS_PROJECT_NAME/build/*.xml" - - "$BEATS_PROJECT_NAME/build/*.json" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: ":linux: Packaging Linux" - key: "packaging-linux" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - env: - PLATFORMS: "${PACKAGING_PLATFORMS}" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux" - - - label: ":linux: Packaging ARM" - key: "packaging-arm" - command: "cd $BEATS_PROJECT_NAME && mage package" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - env: - PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" - PACKAGES: "docker" - notify: - - github_commit_status: - context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" - -YAML -fi - -echo "+++ Printing dynamic steps" -cat $pipelineName | yq . -P - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName