Skip to content

Commit

Permalink
[7.17](backport #38825) Remove old steps from filebeat-pipeline.yaml (#…
Browse files Browse the repository at this point in the history 
…38934)

* Migrate filebeat-pipeline.yaml (#38825)

This commit migrate the filebeat pipeline into the main one

Signed-off-by: Alexandros Sapranidis <[email protected]>
(cherry picked from commit 0f118b1)

# Conflicts:
#	.buildkite/filebeat/filebeat-pipeline.yml
#	.buildkite/scripts/common.sh

* Resolve conflicts

Signed-off-by: Alexandros Sapranidis <[email protected]>

* Remove macos ARM 64 from packaging

Signed-off-by: Alexandros Sapranidis <[email protected]>

---------

Signed-off-by: Alexandros Sapranidis <[email protected]>
Co-authored-by: Alexandros Sapranidis <[email protected]>
  • Loading branch information
@mergify @alexsapran
mergify[bot] and alexsapran authored Apr 15, 2024
1 parent 91a1574 commit a8fa073
Show file tree
Hide file tree
Showing 4 changed files with 306 additions and 290 deletions.
330 changes: 277 additions & 53 deletions .buildkite/filebeat/filebeat-pipeline.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,66 +2,290 @@
name: "beats-filebeat"

env:
BEATS_PROJECT_NAME: "filebeat"
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
IMAGE_MACOS_ARM: "generic-13-ventura-arm"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016"
IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
IMAGE_MACOS_ARM: "generic-13-ventura-arm"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"

#Deps
ASDF_MAGE_VERSION: 1.15.0
# Integration Tests
K8S_VERSION: "v1.29.0"
ASDF_KIND_VERSION: "0.20.0"

steps:
- input: "Input Parameters"
key: "run_filebeat"
fields:
- select: "filebeat - run_filebeat"
key: "run_filebeat"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "filebeat - run_filebeat_macos_tests"
key: "run_filebeat_macos_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "filebeat - run_filebeat_arm_tests"
key: "run_filebeat_arm_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "filebeat - run_filebeat_win_tests"
key: "run_filebeat_win_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
if: "build.source == 'ui'"
- group: "Filebeat Mandatory Tests"
key: "filebeat-mandatory-tests"
steps:
- label: ":ubuntu: Ubuntu Unit Tests"
command: |
cd filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Ubuntu Unit Tests"

- label: ":ubuntu: Ubuntu Go Integration Tests"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet filebeat
echo "~~~ Will run tests with env var MODULE=$$MODULE"
# TODO move this section to base image / pre-command hook
echo "~~~ Installing kind"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION
.buildkite/deploy/kubernetes/scripts/kind-setup.sh
export KUBECONFIG="$$PWD/kubecfg"
echo "~~~ Running tests"
cd filebeat
mage goIntegTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Ubuntu Go Integration Tests"

- label: ":ubuntu: Ubuntu Python Integration Tests"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet filebeat
echo "~~~ Will run tests with env var MODULE=$$MODULE"
# TODO move this section to base image / pre-command hook
echo "~~~ Installing kind"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION
.buildkite/deploy/kubernetes/scripts/kind-setup.sh
export KUBECONFIG="$$PWD/kubecfg"
echo "~~~ Running tests"
cd filebeat
mage pythonIntegTest
agents:
provider: gcp
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Python Integration Tests"

- label: ":windows: Windows 2016 Unit Tests"
key: "windows-2016-unit-tests"
command: |
Set-Location -Path filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_2016}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Windows 2016 Unit Tests"

- label: ":windows: Windows 2022 Unit Tests"
key: "windows-2022-unit-tests"
command: |
Set-Location -Path filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_2022}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Windows 2022 Unit Tests"

- group: "Filebeat Extended Tests"
key: "filebeat-extended-tests"
steps:
- label: ":mac: MacOS x64_64 Unit Tests"
key: "macos-unit-tests-extended"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd filebeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_X86_64}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended MacOS Unit Tests"

- label: ":mac: MacOS arm64 Unit Tests"
key: "macos-arm64-unit-tests-extended"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
command: |
set -euo pipefail
source .buildkite/scripts/install_macos_tools.sh
cd filebeat
mage build unitTest
agents:
provider: "orka"
imagePrefix: "${IMAGE_MACOS_ARM}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended MacOS ARM Unit Tests"

- label: ":linux: Ubuntu ARM Unit Tests"
key: "extended-arm64-unit-test"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
command: |
cd filebeat
mage build unitTest
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended Ubuntu ARM Unit Tests"

- group: "Extended Windows Tests"
key: "filebeat-extended-win-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
steps:
- label: ":windows: Windows 2019 Unit Tests"
key: "windows-extended-2019"
command: |
Set-Location -Path filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_2019}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended Windows 2019 Unit Tests"

- label: ":windows: Windows 11 Unit Tests"
key: "windows-extended-11"
command: |
Set-Location -Path filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended Windows 11 Unit Tests"

- label: ":windows: Windows 10 Unit Tests"
key: "windows-extended-10"
command: |
Set-Location -Path filebeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 200
disk_type: "pd-ssd"
artifact_paths:
- "filebeat/build/*.xml"
- "filebeat/build/*.json"
notify:
- github_commit_status:
context: "filebeat: Extended Windows 10 Unit Tests"

- wait: ~
if: "build.source == 'ui'"
allow_dependency_failure: false

- label: ":linux: Load dynamic filebeat pipeline"
key: "filebeat-pipeline"
command: ".buildkite/filebeat/generate_filebeat_pipeline.sh"
notify:
- github_commit_status:
context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
# with PRs, we want to run packaging only if mandatory tests succeed
# for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests
# this allows building DRA artifacts even if there is flakiness in mandatory tests
if: build.env("BUILDKITE_PULL_REQUEST") != "false"
depends_on: "filebeat-mandatory-tests"

- group: "Filebeat Packaging"
key: "packaging"
steps:
- label: ":linux: Packaging Linux"
key: "packaging-linux"
command: |
cd filebeat
mage package
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64"
notify:
- github_commit_status:
context: "filebeat: Packaging"

- label: ":linux: Packaging ARM"
key: "packaging-arm"
command: |
cd filebeat
mage package
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
env:
PLATFORMS: "linux/arm64"
PACKAGES: "docker"
notify:
- github_commit_status:
context: "filebeat: Packaging ARM"
Loading

0 comments on commit a8fa073

Please sign in to comment.