Skip to content

Commit

Permalink
Merge branch 'main' into issue-8907
Browse files Browse the repository at this point in the history
  • Loading branch information
agithomas authored Apr 16, 2024
2 parents b85e4b5 + 5353767 commit 3f945bf
Show file tree
Hide file tree
Showing 15 changed files with 374 additions and 475 deletions.
26 changes: 0 additions & 26 deletions .buildkite/filebeat/filebeat-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,6 @@ steps:

- label: ":ubuntu: Ubuntu Go Integration Tests"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet filebeat
echo "~~~ Will run tests with env var MODULE=$$MODULE"
# TODO move this section to base image / pre-command hook
echo "~~~ Installing kind"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION
.buildkite/deploy/kubernetes/scripts/kind-setup.sh
export KUBECONFIG="$$PWD/kubecfg"
echo "~~~ Running tests"
cd filebeat
mage goIntegTest
agents:
Expand All @@ -71,19 +58,6 @@ steps:

- label: ":ubuntu: Ubuntu Python Integration Tests"
command: |
set -euo pipefail
# defines the MODULE env var based on what's changed in a PR
source .buildkite/scripts/changesets.sh
defineModuleFromTheChangeSet filebeat
echo "~~~ Will run tests with env var MODULE=$$MODULE"
# TODO move this section to base image / pre-command hook
echo "~~~ Installing kind"
asdf plugin add kind
asdf install kind $ASDF_KIND_VERSION
.buildkite/deploy/kubernetes/scripts/kind-setup.sh
export KUBECONFIG="$$PWD/kubecfg"
echo "~~~ Running tests"
cd filebeat
mage pythonIntegTest
agents:
Expand Down
2 changes: 0 additions & 2 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ fi
ENABLED_BEATS_PIPELINES_SLUGS=(
"auditbeat"
"filebeat"
"beats-libbeat"
"beats-metricbeat"
"beats-packetbeat"
"beats-winlogbeat"
Expand All @@ -70,7 +69,6 @@ ENABLED_BEATS_PIPELINES_SLUGS=(
"beats-xpack-filebeat"
"beats-xpack-metricbeat"
"beats-xpack-heartbeat"
"beats-xpack-osquerybeat"
"deploy-k8s"
)

Expand Down
143 changes: 105 additions & 38 deletions .buildkite/libbeat/pipeline.libbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,50 +4,117 @@ name: "beats-libbeat"
env:
AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64"
BEATS_PROJECT_NAME: "libbeat"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"

#Packaging
PACKAGING_ARM_PLATFORMS: "linux/arm64"
PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"

#Deps
ASDF_MAGE_VERSION: 1.15.0

steps:
- group: "Mandatory Tests"
key: "mandatory-tests"
steps:
- label: ":linux: Ubuntu Unit Tests"
key: "mandatory-linux-unit-test"
command: |
set -euo pipefail
cd libbeat
mage build unitTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths:
- "libbeat/build/*.xml"
- "libbeat/build/*.json"
notify:
- github_commit_status:
context: "libbeat: Ununtu Unit Tests"

- label: ":go: Go Integration Tests"
key: "mandatory-int-test"
command: |
set -euo pipefail
cd libbeat
mage goIntegTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths:
- "libbeat/build/*.xml"
- "libbeat/build/*.json"
notify:
- github_commit_status:
context: "libbeat: Go Integration Tests"

- label: ":python: Python Integration Tests"
key: "mandatory-python-int-test"
command: |
set -euo pipefail
cd libbeat
mage pythonIntegTest
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths:
- "libbeat/build/*.xml"
- "libbeat/build/*.json"
notify:
- github_commit_status:
context: "libbeat: Python Integration Tests"

- label: ":negative_squared_cross_mark: Cross compile"
key: "mandatory-cross-compile"
command: |
set -euo pipefail
cd libbeat
make crosscompile
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
artifact_paths:
- "libbeat/build/*.xml"
- "libbeat/build/*.json"
notify:
- github_commit_status:
context: "libbeat: Cross compile"

- label: ":testengine: Stress Tests"
key: "mandatory-stress-test"
command: |
set -euo pipefail
cd libbeat
make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS=' | go-junit-report > libbeat-stress-test.xml' stress-tests
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths: "libbeat/libbeat-stress-test.xml"
notify:
- github_commit_status:
context: "libbeat: Stress Tests"

- input: "Input Parameters"
key: "input-run-all-stages"
fields:
- select: "Libbeat - run_libbeat"
key: "run_libbeat"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
- select: "Libbeat - run_libbeat_arm_tests"
key: "run_libbeat_arm_tests"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
if: "build.source == 'ui'"

- wait: ~
if: "build.source == 'ui'"
allow_dependency_failure: false

- label: ":linux: Load dynamic Libbeat pipeline"
key: "libbeat-pipeline"
command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
agents:
image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest"
notify:
- github_commit_status:
context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps"
- group: "Extended Tests"
key: "extended-tests"
if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
steps:
- label: ":linux: Ubuntu ARM64 Unit Tests"
key: "extended-arm64-unit-tests"
command: |
set -euo pipefail
cd libbeat
mage build unitTest
agents:
provider: "aws"
imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
instanceType: "${AWS_ARM_INSTANCE_TYPE}"
artifact_paths:
- "libbeat/build/*.xml"
- "libbeat/build/*.json"
notify:
- github_commit_status:
context: "libbeat: Ubuntu ARM64 Unit Tests"
53 changes: 53 additions & 0 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,35 @@ steps:
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger x-pack/osquerybeat"
plugins:
- monorepo-diff#v1.0.1:
diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD"
watch:
- path:
- x-pack/osquerybeat/**
- .buildkite/x-pack/pipeline.xpack.metricbeat.yml
- .buildkite/scripts/**
- .buildkite/hooks/**
# x-pack
- libbeat/**
- x-pack/libbeat/**
#OSS
- go.mod
- pytest.ini
- dev-tools/
- libbeat/**
- testing/**
config:
trigger: "beats-xpack-osquerybeat"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"
env:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger Xpack/Winlogbeat"
plugins:
- monorepo-diff#v1.0.1:
Expand Down Expand Up @@ -205,3 +234,27 @@ steps:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}

- label: "Trigger Libbeat"
plugins:
- monorepo-diff#v1.0.1:
diff: "git diff --name-only ${GITHUB_PR_TARGET_BRANCH}...HEAD"
watch:
- path:
- libbeat/
- .buildkite/libbeat/pipeline.libbeat.yml
- .buildkite/scripts
- .buildkite/hooks
#OSS
- go.mod
- pytest.ini
- dev-tools/
- testing/
config:
trigger: "beats-libbeat"
build:
commit: "${BUILDKITE_COMMIT}"
branch: "${BUILDKITE_BRANCH}"
env:
- BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
- GITHUB_PR_LABELS=${GITHUB_PR_LABELS}
32 changes: 0 additions & 32 deletions .buildkite/pull-requests.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,6 @@
"skip_ci_on_only_changed": [ ],
"always_require_ci_on_changed": [ ]
},
{
"enabled": true,
"pipelineSlug": "beats-libbeat",
"allow_org_users": true,
"allowed_repo_permissions": ["admin", "write"],
"allowed_list": [ ],
"set_commit_status": true,
"build_on_commit": true,
"build_on_comment": true,
"trigger_comment_regex": "^/test libbeat$|^/test filebeat",
"always_trigger_comment_regex": "^/test libbeat$",
"skip_ci_labels": [ ],
"skip_target_branches": [ ],
"skip_ci_on_only_changed": [ ],
"always_require_ci_on_changed": ["^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
},
{
"enabled": true,
"pipelineSlug": "beats-packetbeat",
Expand Down Expand Up @@ -143,22 +127,6 @@
"skip_target_branches": [ ],
"skip_ci_on_only_changed": [ ],
"always_require_ci_on_changed": ["^x-pack/heartbeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
},
{
"enabled": true,
"pipelineSlug": "beats-xpack-osquerybeat",
"allow_org_users": true,
"allowed_repo_permissions": ["admin", "write"],
"allowed_list": [ ],
"set_commit_status": true,
"build_on_commit": true,
"build_on_comment": true,
"trigger_comment_regex": "^/test x-pack/osquerybeat$",
"always_trigger_comment_regex": "^/test x-pack/osquerybeat$",
"skip_ci_labels": [ ],
"skip_target_branches": [ ],
"skip_ci_on_only_changed": [ ],
"always_require_ci_on_changed": ["^x-pack/osquerybeat/.*", "^.buildkite/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*", "^x-pack/libbeat/.*"]
}
]
}
15 changes: 3 additions & 12 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ ONLY_DOCS=${ONLY_DOCS:-"true"}
OSS_MODULE_PATTERN="^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
# define if needed run the whole pipeline for the particular beat
[ -z "${run_libbeat+x}" ] && run_libbeat="$(buildkite-agent meta-data get run_libbeat --default "false")"
[ -z "${run_filebeat+x}" ] && run_filebeat="$(buildkite-agent meta-data get run_filebeat --default "false")"
[ -z "${run_packetbeat+x}" ] && run_packetbeat="$(buildkite-agent meta-data get run_packetbeat --default "false")"
[ -z "${run_winlogbeat+x}" ] && run_winlogbeat="$(buildkite-agent meta-data get run_winlogbeat --default "false")"
[ -z "${run_xpack_libbeat+x}" ] && run_xpack_libbeat="$(buildkite-agent meta-data get run_xpack_libbeat --default "false")"
Expand All @@ -21,10 +21,9 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
[ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")"
[ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")"
[ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")"
[ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")"

# define if needed run ARM platform-specific tests for the particular beat
[ -z "${run_libbeat_arm_tests+x}" ] && run_libbeat_arm_tests="$(buildkite-agent meta-data get run_libbeat_arm_tests --default "false")"
[ -z "${run_filebeat_arm_tests+x}" ] && run_filebeat_arm_tests="$(buildkite-agent meta-data get run_filebeat_arm_tests --default "false")"
[ -z "${run_packetbeat_arm_tests+x}" ] && run_packetbeat_arm_tests="$(buildkite-agent meta-data get run_packetbeat_arm_tests --default "false")"
[ -z "${run_xpack_auditbeat_arm_tests+x}" ] && run_xpack_auditbeat_arm_tests="$(buildkite-agent meta-data get run_xpack_auditbeat_arm_tests --default "false")"
[ -z "${run_xpack_filebeat_arm_tests+x}" ] && run_xpack_filebeat_arm_tests="$(buildkite-agent meta-data get run_xpack_filebeat_arm_tests --default "false")"
Expand All @@ -38,7 +37,6 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*"
[ -z "${run_xpack_metricbeat_macos_tests+x}" ] && run_xpack_metricbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_metricbeat_macos_tests --default "false")"
[ -z "${run_xpack_packetbeat_macos_tests+x}" ] && run_xpack_packetbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_packetbeat_macos_tests --default "false")"
[ -z "${run_xpack_heartbeat_macos_tests+x}" ] && run_xpack_heartbeat_macos_tests="$(buildkite-agent meta-data get run_xpack_heartbeat_macos_tests --default "false")"
[ -z "${run_xpack_osquerybeat_macos_tests+x}" ] && run_xpack_osquerybeat_macos_tests="$(buildkite-agent meta-data get run_xpack_osquerybeat_macos_tests --default "false")"

# define if needed run Windows platform-specific tests for the particular beat
[ -z "${run_auditbeat_win_tests+x}" ] && run_auditbeat_win_tests="$(buildkite-agent meta-data get run_auditbeat_win_tests --default "false")"
Expand Down Expand Up @@ -80,10 +78,6 @@ xpack_libbeat_changeset=(
"^x-pack/libbeat/.*"
)

xpack_osquerybeat_changeset=(
"^x-pack/osquerybeat/.*"
)

xpack_packetbeat_changeset=(
"^x-pack/packetbeat/.*"
)
Expand Down Expand Up @@ -148,9 +142,6 @@ case "${BUILDKITE_PIPELINE_SLUG}" in
"beats-xpack-metricbeat")
BEAT_CHANGESET_REFERENCE=${xpack_metricbeat_changeset[@]}
;;
"beats-xpack-osquerybeat")
BEAT_CHANGESET_REFERENCE=${xpack_osquerybeat_changeset[@]}
;;
"beats-xpack-packetbeat")
BEAT_CHANGESET_REFERENCE=${xpack_packetbeat_changeset[@]}
;;
Expand Down Expand Up @@ -404,7 +395,7 @@ are_conditions_met_arm_tests() {

are_conditions_met_macos_tests() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-osquerybeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-heartbeat"; then
if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_MACOS_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_MACOS_LABEL} || "${!TRIGGER_SPECIFIC_MACOS_TESTS}" == "true" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
return 0
fi
Expand Down
Loading

0 comments on commit 3f945bf

Please sign in to comment.