diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index a8def81f1fa0..edbb600e3852 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -2,21 +2,28 @@ name: "beats-auditbeat" env: - BEATS_PROJECT_NAME: "auditbeat" - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + BEATS_PROJECT_NAME: "auditbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -62,7 +69,9 @@ steps: - label: ":linux: Load dynamic auditbeat pipeline" key: "auditbeat-pipeline" - command: ".buildkite/auditbeat/generate_auditbeat_pipeline.sh" + command: ".buildkite/scripts/generate_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh deleted file mode 100644 index 56befb51d254..000000000000 --- a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2016 Unit Tests" - key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2022 Unit Tests" - key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: Crosscompile" - command: - - "make -C auditbeat crosscompile" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - - steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-2019-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-11 Unit Tests" - key: "windows-11-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-10 Unit Tests" - key: "windows-10-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Auditbeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 9daeef287d91..a22e96c974aa 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -2,20 +2,27 @@ name: "beats-filebeat" env: + AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" BEATS_PROJECT_NAME: "filebeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" - IMAGE_MACOS_ARM: "generic-13-ventura-arm" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: - input: "Input Parameters" @@ -61,7 +68,9 @@ steps: - label: ":linux: Load dynamic filebeat pipeline" key: "filebeat-pipeline" - command: ".buildkite/filebeat/generate_filebeat_pipeline.sh" + command: ".buildkite/scripts/generate_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml index e63fb1c60e2b..9b1ea1984a74 100644 --- a/.buildkite/heartbeat/heartbeat-pipeline.yml +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -4,12 +4,12 @@ env: BEATS_PROJECT_NAME: "heartbeat" IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" - IMAGE_RHEL9: "family/platform-ingest-beats-rhel-9" IMAGE_MACOS_X86_64: "generic-13-ventura-x64" IMAGE_MACOS_ARM: "generic-13-ventura-arm" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" @@ -17,6 +17,10 @@ env: GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "m6g.xlarge" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + steps: - group: "Heartbeat Mandatory Testing" key: "mandatory-tests" @@ -24,7 +28,7 @@ steps: steps: - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: ubuntu/Unit Tests" @@ -37,7 +41,7 @@ steps: - "heartbeat/build/*.json" - label: ":rhel:-9 Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: rhel-9/Unit Tests" @@ -51,10 +55,12 @@ steps: - label: ":windows:-2016 Unit Test" key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2016/Unit Tests" + context: "Heartbeat: Windows 2016/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2016}" @@ -66,10 +72,12 @@ steps: - label: ":windows:-2022 Unit Test" key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: windows-2022/Unit Tests" + context: "Heartbeat: Windows 2022/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2022}" @@ -80,7 +88,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Go Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage goIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" notify: - github_commit_status: context: "Heartbeat: Go Integration Tests" @@ -93,7 +101,7 @@ steps: - "heartbeat/build/*.json" - label: ":ubuntu: Python Integration Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage pythonIntegTest" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" notify: - github_commit_status: context: "Heartbeat: Python Integration Tests" @@ -113,7 +121,7 @@ steps: - label: ":linux: ARM64 Unit Tests" key: "arm-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for arm" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: Unit Tests ARM" @@ -126,7 +134,7 @@ steps: - label: ":mac: MacOS Unit Tests" key: "macos-extended" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS Unit Tests" @@ -138,7 +146,7 @@ steps: - label: ":mac: MacOS ARM Unit Tests" key: "macos-extended-arm" if: build.env("GITHUB_PR_TRIGGER_COMMENT") == "heartbeat for macos" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" notify: - github_commit_status: context: "Heartbeat: MacOS ARM Unit Tests" @@ -154,10 +162,12 @@ steps: steps: - label: ":windows:-2019 Unit Tests" key: "windows-extended-2019" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-2019 Unit Tests" + context: "Heartbeat: Windows 2019/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_2019}" @@ -169,10 +179,12 @@ steps: - label: ":windows:-11 Unit Tests" key: "windows-extended-11" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-11 Unit Tests" + context: "Heartbeat: Windows 11/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_11}" @@ -184,10 +196,12 @@ steps: - label: ":windows:-10 Unit Tests" key: "windows-extended-10" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest notify: - github_commit_status: - context: "Heartbeat: Win-10 Unit Tests" + context: "Heartbeat: Windows 10/Unit Tests" agents: provider: "gcp" image: "${IMAGE_WIN_10}" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 5f625b35c591..52bee3fb674c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,11 +9,13 @@ source .buildkite/env-scripts/util.sh AWS_SERVICE_ACCOUNT_SECRET_PATH="kv/ci-shared/platform-ingest/aws_account_auth" PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account" DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod" -#PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/private_ci_artifacts_gcs_credentials" GITHUB_TOKEN_VAULT_PATH="kv/ci-shared/platform-ingest/github_token" if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "heartbeat" || "$BUILDKITE_PIPELINE_SLUG" == "deploy-k8s" ]]; then source .buildkite/env-scripts/env.sh + if [[ -z "${GO_VERSION-""}" ]]; then + export GO_VERSION=$(cat "${WORKSPACE}/.go-version") + fi # Images with prefix "platform-ingest-beats-*" has Go pre-setup. # Image itself takes care of Go version download/installation based on .go-version file diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index 83fda5662c74..46ddeca92a4e 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -2,13 +2,19 @@ name: "beats-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" - GCP_WIN_MACHINE_TYPE: "n2-standard-8" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" BEATS_PROJECT_NAME: "libbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -40,6 +46,8 @@ steps: - label: ":linux: Load dynamic Libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml index 04f3b44575e4..c7dcc01e625e 100644 --- a/.buildkite/metricbeat/pipeline.yml +++ b/.buildkite/metricbeat/pipeline.yml @@ -2,20 +2,26 @@ name: "beats-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -47,6 +53,8 @@ steps: - label: ":linux: Load dynamic metricbeat pipeline" key: "metricbeat-pipeline" command: ".buildkite/scripts/generate_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index 8e606e16622e..9247b1446676 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -2,20 +2,27 @@ name: "beats-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_RHEL9_X86_64: "family/core-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2004-aarch64" + BEATS_PROJECT_NAME: "packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "packetbeat" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -55,6 +62,8 @@ steps: - label: ":linux: Load dynamic packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/scripts/cloud_tests.sh b/.buildkite/scripts/cloud_tests.sh index d96baf670a9a..3ca52305188b 100755 --- a/.buildkite/scripts/cloud_tests.sh +++ b/.buildkite/scripts/cloud_tests.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -euo pipefail # What Terraform Module will run if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then @@ -7,19 +8,52 @@ elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-filebeat" ]]; then export MODULE_DIR="x-pack/filebeat/input/awss3/_meta/terraform" fi -source .buildkite/scripts/install_tools.sh +teardown() { + popd + # Teardown resources after using them + echo "~~~ Terraform Cleanup" + tf_cleanup "${MODULE_DIR}" #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK -set -euo pipefail + echo "~~~ Docker Compose Cleanup" + docker-compose -f .ci/jobs/docker-compose.yml down -v #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +} + +tf_cleanup() { + DIRECTORY=${1:-.} -trap 'teardown || true; unset_secrets' EXIT + for tfstate in $(find $DIRECTORY -name terraform.tfstate); do + cd $(dirname $tfstate) + terraform init + if ! terraform destroy -auto-approve; then + echo "+++ Failed to Terraform destroy the resources" + fi + cd - + done +} + +trap 'teardown' EXIT # Prepare the cloud resources using Terraform -startCloudTestEnv "${MODULE_DIR}" +#startCloudTestEnv "${MODULE_DIR}" +echo "~~~ Loading creds" +set +o xtrace +export AWS_ACCESS_KEY_ID=$BEATS_AWS_ACCESS_KEY +export AWS_SECRET_ACCESS_KEY=$BEATS_AWS_SECRET_KEY +export TEST_TAGS="${TEST_TAGS:+$TEST_TAGS,}aws" +set -o xtrace + +echo "~~~ Run docker-compose services for emulated cloud env" +docker-compose -f .ci/jobs/docker-compose.yml up -d #TODO: move all docker-compose files from the .ci to .buildkite folder before switching to BK +echo "~~~ Initialize TF cloud resources" +pushd "$MODULE_DIR" +export TF_VAR_BRANCH=$(echo "${BUILDKITE_BRANCH}" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g') +export TF_VAR_BUILD_ID="${BUILDKITE_BUILD_ID}" +export TF_VAR_CREATED_DATE=$(date +%s) +export TF_VAR_ENVIRONMENT="ci" +export TF_VAR_REPO="${REPO}" +terraform init && terraform apply -auto-approve # Run tests -echo "--- Run Cloud Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - +echo "~~~ Run Cloud Tests for $BEATS_PROJECT_NAME" +pushd "${BEATS_PROJECT_NAME}" mage build test - -popd > /dev/null diff --git a/.buildkite/scripts/common.sh b/.buildkite/scripts/common.sh index 3c968af20883..64b36131caf6 100755 --- a/.buildkite/scripts/common.sh +++ b/.buildkite/scripts/common.sh @@ -23,6 +23,7 @@ XPACK_MODULE_PATTERN="^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*" [ -z "${run_xpack_packetbeat+x}" ] && run_xpack_packetbeat="$(buildkite-agent meta-data get run_xpack_packetbeat --default "false")" [ -z "${run_xpack_winlogbeat+x}" ] && run_xpack_winlogbeat="$(buildkite-agent meta-data get run_xpack_winlogbeat --default "false")" [ -z "${run_xpack_auditbeat+x}" ] && run_xpack_auditbeat="$(buildkite-agent meta-data get run_xpack_auditbeat --default "false")" +[ -z "${run_xpack_dockerlogbeat+x}" ] && run_xpack_dockerlogbeat="$(buildkite-agent meta-data get run_xpack_dockerlogbeat --default "false")" [ -z "${run_xpack_filebeat+x}" ] && run_xpack_filebeat="$(buildkite-agent meta-data get run_xpack_filebeat --default "false")" [ -z "${run_xpack_heartbeat+x}" ] && run_xpack_heartbeat="$(buildkite-agent meta-data get run_xpack_heartbeat --default "false")" [ -z "${run_xpack_osquerybeat+x}" ] && run_xpack_osquerybeat="$(buildkite-agent meta-data get run_xpack_osquerybeat --default "false")" @@ -63,7 +64,7 @@ auditbeat_changeset=( filebeat_changeset=( "^filebeat/.*" - ) + ) metricbeat_changeset=( "^metricbeat/.*" @@ -195,12 +196,13 @@ case "${BUILDKITE_PIPELINE_SLUG}" in BEAT_CHANGESET_REFERENCE=${xpack_winlogbeat_changeset[@]} ;; *) - echo "The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." + echo "~~~ The changeset for the ${BUILDKITE_PIPELINE_SLUG} pipeline hasn't been defined yet." ;; esac check_and_set_beat_vars() { - if [[ -n "$BEATS_PROJECT_NAME" && "$BEATS_PROJECT_NAME" == *"x-pack/"* ]]; then + local BEATS_PROJECT_NAME=${BEATS_PROJECT_NAME:=""} + if [[ "${BEATS_PROJECT_NAME:=""}" == *"x-pack/"* ]]; then BEATS_XPACK_PROJECT_NAME=${BEATS_PROJECT_NAME//-/} #remove - BEATS_XPACK_PROJECT_NAME=${BEATS_XPACK_PROJECT_NAME//\//_} #replace / to _ BEATS_XPACK_LABEL_PROJECT_NAME=${BEATS_PROJECT_NAME//\//-} #replace / to - for labels @@ -210,7 +212,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_XPACK_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_XPACK_PROJECT_NAME" + echo "--- Beats project name is $BEATS_XPACK_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${xpack_changeset[@]}" @@ -223,7 +225,7 @@ check_and_set_beat_vars() { TRIGGER_SPECIFIC_AWS_TESTS="run_${BEATS_PROJECT_NAME}_aws_tests" TRIGGER_SPECIFIC_MACOS_TESTS="run_${BEATS_PROJECT_NAME}_macos_tests" TRIGGER_SPECIFIC_WIN_TESTS="run_${BEATS_PROJECT_NAME}_win_tests" - echo "Beats project name is $BEATS_PROJECT_NAME" + echo "--- Beats project name is $BEATS_PROJECT_NAME" mandatory_changeset=( "${BEAT_CHANGESET_REFERENCE[@]}" "${oss_changeset[@]}" @@ -251,19 +253,6 @@ with_docker_compose() { docker-compose version } -with_Terraform() { - echo "Setting up the Terraform environment..." - local path_to_file="${WORKSPACE}/terraform.zip" - create_workspace - check_platform_architeture - retry 5 curl -sSL -o ${path_to_file} "https://releases.hashicorp.com/terraform/${ASDF_TERRAFORM_VERSION}/terraform_${ASDF_TERRAFORM_VERSION}_${platform_type_lowercase}_${go_arch_type}.zip" - unzip -q ${path_to_file} -d ${BIN}/ - rm ${path_to_file} - chmod +x ${BIN}/terraform - export PATH="${BIN}:${PATH}" - terraform version -} - create_workspace() { if [[ ! -d "${BIN}" ]]; then mkdir -p "${BIN}" @@ -466,7 +455,7 @@ are_conditions_met_macos_tests() { } are_conditions_met_win_tests() { - if are_conditions_met_mandatory_tests; then + if are_conditions_met_mandatory_tests; then if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || "$BUILDKITE_PIPELINE_SLUG" == "filebeat" ]]; then if [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "${BEATS_GH_WIN_COMMENT}" || "${GITHUB_PR_LABELS}" =~ ${BEATS_GH_WIN_LABEL} || "${!TRIGGER_SPECIFIC_WIN_TESTS}" == "true" ]]; then return 0 @@ -529,7 +518,11 @@ defineModuleFromTheChangeSet() { fi done if [[ -z "$changed_modules" ]]; then # TODO: remove this condition and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved - export MODULE="aws" + if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then + export MODULE="aws" + else + export MODULE="kubernetes" + fi else export MODULE="${changed_modules}" # TODO: remove this line and uncomment the line below when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved # export MODULE="${changed_modules}" # TODO: uncomment the line when the issue https://github.com/elastic/ingest-dev/issues/2993 is solved diff --git a/.buildkite/scripts/generate_auditbeat_pipeline.sh b/.buildkite/scripts/generate_auditbeat_pipeline.sh new file mode 100644 index 000000000000..d514c5d79d85 --- /dev/null +++ b/.buildkite/scripts/generate_auditbeat_pipeline.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.auditbeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":rhel: RHEL Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_RHEL9}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 2022 Unit Tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":linux: Crosscompile" + command: "make -C $BEATS_PROJECT_NAME crosscompile" + env: + GOX_FLAGS: "-arch amd64" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Crosscompile" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: + +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd ${BEATS_PROJECT_NAME} && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "extended-win-2019-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + + - label: ":windows: Windows 10 Unit Tests" + key: "extended-win-10-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + + - label: ":windows: Windows 11 Unit Tests" + key: "extended-win-11-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName \ No newline at end of file diff --git a/.buildkite/scripts/generate_filebeat_pipeline.sh b/.buildkite/scripts/generate_filebeat_pipeline.sh new file mode 100644 index 000000000000..a3fe1108dc3b --- /dev/null +++ b/.buildkite/scripts/generate_filebeat_pipeline.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash + +source .buildkite/scripts/common.sh + +set -euo pipefail + +pipelineName="pipeline.filebeat-dynamic.yml" + +# TODO: steps: must be always included +echo "Add the mandatory and extended tests without additional conditions into the pipeline" +if are_conditions_met_mandatory_tests; then + cat > $pipelineName <<- YAML + + +steps: + - group: "Mandatory Testing" + key: "mandatory-tests" + + steps: + - label: ":ubuntu: Ubuntu Unit Tests" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Go Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":ubuntu: Ubuntu Python Integration Tests" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" + agents: + provider: gcp + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + artifact_paths: + - "filebeat/build/*.xml" + - "filebeat/build/*.json" + + - label: ":windows: Windows 2016 Unit Tests" + key: "windows-2016-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2016 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2016}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + + - label: ":windows: Windows 2022 Unit Tests" + key: "windows-2022-unit-tests" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2022 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2022}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + +YAML +else + echo "The conditions don't match to requirements for generating pipeline steps." + exit 0 +fi + +echo "Check and add the Extended Tests into the pipeline" + +if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - group: "Extended Tests" + key: "extended-tests" + steps: +YAML +fi + +if are_conditions_met_macos_tests; then + cat >> $pipelineName <<- YAML + + - label: ":mac: MacOS Unit Tests" + key: "macos-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_X86_64}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":mac: MacOS ARM Unit Tests" + key: "macos-arm64-unit-tests-extended" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS ARM Unit Tests" + agents: + provider: "orka" + imagePrefix: "${IMAGE_MACOS_ARM}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_arm_tests; then + cat >> $pipelineName <<- YAML + + - label: ":linux: Ubuntu ARM Unit Tests" + key: "extended-arm64-unit-test" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +if are_conditions_met_win_tests; then + cat >> $pipelineName <<- YAML + + - group: "Windows Extended Testing" + key: "extended-tests-win" + steps: + - label: ":windows: Windows 2019 Unit Tests" + key: "windows-extended-2019" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 2019 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_2019}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 11 Unit Tests" + key: "windows-extended-11" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 11 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_11}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + + - label: ":windows: Windows 10 Unit Tests" + key: "windows-extended-10" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows 10 Unit Tests" + agents: + provider: "gcp" + image: "${IMAGE_WIN_10}" + machine_type: "${GCP_WIN_MACHINE_TYPE}" + disk_size: 200 + disk_type: "pd-ssd" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" +YAML +fi + +echo "Check and add the Packaging into the pipeline" +if are_conditions_met_packaging; then +cat >> $pipelineName <<- YAML + + - group: "Packaging" + key: "packaging" + depends_on: + - "mandatory-tests" + + steps: + - label: ":linux: Packaging Linux" + key: "packaging-linux" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "gcp" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" + env: + PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" + + - label: ":linux: Packaging ARM" + key: "packaging-arm" + command: "cd $BEATS_PROJECT_NAME && mage package" + agents: + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" + env: + PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" + PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" + +YAML +fi + +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P + +echo "--- Loading dynamic steps" +buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_libbeat_pipeline.sh b/.buildkite/scripts/generate_libbeat_pipeline.sh index 0674d0b186b5..37132e74abf5 100755 --- a/.buildkite/scripts/generate_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_libbeat_pipeline.sh @@ -17,48 +17,71 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: " ${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":testengine: Stress Tests" key: "mandatory-stress-test" - command: ".buildkite/scripts/stress_tests.sh" + command: "cd $BEATS_PROJECT_NAME && make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' GOTEST_OUTPUT_OPTIONS='| go-junit-report > libbeat-stress-test.xml' stress-tests" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: "${BEATS_PROJECT_NAME}/libbeat-stress-test.xml" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Stress Tests" YAML else @@ -73,20 +96,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_metricbeat_pipeline.sh b/.buildkite/scripts/generate_metricbeat_pipeline.sh index 477f8fb25a34..a0386aeb1b75 100755 --- a/.buildkite/scripts/generate_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_metricbeat_pipeline.sh @@ -17,42 +17,68 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ununtu Unit Tests" - label: ":go: Go Intergration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" + env: + MODULE: $MODULE agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -65,7 +91,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -73,38 +104,30 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -124,7 +147,13 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" + YAML fi @@ -143,17 +172,22 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" @@ -161,12 +195,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_packetbeat_pipeline.sh b/.buildkite/scripts/generate_packetbeat_pipeline.sh index 97bdb531a6b0..e4870437ac16 100755 --- a/.buildkite/scripts/generate_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_packetbeat_pipeline.sh @@ -17,25 +17,37 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -48,43 +60,39 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - group: "Extended Windowds Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -111,21 +119,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,30 +162,38 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" + disk_size: 100 + disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "aws" - imagePrefix: "${IMAGE_UBUNTU_ARM_64}" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_winlogbeat_pipeline.sh index ce812016e795..ede70538c2c5 100755 --- a/.buildkite/scripts/generate_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_winlogbeat_pipeline.sh @@ -18,15 +18,22 @@ steps: - label: ":negative_squared_cross_mark: Cross compile" key: "mandatory-cross-compile" - command: ".buildkite/scripts/crosscompile.sh" + command: "make -C $BEATS_PROJECT_NAME crosscompile" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cross compile" - label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -40,7 +47,12 @@ steps: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2019}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -48,28 +60,29 @@ steps: - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows 10/11 Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" + image: "{{matrix.image}}" + machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" YAML else echo "The conditions don't match to requirements for generating pipeline steps." @@ -90,20 +103,23 @@ if are_conditions_met_packaging; then steps: - label: ":linux: Packaging Linux" key: "packaging-linux" - command: ".buildkite/scripts/packaging.sh" + command: "cd $BEATS_PROJECT_NAME && mage package" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh index f29e6152e602..eb9c8269f3db 100755 --- a/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_auditbeat_pipeline.sh @@ -25,20 +25,32 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit (MODULE) Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -51,7 +63,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +77,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -74,8 +93,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -102,21 +125,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -144,6 +177,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -155,12 +191,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh index 46e92f8ddf97..f9ca4e7f7677 100755 --- a/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh @@ -22,7 +22,13 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -33,7 +39,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" YAML fi @@ -61,6 +72,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -72,12 +86,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh index 80d28770bf05..78a351a078d9 100755 --- a/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_filebeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,7 +36,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" @@ -42,10 +52,17 @@ steps: machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -58,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -67,7 +89,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -81,7 +105,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,21 +137,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -132,15 +171,22 @@ if are_conditions_met_aws_tests; then - label: ":linux: Cloud Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Temporary disable, additional conditions required, we hit AWS limits" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud Tests" YAML fi @@ -168,6 +214,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -179,12 +228,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh index b51ca0ab3b9e..bec8e9bf96dd 100755 --- a/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_heartbeat_pipeline.sh @@ -17,28 +17,40 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" # ## TODO: there are windows test failures already reported # ## https://github.com/elastic/beats/issues/23957 and https://github.com/elastic/beats/issues/23958 # ## waiting for being fixed. # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "mandatory-win-unit-tests" # agents: # provider: "gcp" @@ -51,7 +63,9 @@ steps: # image: # - "${IMAGE_WIN_2016}" # - "${IMAGE_WIN_2022}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" # ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -60,7 +74,10 @@ steps: # steps: # - label: ":windows: Windows Unit Tests - {{matrix.image}}" -# command: ".buildkite/scripts/win_unit_tests.ps1" +# command: +# - "Set-Location -Path $BEATS_PROJECT_NAME" +# - "New-Item -ItemType Directory -Force -Path 'build'" +# - "mage unitTest" # key: "extended-win-unit-tests" # agents: # provider: "gcp" @@ -74,7 +91,9 @@ steps: # - "${IMAGE_WIN_10}" # - "${IMAGE_WIN_11}" # - "${IMAGE_WIN_2019}" -# artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -95,7 +114,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -123,6 +147,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -134,12 +161,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh index 66f0750ab6fa..7e3d5086288f 100755 --- a/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_libbeat_pipeline.sh @@ -17,33 +17,50 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage goIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":python: Python Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage pythonIntegTest" agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader/etw build goUnitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -56,44 +73,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ### TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage -w reader\etw build goUnitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_11}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -108,20 +121,25 @@ if are_conditions_met_arm_tests; then - group: "Extended Tests" key: "extended-tests" steps: - - label: ":linux: Arm64 Unit Tests" + - label: ":linux: Ubuntu ARM64 Unit Tests" key: "extended-arm64-unit-tests" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM64 Unit Tests" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh index ddc3ce2c8f20..d38c31236562 100755 --- a/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_metricbeat_pipeline.sh @@ -17,37 +17,60 @@ steps: steps: - label: ":linux: Ubuntu Unit Tests" key: "mandatory-linux-unit-test" - command: "cd $BEATS_PROJECT_NAME && mage build unitTest" + command: | + cd $BEATS_PROJECT_NAME + mage build unitTest agents: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":go: Go Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" + + - label: ":go: Go (MODULE) Integration Tests" key: "mandatory-int-test" - command: ".buildkite/scripts/go_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage goIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":python: Python Integration Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go (MODULE) Integration Tests" + + - label: ":python: Python (MODULE) Integration Tests" key: "mandatory-python-int-test" - command: ".buildkite/scripts/py_int_tests.sh" + command: | + cd $BEATS_PROJECT_NAME + mage pythonIntegTest env: MODULE: $MODULE agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Python (MODULE) Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -60,44 +83,40 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h - group: "Extended Windows Tests" key: "extended-win-tests" steps: - - label: ":windows: Windows 10 Unit Tests" - key: "extended-win-10-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Windows 11 Unit Tests" - key: "extended-win-11-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machineType: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows: Win 2019 Unit Tests" - key: "extended-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + - label: ":windows: Windows Unit Tests - {{matrix.image}}" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest + key: "extended-win-unit-tests" agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" + image: "{{matrix.image}}" machineType: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + matrix: + setup: + image: + - "${IMAGE_WIN_10}" + - "${IMAGE_WIN_11}" + - "${IMAGE_WIN_2019}" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -125,23 +144,35 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_aws_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: Cloud Tests" + - label: ":linux: Cloud (MODULE) Tests" key: "extended-cloud-test" command: ".buildkite/scripts/cloud_tests.sh" + skip: "Does not belong to a stage, exists but not run" env: MODULE: $MODULE + ASDF_TERRAFORM_VERSION: 1.0.2 agents: provider: "gcp" - image: "${DEFAULT_UBUNTU_X86_64_IMAGE}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Cloud (MODULE) Tests" YAML fi @@ -169,6 +200,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -180,12 +214,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P -echo "--- Loading dynamic steps" +echo "~~~ Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh index d6887c4c3826..5372d7f3faa7 100755 --- a/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":go: Go Integration Tests" key: "mandatory-int-test" @@ -31,10 +36,17 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Go Integration Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -47,7 +59,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -56,7 +73,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -70,7 +89,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -91,7 +115,12 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi @@ -119,12 +148,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh index 4eb2a1c3e049..c29b6c5d9bc1 100644 --- a/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_packetbeat_pipeline.sh @@ -22,7 +22,12 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu Unit Tests" - label: ":linux: Ubuntu System Tests" key: "mandatory-linux-system-test" @@ -31,20 +36,33 @@ steps: provider: "gcp" image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.xml" - - - label: ":rhel: RHEL-9 Unit Tests" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu System Tests" + + - label: ":rhel: RHEL9 Unit Tests" key: "mandatory-rhel9-unit-test" - command: ".buildkite/scripts/unit_tests.sh" + command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "gcp" image: "${IMAGE_RHEL9_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: RHEL9 Unit Tests" - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: + - "Set-Location -Path $BEATS_PROJECT_NAME" + - "New-Item -ItemType Directory -Force -Path 'build'" + - "mage unitTest" key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -57,7 +75,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 2022 System Tests" @@ -69,7 +92,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" ## TODO: this condition will be changed in the Phase 3 of the Migration Plan https://docs.google.com/document/d/1IPNprVtcnHlem-uyGZM0zGzhfUuFAh4LeSl9JFHMSZQ/edit#heading=h.sltz78yy249h @@ -78,7 +103,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -92,7 +119,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" ## TODO: uncomment when the issue https://github.com/elastic/beats/issues/38142 is solved # - label: ":windows: Windows 10 System Tests" @@ -104,7 +136,9 @@ steps: # machineType: "${GCP_WIN_MACHINE_TYPE}" # disk_size: 100 # disk_type: "pd-ssd" - # artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + # artifact_paths: + # - "$BEATS_PROJECT_NAME/build/*.xml" + # - "$BEATS_PROJECT_NAME/build/*.json" YAML else @@ -131,21 +165,31 @@ if are_conditions_met_macos_tests; then agents: provider: "orka" imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: MacOS Unit Tests" YAML fi if are_conditions_met_arm_tests; then cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" + - label: ":linux: Ubuntu ARM Unit Tests" key: "extended-arm64-unit-test" command: "cd $BEATS_PROJECT_NAME && mage build unitTest" agents: provider: "aws" imagePrefix: "${IMAGE_UBUNTU_ARM_64}" instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Ubuntu ARM Unit Tests" YAML fi @@ -173,6 +217,9 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" - label: ":linux: Packaging ARM" key: "packaging-arm" @@ -184,12 +231,15 @@ if are_conditions_met_packaging; then env: PLATFORMS: "${PACKAGING_ARM_PLATFORMS}" PACKAGES: "docker" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux ARM" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh index 108a70c15622..ad5ebd445837 100755 --- a/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh +++ b/.buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh @@ -18,7 +18,9 @@ steps: - label: ":windows: Windows 2019 Unit (MODULE) Tests" key: "mandatory-win-2019-unit-tests" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest env: MODULE: $MODULE agents: @@ -27,10 +29,17 @@ steps: machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows (MODULE) {{matrix.image}} Unit Tests" - label: ":windows: Windows 2016/2022 Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "mandatory-win-unit-tests" agents: provider: "gcp" @@ -43,7 +52,12 @@ steps: image: - "${IMAGE_WIN_2016}" - "${IMAGE_WIN_2022}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" # echo "Add the extended windows tests into the pipeline" # TODO: ADD conditions from the main pipeline @@ -53,7 +67,9 @@ steps: steps: - label: ":windows: Windows Unit Tests - {{matrix.image}}" - command: ".buildkite/scripts/win_unit_tests.ps1" + command: | + Set-Location -Path $BEATS_PROJECT_NAME + mage build unitTest key: "extended-win-unit-tests" agents: provider: "gcp" @@ -67,7 +83,12 @@ steps: - "${IMAGE_WIN_10}" - "${IMAGE_WIN_11}" - "${IMAGE_WIN_2019}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" + artifact_paths: + - "$BEATS_PROJECT_NAME/build/*.xml" + - "$BEATS_PROJECT_NAME/build/*.json" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Windows {{matrix.image}} Unit Tests" YAML else @@ -99,12 +120,15 @@ if are_conditions_met_packaging; then disk_type: "pd-ssd" env: PLATFORMS: "${PACKAGING_PLATFORMS}" + notify: + - github_commit_status: + context: "$BEATS_PROJECT_NAME: Packaging Linux" YAML fi -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName +echo "+++ Printing dynamic steps" +cat $pipelineName | yq . -P echo "--- Loading dynamic steps" buildkite-agent pipeline upload $pipelineName diff --git a/.buildkite/scripts/go_int_tests.sh b/.buildkite/scripts/go_int_tests.sh deleted file mode 100755 index b4c519f45126..000000000000 --- a/.buildkite/scripts/go_int_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Go Intergration Tests for $BEATS_PROJECT_NAME" -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage goIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/packaging/package-step.sh b/.buildkite/scripts/packaging/package-step.sh index 9eddfafcfba6..5f04ed3849bc 100755 --- a/.buildkite/scripts/packaging/package-step.sh +++ b/.buildkite/scripts/packaging/package-step.sh @@ -18,7 +18,7 @@ if are_files_changed "$changeset"; then - label: ":ubuntu: ${BEATS_PROJECT_NAME}/Packaging Linux X86" key: "package-linux-x86" env: - PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PLATFORMS: $PACKAGING_PLATFORMS SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" notify: @@ -32,7 +32,7 @@ if are_files_changed "$changeset"; then - label: ":linux: ${BEATS_PROJECT_NAME}/Packaging Linux ARM" key: "package-linux-arm" env: - PLATFORMS: "linux/arm64" + PLATFORMS: $PACKAGING_ARM_PLATFORMS PACKAGES: "docker" SNAPSHOT: true command: ".buildkite/scripts/packaging/package.sh" diff --git a/.buildkite/scripts/py_int_tests.sh b/.buildkite/scripts/py_int_tests.sh deleted file mode 100755 index 19fa8796c3e7..000000000000 --- a/.buildkite/scripts/py_int_tests.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/install_tools.sh - -set -euo pipefail - -echo "--- Run Python Intergration Tests for $BEATS_PROJECT_NAME" - -pushd "${BEATS_PROJECT_NAME}" > /dev/null - -mage pythonIntegTest - -popd > /dev/null diff --git a/.buildkite/scripts/setenv.sh b/.buildkite/scripts/setenv.sh index f0116c6b3086..f6b03f53f9bd 100755 --- a/.buildkite/scripts/setenv.sh +++ b/.buildkite/scripts/setenv.sh @@ -2,37 +2,19 @@ set -euo pipefail -REPO="beats" -TMP_FOLDER="tmp.${REPO}" -DOCKER_REGISTRY="docker.elastic.co" -SETUP_GVM_VERSION="v0.5.1" -DOCKER_COMPOSE_VERSION="1.21.0" -DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" -SETUP_WIN_PYTHON_VERSION="3.11.0" -NMAP_WIN_VERSION="7.12" # Earlier versions of NMap provide WinPcap (the winpcap packages don't install nicely because they pop-up a UI) -GO_VERSION=$(cat .go-version) -ASDF_MAGE_VERSION="1.15.0" -PACKAGING_PLATFORMS="+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" -PACKAGING_ARM_PLATFORMS="linux/arm64" -ASDF_TERRAFORM_VERSION="1.0.2" -AWS_REGION="eu-central-1" -NODEJS_VERSION="18.17.1" +export REPO="beats" +export DOCKER_REGISTRY="docker.elastic.co" +export SETUP_GVM_VERSION="v0.5.1" +export DOCKER_COMPOSE_VERSION="1.21.0" +export DOCKER_COMPOSE_VERSION_AARCH64="v2.21.0" +export ASDF_NODEJS_VERSION="18.17.1" +export AWS_REGION="eu-central-1" -export SETUP_GVM_VERSION -export DOCKER_COMPOSE_VERSION -export DOCKER_COMPOSE_VERSION_AARCH64 -export SETUP_WIN_PYTHON_VERSION -export NMAP_WIN_VERSION +WORKSPACE=${WORKSPACE:-"$(pwd)"} +export WORKSPACE +GO_VERSION=$(cat .go-version) export GO_VERSION -export ASDF_MAGE_VERSION -export PACKAGING_PLATFORMS -export PACKAGING_ARM_PLATFORMS -export REPO -export TMP_FOLDER -export DOCKER_REGISTRY -export ASDF_TERRAFORM_VERSION -export AWS_REGION -export NODEJS_VERSION + exportVars() { local platform_type="$(uname)" diff --git a/.buildkite/scripts/win_unit_tests.ps1 b/.buildkite/scripts/win_unit_tests.ps1 deleted file mode 100644 index 6c3af6e9321c..000000000000 --- a/.buildkite/scripts/win_unit_tests.ps1 +++ /dev/null @@ -1,188 +0,0 @@ -param( - [string]$testType = "unittest" -) - -$ErrorActionPreference = "Stop" # set -e -$WorkFolder = $env:BEATS_PROJECT_NAME -$WORKSPACE = Get-Location -# Forcing to checkout again all the files with a correct autocrlf. -# Doing this here because we cannot set git clone options before. -function fixCRLF { - Write-Host "-- Fixing CRLF in git checkout --" - git config core.autocrlf false - git rm --quiet --cached -r . - git reset --quiet --hard -} - -function retry { - param( - [int]$retries, - [ScriptBlock]$scriptBlock - ) - $count = 0 - while ($count -lt $retries) { - $count++ - try { - & $scriptBlock - return - } catch { - $exitCode = $_.Exception.ErrorCode - Write-Host "Retry $count/$retries exited $exitCode, retrying..." - Start-Sleep -Seconds ([Math]::Pow(2, $count)) - } - } - Write-Host "Retry $count/$retries exited, no more retries left." -} - -function verifyFileChecksum { - param ( - [string]$filePath, - [string]$checksumFilePath - ) - $actualHash = (Get-FileHash -Algorithm SHA256 -Path $filePath).Hash - $checksumData = Get-Content -Path $checksumFilePath - $expectedHash = ($checksumData -split "\s+")[0] - if ($actualHash -eq $expectedHash) { - Write-Host "CheckSum is checked. File is correct. Original checkSum is: $expectedHash " - return $true - } else { - Write-Host "CheckSum is wrong. File can be corrupted or modified. Current checksum is: $actualHash, the original checksum is: $expectedHash" - return $false - } -} - -function withGolang($version) { - Write-Host "-- Installing Go $version --" - $goDownloadPath = Join-Path $env:TEMP "go_installer.msi" - $goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $goInstallerUrl -OutFile $goDownloadPath - } - Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $goDownloadPath /quiet" -Wait - $env:GOPATH = "${env:ProgramFiles}\Go" - $env:GOBIN = "${env:GOPATH}\bin" - $env:Path += ";$env:GOPATH;$env:GOBIN" - go version - installGoDependencies -} - -function withPython($version) { - Write-Host "-- Installing Python $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $pyDownloadPath = Join-Path $env:TEMP "python-$version-amd64.exe" - $pyInstallerUrl = "https://www.python.org/ftp/python/$version/python-$version-amd64.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $pyInstallerUrl -OutFile $pyDownloadPath - } - Start-Process -FilePath $pyDownloadPath -ArgumentList "/quiet", "InstallAllUsers=1", "PrependPath=1", "Include_test=0" -Wait - $pyBinPath = "${env:ProgramFiles}\Python311" - $env:Path += ";$pyBinPath" - python --version -} - -function withMinGW { - Write-Host "-- Installing MinGW --" - [Net.ServicePointManager]::SecurityProtocol = "tls11, tls12, ssl3" - $gwInstallerUrl = "https://github.com/brechtsanders/winlibs_mingw/releases/download/12.1.0-14.0.6-10.0.0-ucrt-r3/winlibs-x86_64-posix-seh-gcc-12.1.0-llvm-14.0.6-mingw-w64ucrt-10.0.0-r3.zip" - $gwInstallerCheckSumUrl = "$gwInstallerUrl.sha256" - $gwDownloadPath = "$env:TEMP\winlibs-x86_64.zip" - $gwDownloadCheckSumPath = "$env:TEMP\winlibs-x86_64.zip.sha256" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -Uri $gwInstallerUrl -OutFile $gwDownloadPath - Invoke-WebRequest -Uri $gwInstallerCheckSumUrl -OutFile $gwDownloadCheckSumPath - } - $comparingResult = verifyFileChecksum -filePath $gwDownloadPath -checksumFilePath $gwDownloadCheckSumPath - if ($comparingResult) { - Expand-Archive -Path $gwDownloadPath -DestinationPath "$env:TEMP" - $gwBinPath = "$env:TEMP\mingw64\bin" - $env:Path += ";$gwBinPath" - } else { - exit 1 - } - -} -function installGoDependencies { - $installPackages = @( - "github.com/magefile/mage" - "github.com/elastic/go-licenser" - "golang.org/x/tools/cmd/goimports" - "github.com/jstemmer/go-junit-report/v2" - "gotest.tools/gotestsum" - ) - foreach ($pkg in $installPackages) { - go install "$pkg@latest" - } -} - -function withNmap($version) { - Write-Host "-- Installing Nmap $version --" - [Net.ServicePointManager]::SecurityProtocol = "tls, tls11, tls12, ssl3" - $nmapInstallerUrl = "https://nmap.org/dist/nmap-$version-setup.exe" - $nmapDownloadPath = "$env:TEMP\nmap-$version-setup.exe" - retry -retries 5 -scriptBlock { - Invoke-WebRequest -UseBasicParsing -Uri $nmapInstallerUrl -OutFile $nmapDownloadPath - } - Start-Process -FilePath $nmapDownloadPath -ArgumentList "/S" -Wait -} -function google_cloud_auth { - $tempFileName = "google-cloud-credentials.json" - $secretFileLocation = Join-Path $env:TEMP $tempFileName - $null = New-Item -ItemType File -Path $secretFileLocation - Set-Content -Path $secretFileLocation -Value $env:PRIVATE_CI_GCS_CREDENTIALS_SECRET - gcloud auth activate-service-account --key-file $secretFileLocation > $null 2>&1 - $env:GOOGLE_APPLICATION_CREDENTIALS = $secretFileLocation -} - -function google_cloud_auth_cleanup { - if (Test-Path $env:GOOGLE_APPLICATION_CREDENTIALS) { - Remove-Item $env:GOOGLE_APPLICATION_CREDENTIALS -Force - Remove-Item Env:\GOOGLE_APPLICATION_CREDENTIALS - } else { - Write-Host "No GCP credentials were added" - } -} - -fixCRLF - -withGolang $env:GO_VERSION - -withPython $env:SETUP_WIN_PYTHON_VERSION - -withMinGW - -if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-packetbeat" -or $env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-filebeat") { - withNmap $env:NMAP_WIN_VERSION -} - -$ErrorActionPreference = "Continue" # set +e - -Set-Location -Path $WorkFolder - -$magefile = "$WORKSPACE\$WorkFolder\.magefile" -$env:MAGEFILE_CACHE = $magefile - -New-Item -ItemType Directory -Force -Path "build" - -if ($testType -eq "unittest") { - if ($env:BUILDKITE_PIPELINE_SLUG -eq "beats-xpack-libbeat") { - mage -w reader/etw build goUnitTest - } else { - mage build unitTest - } -} -elseif ($testType -eq "systemtest") { - try { - google_cloud_auth - mage systemTest - } finally { - google_cloud_auth_cleanup - } -} -else { - Write-Host "Unknown test type. Please specify 'unittest' or 'systemtest'." -} - -$EXITCODE=$LASTEXITCODE -$ErrorActionPreference = "Stop" - -Exit $EXITCODE diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index 7c5ee4d16090..254370db030b 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -2,16 +2,23 @@ name: "beats-winlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + BEATS_PROJECT_NAME: "winlogbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "winlogbeat" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -35,6 +42,8 @@ steps: - label: ":linux: Load dynamic winlogbeat pipeline" key: "winlogbeat-pipeline" command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml index d88bf2a4ff05..c0c92cc3ae13 100644 --- a/.buildkite/x-pack/pipeline.xpack.auditbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.auditbeat.yml @@ -2,21 +2,28 @@ name: "beats-xpack-auditbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/auditbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/auditbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -57,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack auditbeat pipeline" key: "xpack-auditbeat-pipeline" command: ".buildkite/scripts/generate_xpack_auditbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml index bcc2610e175b..f55b812cd173 100644 --- a/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml @@ -2,13 +2,28 @@ name: "beats-xpack-dockerlogbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" - GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" BEATS_PROJECT_NAME: "x-pack/dockerlogbeat" + GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" + GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" + GCP_WIN_MACHINE_TYPE: "n2-standard-8" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -33,6 +48,8 @@ steps: - label: ":linux: Load dynamic x-pack dockerlogbeat pipeline" key: "xpack-dockerlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_dockerlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index a324353b65f9..02b1cf0ddead 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-filebeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/filebeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/filebeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -64,6 +72,8 @@ steps: - label: ":linux: Load dynamic x-pack filebeat pipeline" key: "xpack-filebeat-pipeline" command: ".buildkite/scripts/generate_xpack_filebeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml index 2804e98996a1..1b04fcf99225 100644 --- a/.buildkite/x-pack/pipeline.xpack.heartbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.heartbeat.yml @@ -1,22 +1,29 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json name: "beats-xpack-heartbeat" - env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/heartbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/heartbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -49,6 +56,8 @@ steps: - label: ":linux: Load dynamic x-pack heartbeat pipeline" key: "xpack-heartbeat-pipeline" command: ".buildkite/scripts/generate_xpack_heartbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 0c745b1a09d6..c029951f0107 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -2,18 +2,28 @@ name: "beats-xpack-libbeat" env: - IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/libbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/libbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +55,8 @@ steps: - label: ":linux: Load dynamic x-pack libbeat pipeline" key: "libbeat-pipeline" command: ".buildkite/scripts/generate_xpack_libbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml index 216f31343446..855c59b8bac2 100644 --- a/.buildkite/x-pack/pipeline.xpack.metricbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.metricbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-metricbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - DEFAULT_UBUNTU_X86_64_IMAGE: "family/core-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/metricbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/metricbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack metricbeat pipeline" key: "xpack-metricbeat-pipeline" command: ".buildkite/scripts/generate_xpack_metricbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml index 57726c038280..7cd699b53319 100644 --- a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml @@ -2,17 +2,28 @@ name: "beats-xpack-osquerybeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/osquerybeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - BEATS_PROJECT_NAME: "x-pack/osquerybeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -45,6 +56,8 @@ steps: - label: ":linux: Load dynamic x-pack osquerybeat pipeline" key: "xpack-osquerybeat-pipeline" command: ".buildkite/scripts/generate_xpack_osquerybeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 750b59e716d8..5e3ce87e2bf4 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -2,20 +2,28 @@ name: "beats-xpack-packetbeat" env: - IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" - IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" - IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" + BEATS_PROJECT_NAME: "x-pack/packetbeat" GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" - AWS_ARM_INSTANCE_TYPE: "t4g.xlarge" - BEATS_PROJECT_NAME: "x-pack/packetbeat" + IMAGE_MACOS_ARM: "generic-13-ventura-arm" + IMAGE_MACOS_X86_64: "generic-13-ventura-x64" + IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9" + IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64" + IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" + + #Packaging + PACKAGING_ARM_PLATFORMS: "linux/arm64" + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 steps: @@ -56,6 +64,8 @@ steps: - label: ":linux: Load dynamic x-pack packetbeat pipeline" key: "packetbeat-pipeline" command: ".buildkite/scripts/generate_xpack_packetbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 5c8acefd6989..80b70e287d96 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -3,15 +3,22 @@ name: "beats-xpack-winlogbeat" env: IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204" - IMAGE_WIN_10: "family/general-windows-10" - IMAGE_WIN_11: "family/general-windows-11" - IMAGE_WIN_2016: "family/core-windows-2016" - IMAGE_WIN_2019: "family/core-windows-2019" - IMAGE_WIN_2022: "family/core-windows-2022" + IMAGE_WIN_10: "family/platform-ingest-beats-windows-10" + IMAGE_WIN_11: "family/platform-ingest-beats-windows-11" + IMAGE_WIN_2016: "family/platform-ingest-beats-windows-2016" + IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019" + IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022" GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16" GCP_WIN_MACHINE_TYPE: "n2-standard-8" BEATS_PROJECT_NAME: "x-pack/winlogbeat" + #Packaging + PACKAGING_PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64" + PACKAGING_ARM_PLATFORMS: "linux/arm64" + + #Deps + ASDF_MAGE_VERSION: 1.15.0 + steps: - input: "Input Parameters" @@ -34,6 +41,8 @@ steps: - label: ":linux: Load dynamic x-pack winlogbeat pipeline" key: "xpack-winlogbeat-pipeline" command: ".buildkite/scripts/generate_xpack_winlogbeat_pipeline.sh" + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci:latest" notify: - github_commit_status: context: "${BEATS_PROJECT_NAME}: Load dynamic pipeline's steps" diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 78801d4a00ea..525dee11d47f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Auditbeat* +- Added `add_session_metadata` processor, which enables session viewer on Auditbeat data. {pull}37640[37640] +- Add linux capabilities to processes in the system/process. {pull}37453[37453] +- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223] +- Add linux capabilities to processes in the system/process. {pull}37453[37453] +- Add opt-in eBPF backend for file_integrity module. {pull}37223[37223] +- Add process data to file events (Linux only, eBPF backend). {pull}38199[38199] *Filebeat* @@ -176,6 +182,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Prevent complete loss of long request trace data. {issue}37826[37826] {pull}37836[37836] - Added experimental version of the Websocket Input. {pull}37774[37774] - Add support for PEM-based Okta auth in CEL. {pull}37813[37813] +- Add Salesforce input. {pull}37331[37331] - Add ETW input. {pull}36915[36915] - Update CEL mito extensions to v1.9.0 to add keys/values helper. {pull}37971[37971] - Add logging for cache processor file reads and writes. {pull}38052[38052] diff --git a/NOTICE.txt b/NOTICE.txt index c4ae3f617b57..f846edfc9ec5 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12287,11 +12287,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/ebpfevents -Version: v0.4.0 +Version: v0.5.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.4.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/ebpfevents@v0.5.0/LICENSE.txt: The https://github.com/elastic/ebpfevents repository contains source code under various licenses: @@ -16447,6 +16447,37 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/go-sfdc +Version: v0.0.0-20201201191151-3190c381b3e1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-sfdc@v0.0.0-20201201191151-3190c381b3e1/LICENSE.txt: + +MIT License + +Copyright (c) 2019 Robert Sean Justice + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-ldap/ldap/v3 Version: v3.4.6 @@ -17400,6 +17431,25 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/golang-jwt/jwt +Version: v3.2.1+incompatible +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/golang-jwt/jwt@v3.2.1+incompatible/LICENSE: + +Copyright (c) 2012 Dave Grijalva +Copyright (c) 2021 golang-jwt maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + -------------------------------------------------------------------------------- Dependency : github.com/golang/mock Version: v1.6.0 @@ -22921,6 +22971,45 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/tklauser/go-sysconf +Version: v0.3.10 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: + +BSD 3-Clause License + +Copyright (c) 2018-2021, Tobias Klauser +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/tsg/go-daemon Version: v0.0.0-20200207173439-e704b93fd89b @@ -25019,6 +25108,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/exp +Version: v0.0.0-20231127185646-65229373498e +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/lint Version: v0.0.0-20210508222113-6edffad5e616 @@ -36661,11 +36787,11 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/cilium/ebpf -Version: v0.12.3 +Version: v0.13.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.12.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cilium/ebpf@v0.13.2/LICENSE: MIT License @@ -37084,6 +37210,24 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/ristretto@v0.1. END OF TERMS AND CONDITIONS +-------------------------------------------------------------------------------- +Dependency : github.com/dgrijalva/jwt-go +Version: v3.2.0+incompatible +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/dgrijalva/jwt-go@v3.2.0+incompatible/LICENSE: + +Copyright (c) 2012 Dave Grijalva + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + -------------------------------------------------------------------------------- Dependency : github.com/dgryski/go-farm Version: v0.0.0-20190423205320-6a90982ecee2 @@ -38697,11 +38841,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/frankban/quicktest -Version: v1.14.5 +Version: v1.14.3 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.14.3/LICENSE: MIT License @@ -39304,6 +39448,37 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/go-quicktest/qt +Version: v1.101.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-quicktest/qt@v1.101.0/LICENSE: + +MIT License + +Copyright (c) 2017 Canonical Ltd. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/go-sourcemap/sourcemap Version: v2.1.2+incompatible @@ -49663,27 +49838,6 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : github.com/pkg/diff -Version: v0.0.0-20210226163009-20ebb0f2a09e -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/pkg/diff@v0.0.0-20210226163009-20ebb0f2a09e/LICENSE: - -Copyright 2018 Joshua Bleecher Snyder - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/pmezard/go-difflib Version: v1.0.0 @@ -49967,11 +50121,11 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golan -------------------------------------------------------------------------------- Dependency : github.com/rogpeppe/go-internal -Version: v1.9.0 +Version: v1.11.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.9.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.11.0/LICENSE: Copyright (c) 2018 The Go Authors. All rights reserved. @@ -50873,45 +51027,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/tklauser/go-sysconf -Version: v0.3.10 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/tklauser/go-sysconf@v0.3.10/LICENSE: - -BSD 3-Clause License - -Copyright (c) 2018-2021, Tobias Klauser -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : github.com/tklauser/numcpus Version: v0.4.0 @@ -53855,43 +53970,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/exp -Version: v0.0.0-20231127185646-65229373498e -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: - -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/term Version: v0.18.0 diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index 22813a47f22e..b282aaaf3d29 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -134,6 +134,7 @@ type Event struct { Action Action `json:"action"` // Action (like created, updated). Hashes map[HashType]Digest `json:"hash,omitempty"` // File hashes. ParserResults mapstr.M `json:"file,omitempty"` // Results from running file parsers. + Process *Process `json:"process,omitempty"` // Process data. Available only on Linux when using the eBPF backend. // Metadata rtt time.Duration // Time taken to collect the info. @@ -141,6 +142,33 @@ type Event struct { hashFailed bool // Set when hashing the file failed. } +// Process contain information about a process. +// These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +type Process struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + // Process name. Sometimes called program name or similar. + Name string `json:"name,omitempty"` + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + // Process id. + PID uint32 `json:"pid,omitempty"` +} + // Metadata contains file metadata. type Metadata struct { Inode uint64 `json:"inode"` @@ -354,6 +382,24 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { } } + if e.Process != nil { + process := mapstr.M{ + "pid": e.Process.PID, + "name": e.Process.Name, + "entity_id": e.Process.EntityID, + "user": mapstr.M{ + "id": e.Process.User.ID, + "name": e.Process.User.Name, + }, + "group": mapstr.M{ + "id": e.Process.Group.ID, + "name": e.Process.Group.Name, + }, + } + + out.MetricSetFields.Put("process", process) + } + if len(e.Hashes) > 0 { hashes := make(mapstr.M, len(e.Hashes)) for hashType, digest := range e.Hashes { diff --git a/auditbeat/module/file_integrity/event_linux.go b/auditbeat/module/file_integrity/event_linux.go index 7643d03a6b42..c0eb2d57b159 100644 --- a/auditbeat/module/file_integrity/event_linux.go +++ b/auditbeat/module/file_integrity/event_linux.go @@ -26,6 +26,7 @@ import ( "strconv" "time" + "github.com/elastic/beats/v7/libbeat/ebpf/sys" "github.com/elastic/ebpfevents" ) @@ -41,7 +42,9 @@ func NewEventFromEbpfEvent( path, target string action Action metadata Metadata + process Process err error + errors []error ) switch ee.Type { case ebpfevents.EventTypeFileCreate: @@ -54,7 +57,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileCreateEvent.SymlinkTargetPath + metadata, err = metadataFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileCreate(fileCreateEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileRename: action = Moved @@ -65,7 +77,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileRenameEvent.SymlinkTargetPath + metadata, err = metadataFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileRename(fileRenameEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileDelete: action = Deleted @@ -76,6 +97,11 @@ func NewEventFromEbpfEvent( return event, false } target = fileDeleteEvent.SymlinkTargetPath + + process, err = processFromFileDelete(fileDeleteEvent) + if err != nil { + errors = append(errors, err) + } case ebpfevents.EventTypeFileModify: fileModifyEvent := ee.Body.(*ebpfevents.FileModify) @@ -92,7 +118,16 @@ func NewEventFromEbpfEvent( return event, false } target = fileModifyEvent.SymlinkTargetPath + metadata, err = metadataFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } + + process, err = processFromFileModify(fileModifyEvent) + if err != nil { + errors = append(errors, err) + } } event := Event{ @@ -102,10 +137,8 @@ func NewEventFromEbpfEvent( Info: &metadata, Source: SourceEBPF, Action: action, - errors: make([]error, 0), - } - if err != nil { - event.errors = append(event.errors, err) + Process: &process, + errors: errors, } if event.Action == Deleted { @@ -115,7 +148,6 @@ func NewEventFromEbpfEvent( case FileType: fillHashes(&event, path, maxFileSize, hashTypes, fileParsers) case SymlinkType: - var err error event.TargetPath, err = filepath.EvalSymlinks(event.Path) if err != nil { event.errors = append(event.errors, err) @@ -147,6 +179,59 @@ func metadataFromFileModify(evt *ebpfevents.FileModify) (Metadata, error) { return md, err } +func newProcess(pid uint32, start uint64, comm string, euid, egid uint32) (Process, error) { + var ( + p Process + err error + ) + + t, err := sys.TimeFromNsSinceBoot(start) + if err != nil { + return p, err + } + + p.EntityID, err = sys.EntityID(pid, t) + if err != nil { + return p, err + } + p.Name = comm + p.PID = pid + + p.User.ID = strconv.FormatUint(uint64(euid), 10) + u, err := user.LookupId(p.User.ID) + if err == nil { + p.User.Name = u.Username + } else { + p.User.Name = "n/a" + } + + p.Group.ID = strconv.FormatUint(uint64(egid), 10) + g, err := user.LookupGroupId(p.Group.ID) + if err == nil { + p.Group.Name = g.Name + } else { + p.Group.Name = "n/a" + } + + return p, nil +} + +func processFromFileCreate(evt *ebpfevents.FileCreate) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileRename(evt *ebpfevents.FileRename) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileModify(evt *ebpfevents.FileModify) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + +func processFromFileDelete(evt *ebpfevents.FileDelete) (Process, error) { + return newProcess(evt.Pids.Tgid, evt.Pids.StartTimeNs, evt.Comm, evt.Creds.Euid, evt.Creds.Egid) +} + func fillFileInfo(md *Metadata, finfo ebpfevents.FileInfo) error { md.Inode = finfo.Inode md.UID = finfo.Uid diff --git a/auditbeat/module/file_integrity/event_linux_test.go b/auditbeat/module/file_integrity/event_linux_test.go index 1a440afb8f17..beac98789099 100644 --- a/auditbeat/module/file_integrity/event_linux_test.go +++ b/auditbeat/module/file_integrity/event_linux_test.go @@ -21,7 +21,6 @@ package file_integrity import ( "os" - "os/user" "testing" "github.com/stretchr/testify/assert" @@ -40,13 +39,25 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Inode: 1234, Mode: os.FileMode(0o644), Size: 2345, - Uid: 3456, - Gid: 4567, + Uid: uint32(os.Geteuid()), + Gid: uint32(os.Getegid()), }, Path: "/foo", SymlinkTargetPath: "/bar", + Creds: ebpfevents.CredInfo{ + Ruid: 1, + Rgid: 2, + Euid: uint32(os.Geteuid()), + Egid: uint32(os.Getegid()), + Suid: 5, + Sgid: 6, + }, }, } + event, ok := NewEventFromEbpfEvent( + ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) + assert.True(t, ok) + expectedEvent := Event{ Action: Created, Path: "/foo", @@ -54,21 +65,22 @@ func TestNewEventFromEbpfEvent(t *testing.T) { Info: &Metadata{ Type: FileType, Inode: 1234, - UID: 3456, - GID: 4567, + UID: uint32(os.Geteuid()), + GID: uint32(os.Getegid()), Size: 2345, - Owner: "n/a", - Group: "n/a", + Owner: event.Info.Owner, + Group: event.Info.Group, Mode: os.FileMode(0o644), }, - Source: SourceEBPF, - errors: []error{user.UnknownUserIdError(3456)}, + Process: event.Process, // 1:1 copy this as it changes on every machine + Source: SourceEBPF, + errors: nil, } - - event, ok := NewEventFromEbpfEvent( - ebpfEvent, 0, []HashType{}, []FileParser{}, func(path string) bool { return false }) - assert.True(t, ok) event.Timestamp = expectedEvent.Timestamp assert.Equal(t, expectedEvent, event) + assert.NotEqual(t, "", event.Process.EntityID) + assert.NotEqual(t, 0, event.Process.PID) + assert.NotEqual(t, 0, event.Process.User.ID) + assert.NotEqual(t, "", event.Process.User.Name) } diff --git a/auditbeat/tests/system/test_file_integrity.py b/auditbeat/tests/system/test_file_integrity.py index e6b03306c3a2..64062dd0a772 100644 --- a/auditbeat/tests/system/test_file_integrity.py +++ b/auditbeat/tests/system/test_file_integrity.py @@ -5,6 +5,10 @@ from auditbeat import * +if platform.platform().split('-')[0] == 'Linux': + import pwd + + def is_root(): if 'geteuid' not in dir(os): return False @@ -100,6 +104,16 @@ def wait_startup(self, backend, dir): # may differ self.wait_log_contains(escape_path(dir), max_timeout=30, ignore_case=True) + def _assert_process_data(self, event, backend): + if backend != "ebpf": + return + assert event["process.entity_id"] != "" + assert event["process.executable"] == "pytest" + assert event["process.pid"] == os.getpid() + assert int(event["process.user.id"]) == os.geteuid() + assert event["process.user.name"] == pwd.getpwuid(os.geteuid()).pw_name + assert int(event["process.group.id"]) == os.getegid() + def _test_non_recursive(self, backend): """ file_integrity monitors watched directories (non recursive). @@ -172,6 +186,8 @@ def _test_non_recursive(self, backend): # assert file inside subdir is not reported assert self.log_contains(file3) is False + self._assert_process_data(objs[0], backend) + @unittest.skipIf(os.getenv("CI") is not None and platform.system() == 'Darwin', 'Flaky test: https://github.com/elastic/beats/issues/24678') def test_non_recursive__fsnotify(self): @@ -252,6 +268,8 @@ def _test_recursive(self, backend): file_events(objs, file1, ['created']) file_events(objs, file2, ['created']) + self._assert_process_data(objs[0], backend) + def test_recursive__fsnotify(self): self._test_recursive("fsnotify") diff --git a/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml b/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml index 4c75de47c99d..e506b0bc97bc 100644 --- a/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml +++ b/filebeat/module/logstash/log/ingest/pipeline-plaintext.yml @@ -13,6 +13,8 @@ processors: (.| )* patterns: + - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\]\[%{NOTSPACE:logstash.log.plugin_id}\] + %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\]\[%{NOTSPACE:logstash.log.pipeline_id}\] %{GREEDYMULTILINE:message} - \[%{TIMESTAMP_ISO8601:logstash.log.timestamp}\]\[%{LOGSTASH_LOGLEVEL:log.level}\s?\]\[%{LOGSTASH_CLASS_MODULE:logstash.log.module}\s*\] diff --git a/go.mod b/go.mod index 745d47673b61..a080de26a98f 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elastic/beats/v7 -go 1.21 +go 1.21.8 require ( cloud.google.com/go/bigquery v1.55.0 @@ -201,7 +201,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/ebpfevents v0.4.0 + github.com/elastic/ebpfevents v0.5.0 github.com/elastic/elastic-agent-autodiscover v0.6.8 github.com/elastic/elastic-agent-libs v0.7.5 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 @@ -211,7 +211,9 @@ require ( github.com/elastic/tk-btf v0.1.0 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15 + github.com/g8rswimmer/go-sfdc v0.0.0-00010101000000-000000000000 github.com/go-ldap/ldap/v3 v3.4.6 + github.com/golang-jwt/jwt v3.2.1+incompatible github.com/google/cel-go v0.19.0 github.com/googleapis/gax-go/v2 v2.12.0 github.com/gorilla/handlers v1.5.1 @@ -224,10 +226,12 @@ require ( github.com/pkg/xattr v0.4.9 github.com/sergi/go-diff v1.3.1 github.com/shirou/gopsutil/v3 v3.22.10 + github.com/tklauser/go-sysconf v0.3.10 go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.8 go.elastic.co/apm/module/apmhttp/v2 v2.4.8 go.elastic.co/apm/v2 v2.4.8 go.mongodb.org/mongo-driver v1.5.1 + golang.org/x/exp v0.0.0-20231127185646-65229373498e golang.org/x/tools/go/vcs v0.1.0-deprecated google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb gopkg.in/natefinch/lumberjack.v2 v2.0.0 @@ -271,11 +275,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect - github.com/cilium/ebpf v0.12.3 // indirect + github.com/cilium/ebpf v0.13.2 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect @@ -360,7 +365,6 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect @@ -375,7 +379,6 @@ require ( go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/otel/trace v1.21.0 // indirect - golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -415,6 +418,7 @@ replace ( github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c + github.com/g8rswimmer/go-sfdc => github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1 github.com/godror/godror => github.com/godror/godror v0.33.2 // updating to v0.24.2 caused a breaking change github.com/golang/glog => github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4 github.com/google/gopacket => github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a diff --git a/go.sum b/go.sum index ff0abc92363a..7900b6758365 100644 --- a/go.sum +++ b/go.sum @@ -442,8 +442,8 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4= -github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM= +github.com/cilium/ebpf v0.13.2 h1:uhLimLX+jF9BTPPvoCUYh/mBeoONkjgaJ9w9fn0mRj4= +github.com/cilium/ebpf v0.13.2/go.mod h1:DHp1WyrLeiBh19Cf/tfiSMhqheEiK8fXFZ4No0P1Hso= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -598,6 +598,7 @@ github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2/go.mod h1:XG9mPq0dF github.com/dgraph-io/badger/v3 v3.2103.1 h1:zaX53IRg7ycxVlkd5pYdCeFp1FynD6qBGQoQql3R3Hk= github.com/dgraph-io/badger/v3 v3.2103.1/go.mod h1:dULbq6ehJ5K0cGW/1TQ9iSfUk0gbSiToDWmWmTsJ53E= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= @@ -671,8 +672,8 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/ebpfevents v0.4.0 h1:M80eAeJnzvGQgU9cjJqkjFca9pjM3aq/TuZxJeom4bI= -github.com/elastic/ebpfevents v0.4.0/go.mod h1:o21z5xup/9dK8u0Hg9bZRflSqqj1Zu5h2dg2hSTcUPQ= +github.com/elastic/ebpfevents v0.5.0 h1:QkyMAYWo3fXFbYtXAXU8sZu2SQ4LXVYC6gLXIWXy02E= +github.com/elastic/ebpfevents v0.5.0/go.mod h1:ESG9gw7N+n5yCCMgdg1IIJENKWSmX7+X0Fi9GUs9nvU= github.com/elastic/elastic-agent-autodiscover v0.6.8 h1:BSXz+QwjZAEt08G+T3GDGl14Bh9a6zD8luNCvZut/b8= github.com/elastic/elastic-agent-autodiscover v0.6.8/go.mod h1:hFeFqneS2r4jD0/QzGkrNk0YVdN0JGh7lCWdsH7zcI4= github.com/elastic/elastic-agent-client/v7 v7.8.1 h1:J9wZc/0mUvSEok0X5iR5+n60Jgb+AWooKddb3XgPWqM= @@ -706,6 +707,8 @@ github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f h1:Fvsq github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f/go.mod h1:OPGqFNdTS34kMReS5hPFtBhD9J8itmSDurs1ix2wx7c= github.com/elastic/go-seccomp-bpf v1.4.0 h1:6y3lYrEHrLH9QzUgOiK8WDqmPaMnnB785WxibCNIOH4= github.com/elastic/go-seccomp-bpf v1.4.0/go.mod h1:wIMxjTbKpWGQk4CV9WltlG6haB4brjSH/dvAohBPM1I= +github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1 h1:KS+lvT/rUS8Z4++RoiM2pHOKmBv8mLERmgiX04VEgwk= +github.com/elastic/go-sfdc v0.0.0-20201201191151-3190c381b3e1/go.mod h1:/FB/tWFyF33vmdjwIwqAKu9QMVFVEjeoWi9V6eUcQEQ= github.com/elastic/go-structform v0.0.10 h1:oy08o/Ih2hHTkNcRY/1HhaYvIp5z6t8si8gnCJPDo1w= github.com/elastic/go-structform v0.0.10/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4= github.com/elastic/go-sysinfo v1.13.1 h1:U5Jlx6c/rLkR72O8wXXXo1abnGlWGJU/wbzNJ2AfQa4= @@ -772,8 +775,8 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= -github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= -github.com/frankban/quicktest v1.14.5/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -919,6 +922,8 @@ github.com/go-openapi/validate v0.20.1/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE github.com/go-openapi/validate v0.20.2/go.mod h1:e7OJoKNgd0twXZwIn0A43tHbvIcr/rZIVCbJBpTUoY0= github.com/go-pdf/fpdf v0.5.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= github.com/go-pdf/fpdf v0.6.0/go.mod h1:HzcnA+A23uwogo0tp9yU+l3V+KXhiESpt1PMayhOh5M= +github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI= +github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= @@ -993,6 +998,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= @@ -1639,7 +1645,6 @@ github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqggOuDwUaBQpluzLNDqif3kq9z2wpdYEfQ= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1732,8 +1737,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rootless-containers/rootlesskit v1.1.0 h1:cRaRIYxY8oce4eE/zeAUZhgKu/4tU1p9YHN4+suwV7M= github.com/rootless-containers/rootlesskit v1.1.0/go.mod h1:H+o9ndNe7tS91WqU0/+vpvc+VaCd7TCIWaJjnV0ujUo= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= diff --git a/libbeat/ebpf/sys/sys.go b/libbeat/ebpf/sys/sys.go new file mode 100644 index 000000000000..4156321a3214 --- /dev/null +++ b/libbeat/ebpf/sys/sys.go @@ -0,0 +1,68 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "sync" + "time" + + "github.com/elastic/go-sysinfo" + "github.com/elastic/go-sysinfo/types" +) + +var ( + hostInfoOnce = sync.OnceValues(func() (types.HostInfo, error) { + host, err := sysinfo.Host() + + if host == nil { + return types.HostInfo{}, err + } + + return host.Info(), err + }) +) + +// EntityID creates an ID that uniquely identifies this process across machines. +func EntityID(pid uint32, start time.Time) (string, error) { + info, err := hostInfoOnce() + if err != nil { + return "", err + } + + h := sha256.New() + if _, err := h.Write([]byte(info.UniqueID)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(pid)); err != nil { + return "", err + } + if err := binary.Write(h, binary.LittleEndian, int64(start.Nanosecond())); err != nil { + return "", err + } + + sum := h.Sum(nil) + if len(sum) > 12 { + sum = sum[:12] + } + return base64.RawStdEncoding.EncodeToString(sum), nil +} diff --git a/libbeat/ebpf/sys/time.go b/libbeat/ebpf/sys/time.go new file mode 100644 index 000000000000..7dca6454c328 --- /dev/null +++ b/libbeat/ebpf/sys/time.go @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux + +package sys + +import ( + "sync" + "time" + + "github.com/tklauser/go-sysconf" +) + +var ( + ticksPerSecondOnce = sync.OnceValues(func() (uint64, error) { + ticks, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) + if err != nil { + return 0, err + } + + return uint64(ticks), err + }) +) + +func TicksToNs(ticks uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ticks * uint64(time.Second.Nanoseconds()) / tps, nil +} + +func TimeFromNsSinceBoot(ns uint64) (time.Time, error) { + info, err := hostInfoOnce() + if err != nil { + return time.Time{}, err + } + + reduced, err := reduceTimestampPrecision(ns) + if err != nil { + return time.Time{}, err + } + + return info.BootTime.Add(time.Duration(reduced)), nil +} + +// When generating an `entity_id` in ECS we need to reduce the precision of a +// process's start time to that of procfs. Process start times can come from either +// eBPF (high precision) or other sources. We must reduce them all to the +// lowest common denominator such that entity ID's generated are always consistent. +func reduceTimestampPrecision(ns uint64) (uint64, error) { + tps, err := ticksPerSecondOnce() + if err != nil { + return 0, err + } + + return ns - (ns % (uint64(time.Second.Nanoseconds()) / tps)), nil +} diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 8aeef2c623e7..936bbea8ca4a 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -49,22 +49,28 @@ var ( type Client struct { conn eslegclient.Connection - index outputs.IndexSelector - pipeline *outil.Selector + indexSelector outputs.IndexSelector + pipelineSelector *outil.Selector - observer outputs.Observer - NonIndexableAction string + observer outputs.Observer + + // If deadLetterIndex is set, events with bulk-ingest errors will be + // forwarded to this index. Otherwise, they will be dropped. + deadLetterIndex string log *logp.Logger } -// ClientSettings contains the settings for a client. -type ClientSettings struct { - eslegclient.ConnectionSettings - Index outputs.IndexSelector - Pipeline *outil.Selector - Observer outputs.Observer - NonIndexableAction string +// clientSettings contains the settings for a client. +type clientSettings struct { + connection eslegclient.ConnectionSettings + indexSelector outputs.IndexSelector + pipelineSelector *outil.Selector + observer outputs.Observer + + // If deadLetterIndex is set, events with bulk-ingest errors will be + // forwarded to this index. Otherwise, they will be dropped. + deadLetterIndex string } type bulkResultStats struct { @@ -81,29 +87,15 @@ const ( // NewClient instantiates a new client. func NewClient( - s ClientSettings, + s clientSettings, onConnect *callbacksRegistry, ) (*Client, error) { - pipeline := s.Pipeline + pipeline := s.pipelineSelector if pipeline != nil && pipeline.IsEmpty() { pipeline = nil } - conn, err := eslegclient.NewConnection(eslegclient.ConnectionSettings{ - URL: s.URL, - Beatname: s.Beatname, - Username: s.Username, - Password: s.Password, - APIKey: s.APIKey, - Headers: s.Headers, - Kerberos: s.Kerberos, - Observer: s.Observer, - Parameters: s.Parameters, - CompressionLevel: s.CompressionLevel, - EscapeHTML: s.EscapeHTML, - Transport: s.Transport, - IdleConnTimeout: s.IdleConnTimeout, - }) + conn, err := eslegclient.NewConnection(s.connection) if err != nil { return nil, err } @@ -134,11 +126,11 @@ func NewClient( } client := &Client{ - conn: *conn, - index: s.Index, - pipeline: pipeline, - observer: s.Observer, - NonIndexableAction: s.NonIndexableAction, + conn: *conn, + indexSelector: s.indexSelector, + pipelineSelector: pipeline, + observer: s.observer, + deadLetterIndex: s.deadLetterIndex, log: logp.NewLogger("elasticsearch"), } @@ -174,11 +166,11 @@ func (client *Client) Clone() *Client { client.conn.Transport.Proxy.Disable = client.conn.Transport.Proxy.URL == nil c, _ := NewClient( - ClientSettings{ - ConnectionSettings: connection, - Index: client.index, - Pipeline: client.pipeline, - NonIndexableAction: client.NonIndexableAction, + clientSettings{ + connection: connection, + indexSelector: client.indexSelector, + pipelineSelector: client.pipelineSelector, + deadLetterIndex: client.deadLetterIndex, }, nil, // XXX: do not pass connection callback? ) @@ -296,10 +288,7 @@ func (client *Client) publishEvents(ctx context.Context, data []publisher.Event) } if failed > 0 { - if sendErr == nil { - sendErr = eslegclient.ErrTempBulkFailure - } - return failedEvents, sendErr + return failedEvents, eslegclient.ErrTempBulkFailure } return nil, nil } @@ -339,7 +328,7 @@ func (client *Client) createEventBulkMeta(version version.V, event *beat.Event) return nil, err } - index, err := client.index.Select(event) + index, err := client.getIndex(event) if err != nil { err := fmt.Errorf("failed to select event index: %w", err) return nil, err @@ -371,6 +360,16 @@ func (client *Client) createEventBulkMeta(version version.V, event *beat.Event) return eslegclient.BulkIndexAction{Index: meta}, nil } +func (client *Client) getIndex(event *beat.Event) (string, error) { + // If this event has been dead-lettered, override its index + if event.Meta != nil { + if deadLetter, _ := event.Meta.HasKey(dead_letter_marker_field); deadLetter { + return client.deadLetterIndex, nil + } + } + return client.indexSelector.Select(event) +} + func (client *Client) getPipeline(event *beat.Event) (string, error) { if event.Meta != nil { pipeline, err := events.GetMetaStringValue(*event, events.FieldMetaPipeline) @@ -384,8 +383,8 @@ func (client *Client) getPipeline(event *beat.Event) (string, error) { return strings.ToLower(pipeline), nil } - if client.pipeline != nil { - return client.pipeline.Select(event) + if client.pipelineSelector != nil { + return client.pipelineSelector.Select(event) } return "", nil } @@ -434,7 +433,7 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat client.log.Errorf("Can't deliver to dead letter index event (status=%v). Enable debug logs to view the event and cause.", status) client.log.Debugf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg) // poison pill - this will clog the pipeline if the underlying failure is non transient. - } else if client.NonIndexableAction == dead_letter_index { + } else if client.deadLetterIndex != "" { client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Enable debug logs to view the event and cause.", status) client.log.Debugf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg) if data[i].Content.Meta == nil { diff --git a/libbeat/outputs/elasticsearch/client_proxy_test.go b/libbeat/outputs/elasticsearch/client_proxy_test.go index e3fd914bbe7d..c2f23f34052a 100644 --- a/libbeat/outputs/elasticsearch/client_proxy_test.go +++ b/libbeat/outputs/elasticsearch/client_proxy_test.go @@ -165,7 +165,9 @@ func TestProxyDisableOverridesProxySettings(t *testing.T) { func execClient(t *testing.T, env ...string) { // The child process always runs only the TestClientPing test, which pings // the server at TEST_SERVER_URL and then terminates. - cmd := exec.Command(os.Args[0], "-test.run=TestClientPing") + executable, err := os.Executable() + require.NoError(t, err, "couldn't get current executable") + cmd := exec.Command(executable, "-test.run=TestClientPing") cmd.Env = append(append(os.Environ(), "TEST_START_CLIENT=1"), env...) @@ -173,7 +175,7 @@ func execClient(t *testing.T, env ...string) { cmd.Stderr = cmdOutput cmd.Stdout = cmdOutput - err := cmd.Run() + err = cmd.Run() if err != nil { t.Error("Error executing client:\n" + cmdOutput.String()) } @@ -185,8 +187,8 @@ func doClientPing(t *testing.T) { proxy := os.Getenv("TEST_PROXY_URL") // if TEST_PROXY_DISABLE is nonempty, set ClientSettings.ProxyDisable. proxyDisable := os.Getenv("TEST_PROXY_DISABLE") - clientSettings := ClientSettings{ - ConnectionSettings: eslegclient.ConnectionSettings{ + clientSettings := clientSettings{ + connection: eslegclient.ConnectionSettings{ URL: serverURL, Headers: map[string]string{headerTestField: headerTestValue}, Transport: httpcommon.HTTPTransportSettings{ @@ -195,14 +197,14 @@ func doClientPing(t *testing.T) { }, }, }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), + indexSelector: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), } if proxy != "" { u, err := url.Parse(proxy) require.NoError(t, err) proxyURL := httpcommon.ProxyURI(*u) - clientSettings.Transport.Proxy.URL = &proxyURL + clientSettings.connection.Transport.Proxy.URL = &proxyURL } client, err := NewClient(clientSettings, nil) require.NoError(t, err) @@ -210,7 +212,7 @@ func doClientPing(t *testing.T) { // This ping won't succeed; we aren't testing end-to-end communication // (which would require a lot more setup work), we just want to make sure // the client is pointed at the right server or proxy. - client.Connect() + _ = client.Connect() } // serverState contains the state of the http listeners for proxy tests, diff --git a/libbeat/outputs/elasticsearch/client_test.go b/libbeat/outputs/elasticsearch/client_test.go index 58e5f3ee5e20..12493e28d028 100644 --- a/libbeat/outputs/elasticsearch/client_test.go +++ b/libbeat/outputs/elasticsearch/client_test.go @@ -90,10 +90,10 @@ func (bm *batchMock) RetryEvents(events []publisher.Event) { func TestPublish(t *testing.T) { makePublishTestClient := func(t *testing.T, url string) *Client { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - ConnectionSettings: eslegclient.ConnectionSettings{URL: url}, - Index: testIndexSelector{}, + clientSettings{ + observer: outputs.NewNilObserver(), + connection: eslegclient.ConnectionSettings{URL: url}, + indexSelector: testIndexSelector{}, }, nil, ) @@ -248,9 +248,8 @@ func TestPublish(t *testing.T) { func TestCollectPublishFailsNone(t *testing.T) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -272,9 +271,8 @@ func TestCollectPublishFailsNone(t *testing.T) { func TestCollectPublishFailMiddle(t *testing.T) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -302,9 +300,9 @@ func TestCollectPublishFailMiddle(t *testing.T) { func TestCollectPublishFailDeadLetterQueue(t *testing.T) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "dead_letter_index", + clientSettings{ + observer: outputs.NewNilObserver(), + deadLetterIndex: "test_index", }, nil, ) @@ -361,9 +359,9 @@ func TestCollectPublishFailDeadLetterQueue(t *testing.T) { func TestCollectPublishFailDrop(t *testing.T) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), + deadLetterIndex: "", }, nil, ) @@ -405,9 +403,8 @@ func TestCollectPublishFailDrop(t *testing.T) { func TestCollectPublishFailAll(t *testing.T) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -434,9 +431,8 @@ func TestCollectPipelinePublishFail(t *testing.T) { logp.TestingSetup(logp.WithSelectors("elasticsearch")) client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -481,9 +477,9 @@ func TestCollectPipelinePublishFail(t *testing.T) { func BenchmarkCollectPublishFailsNone(b *testing.B) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), + deadLetterIndex: "", }, nil, ) @@ -510,9 +506,8 @@ func BenchmarkCollectPublishFailsNone(b *testing.B) { func BenchmarkCollectPublishFailMiddle(b *testing.B) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -540,9 +535,8 @@ func BenchmarkCollectPublishFailMiddle(b *testing.B) { func BenchmarkCollectPublishFailAll(b *testing.B) { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - NonIndexableAction: "drop", + clientSettings{ + observer: outputs.NewNilObserver(), }, nil, ) @@ -589,16 +583,16 @@ func TestClientWithHeaders(t *testing.T) { })) defer ts.Close() - client, err := NewClient(ClientSettings{ - Observer: outputs.NewNilObserver(), - ConnectionSettings: eslegclient.ConnectionSettings{ + client, err := NewClient(clientSettings{ + observer: outputs.NewNilObserver(), + connection: eslegclient.ConnectionSettings{ URL: ts.URL, Headers: map[string]string{ "host": "myhost.local", "X-Test": "testing value", }, }, - Index: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), + indexSelector: outil.MakeSelector(outil.ConstSelectorExpr("test", outil.SelectorLowerCase)), }, nil) assert.NoError(t, err) @@ -667,10 +661,10 @@ func TestBulkEncodeEvents(t *testing.T) { } client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - Index: index, - Pipeline: pipeline, + clientSettings{ + observer: outputs.NewNilObserver(), + indexSelector: index, + pipelineSelector: pipeline, }, nil, ) @@ -743,10 +737,10 @@ func TestBulkEncodeEventsWithOpType(t *testing.T) { } client, _ := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - Index: index, - Pipeline: pipeline, + clientSettings{ + observer: outputs.NewNilObserver(), + indexSelector: index, + pipelineSelector: pipeline, }, nil, ) @@ -786,9 +780,9 @@ func TestClientWithAPIKey(t *testing.T) { })) defer ts.Close() - client, err := NewClient(ClientSettings{ - Observer: outputs.NewNilObserver(), - ConnectionSettings: eslegclient.ConnectionSettings{ + client, err := NewClient(clientSettings{ + observer: outputs.NewNilObserver(), + connection: eslegclient.ConnectionSettings{ URL: ts.URL, APIKey: "hyokHG4BfWk5viKZ172X:o45JUkyuS--yiSAuuxl8Uw", }, @@ -806,13 +800,13 @@ func TestClientWithAPIKey(t *testing.T) { func TestPublishEventsWithBulkFiltering(t *testing.T) { makePublishTestClient := func(t *testing.T, url string, configParams map[string]string) *Client { client, err := NewClient( - ClientSettings{ - Observer: outputs.NewNilObserver(), - ConnectionSettings: eslegclient.ConnectionSettings{ + clientSettings{ + observer: outputs.NewNilObserver(), + connection: eslegclient.ConnectionSettings{ URL: url, Parameters: configParams, }, - Index: testIndexSelector{}, + indexSelector: testIndexSelector{}, }, nil, ) @@ -926,3 +920,23 @@ func TestPublishEventsWithBulkFiltering(t *testing.T) { require.Equal(t, len(recParams), 1) }) } + +func TestGetIndex(t *testing.T) { + dead_letter_index := "dead_index" + client := &Client{ + deadLetterIndex: dead_letter_index, + indexSelector: testIndexSelector{}, + } + + event := &beat.Event{ + Meta: make(map[string]interface{}), + } + index, err := client.getIndex(event) + require.NoError(t, err, "getIndex call must succeed") + assert.Equal(t, "test", index, "Event with no dead letter marker should use the client's index selector") + + event.Meta[dead_letter_marker_field] = true + index, err = client.getIndex(event) + require.NoError(t, err, "getIndex call must succeed") + assert.Equal(t, dead_letter_index, index, "Event with dead letter marker should use the client's dead letter index") +} diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go index 32cb90c904c4..c69cf229856a 100644 --- a/libbeat/outputs/elasticsearch/config_test.go +++ b/libbeat/outputs/elasticsearch/config_test.go @@ -34,11 +34,11 @@ non_indexable_policy.drop: ~ if err != nil { t.Fatalf("Can't create test configuration from valid input") } - policy, err := newNonIndexablePolicy(elasticsearchOutputConfig.NonIndexablePolicy) + index, err := deadLetterIndexForPolicy(elasticsearchOutputConfig.NonIndexablePolicy) if err != nil { - t.Fatalf("Can't create test configuration from valid input") + t.Fatalf("Can't read non-indexable policy: %v", err.Error()) } - assert.Equal(t, drop, policy.action(), "action should be drop") + assert.Equal(t, "", index, "dead letter index should be empty string") } func TestDeadLetterIndexPolicyConfig(t *testing.T) { @@ -51,11 +51,11 @@ non_indexable_policy.dead_letter_index: if err != nil { t.Fatalf("Can't create test configuration from valid input") } - policy, err := newNonIndexablePolicy(elasticsearchOutputConfig.NonIndexablePolicy) + index, err := deadLetterIndexForPolicy(elasticsearchOutputConfig.NonIndexablePolicy) if err != nil { - t.Fatalf("Can't create test configuration from valid input") + t.Fatalf("Can't read non-indexable policy: %v", err.Error()) } - assert.Equal(t, "my-dead-letter-index", policy.index(), "index should match config") + assert.Equal(t, "my-dead-letter-index", index, "index should match config") } func TestInvalidNonIndexablePolicyConfig(t *testing.T) { @@ -88,9 +88,10 @@ non_indexable_policy.dead_letter_index: if err != nil { t.Fatalf("Can't create test configuration from valid input") } - _, err = newNonIndexablePolicy(elasticsearchOutputConfig.NonIndexablePolicy) + + _, err = deadLetterIndexForPolicy(elasticsearchOutputConfig.NonIndexablePolicy) if err == nil { - t.Fatalf("Can create test configuration from invalid input") + t.Fatalf("Invalid non-indexable policy config should produce an error") } t.Logf("error %s", err.Error()) }) diff --git a/libbeat/outputs/elasticsearch/dead_letter_index.go b/libbeat/outputs/elasticsearch/dead_letter_index.go new file mode 100644 index 000000000000..d27882056919 --- /dev/null +++ b/libbeat/outputs/elasticsearch/dead_letter_index.go @@ -0,0 +1,56 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/elastic-agent-libs/config" +) + +const ( + dead_letter_marker_field = "deadlettered" + drop = "drop" + dead_letter_index = "dead_letter_index" +) + +func deadLetterIndexForConfig(config *config.C) (string, error) { + var indexConfig struct { + Index string + } + err := config.Unpack(&indexConfig) + if err != nil { + return "", err + } + if indexConfig.Index == "" { + return "", fmt.Errorf("%s policy requires an `index` to be specified", dead_letter_index) + } + return indexConfig.Index, nil +} + +func deadLetterIndexForPolicy(configNamespace *config.Namespace) (string, error) { + if configNamespace == nil || configNamespace.Name() == drop { + return "", nil + } + if configNamespace.Name() == dead_letter_index { + cfgwarn.Beta("The non_indexable_policy dead_letter_index is beta.") + return deadLetterIndexForConfig(configNamespace.Config()) + } + return "", fmt.Errorf("no such policy type: %s", configNamespace.Name()) +} diff --git a/libbeat/outputs/elasticsearch/death_letter_selector.go b/libbeat/outputs/elasticsearch/death_letter_selector.go deleted file mode 100644 index 02bd3780cab7..000000000000 --- a/libbeat/outputs/elasticsearch/death_letter_selector.go +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package elasticsearch - -import ( - "github.com/elastic/beats/v7/libbeat/beat" - "github.com/elastic/beats/v7/libbeat/outputs" -) - -type DeadLetterSelector struct { - Selector outputs.IndexSelector - DeadLetterIndex string -} - -func (d DeadLetterSelector) Select(event *beat.Event) (string, error) { - result, _ := event.Meta.HasKey(dead_letter_marker_field) - if result { - return d.DeadLetterIndex, nil - } - return d.Selector.Select(event) -} diff --git a/libbeat/outputs/elasticsearch/elasticsearch.go b/libbeat/outputs/elasticsearch/elasticsearch.go index 649168eb11b4..616e08e08f67 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch.go +++ b/libbeat/outputs/elasticsearch/elasticsearch.go @@ -73,9 +73,9 @@ func makeES( return outputs.Fail(err) } - policy, err := newNonIndexablePolicy(esConfig.NonIndexablePolicy) + deadLetterIndex, err := deadLetterIndexForPolicy(esConfig.NonIndexablePolicy) if err != nil { - log.Errorf("error while creating file identifier: %v", err) + log.Errorf("error in non_indexable_policy: %v", err) return outputs.Fail(err) } @@ -94,13 +94,6 @@ func makeES( params = nil } - if policy.action() == dead_letter_index { - index = DeadLetterSelector{ - Selector: index, - DeadLetterIndex: policy.index(), - } - } - clients := make([]outputs.NetworkClient, len(hosts)) for i, host := range hosts { esURL, err := common.MakeURL(esConfig.Protocol, esConfig.Path, host, 9200) @@ -110,8 +103,8 @@ func makeES( } var client outputs.NetworkClient - client, err = NewClient(ClientSettings{ - ConnectionSettings: eslegclient.ConnectionSettings{ + client, err = NewClient(clientSettings{ + connection: eslegclient.ConnectionSettings{ URL: esURL, Beatname: beat.Beat, Kerberos: esConfig.Kerberos, @@ -126,10 +119,10 @@ func makeES( Transport: esConfig.Transport, IdleConnTimeout: esConfig.Transport.IdleConnTimeout, }, - Index: index, - Pipeline: pipeline, - Observer: observer, - NonIndexableAction: policy.action(), + indexSelector: index, + pipelineSelector: pipeline, + observer: observer, + deadLetterIndex: deadLetterIndex, }, &connectCallbackRegistry) if err != nil { return outputs.Fail(err) @@ -144,7 +137,7 @@ func makeES( func buildSelectors( im outputs.IndexManager, - beat beat.Info, + _ beat.Info, cfg *config.C, ) (index outputs.IndexSelector, pipeline *outil.Selector, err error) { index, err = im.BuildSelector(cfg) diff --git a/libbeat/outputs/elasticsearch/elasticsearch_test.go b/libbeat/outputs/elasticsearch/elasticsearch_test.go index 45db313d903c..7950bdd63234 100644 --- a/libbeat/outputs/elasticsearch/elasticsearch_test.go +++ b/libbeat/outputs/elasticsearch/elasticsearch_test.go @@ -18,7 +18,6 @@ package elasticsearch import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -30,9 +29,9 @@ import ( ) func TestConnectCallbacksManagement(t *testing.T) { - f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { return nil } + f1 := func(client *eslegclient.Connection) error { return nil } + f2 := func(client *eslegclient.Connection) error { return nil } _, err := RegisterConnectCallback(f0) if err != nil { @@ -55,9 +54,9 @@ func TestConnectCallbacksManagement(t *testing.T) { } func TestGlobalConnectCallbacksManagement(t *testing.T) { - f0 := func(client *eslegclient.Connection) error { fmt.Println("i am function #0"); return nil } - f1 := func(client *eslegclient.Connection) error { fmt.Println("i am function #1"); return nil } - f2 := func(client *eslegclient.Connection) error { fmt.Println("i am function #2"); return nil } + f0 := func(client *eslegclient.Connection) error { return nil } + f1 := func(client *eslegclient.Connection) error { return nil } + f2 := func(client *eslegclient.Connection) error { return nil } _, err := RegisterGlobalCallback(f0) if err != nil { @@ -116,13 +115,16 @@ func TestPipelineSelection(t *testing.T) { }, } - for name, test := range cases { + for name, _test := range cases { + // de-alias loop variable + test := _test t.Run(name, func(t *testing.T) { selector, err := buildPipelineSelector(config.MustNewConfigFrom(test.cfg)) + assert.NoError(t, err) client, err := NewClient( - ClientSettings{ - Pipeline: &selector, + clientSettings{ + pipelineSelector: &selector, }, nil, ) diff --git a/libbeat/outputs/elasticsearch/non_indexable_policy.go b/libbeat/outputs/elasticsearch/non_indexable_policy.go deleted file mode 100644 index 5b588efb93fb..000000000000 --- a/libbeat/outputs/elasticsearch/non_indexable_policy.go +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package elasticsearch - -import ( - "fmt" - - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/elastic-agent-libs/config" -) - -const ( - dead_letter_marker_field = "deadlettered" - drop = "drop" - dead_letter_index = "dead_letter_index" -) - -type DropPolicy struct{} - -func (d DropPolicy) action() string { - return drop -} - -func (d DropPolicy) index() string { - panic("drop policy doesn't have an target index") -} - -type DeadLetterIndexPolicy struct { - Index string -} - -func (d DeadLetterIndexPolicy) action() string { - return dead_letter_index -} - -func (d DeadLetterIndexPolicy) index() string { - return d.Index -} - -type nonIndexablePolicy interface { - action() string - index() string -} - -var ( - policyFactories = map[string]policyFactory{ - drop: newDropPolicy, - dead_letter_index: newDeadLetterIndexPolicy, - } -) - -func newDeadLetterIndexPolicy(config *config.C) (nonIndexablePolicy, error) { - cfgwarn.Beta("The non_indexable_policy dead_letter_index is beta.") - policy := DeadLetterIndexPolicy{} - err := config.Unpack(&policy) - if policy.index() == "" { - return nil, fmt.Errorf("%s policy requires an `index` to be specified specified", dead_letter_index) - } - return policy, err -} - -func newDropPolicy(*config.C) (nonIndexablePolicy, error) { - return defaultDropPolicy(), nil -} - -func defaultPolicy() nonIndexablePolicy { - return defaultDropPolicy() -} - -func defaultDropPolicy() nonIndexablePolicy { - return &DropPolicy{} -} - -type policyFactory func(config *config.C) (nonIndexablePolicy, error) - -func newNonIndexablePolicy(configNamespace *config.Namespace) (nonIndexablePolicy, error) { - if configNamespace == nil { - return defaultPolicy(), nil - } - - policyType := configNamespace.Name() - factory, ok := policyFactories[policyType] - if !ok { - return nil, fmt.Errorf("no such policy type: %s", policyType) - } - - return factory(configNamespace.Config()) -} diff --git a/libbeat/outputs/shipper/shipper.go b/libbeat/outputs/shipper/shipper.go index fe19a36b31d2..83955a80f4ce 100644 --- a/libbeat/outputs/shipper/shipper.go +++ b/libbeat/outputs/shipper/shipper.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/outputs" "github.com/elastic/beats/v7/libbeat/publisher" - proxyqueue "github.com/elastic/beats/v7/libbeat/publisher/queue/proxy" + "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" "github.com/elastic/elastic-agent-shipper-client/pkg/helpers" sc "github.com/elastic/elastic-agent-shipper-client/pkg/proto" @@ -111,8 +111,12 @@ func makeShipper( return outputs.Group{ Clients: []outputs.Client{swb}, Retry: config.MaxRetries, - QueueFactory: proxyqueue.FactoryForSettings( - proxyqueue.Settings{BatchSize: config.BulkMaxSize}), + QueueFactory: memqueue.FactoryForSettings( + memqueue.Settings{ + Events: config.BulkMaxSize * 2, + MaxGetRequest: config.BulkMaxSize, + FlushTimeout: 0, + }), }, nil } diff --git a/libbeat/publisher/queue/proxy/README.md b/libbeat/publisher/queue/proxy/README.md deleted file mode 100644 index 0ff611fb2389..000000000000 --- a/libbeat/publisher/queue/proxy/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Beats Proxy Queue - -The proxy queue is an implementation of the [beats Queue interface](https://github.com/elastic/beats/blob/main/libbeat/publisher/queue/queue.go) meant to work with the Shipper output. The Shipper output is unique because rather than sending events to a remote server it sends them to the Shipper, a local process that has its own queue where it stages events for delivery to their true destination upstream. This means that if the Shipper output is used with a conventional queue, events will remain queued in both Beats _and_ the shipper until they receive upstream acknowledgment, potentially doubling the memory needed for a given set of events. - -The solution to this is the proxy queue: from the perspective of the Beats pipeline, it behaves like a normal (albeit small) queue, but its buffer is immediately cleared on being read, and it provides a hook in its event batches for the output to free its contents once sent, while still preserving metadata so that inputs that require end-to-end acknowledgment of their events can receive the acknowledgments later, after the Shipper confirms upstream ingestion. - -## Limitations - -Some features present in other queues are unimplemented or ignored by the proxy queue since they are unneeded when ingesting via the Shipper output: - -- `queue.EntryID`: a `Publish` call to a normal queue returns an `EntryID`, a unique integer that is incremented with each event. This data is only used internally in the Shipper to track acknowledgments, and is unused by Beats. -- Producer cancel: When a `queue.Producer` (the API interface for adding data to a queue) is cancelled, the memory queue attempts to remove from its buffer any events sent by that producer that have not yet been consumed. This feature is only ever used during Beats shutdown, and since the proxy queue in particular never accumulates events itself but instead stores them in the Shipper's queue, it has no mechanism to cancel most outstanding events. -- Requested batch size: The queue interface reads event batches by specifying the desired number of events, which the queue will try to satisfy. Because batches from the proxy queue are being sent to a local process rather than over a network, there is less performance sensitivity to the batch size. Because the proxy queue optimizes its buffer by using it to directly store the batch contents, we can get simpler and more consistent performance by accumulating up to a maximum size and then sending that immediately when a batch is requested. Therefore the proxy queue has its own configurable target batch size, and ignores the parameter given by the consumer. -- Metrics: The proxy queue implements the usual queue metrics for the Beats pipeline, however it doesn't implement the `Metrics()` call, as that is only used by the Shipper (and its contents would be mostly meaningless in the proxy case since events are not allowed to accumulate). - -## Implementation - -The proxy queue is loosely based on the implementation of the memory queue, but with many simplifications enabled by its more limited scope. It has three control channels, `getChan`, `pushChan`, and `doneChan`, all unbuffered. Its internal state can only be changed by sending requests to those channels (or closing the channel in the case of `doneChan`), or by closing the done channel on batches it has returned. - -### The pipeline - -Here is the event data flow through the proxy queue, in the context of the Beats pipeline: - -![The proxy queue in context](diagrams/broker.svg) - -An input adds an event to the proxy queue by creating a `queue.Producer` via the queue's API and calling its `Publish` function. If the producer was created with an acknowledgment callback, then a pointer to the producer will be included in its event metadata so later stages of the pipeline can notify it when ingestion is complete. - -The producer passes an incoming event on to the queue by sending a `pushRequest` to the queue's `pushChan`. The request includes the event, the producer (if acknowledgment is required), a channel on which to receive the response (boolean indicating success or failure), and a flag indicating whether a full queue should block the request until there is room or return immediately with failure. `pushChan` is unbuffered, and any request sent through it is guaranteed to receive a response. If the request's `canBlock` flag is false, that response is guaranteed not to block. If `canBlock` is true, the response is guaranteed to be success unless the queue has been closed. - -On the other side of the queue, a worker routine (`queueReader`) requests batches from the queue via its `Get` function, which sends a `getRequest` to the queue's `getChan`. A `getRequest` always blocks until there is data to read or until the queue is closed; as with `pushRequest`, once it is accepted it always returns a response. If the request is successful, the response will be a `proxyqueue.batch` (implementing the `queue.Batch` interface). The `queueReader`'s job is to collect batches from the queue and wrap them in a `publisher.Batch` interface (concrete type `ttlBatch`) that tracks retry metadata used in the final stages of the pipeline. - -The wrapped batches generated by the `queueReader` are received by the `eventConsumer`, which is the worker that distributes pipeline batches among the output workers via their shared input channel, and handles retries for output workers that encounter errors. - -Only an output worker can complete the life cycle of a batch. In the proxy queue this happens in two stages: when the batch is successfully sent to the Shipper, its `FreeEntries` function is called, which clears the internal reference to the events -- once these are sent, they are no longer needed since they are already enqueued in the Shipper. Then, when the Shipper confirms (via its `PersistedIndex` API, see the Shipper repository for details) that all events from the batch have been processed, the batch's `Done` function is called, which closes the batch's internal channel, `doneChan`. - -Finally, the queue's broker routine monitors the `doneChan` of the oldest outstanding batch; when it is closed, the broker invokes the appropriate acknowledgment callbacks and advances to the next oldest batch. - -### Acknowledgment tracking - -As with other queues, acknowledgments of batches must be globally synchronized by the queue broker, since the pipeline API requires that acknowledgments are sent to producers in the same order the events were generated (out-of-order acknowledgments can cause data loss). The acknowledgments required by any one batch are stored within the batch itself (in the `producerACKs` helper object). The queue broker maintains an ordered linked list of all batches awaiting acknowledgment, and the `select` call in its main loop checks the oldest outstanding batch, calling the appropriate callbacks as it advances. - -### The broker loop - -All internal control logic is handled in the run loop `broker.run()` in `broker.go`. Its state is stored in these fields: - -```go - queuedEntries []queueEntry - blockedRequests blockedRequests - outstandingBatches batchList -``` - -- `queuedEntries` is a list of the events (and producers, if appropriate) currently stored by the queue. Its length is at most `batchSize`. -- `blockedRequests` is a linked list of pending `pushRequest`s from producers that could not be immediately handled because the queue was full. Each one contains a response channel, and the originating producer is listening on that channel waiting for space in the queue. When space is available, events in these requests will be added to `queuedEntries` and the result will be sent to their response channels. -- `outstandingBatches` is a linked list of batches that have been consumed from this queue but not yet acknowledged. It is in the same order as the batches were originally created, so the first entry in the list is always the oldest batch awaiting acknowledgment. - -The core loop calls `select` across up to four channels: - -- `putChan` accepts requests to add entries to the queue. If the queue is already full (`len(queuedEntries) == batchSize`), the request is either added to `blockedRequests` or returns with immediate failure (depending on the value of `canBlock`). Otherwise, the new entry is added to `queuedEntries` to be included in the next batch. -- `getChan` is enabled only if `queuedEntries` isn't empty (otherwise there would be nothing to return). In that case, a new batch is created with the contents of `queuedEntries`, and metadata for any required future acknowledgments is computed (so that acknowledgment data can persist after the events themselves are freed). -- `outstandingBatches.nextDoneChan()` returns the acknowledgment channel for the oldest outstanding batch; if a read on this channel goes through, it means the channel was closed and the batch has been acknowledged, so the producer and pipeline callbacks are invoked and we advance to the next outstanding batch. -- `doneChan` indicates closure of the queue. In this case we reject any remaining requests in `blockedRequests` and return. (We do not do anything with `outstandingBatches`, since batches that are still unacknowledged at this point should be considered dropped, so we do not want producers to believe they have sent successfully.) - -## Possible improvements - -The proxy queue is designed to minimize memory use while respecting the established API for the Beats pipeline. However, its inability to buffer incoming events means that raw latency may increase in some scenarios. If benchmarks show that the proxy queue is a CPU or latency bottleneck, there are some natural improvements that would likely yield significant improvements: - -- The proxy queue currently buffers at most one batch at a time. Buffering a small constant number of batches instead would potentially block the inputs less often, leading to steadier throughput. -- Unlike the other queues, the proxy queue handles acknowledgments on its main work loop. This may increase latency of control signals if it is given acknowledgment callbacks that perform significant work. In that case, we could add a standalone acknowledgment routine similar to the other queues, so slow acknowledgments do not delay the core control logic. \ No newline at end of file diff --git a/libbeat/publisher/queue/proxy/batch.go b/libbeat/publisher/queue/proxy/batch.go deleted file mode 100644 index 1747af527ecd..000000000000 --- a/libbeat/publisher/queue/proxy/batch.go +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package proxyqueue - -type batch struct { - entries []queueEntry - - // Original number of entries (persists even if entries are freed). - originalEntryCount int - - producerACKs []producerACKData - - // When a batch is acknowledged, doneChan is closed to tell - // the queue to call the appropriate producer and metrics callbacks. - doneChan chan struct{} - - // Batches are collected in linked lists to preserve the order of - // acknowledgments. This field should only be used by batchList. - next *batch -} - -type batchList struct { - first *batch - last *batch -} - -// producerACKData tracks the number of events that need to be acknowledged -// from a single batch targeting a single producer. -type producerACKData struct { - producer *producer - count int -} - -func (b *batch) Count() int { - return b.originalEntryCount -} - -func (b *batch) Entry(i int) interface{} { - return b.entries[i].event -} - -func (b *batch) FreeEntries() { - b.entries = nil -} - -func (b *batch) Done() { - close(b.doneChan) -} - -func acksForEntries(entries []queueEntry) []producerACKData { - results := []producerACKData{} - // We traverse the list back to front, so we can coalesce multiple events - // into a single entry in the ACK data. - for i := len(entries) - 1; i >= 0; i-- { - entry := entries[i] - if producer := entry.producer; producer != nil { - if producer.producedCount > producer.consumedCount { - results = append(results, producerACKData{ - producer: producer, - count: int(producer.producedCount - producer.consumedCount), - }) - producer.consumedCount = producer.producedCount - } - } - } - return results -} - -func (l *batchList) add(b *batch) { - b.next = nil // Should be unneeded but let's be cautious - if l.last != nil { - l.last.next = b - } else { - l.first = b - } - l.last = b -} - -func (l *batchList) remove() *batch { - result := l.first - if l.first != nil { - l.first = l.first.next - if l.first == nil { - l.last = nil - } - } - return result -} - -func (l *batchList) nextDoneChan() chan struct{} { - if l.first != nil { - return l.first.doneChan - } - return nil -} diff --git a/libbeat/publisher/queue/proxy/broker.go b/libbeat/publisher/queue/proxy/broker.go deleted file mode 100644 index 832739cc26d9..000000000000 --- a/libbeat/publisher/queue/proxy/broker.go +++ /dev/null @@ -1,286 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package proxyqueue - -import ( - "io" - "sync" - - "github.com/elastic/beats/v7/libbeat/publisher/queue" - "github.com/elastic/elastic-agent-libs/logp" -) - -type broker struct { - doneChan chan struct{} - - logger *logp.Logger - - // The maximum number of events in any pending batch - batchSize int - - /////////////////////////// - // api channels - - // Producers send queue entries to pushChan to add them to the next batch. - pushChan chan *pushRequest - - // Consumers send requests to getChan to read entries from the queue. - getChan chan getRequest - - // A callback that should be invoked when ACKs are processed. - // This is used to forward notifications back to the pipeline observer, - // which updates the beats registry if needed. This callback is included - // in batches created by the proxy queue, so they can invoke it when they - // receive a Done call. - ackCallback func(eventCount int) - - // Internal state for the broker's run loop. - queuedEntries []queueEntry - blockedRequests blockedRequests - outstandingBatches batchList - - // wait group for worker shutdown - wg sync.WaitGroup -} - -type Settings struct { - BatchSize int -} - -type queueEntry struct { - event interface{} - - // The producer that generated this event, or nil if this producer does - // not require ack callbacks. - producer *producer -} - -type blockedRequest struct { - next *blockedRequest - request *pushRequest -} - -// linked list helper to store an ordered list of blocked requests -type blockedRequests struct { - first *blockedRequest - last *blockedRequest -} - -const QueueType = "proxy" - -// FactoryForSettings is a simple wrapper around NewQueue so a concrete -// Settings object can be wrapped in a queue-agnostic interface for -// later use by the pipeline. -func FactoryForSettings(settings Settings) queue.QueueFactory { - return func( - logger *logp.Logger, - ackCallback func(eventCount int), - inputQueueSize int, - ) (queue.Queue, error) { - return NewQueue(logger, ackCallback, settings), nil - } -} - -// NewQueue creates a new broker based in-memory queue holding up to sz number of events. -// If waitOnClose is set to true, the broker will block on Close, until all internal -// workers handling incoming messages and ACKs have been shut down. -func NewQueue( - logger *logp.Logger, - ackCallback func(eventCount int), - settings Settings, -) *broker { - if logger == nil { - logger = logp.NewLogger("proxyqueue") - } - - b := &broker{ - doneChan: make(chan struct{}), - logger: logger, - batchSize: settings.BatchSize, - - // broker API channels - pushChan: make(chan *pushRequest), - getChan: make(chan getRequest), - - ackCallback: ackCallback, - } - - b.wg.Add(1) - go func() { - defer b.wg.Done() - b.run() - }() - - return b -} - -func (b *broker) Close() error { - close(b.doneChan) - b.wg.Wait() - return nil -} - -func (b *broker) QueueType() string { - return QueueType -} - -func (b *broker) BufferConfig() queue.BufferConfig { - return queue.BufferConfig{} -} - -func (b *broker) Producer(cfg queue.ProducerConfig) queue.Producer { - return newProducer(b, cfg.ACK) -} - -func (b *broker) Get(_ int) (queue.Batch, error) { - // The response channel needs a buffer size of 1 to guarantee that the - // broker routine will not block when sending the response. - responseChan := make(chan *batch, 1) - select { - case <-b.doneChan: - return nil, io.EOF - case b.getChan <- getRequest{responseChan: responseChan}: - } - - // if request has been sent, we are guaranteed a response - return <-responseChan, nil -} - -// Metrics returns an empty response because the proxy queue -// doesn't accumulate batches; for the real metadata, use either the -// Beats pipeline metrics, or the queue metrics in the shipper, which -// is where pending events are really queued when the proxy queue is -// in use. -func (b *broker) Metrics() (queue.Metrics, error) { - return queue.Metrics{}, nil -} - -func (b *broker) run() { - for { - var getChan chan getRequest - // Get requests are enabled if the current pending batch is nonempty. - if len(b.queuedEntries) > 0 { - getChan = b.getChan - } - - select { - case <-b.doneChan: - // The queue is closing, reject any requests that were blocked - // waiting for space in the queue. - blocked := b.blockedRequests - for req := blocked.next(); req != nil; req = blocked.next() { - req.responseChan <- false - } - return - - case req := <-b.pushChan: // producer pushing new event - b.handlePushRequest(req) - - case req := <-getChan: // consumer asking for next batch - b.handleGetRequest(req) - - case <-b.outstandingBatches.nextDoneChan(): - ackedBatch := b.outstandingBatches.remove() - // Notify any listening producers - for _, ack := range ackedBatch.producerACKs { - ack.producer.ackHandler(ack.count) - } - // Notify the pipeline's metrics reporter - //nolint:typecheck // this nil check is ok - if b.ackCallback != nil { - b.ackCallback(ackedBatch.originalEntryCount) - } - } - } -} - -func (b *broker) handlePushRequest(req *pushRequest) { - if len(b.queuedEntries) < b.batchSize { - b.queuedEntries = append(b.queuedEntries, - queueEntry{event: req.event, producer: req.producer}) - if req.producer != nil { - req.producer.producedCount++ - } - req.responseChan <- true - } else if req.canBlock { - // If there isn't room for the event, but the producer wants - // to block until there is, add it to the queue. - b.blockedRequests.add(req) - } else { - // The pending batch is full, the producer doesn't want to - // block, so return immediate failure. - req.responseChan <- false - } -} - -func (b *broker) handleGetRequest(req getRequest) { - acks := acksForEntries(b.queuedEntries) - - newBatch := &batch{ - entries: b.queuedEntries, - originalEntryCount: len(b.queuedEntries), - producerACKs: acks, - doneChan: make(chan struct{}), - } - b.outstandingBatches.add(newBatch) - req.responseChan <- newBatch - - // Unblock any pending requests we can fit into the new batch. - entries := []queueEntry{} - for len(entries) < b.batchSize { - req := b.blockedRequests.next() - if req == nil { - // No more blocked requests - break - } - - entries = append(entries, - queueEntry{event: req.event, producer: req.producer}) - if req.producer != nil { - req.producer.producedCount++ - } - req.responseChan <- true - } - - // Reset the pending entries - b.queuedEntries = entries -} - -// Adds a new request to the end of the current list. -func (b *blockedRequests) add(request *pushRequest) { - blockedReq := &blockedRequest{request: request} - if b.first == nil { - b.first = blockedReq - } else { - b.last.next = blockedReq - } - b.last = blockedReq -} - -// Removes the oldest request from the list and returns it. -func (b *blockedRequests) next() *pushRequest { - var result *pushRequest - if b.first != nil { - result = b.first.request - b.first = b.first.next - if b.first == nil { - b.last = nil - } - } - return result -} diff --git a/libbeat/publisher/queue/proxy/diagrams/broker.d2 b/libbeat/publisher/queue/proxy/diagrams/broker.d2 deleted file mode 100644 index 7b2f1cccb327..000000000000 --- a/libbeat/publisher/queue/proxy/diagrams/broker.d2 +++ /dev/null @@ -1,54 +0,0 @@ -# A diagram of the Beats pipeline and the proxy queue's interaction -# with it. -# To regenerate the image after changing this file, run: -# d2 broker.d2 broker.svg -# To live-edit this file with immediate regeneration of the diagram, run: -# d2 --watch broker.d2 broker.svg - -Input - -producer { - ackHandler -} - -queue: Proxy Queue (broker) { - pushChan - getChan -} - -Input -> producer: Publish -producer.ackHandler -> Input: Event acknowledgment -producer -> queue.pushChan: pushRequest -queue.pushChan -> producer: result - -queueReader { - explanation: |md - `queueReader` is a worker that reads raw batches (satisfying the - `queue.Batch` interface, in this case via `proxyqueue.batch`) - from the queue, wraps them in a `publisher.Batch` interface - (`ttlBatch`) to support pipeline operations like retry/error - handling, and hands them off to `eventConsumer` which - distributes them to output workers. - | -} - -queueReader -> queue.getChan: getRequest -queue.getChan -> queueReader: proxyqueue\.batch - -eventConsumer -> queueReader: queueReaderRequest -queueReader -> eventConsumer: ttlBatch - -eventConsumer { - explanation: |md - `eventConsumer` is a worker that distributes event batches to - (potentially) multiple output workers. When a batch fails, the - output sends it back to `eventConsumer` for redistribution. - | -} - -out1: Output Worker -out2: ... - -eventConsumer -> out1: Publish(ttlBatch) -eventConsumer -> out2 - diff --git a/libbeat/publisher/queue/proxy/diagrams/broker.svg b/libbeat/publisher/queue/proxy/diagrams/broker.svg deleted file mode 100644 index 9e89cf655505..000000000000 --- a/libbeat/publisher/queue/proxy/diagrams/broker.svg +++ /dev/null @@ -1,847 +0,0 @@ - -InputproducerProxy Queue (broker)queueReadereventConsumerOutput Worker...ackHandlerpushChangetChan

queueReader is a worker that reads raw batches (satisfying the
-queue.Batch interface, in this case via proxyqueue.batch)
-from the queue, wraps them in a publisher.Batch interface
-(ttlBatch) to support pipeline operations like retry/error
-handling, and hands them off to eventConsumer which
-distributes them to output workers.

-

eventConsumer is a worker that distributes event batches to
-(potentially) multiple output workers. When a batch fails, the
-output sends it back to eventConsumer for redistribution.

-
 PublishEvent acknowledgmentpushRequestresultgetRequestproxyqueue.batchqueueReaderRequestttlBatchPublish(ttlBatch) - - - - - - - - - - - diff --git a/libbeat/publisher/queue/proxy/internal_api.go b/libbeat/publisher/queue/proxy/internal_api.go deleted file mode 100644 index 8e7e972ac219..000000000000 --- a/libbeat/publisher/queue/proxy/internal_api.go +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package proxyqueue - -// producer -> broker API - -type pushRequest struct { - event interface{} - producer *producer - - // After receiving a request, the broker will respond on this channel - // with whether the new entry was accepted or not. - responseChan chan bool - - // If canBlock is true, then the broker will store this request until - // either the request can be accepted or the queue itself is closed. - // Otherwise it will immediately reject the requst if there is no - // space in the pending buffer. - canBlock bool -} - -// consumer -> broker API - -type getRequest struct { - responseChan chan *batch // channel to send response to -} diff --git a/libbeat/publisher/queue/proxy/produce.go b/libbeat/publisher/queue/proxy/produce.go deleted file mode 100644 index 87a01450e546..000000000000 --- a/libbeat/publisher/queue/proxy/produce.go +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package proxyqueue - -import ( - "github.com/elastic/beats/v7/libbeat/publisher/queue" -) - -type producer struct { - broker *broker - cancelled bool - // If ackHandler is nil then this producer does not listen to acks. - ackHandler func(count int) - - // producedCount and consumedCount are used to assemble batches and - // should only be accessed by the broker's main loop. - producedCount uint64 - consumedCount uint64 -} - -func newProducer(b *broker, ackHandler func(count int)) queue.Producer { - return &producer{ - broker: b, - ackHandler: ackHandler} -} - -func (p *producer) makePushRequest(event interface{}, canBlock bool) *pushRequest { - req := &pushRequest{ - event: event, - responseChan: make(chan bool, 1), - canBlock: canBlock, - } - if p.ackHandler != nil { - req.producer = p - } - return req -} - -func (p *producer) Publish(event interface{}) (queue.EntryID, bool) { - if p.cancelled { - return 0, false - } - return 0, p.publish(p.makePushRequest(event, true)) -} - -func (p *producer) TryPublish(event interface{}) (queue.EntryID, bool) { - if p.cancelled { - return 0, false - } - return 0, p.publish(p.makePushRequest(event, false)) -} - -func (p *producer) Cancel() int { - p.cancelled = true - return 0 -} - -func (p *producer) publish(req *pushRequest) bool { - select { - case p.broker.pushChan <- req: - return <-req.responseChan - case <-p.broker.doneChan: - // The queue is shutting down - return false - } -} diff --git a/libbeat/publisher/queue/proxy/queue_test.go b/libbeat/publisher/queue/proxy/queue_test.go deleted file mode 100644 index 437216e2d7a5..000000000000 --- a/libbeat/publisher/queue/proxy/queue_test.go +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package proxyqueue - -import ( - "fmt" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/elastic-agent-libs/logp" - - "github.com/elastic/beats/v7/libbeat/publisher/queue" -) - -// Because acknowledgments are partially asynchronous (acknowledging -// a batch notifies the queue, which then notifies the original producer -// callback), we can't make a fully deterministic test for ACK counts -// since in principle it depends on the scheduler. -// Nevertheless, in practice the latency should be very low. testACKListener -// is a helper object to track ACK state while allowing for timeouts when -// some propagation delay is unavoidable. -type testACKListener struct { - sync.Mutex - - ackedCount int - - // If not enough ACKs have been received yet, waitForTotalACKs sets - // waiting to true and listens on updateChan. - // If waiting is set when the ACK callback is called, then it sends - // on updateChan to wake up waitForTotalACKs. - waiting bool - updateChan chan struct{} -} - -func TestBasicEventFlow(t *testing.T) { - logger := logp.NewLogger("proxy-queue-tests") - - // Create a proxy queue where each batch is at most 2 events - testQueue := NewQueue(logger, nil, Settings{BatchSize: 2}) - defer testQueue.Close() - - listener := newTestACKListener() - producer := testQueue.Producer(queue.ProducerConfig{ - ACK: listener.ACK, - }) - // Try to publish 3 events, only the first two should succeed until we read a batch - _, success := producer.TryPublish(1) - assert.True(t, success) - _, success = producer.TryPublish(2) - assert.True(t, success) - _, success = producer.TryPublish(3) - assert.False(t, success, "Current batch should only fit two events") - - batch, err := testQueue.Get(0) - assert.NoError(t, err, "Should be able to read a batch") - assert.Equal(t, 0, listener.ackedCount, "No batches have been acked yet") - batch.Done() - assert.NoError(t, listener.waitForTotalACKs(2, time.Second)) - - // Make sure that reading an event unblocked the queue - _, success = producer.TryPublish(4) - assert.True(t, success, "Queue should accept incoming event") -} - -func TestBlockedProducers(t *testing.T) { - logger := logp.NewLogger("proxy-queue-tests") - - // Create a proxy queue where each batch is at most 2 events - testQueue := NewQueue(logger, nil, Settings{BatchSize: 2}) - defer testQueue.Close() - - listener := newTestACKListener() - - // Create many producer goroutines and send an event through each - // one. Only two events can be in the queue at any one time, so - // the rest of the producers will block until we read enough batches - // from the queue. - const PRODUCER_COUNT = 10 - for i := 0; i < PRODUCER_COUNT; i++ { - go func(producerID int) { - producer := testQueue.Producer(queue.ProducerConfig{ - ACK: listener.ACK, - }) - producer.Publish(producerID) - }(i) - } - - consumedEventCount := 0 - batches := []queue.Batch{} - // First, read all the events. We should be able to do this successfully - // even before any have been acknowledged. - for consumedEventCount < PRODUCER_COUNT { - batch, err := testQueue.Get(0) - assert.NoError(t, err) - consumedEventCount += batch.Count() - batches = append(batches, batch) - } - - assert.Equal(t, 0, listener.ackedCount, "No batches have been acked yet") - for _, batch := range batches { - batch.Done() - } - assert.NoError(t, listener.waitForTotalACKs(PRODUCER_COUNT, time.Second)) -} - -func TestOutOfOrderACK(t *testing.T) { - logger := logp.NewLogger("proxy-queue-tests") - - // Create a proxy queue where each batch is at most 2 events - testQueue := NewQueue(logger, nil, Settings{BatchSize: 2}) - defer testQueue.Close() - - listener := newTestACKListener() - producer := testQueue.Producer(queue.ProducerConfig{ - ACK: listener.ACK, - }) - - const BATCH_COUNT = 10 - batches := []queue.Batch{} - for i := 0; i < BATCH_COUNT; i++ { - // Publish two events - _, success := producer.Publish(0) - assert.True(t, success, "Publish should succeed") - _, success = producer.Publish(0) - assert.True(t, success, "Publish should succeed") - - // Consume a batch, which should contain the events we just published - batch, err := testQueue.Get(0) - assert.NoError(t, err) - batch.FreeEntries() - assert.Equal(t, 2, batch.Count()) - - batches = append(batches, batch) - } - - // Acknowledge all except the first batch - for _, batch := range batches[1:] { - batch.Done() - } - // Make sure that no ACKs come in even if we wait a bit - err := listener.waitForTotalACKs(1, 50*time.Millisecond) - assert.Error(t, err, "No ACK callbacks should have been called yet") - - // ACKing the first batch should unblock all the rest - batches[0].Done() - assert.NoError(t, listener.waitForTotalACKs(BATCH_COUNT*2, time.Second)) -} - -func TestWriteAfterClose(t *testing.T) { - logger := logp.NewLogger("proxy-queue-tests") - - testQueue := NewQueue(logger, nil, Settings{BatchSize: 2}) - producer := testQueue.Producer(queue.ProducerConfig{}) - testQueue.Close() - - // Make sure Publish fails instead of blocking - _, success := producer.Publish(1) - assert.False(t, success, "Publish should fail since queue is closed") -} - -func newTestACKListener() *testACKListener { - return &testACKListener{ - updateChan: make(chan struct{}, 1), - } -} - -// ACK should be provided to the queue producer. It can be safely called from -// multiple goroutines. -func (l *testACKListener) ACK(count int) { - l.Lock() - l.ackedCount += count - if l.waiting { - // If waitFortotalACKs is waiting on something, wake it up so it can retry. - l.waiting = false - l.updateChan <- struct{}{} - } - l.Unlock() -} - -// flush should be called on timeout, to clear updateChan if needed. -func (l *testACKListener) flush() { - l.Lock() - select { - case <-l.updateChan: - default: - } - l.waiting = false - l.Unlock() -} - -// waitForTotalACKs waits until the specified number of total ACKs have been -// received, or the timeout interval is exceeded. It should only be called -// from a single goroutine at once. -func (l *testACKListener) waitForTotalACKs(targetCount int, timeout time.Duration) error { - timeoutChan := time.After(timeout) - for { - l.Lock() - if l.ackedCount >= targetCount { - l.Unlock() - return nil - } - // Not enough ACKs have been sent yet, so we have to wait. - l.waiting = true - l.Unlock() - select { - case <-l.updateChan: - // New ACKs came in, retry - continue - case <-timeoutChan: - l.flush() - return fmt.Errorf("timed out waiting for acknowledgments: have %d, wanted %d", l.ackedCount, targetCount) - } - } -} diff --git a/libbeat/tests/system/requirements.txt b/libbeat/tests/system/requirements.txt index fc4227738c32..87133cda11dc 100644 --- a/libbeat/tests/system/requirements.txt +++ b/libbeat/tests/system/requirements.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/libbeat/tests/system/requirements_aix.txt b/libbeat/tests/system/requirements_aix.txt index fc4227738c32..87133cda11dc 100644 --- a/libbeat/tests/system/requirements_aix.txt +++ b/libbeat/tests/system/requirements_aix.txt @@ -1,3 +1,7 @@ +requests==2.31.0 +urllib3==1.26.18 +docker==6.1.3 +docker-compose @ git+https://github.com/pkoutsovasilis/compose@v1_fix async-timeout==4.0.3 attrs==19.3.0 autopep8==1.5.4 @@ -13,8 +17,6 @@ cryptography==42.0.4 deepdiff==4.2.0 Deprecated==1.2.14 distro==1.9.0 -docker==6.0.1 -docker-compose==1.29.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 @@ -58,7 +60,6 @@ pytest-timeout==1.4.2 python-dotenv==0.21.1 PyYAML==5.3.1 redis==4.4.4 -requests==2.31.0 semver==2.8.1 six==1.15.0 stomp.py==4.1.22 @@ -67,7 +68,6 @@ texttable==0.9.1 toml==0.10.1 tomli==2.0.1 typing_extensions==4.9.0 -urllib3==1.26.18 wcwidth==0.2.5 websocket-client==0.47.0 wrapt==1.16.0 diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index b6da04173167..a16b942032d8 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.21.8 +COPY --from=docker:26.0.0-alpine3.19 /usr/local/bin/docker /usr/local/bin/ RUN \ apt update \ @@ -10,6 +11,8 @@ RUN \ python3-venv \ libaio-dev \ unzip \ + libssl-dev \ + libffi-dev \ && rm -rf /var/lib/apt/lists/* # Use a virtualenv to avoid the PEP668 "externally managed environment" error caused by conflicts @@ -19,9 +22,12 @@ RUN python3 -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" RUN pip3 install --upgrade pip==20.1.1 -RUN pip3 install --upgrade docker-compose==1.23.2 RUN pip3 install --upgrade setuptools==47.3.2 RUN pip3 install --upgrade PyYAML==5.3.1 +RUN pip3 install requests==2.31.0 +RUN pip3 install urllib3==1.26.18 +RUN pip3 install docker==6.1.3 +RUN pip3 install git+https://github.com/pkoutsovasilis/compose@v1_fix # Oracle instant client RUN cd /usr/lib \ diff --git a/x-pack/auditbeat/cmd/root.go b/x-pack/auditbeat/cmd/root.go index e6793857cb05..e410e3884ad3 100644 --- a/x-pack/auditbeat/cmd/root.go +++ b/x-pack/auditbeat/cmd/root.go @@ -30,6 +30,9 @@ import ( // Register Auditbeat x-pack modules. _ "github.com/elastic/beats/v7/x-pack/auditbeat/include" _ "github.com/elastic/beats/v7/x-pack/libbeat/include" + + // Import processors + _ "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd" ) // Name of the beat diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go new file mode 100644 index 000000000000..50636f9d476c --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata.go @@ -0,0 +1,215 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package sessionmd + +import ( + "context" + "fmt" + "reflect" + "strconv" + "time" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/processors" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider" + cfg "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const ( + processorName = "add_session_metadata" + logName = "processor." + processorName +) + +func init() { + processors.RegisterPlugin(processorName, New) +} + +type addSessionMetadata struct { + config config + logger *logp.Logger + db *processdb.DB + provider provider.Provider +} + +func New(cfg *cfg.C) (beat.Processor, error) { + c := defaultConfig() + if err := cfg.Unpack(&c); err != nil { + return nil, fmt.Errorf("fail to unpack the %v configuration: %w", processorName, err) + } + + logger := logp.NewLogger(logName) + + ctx := context.Background() + reader := procfs.NewProcfsReader(*logger) + db, err := processdb.NewDB(reader, *logger) + if err != nil { + return nil, fmt.Errorf("failed to create DB: %w", err) + } + + backfilledPIDs := db.ScrapeProcfs() + logger.Debugf("backfilled %d processes", len(backfilledPIDs)) + + switch c.Backend { + case "auto": + // "auto" always uses ebpf, as it's currently the only backend + fallthrough + case "ebpf": + p, err := ebpf_provider.NewProvider(ctx, logger, db) + if err != nil { + return nil, fmt.Errorf("failed to create ebpf provider: %w", err) + } + return &addSessionMetadata{ + config: c, + logger: logger, + db: db, + provider: p, + }, nil + default: + return nil, fmt.Errorf("unknown backend configuration") + } +} + +func (p *addSessionMetadata) Run(ev *beat.Event) (*beat.Event, error) { + _, err := ev.GetValue(p.config.PIDField) + if err != nil { + // Do not attempt to enrich events without PID; it's not a supported event + return ev, nil //nolint:nilerr // Running on events without PID is expected + } + + err = p.provider.UpdateDB(ev) + if err != nil { + return ev, err + } + + result, err := p.enrich(ev) + if err != nil { + return ev, fmt.Errorf("enriching event: %w", err) + } + return result, nil +} + +func (p *addSessionMetadata) String() string { + return fmt.Sprintf("%v=[backend=%s, pid_field=%s, replace_fields=%t]", + processorName, p.config.Backend, p.config.PIDField, p.config.ReplaceFields) +} + +func (p *addSessionMetadata) enrich(ev *beat.Event) (*beat.Event, error) { + pidIf, err := ev.GetValue(p.config.PIDField) + if err != nil { + return nil, err + } + pid, err := pidToUInt32(pidIf) + if err != nil { + return nil, fmt.Errorf("cannot parse pid field '%s': %w", p.config.PIDField, err) + } + + fullProcess, err := p.db.GetProcess(pid) + if err != nil { + return nil, fmt.Errorf("pid %v not found in db: %w", pid, err) + } + + processMap := fullProcess.ToMap() + + if b, err := ev.Fields.HasKey("process"); !b || err != nil { + return nil, fmt.Errorf("no process field in event") + } + m, ok := tryToMapStr(ev.Fields["process"]) + if !ok { + return nil, fmt.Errorf("process field type not supported") + } + + result := ev.Clone() + err = mapstr.MergeFieldsDeep(m, processMap, true) + if err != nil { + return nil, fmt.Errorf("merging enriched fields with event: %w", err) + } + result.Fields["process"] = m + + if p.config.ReplaceFields { + if err := p.replaceFields(result); err != nil { + return nil, fmt.Errorf("replace fields: %w", err) + } + } + return result, nil +} + +// pidToUInt32 converts PID value to uint32 +func pidToUInt32(value interface{}) (pid uint32, err error) { + switch v := value.(type) { + case string: + nr, err := strconv.Atoi(v) + if err != nil { + return 0, fmt.Errorf("error converting string to integer: %w", err) + } + pid = uint32(nr) + case uint32: + pid = v + case int, int8, int16, int32, int64: + pid64 := reflect.ValueOf(v).Int() + if pid = uint32(pid64); int64(pid) != pid64 { + return 0, fmt.Errorf("integer out of range: %d", pid64) + } + case uint, uintptr, uint8, uint16, uint64: + pidu64 := reflect.ValueOf(v).Uint() + if pid = uint32(pidu64); uint64(pid) != pidu64 { + return 0, fmt.Errorf("integer out of range: %d", pidu64) + } + default: + return 0, fmt.Errorf("not an integer or string, but %T", v) + } + return pid, nil +} + +// replaceFields replaces event fields with values suitable user with the session viewer in Kibana +// The current version of session view in Kibana expects different values than what are used by auditbeat +// for some fields. This function converts these field to have values that will work with session view. +// +// This function is temporary, and can be removed when this Kibana issue is completed: https://github.com/elastic/kibana/issues/179396. +func (p *addSessionMetadata) replaceFields(ev *beat.Event) error { + kind, err := ev.Fields.GetValue("event.kind") + if err != nil { + return err + } + isAuditdEvent, err := ev.Fields.HasKey("auditd") + if err != nil { + return err + } + if kind == "event" && isAuditdEvent { + // process start + syscall, err := ev.Fields.GetValue("auditd.data.syscall") + if err != nil { + return nil //nolint:nilerr // processor can be called on unsupported events; not an error + } + switch syscall { + case "execveat", "execve": + ev.Fields.Put("event.action", []string{"exec", "fork"}) + ev.Fields.Put("event.type", []string{"start"}) + + case "exit_group": + ev.Fields.Put("event.action", []string{"end"}) + ev.Fields.Put("event.type", []string{"end"}) + ev.Fields.Put("process.end", time.Now()) + } + } + return nil +} + +func tryToMapStr(v interface{}) (mapstr.M, bool) { + switch m := v.(type) { + case mapstr.M: + return m, true + case map[string]interface{}: + return mapstr.M(m), true + default: + return nil, false + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go new file mode 100644 index 000000000000..4890505aac48 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/add_session_metadata_test.go @@ -0,0 +1,378 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package sessionmd + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var ( + enrichTests = []struct { + testName string + mockProcesses []types.ProcessExecEvent + config config + input beat.Event + expected beat.Event + expect_error bool + }{ + { + testName: "enrich process", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + mockProcesses: []types.ProcessExecEvent{ + { + PIDs: types.PIDInfo{ + Tid: uint32(100), + Tgid: uint32(100), + Ppid: uint32(50), + Pgid: uint32(100), + Sid: uint32(40), + }, + CWD: "/", + Filename: "/bin/ls", + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(50), + Tgid: uint32(50), + Ppid: uint32(40), + Sid: uint32(40), + }, + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(40), + Tgid: uint32(40), + Ppid: uint32(1), + Sid: uint32(1), + }, + }, + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + expected: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "/bin/ls", + "working_directory": "/", + "pid": uint32(100), + "parent": mapstr.M{ + "pid": uint32(50), + }, + "session_leader": mapstr.M{ + "pid": uint32(40), + }, + "group_leader": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: false, + }, + { + testName: "no PID field in event", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: true, + }, + { + testName: "PID not number", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": "xyz", + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(50), + }, + }, + }, + }, + expect_error: true, + }, + { + testName: "PID not in DB", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "pid": "100", + "executable": "ls", + "working_directory": "/", + "parent": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: true, + }, + { + testName: "process field not in event", + // This event, without a "process" field, is not supported by enrich, it should be handled gracefully + config: config{ + ReplaceFields: false, + PIDField: "action.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "action": mapstr.M{ + "pid": "1010", + }, + }, + }, + expect_error: true, + }, + { + testName: "process field not mapstr", + // Unsupported process field type should be handled gracefully + config: config{ + ReplaceFields: false, + PIDField: "action.pid", + }, + input: beat.Event{ + Fields: mapstr.M{ + "action": mapstr.M{ + "pid": "100", + }, + "process": map[int]int{ + 10: 100, + 20: 200, + }, + }, + }, + expect_error: true, + }, + { + testName: "enrich event with map[string]any process field", + config: config{ + ReplaceFields: false, + PIDField: "process.pid", + }, + mockProcesses: []types.ProcessExecEvent{ + { + PIDs: types.PIDInfo{ + Tid: uint32(100), + Tgid: uint32(100), + Ppid: uint32(50), + Pgid: uint32(100), + Sid: uint32(40), + }, + CWD: "/", + Filename: "/bin/ls", + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(50), + Tgid: uint32(50), + Ppid: uint32(40), + Sid: uint32(40), + }, + }, + { + PIDs: types.PIDInfo{ + Tid: uint32(40), + Tgid: uint32(40), + Ppid: uint32(1), + Sid: uint32(1), + }, + }, + }, + input: beat.Event{ + Fields: map[string]any{ + "process": map[string]any{ + "pid": uint32(100), + }, + }, + }, + expected: beat.Event{ + Fields: mapstr.M{ + "process": mapstr.M{ + "executable": "/bin/ls", + "working_directory": "/", + "pid": uint32(100), + "parent": mapstr.M{ + "pid": uint32(50), + }, + "session_leader": mapstr.M{ + "pid": uint32(40), + }, + "group_leader": mapstr.M{ + "pid": uint32(100), + }, + }, + }, + }, + expect_error: false, + }, + } + + filterTests = []struct { + testName string + mx mapstr.M + my mapstr.M + expected bool + }{ + { + testName: "equal", + mx: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + my: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + expected: true, + }, + { + testName: "mismatched values", + mx: mapstr.M{ + "key1": "A", + "key2": "B", + "key3": "C", + }, + my: mapstr.M{ + "key1": "A", + "key2": "X", + "key3": "C", + }, + expected: false, + }, + { + testName: "ignore key only in 2nd map", + mx: mapstr.M{ + "key1": "A", + "key2": "B", + }, + my: mapstr.M{ + "key1": "A", + "key2": "B", + "key3": "C", + }, + expected: true, + }, + { + testName: "nested mismatch", + mx: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": "B", + }, + }, + my: mapstr.M{ + "key1": "A", + "key2": mapstr.M{ + "key2_2": 2.0, + }, + "key3": 1, + }, + expected: false, + }, + } + + logger = logp.NewLogger("add_session_metadata_test") +) + +func TestEnrich(t *testing.T) { + for _, tt := range enrichTests { + reader := procfs.NewMockReader() + db, err := processdb.NewDB(reader, *logger) + assert.Nil(t, err) + + for _, ev := range tt.mockProcesses { + db.InsertExec(ev) + } + s := addSessionMetadata{ + logger: logger, + db: db, + config: tt.config, + } + + // avoid taking address of loop variable + i := tt.input + actual, err := s.enrich(&i) + if tt.expect_error { + assert.Error(t, err, "%s: error unexpectedly nil", tt.testName) + } else { + assert.Nil(t, err, "%s: enrich error: %w", tt.testName, err) + assert.NotNil(t, actual, "%s: returned nil event", tt.testName) + + //Validate output + if diff := cmp.Diff(tt.expected.Fields, actual.Fields, ignoreMissingFrom(tt.expected.Fields)); diff != "" { + t.Errorf("field mismatch:\n%s", diff) + } + } + } +} + +// IgnoreMissingFrom returns a filter that will ignore all fields missing from m +func ignoreMissingFrom(m mapstr.M) cmp.Option { + return cmp.FilterPath(func(p cmp.Path) bool { + mi, ok := p.Index(-1).(cmp.MapIndex) + if !ok { + return false + } + vx, _ := mi.Values() + return !vx.IsValid() + }, cmp.Ignore()) +} + +// TestFilter ensures `ignoreMissingFrom` filter is working as expected +// Note: This validates test code only +func TestFilter(t *testing.T) { + for _, tt := range filterTests { + if eq := cmp.Equal(tt.mx, tt.my, ignoreMissingFrom(tt.mx)); eq != tt.expected { + t.Errorf("%s: unexpected comparator result", tt.testName) + } + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/config.go b/x-pack/auditbeat/processors/sessionmd/config.go new file mode 100644 index 000000000000..31c07c9065f1 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/config.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package sessionmd + +// Config for add_session_metadata processor. +type config struct { + Backend string `config:"backend"` + ReplaceFields bool `config:"replace_fields"` + PIDField string `config:"pid_field"` +} + +func defaultConfig() config { + return config{ + Backend: "auto", + ReplaceFields: false, + PIDField: "process.pid", + } +} diff --git a/x-pack/auditbeat/processors/sessionmd/doc.go b/x-pack/auditbeat/processors/sessionmd/doc.go new file mode 100644 index 000000000000..6067081c82cb --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/doc.go @@ -0,0 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// sessionmd provides a Beat processor that can enrich process event documents with +// additional session metadata for the processes. +package sessionmd diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db.go b/x-pack/auditbeat/processors/sessionmd/processdb/db.go new file mode 100644 index 000000000000..6b2de897973b --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db.go @@ -0,0 +1,668 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package processdb + +import ( + "encoding/base64" + "errors" + "fmt" + "os" + "path" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/elastic/beats/v7/libbeat/common/capabilities" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/elastic-agent-libs/logp" +) + +type TTYType int + +const ( + TTYUnknown TTYType = iota + Pts + TTY + TTYConsole +) + +type EntryType string + +const ( + Init EntryType = "init" + Sshd EntryType = "sshd" + Ssm EntryType = "ssm" + Container EntryType = "container" + Terminal EntryType = "terminal" + EntryConsole EntryType = "console" + EntryUnknown EntryType = "unknown" +) + +var containerRuntimes = [...]string{ + "containerd-shim", + "runc", + "conmon", +} + +// "filtered" executables are executables that relate to internal +// implementation details of entry mechanisms. The set of circumstances under +// which they can become an entry leader are reduced compared to other binaries +// (see implementation and unit tests). +var filteredExecutables = [...]string{ + "runc", + "containerd-shim", + "calico-node", + "check-status", + "conmon", +} + +const ( + ptsMinMajor = 136 + ptsMaxMajor = 143 + ttyMajor = 4 + consoleMaxMinor = 63 + ttyMaxMinor = 255 +) + +type Process struct { + PIDs types.PIDInfo + Creds types.CredInfo + CTTY types.TTYDev + Argv []string + Cwd string + Env map[string]string + Filename string +} + +var ( + bootID string + pidNsInode uint64 + initError error + once sync.Once +) + +func readBootID() (string, error) { + bootID, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + panic(fmt.Sprintf("could not read /proc/sys/kernel/random/boot_id: %v", err)) + } + + return strings.TrimRight(string(bootID), "\n"), nil +} + +func readPIDNsInode() (uint64, error) { + var ret uint64 + + pidNsInodeRaw, err := os.Readlink("/proc/self/ns/pid") + if err != nil { + panic(fmt.Sprintf("could not read /proc/self/ns/pid: %v", err)) + } + + if _, err = fmt.Sscanf(pidNsInodeRaw, "pid:[%d]", &ret); err != nil { + panic(fmt.Sprintf("could not parse contents of /proc/self/ns/pid (%s): %v", pidNsInodeRaw, err)) + } + + return ret, nil +} + +func pidInfoFromProto(p types.PIDInfo) types.PIDInfo { + return types.PIDInfo{ + StartTimeNS: p.StartTimeNS, + Tid: p.Tid, + Tgid: p.Tgid, + Vpid: p.Vpid, + Ppid: p.Ppid, + Pgid: p.Pgid, + Sid: p.Sid, + } +} + +func credInfoFromProto(p types.CredInfo) types.CredInfo { + return types.CredInfo{ + Ruid: p.Ruid, + Rgid: p.Rgid, + Euid: p.Euid, + Egid: p.Egid, + Suid: p.Suid, + Sgid: p.Sgid, + CapPermitted: p.CapPermitted, + CapEffective: p.CapEffective, + } +} + +func ttyTermiosFromProto(p types.TTYTermios) types.TTYTermios { + return types.TTYTermios{ + CIflag: p.CIflag, + COflag: p.COflag, + CLflag: p.CLflag, + CCflag: p.CCflag, + } +} + +func ttyWinsizeFromProto(p types.TTYWinsize) types.TTYWinsize { + return types.TTYWinsize{ + Rows: p.Rows, + Cols: p.Cols, + } +} + +func ttyDevFromProto(p types.TTYDev) types.TTYDev { + return types.TTYDev{ + Major: p.Major, + Minor: p.Minor, + Winsize: ttyWinsizeFromProto(p.Winsize), + Termios: ttyTermiosFromProto(p.Termios), + } +} + +func initialize() { + var err error + bootID, err = readBootID() + if err != nil { + initError = err + return + } + pidNsInode, err = readPIDNsInode() + if err != nil { + initError = err + } +} + +type DB struct { + mutex sync.RWMutex + logger *logp.Logger + processes map[uint32]Process + entryLeaders map[uint32]EntryType + entryLeaderRelationships map[uint32]uint32 + procfs procfs.Reader +} + +func NewDB(reader procfs.Reader, logger logp.Logger) (*DB, error) { + once.Do(initialize) + if initError != nil { + return &DB{}, initError + } + return &DB{ + logger: logp.NewLogger("processdb"), + processes: make(map[uint32]Process), + entryLeaders: make(map[uint32]EntryType), + entryLeaderRelationships: make(map[uint32]uint32), + procfs: reader, + }, nil +} + +func (db *DB) calculateEntityIDv1(pid uint32, startTime time.Time) string { + return base64.StdEncoding.EncodeToString( + []byte( + fmt.Sprintf("%d__%s__%d__%d", + pidNsInode, + bootID, + uint64(pid), + uint64(startTime.Unix()), + ), + ), + ) +} + +// `path.Base` returns a '.' for empty strings, this just special cases that +// situation to return an empty string +func basename(pathStr string) string { + if pathStr == "" { + return "" + } + + return path.Base(pathStr) +} + +func (db *DB) InsertFork(fork types.ProcessForkEvent) { + db.mutex.Lock() + defer db.mutex.Unlock() + + pid := fork.ChildPIDs.Tgid + ppid := fork.ParentPIDs.Tgid + if entry, ok := db.processes[ppid]; ok { + entry.PIDs = pidInfoFromProto(fork.ChildPIDs) + entry.Creds = credInfoFromProto(fork.Creds) + db.processes[pid] = entry + if entryPID, ok := db.entryLeaderRelationships[ppid]; ok { + db.entryLeaderRelationships[pid] = entryPID + } + } else { + db.processes[pid] = Process{ + PIDs: pidInfoFromProto(fork.ChildPIDs), + Creds: credInfoFromProto(fork.Creds), + } + } +} + +func (db *DB) insertProcess(process Process) { + pid := process.PIDs.Tgid + db.processes[pid] = process + entryLeaderPID := db.evaluateEntryLeader(process) + if entryLeaderPID != nil { + db.entryLeaderRelationships[pid] = *entryLeaderPID + db.logger.Debugf("%v name: %s, entry_leader: %d, entry_type: %s", process.PIDs, process.Filename, *entryLeaderPID, string(db.entryLeaders[*entryLeaderPID])) + } else { + db.logger.Debugf("%v name: %s, NO ENTRY LEADER", process.PIDs, process.Filename) + } +} + +func (db *DB) InsertExec(exec types.ProcessExecEvent) { + db.mutex.Lock() + defer db.mutex.Unlock() + + proc := Process{ + PIDs: pidInfoFromProto(exec.PIDs), + Creds: credInfoFromProto(exec.Creds), + CTTY: ttyDevFromProto(exec.CTTY), + Argv: exec.Argv, + Cwd: exec.CWD, + Env: exec.Env, + Filename: exec.Filename, + } + + db.processes[exec.PIDs.Tgid] = proc + entryLeaderPID := db.evaluateEntryLeader(proc) + if entryLeaderPID != nil { + db.entryLeaderRelationships[exec.PIDs.Tgid] = *entryLeaderPID + } +} + +func (db *DB) createEntryLeader(pid uint32, entryType EntryType) { + db.entryLeaders[pid] = entryType + db.logger.Debugf("created entry leader %d: %s, name: %s", pid, string(entryType), db.processes[pid].Filename) +} + +// pid returned is a pointer type because its possible for no +func (db *DB) evaluateEntryLeader(p Process) *uint32 { + pid := p.PIDs.Tgid + + // init never has an entry leader or meta type + if p.PIDs.Tgid == 1 { + db.logger.Debugf("entry_eval %d: process is init, no entry type", p.PIDs.Tgid) + return nil + } + + // kernel threads also never have an entry leader or meta type kthreadd + // (always pid 2) is the parent of all kernel threads, by filtering pid == + // 2 || ppid == 2, we get rid of all of them + if p.PIDs.Tgid == 2 || p.PIDs.Ppid == 2 { + db.logger.Debugf("entry_eval %d: kernel threads never an entry type (parent is pid 2)", p.PIDs.Tgid) + return nil + } + + // could be an entry leader + if p.PIDs.Tgid == p.PIDs.Sid { + ttyType := getTTYType(p.CTTY.Major, p.CTTY.Minor) + + procBasename := basename(p.Filename) + switch { + case ttyType == TTY: + db.createEntryLeader(pid, Terminal) + db.logger.Debugf("entry_eval %d: entry type is terminal", p.PIDs.Tgid) + return &pid + case ttyType == TTYConsole && procBasename == "login": + db.createEntryLeader(pid, EntryConsole) + db.logger.Debugf("entry_eval %d: entry type is console", p.PIDs.Tgid) + return &pid + case p.PIDs.Ppid == 1: + db.createEntryLeader(pid, Init) + db.logger.Debugf("entry_eval %d: entry type is init", p.PIDs.Tgid) + return &pid + case !isFilteredExecutable(procBasename): + if parent, ok := db.processes[p.PIDs.Ppid]; ok { + parentBasename := basename(parent.Filename) + if ttyType == Pts && parentBasename == "ssm-session-worker" { + db.createEntryLeader(pid, Ssm) + db.logger.Debugf("entry_eval %d: entry type is ssm", p.PIDs.Tgid) + return &pid + } else if parentBasename == "sshd" && procBasename != "sshd" { + // TODO: get ip from env vars + db.createEntryLeader(pid, Sshd) + db.logger.Debugf("entry_eval %d: entry type is sshd", p.PIDs.Tgid) + return &pid + } else if isContainerRuntime(parentBasename) { + db.createEntryLeader(pid, Container) + db.logger.Debugf("entry_eval %d: entry type is container", p.PIDs.Tgid) + return &pid + } + } + default: + db.logger.Debugf("entry_eval %d: is a filtered executable: %s", p.PIDs.Tgid, procBasename) + } + } + + // if not a session leader or was not determined to be an entry leader, get + // it via parent, session leader, group leader (in that order) + relations := []struct { + pid uint32 + name string + }{ + { + pid: p.PIDs.Ppid, + name: "parent", + }, + { + pid: p.PIDs.Sid, + name: "session_leader", + }, + { + pid: p.PIDs.Pgid, + name: "group_leader", + }, + } + + for _, relation := range relations { + if entry, ok := db.entryLeaderRelationships[relation.pid]; ok { + entryType := db.entryLeaders[entry] + db.logger.Debugf("entry_eval %d: got entry_leader: %d (%s), from relative: %d (%s)", p.PIDs.Tgid, entry, string(entryType), relation.pid, relation.name) + return &entry + } else { + db.logger.Debugf("entry_eval %d: failed to find relative: %d (%s)", p.PIDs.Tgid, relation.pid, relation.name) + } + } + + // if it's a session leader, then make it its own entry leader with unknown + // entry type + if p.PIDs.Tgid == p.PIDs.Sid { + db.createEntryLeader(pid, EntryUnknown) + db.logger.Debugf("entry_eval %d: this is a session leader and no relative has an entry leader. entry type is unknown", p.PIDs.Tgid) + return &pid + } + + db.logger.Debugf("entry_eval %d: this is not a session leader and no relative has an entry leader, entry_leader will be unset", p.PIDs.Tgid) + return nil +} + +func (db *DB) InsertSetsid(setsid types.ProcessSetsidEvent) { + db.mutex.Lock() + defer db.mutex.Unlock() + + if entry, ok := db.processes[setsid.PIDs.Tgid]; ok { + entry.PIDs = pidInfoFromProto(setsid.PIDs) + db.processes[setsid.PIDs.Tgid] = entry + } else { + db.processes[setsid.PIDs.Tgid] = Process{ + PIDs: pidInfoFromProto(setsid.PIDs), + } + } +} + +func (db *DB) InsertExit(exit types.ProcessExitEvent) { + db.mutex.Lock() + defer db.mutex.Unlock() + + pid := exit.PIDs.Tgid + delete(db.processes, pid) + delete(db.entryLeaders, pid) + delete(db.entryLeaderRelationships, pid) +} + +func interactiveFromTTY(tty types.TTYDev) bool { + return TTYUnknown != getTTYType(tty.Major, tty.Minor) +} + +func fullProcessFromDBProcess(p Process) types.Process { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(p.PIDs.StartTimeNS) + interactive := interactiveFromTTY(p.CTTY) + + ret := types.Process{ + PID: p.PIDs.Tgid, + Start: timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime), + Name: basename(p.Filename), + Executable: p.Filename, + Args: p.Argv, + WorkingDirectory: p.Cwd, + Interactive: &interactive, + } + + euid := p.Creds.Euid + egid := p.Creds.Egid + ret.User.ID = strconv.FormatUint(uint64(euid), 10) + ret.Group.ID = strconv.FormatUint(uint64(egid), 10) + ret.Thread.Capabilities.Permitted, _ = capabilities.FromUint64(p.Creds.CapPermitted) + ret.Thread.Capabilities.Effective, _ = capabilities.FromUint64(p.Creds.CapEffective) + + return ret +} + +func fillParent(process *types.Process, parent Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(parent.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(parent.CTTY) + euid := parent.Creds.Euid + egid := parent.Creds.Egid + process.Parent.PID = parent.PIDs.Tgid + process.Parent.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.Parent.Name = basename(parent.Filename) + process.Parent.Executable = parent.Filename + process.Parent.Args = parent.Argv + process.Parent.WorkingDirectory = parent.Cwd + process.Parent.Interactive = &interactive + process.Parent.User.ID = strconv.FormatUint(uint64(euid), 10) + process.Parent.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillGroupLeader(process *types.Process, groupLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(groupLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(groupLeader.CTTY) + euid := groupLeader.Creds.Euid + egid := groupLeader.Creds.Egid + process.GroupLeader.PID = groupLeader.PIDs.Tgid + process.GroupLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.GroupLeader.Name = basename(groupLeader.Filename) + process.GroupLeader.Executable = groupLeader.Filename + process.GroupLeader.Args = groupLeader.Argv + process.GroupLeader.WorkingDirectory = groupLeader.Cwd + process.GroupLeader.Interactive = &interactive + process.GroupLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.GroupLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillSessionLeader(process *types.Process, sessionLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(sessionLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(sessionLeader.CTTY) + euid := sessionLeader.Creds.Euid + egid := sessionLeader.Creds.Egid + process.SessionLeader.PID = sessionLeader.PIDs.Tgid + process.SessionLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.SessionLeader.Name = basename(sessionLeader.Filename) + process.SessionLeader.Executable = sessionLeader.Filename + process.SessionLeader.Args = sessionLeader.Argv + process.SessionLeader.WorkingDirectory = sessionLeader.Cwd + process.SessionLeader.Interactive = &interactive + process.SessionLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.SessionLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) +} + +func fillEntryLeader(process *types.Process, entryType EntryType, entryLeader Process) { + reducedPrecisionStartTime := timeutils.ReduceTimestampPrecision(entryLeader.PIDs.StartTimeNS) + + interactive := interactiveFromTTY(entryLeader.CTTY) + euid := entryLeader.Creds.Euid + egid := entryLeader.Creds.Egid + process.EntryLeader.PID = entryLeader.PIDs.Tgid + process.EntryLeader.Start = timeutils.TimeFromNsSinceBoot(reducedPrecisionStartTime) + process.EntryLeader.Name = basename(entryLeader.Filename) + process.EntryLeader.Executable = entryLeader.Filename + process.EntryLeader.Args = entryLeader.Argv + process.EntryLeader.WorkingDirectory = entryLeader.Cwd + process.EntryLeader.Interactive = &interactive + process.EntryLeader.User.ID = strconv.FormatUint(uint64(euid), 10) + process.EntryLeader.Group.ID = strconv.FormatUint(uint64(egid), 10) + + process.EntryLeader.EntryMeta.Type = string(entryType) +} + +func (db *DB) setEntityID(process *types.Process) { + if process.PID != 0 && process.Start != nil { + process.EntityID = db.calculateEntityIDv1(process.PID, *process.Start) + } + + if process.Parent.PID != 0 && process.Parent.Start != nil { + process.Parent.EntityID = db.calculateEntityIDv1(process.Parent.PID, *process.Parent.Start) + } + + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + process.GroupLeader.EntityID = db.calculateEntityIDv1(process.GroupLeader.PID, *process.GroupLeader.Start) + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + process.SessionLeader.EntityID = db.calculateEntityIDv1(process.SessionLeader.PID, *process.SessionLeader.Start) + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + process.EntryLeader.EntityID = db.calculateEntityIDv1(process.EntryLeader.PID, *process.EntryLeader.Start) + } +} + +func setSameAsProcess(process *types.Process) { + if process.GroupLeader.PID != 0 && process.GroupLeader.Start != nil { + sameAsProcess := process.PID == process.GroupLeader.PID + process.GroupLeader.SameAsProcess = &sameAsProcess + } + + if process.SessionLeader.PID != 0 && process.SessionLeader.Start != nil { + sameAsProcess := process.PID == process.SessionLeader.PID + process.SessionLeader.SameAsProcess = &sameAsProcess + } + + if process.EntryLeader.PID != 0 && process.EntryLeader.Start != nil { + sameAsProcess := process.PID == process.EntryLeader.PID + process.EntryLeader.SameAsProcess = &sameAsProcess + } +} + +func (db *DB) GetProcess(pid uint32) (types.Process, error) { + db.mutex.RLock() + defer db.mutex.RUnlock() + + process, ok := db.processes[pid] + if !ok { + return types.Process{}, errors.New("process not found") + } + + ret := fullProcessFromDBProcess(process) + + if parent, ok := db.processes[process.PIDs.Ppid]; ok { + fillParent(&ret, parent) + } + + if groupLeader, ok := db.processes[process.PIDs.Pgid]; ok { + fillGroupLeader(&ret, groupLeader) + } + + if sessionLeader, ok := db.processes[process.PIDs.Sid]; ok { + fillSessionLeader(&ret, sessionLeader) + } + + if entryLeaderPID, foundEntryLeaderPID := db.entryLeaderRelationships[process.PIDs.Tgid]; foundEntryLeaderPID { + if entryLeader, foundEntryLeader := db.processes[entryLeaderPID]; foundEntryLeader { + // if there is an entry leader then there is a matching member in the entryLeaders table + fillEntryLeader(&ret, db.entryLeaders[entryLeaderPID], entryLeader) + } else { + db.logger.Errorf("failed to find entry leader entry %d for %d (%s)", entryLeaderPID, pid, db.processes[pid].Filename) + } + } else { + db.logger.Errorf("failed to find entry leader for %d (%s)", pid, db.processes[pid].Filename) + } + + db.setEntityID(&ret) + setSameAsProcess(&ret) + + return ret, nil +} + +func (db *DB) GetEntryType(pid uint32) (EntryType, error) { + db.mutex.RLock() + defer db.mutex.RUnlock() + + if entryType, ok := db.entryLeaders[pid]; ok { + return entryType, nil + } + return EntryUnknown, nil +} + +func (db *DB) ScrapeProcfs() []uint32 { + db.mutex.Lock() + defer db.mutex.Unlock() + + procs, err := db.procfs.GetAllProcesses() + if err != nil { + db.logger.Errorf("failed to get processes from procfs: %v", err) + return make([]uint32, 0) + } + + // sorting the slice to make sure that parents, session leaders, group + // leaders come first in the queue + sort.Slice(procs, func(i, j int) bool { + return procs[i].PIDs.Tgid == procs[j].PIDs.Ppid || + procs[i].PIDs.Tgid == procs[j].PIDs.Sid || + procs[i].PIDs.Tgid == procs[j].PIDs.Pgid + }) + + pids := make([]uint32, 0) + for _, procInfo := range procs { + process := Process{ + PIDs: pidInfoFromProto(procInfo.PIDs), + Creds: credInfoFromProto(procInfo.Creds), + CTTY: ttyDevFromProto(procInfo.CTTY), + Argv: procInfo.Argv, + Cwd: procInfo.Cwd, + Env: procInfo.Env, + Filename: procInfo.Filename, + } + + db.insertProcess(process) + pids = append(pids, process.PIDs.Tgid) + } + + return pids +} + +func stringStartsWithEntryInList(str string, list []string) bool { + for _, entry := range list { + if strings.HasPrefix(str, entry) { + return true + } + } + + return false +} + +func isContainerRuntime(executable string) bool { + return slices.ContainsFunc(containerRuntimes[:], func(s string) bool { + return strings.HasPrefix(executable, s) + }) +} + +func isFilteredExecutable(executable string) bool { + return stringStartsWithEntryInList(executable, filteredExecutables[:]) +} + +func getTTYType(major uint16, minor uint16) TTYType { + if major >= ptsMinMajor && major <= ptsMaxMajor { + return Pts + } + + if ttyMajor == major { + if minor <= consoleMaxMinor { + return TTYConsole + } else if minor > consoleMaxMinor && minor <= ttyMaxMinor { + return TTY + } + } + + return TTYUnknown +} diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go new file mode 100644 index 000000000000..5e8001a68e54 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/processdb/db_test.go @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package processdb + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" +) + +var logger = logp.NewLogger("processdb") + +func TestGetTTYType(t *testing.T) { + assert.Equal(t, TTYConsole, getTTYType(4, 0)) + assert.Equal(t, Pts, getTTYType(136, 0)) + assert.Equal(t, TTY, getTTYType(4, 64)) + assert.Equal(t, TTYUnknown, getTTYType(1000, 1000)) +} diff --git a/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go new file mode 100644 index 000000000000..15f98250f55d --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/processdb/entry_leader_test.go @@ -0,0 +1,1244 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package processdb + +import ( + "path" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/procfs" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" +) + +const ( + containerdShimPath = "/bin/containerd-shim-runc-v2" + containerdPath = "/bin/containerd" + sshdPath = "/usr/bin/sshd" + lsPath = "/usr/bin/ls" + bashPath = "/usr/bin/bash" + grepPath = "/usr/bin/grep" +) + +// Entry evaluation tests +// +// The entry leader isn't an entirely rigorous conceptual framework but that +// shortcoming is outweighted by the large and immediate value it provides. +// +// The idea is to assign two pieces of data to each process, the "entry meta" +// and "entry leader", the former of which describes how the user or system +// that was ultimately responsible for executing this process got into to the +// box (e.g. ssh, ssm, kubectl exec) and the latter of which describes the +// process associated with the user or system's initial entry into the "box" +// (be it a container, VM or otherwise). +// +// Generally speaking, the first session leader in a process lineage of an +// interactive session is an entry leader having an entry meta type depending +// on its lineage. For example, in the following process tree, "bash" is an +// entry leader with entry meta type "sshd": +// +// systemd (pid 1 sid 1) +// \___ sshd (pid 100 sid 100) +// \___ bash (pid 1000 sid 1000) +// \___ vim (pid 1001 sid 1000) +// +// Further entry meta types exist for ssm, container runtimes, serial consoles +// and other ways to get into a "box" (be it a container or actual machine). +// The entry meta type "init" is assigned to system processes created by the +// init service (e.g. rsyslogd, sshd). +// +// As should probably be apparent, the code to assign an entry meta type to a +// process is essentially a large amount of conditional logic with a ton of +// edge cases. It's something we "bolt on" to the linux process model, and thus +// finicky and highly subject to bugs. +// +// Thankfully, writing unit tests for entry leader evaluation is rather +// straightforward as it's basically a pure function that requires no external +// infrastructure to test (just create a mock process event with your desired +// fields set and pass it in). +// +// These tests should effectively serve as the spec for how we assign entry +// leaders. When further entry meta types or cases are added, tests should be + +func requireProcess(t *testing.T, db *DB, pid uint32, processPath string) { + t.Helper() + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, pid, process.PID) + require.Equal(t, processPath, process.Executable) + if processPath == "" { + require.Equal(t, "", process.Name) + } else { + require.Equal(t, path.Base(processPath), process.Name) + } +} + +func requireParent(t *testing.T, db *DB, pid uint32, ppid uint32) { + t.Helper() + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, ppid, process.Parent.PID) +} + +func requireParentUnset(t *testing.T, process types.Process) { + t.Helper() + require.Equal(t, "", process.Parent.EntityID) + require.Equal(t, uint32(0), process.Parent.PID) + require.Nil(t, process.Parent.Start) +} + +func requireSessionLeader(t *testing.T, db *DB, pid uint32, sid uint32) { + t.Helper() + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, sid, process.SessionLeader.PID) + require.NotNil(t, process.SessionLeader.SameAsProcess) + require.Equal(t, pid == sid, *process.SessionLeader.SameAsProcess) +} + +func requireSessionLeaderUnset(t *testing.T, process types.Process) { + t.Helper() + require.Equal(t, "", process.SessionLeader.EntityID) + require.Equal(t, uint32(0), process.SessionLeader.PID) + require.Nil(t, process.SessionLeader.Start) +} + +func requireGroupLeader(t *testing.T, db *DB, pid uint32, pgid uint32) { + t.Helper() + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, pgid, process.GroupLeader.PID) + require.NotNil(t, process.GroupLeader.SameAsProcess) + require.Equal(t, pid == pgid, *process.GroupLeader.SameAsProcess) +} + +func requireEntryLeader(t *testing.T, db *DB, pid uint32, entryPID uint32, expectedEntryType EntryType) { + t.Helper() + process, err := db.GetProcess(pid) + require.Nil(t, err) + require.Equal(t, entryPID, process.EntryLeader.PID) + require.NotNil(t, process.EntryLeader.SameAsProcess) + require.Equal(t, pid == entryPID, *process.EntryLeader.SameAsProcess) + + entryType, err := db.GetEntryType(entryPID) + require.Nil(t, err) + require.Equal(t, expectedEntryType, entryType) +} + +func requireEntryLeaderUnset(t *testing.T, process types.Process) { + t.Helper() + require.Equal(t, "", process.EntryLeader.EntityID) + require.Equal(t, uint32(0), process.EntryLeader.PID) + require.Nil(t, process.EntryLeader.Start) +} + +// tries to construct fork event from what's in the db +func insertForkAndExec(t *testing.T, db *DB, exec types.ProcessExecEvent) { + t.Helper() + var fork types.ProcessForkEvent + fork.ChildPIDs = exec.PIDs + parent, err := db.GetProcess(exec.PIDs.Ppid) + if err != nil { + fork.ParentPIDs = exec.PIDs + fork.ParentPIDs.Tgid = exec.PIDs.Ppid + fork.ParentPIDs.Ppid = 0 + fork.ParentPIDs.Pgid = 0 + + fork.ChildPIDs.Pgid = exec.PIDs.Ppid + + // if the exec makes itself a session and the parent is no where to be + // found we'll make the parent its own session + if exec.PIDs.Tgid == exec.PIDs.Sid { + fork.ParentPIDs.Sid = exec.PIDs.Ppid + } + } else { + fork.ParentPIDs.Tgid = parent.PID + fork.ParentPIDs.Ppid = parent.Parent.PID + fork.ParentPIDs.Sid = parent.SessionLeader.PID + + // keep group leader the same for now + fork.ParentPIDs.Pgid = exec.PIDs.Pgid + } + + if fork.ParentPIDs.Tgid != 0 { + db.InsertFork(fork) + } + + db.InsertExec(exec) +} + +var systemdPath = "/sbin/systemd" + +func populateProcfsWithInit(reader *procfs.MockReader) { + reader.AddEntry(1, procfs.ProcessInfo{ + PIDs: types.PIDInfo{ + Tid: 1, + Tgid: 1, + Pgid: 0, + Sid: 1, + }, + Filename: systemdPath, + }) +} + +func TestSingleProcessSessionLeaderEntryTypeTerminal(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + pid := uint32(1234) + procPath := "/bin/noproc" + db.InsertExec(types.ProcessExecEvent{ + Filename: procPath, + PIDs: types.PIDInfo{ + Tgid: pid, + Sid: pid, + }, + CTTY: types.TTYDev{ + Major: 4, + Minor: 64, + }, + }) + + requireProcess(t, db, 1234, procPath) + requireEntryLeader(t, db, 1234, 1234, Terminal) +} + +func TestSingleProcessSessionLeaderLoginProcess(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + pid := uint32(1234) + loginPath := "/bin/login" + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: loginPath, + PIDs: types.PIDInfo{ + Tgid: pid, + Sid: pid, + }, + CTTY: types.TTYDev{ + Major: 4, + Minor: 62, + }, + }) + + process, err := db.GetProcess(1234) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, "/bin/login") + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, EntryConsole) +} + +func TestSingleProcessSessionLeaderChildOfInit(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + pid := uint32(100) + rsyslogdPath := "/bin/rsyslogd" + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: rsyslogdPath, + PIDs: types.PIDInfo{ + Tgid: pid, + Sid: pid, + Ppid: 1, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + process, err := db.GetProcess(1234) + require.NotNil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, rsyslogdPath) + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, Init) +} + +func TestSingleProcessSessionLeaderChildOfSsmSessionWorker(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + ssmPID := uint32(999) + bashPID := uint32(1000) + ssmPath := "/usr/bin/ssm-session-worker" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: ssmPath, + PIDs: types.PIDInfo{ + Tgid: ssmPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: ssmPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, ssmPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Ssm) +} + +func TestSingleProcessSessionLeaderChildOfSshd(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshdPID := uint32(999) + bashPID := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshdPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshdPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshdPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) +} + +func TestSingleProcessSessionLeaderChildOfContainerdShim(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + containerdShimPID := uint32(999) + bashPID := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + PIDs: types.PIDInfo{ + Tgid: containerdShimPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: containerdShimPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, containerdShimPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Container) +} + +func TestSingleProcessSessionLeaderChildOfRunc(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + runcPID := uint32(999) + bashPID := uint32(1000) + runcPath := "/bin/runc" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: runcPath, + PIDs: types.PIDInfo{ + Tgid: runcPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: runcPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, runcPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Container) +} + +func TestSingleProcessEmptyProcess(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + // No information in proc at all, entry type should be "unknown" + // and entry leader pid should be unset (since pid is not set) + pid := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: pid, + Sid: pid, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + process, err := db.GetProcess(pid) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, pid, bashPath) + requireSessionLeader(t, db, pid, pid) + requireEntryLeader(t, db, pid, pid, EntryUnknown) +} + +// Entry evaluation code should overwrite an old EntryLeaderPID and +// EntryLeaderEntryMetaType +func TestSingleProcessOverwriteOldEntryLeader(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + ssmPID := uint32(999) + bashPID := uint32(1000) + ssmPath := "/usr/bin/ssm-session-worker" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: ssmPath, + PIDs: types.PIDInfo{ + Tgid: ssmPID, + Sid: ssmPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: ssmPID, + Ppid: ssmPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + // bash is not a session leader so it shouldn't be an entry leader. Its + // entry leader should be ssm, which is an init entry leader + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, ssmPID) + requireSessionLeader(t, db, bashPID, ssmPID) + requireEntryLeader(t, db, bashPID, ssmPID, Init) + + // skiping setsid event and assuming the pids will be updated in this exec + db.InsertExec(types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: ssmPID, + }, + CTTY: types.TTYDev{ + Major: 136, + Minor: 62, + }, + }) + + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, ssmPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Ssm) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// +// This is unrealistic, sshd usually forks a bunch of sshd children before +// exec'ing bash (see subsequent tests) but is theoretically possible and +// thus something we should handle. +func TestInitSshdBashLs(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshdPID := uint32(100) + bashPID := uint32(1000) + lsPID := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshdPID, + Sid: sshdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshdPID, + Pgid: bashPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + PIDs: types.PIDInfo{ + Tgid: lsPID, + Sid: bashPID, + Ppid: bashPID, + Pgid: lsPID, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd + requireProcess(t, db, sshdPID, sshdPath) + requireParent(t, db, sshdPID, 1) + requireSessionLeader(t, db, sshdPID, sshdPID) + requireEntryLeader(t, db, sshdPID, sshdPID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshdPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) + requireGroupLeader(t, db, bashPID, bashPID) + + // ls + requireProcess(t, db, lsPID, lsPath) + requireParent(t, db, lsPID, bashPID) + requireSessionLeader(t, db, lsPID, bashPID) + requireEntryLeader(t, db, lsPID, bashPID, Sshd) + requireGroupLeader(t, db, lsPID, lsPID) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ sshd (101, 101, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// +// sshd will usually fork a bunch of sshd children before invoking a shell +// usually 2 if it's a root shell, or 3 if it's a non-root shell. All +// "intermediate" sshd's should have entry meta "init" and an entry leader +// pid of the topmost sshd. +func TestInitSshdSshdBashLs(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshd0PID := uint32(100) + sshd1PID := uint32(101) + bashPID := uint32(1000) + lsPID := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshd0PID, + Sid: sshd0PID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshd1PID, + Sid: sshd1PID, + Ppid: sshd0PID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshd1PID, + Pgid: bashPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + PIDs: types.PIDInfo{ + Tgid: lsPID, + Sid: bashPID, + Ppid: bashPID, + Pgid: lsPID, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd0 + requireProcess(t, db, sshd0PID, sshdPath) + requireParent(t, db, sshd0PID, 1) + requireSessionLeader(t, db, sshd0PID, sshd0PID) + requireEntryLeader(t, db, sshd0PID, sshd0PID, Init) + + // sshd1 + requireProcess(t, db, sshd1PID, sshdPath) + requireParent(t, db, sshd1PID, sshd0PID) + requireSessionLeader(t, db, sshd1PID, sshd1PID) + requireEntryLeader(t, db, sshd1PID, sshd0PID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshd1PID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) + + // ls + requireProcess(t, db, lsPID, lsPath) + requireParent(t, db, lsPID, bashPID) + requireSessionLeader(t, db, lsPID, bashPID) + requireEntryLeader(t, db, lsPID, bashPID, Sshd) +} + +// / (pid, sid, entry meta, entry leader) +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ sshd (101, 101, "init", 100) +// \___ sshd (102, 101, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +func TestInitSshdSshdSshdBashLs(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshd0PID := uint32(100) + sshd1PID := uint32(101) + sshd2PID := uint32(102) + bashPID := uint32(1000) + lsPID := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshd0PID, + Sid: sshd0PID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshd1PID, + Sid: sshd1PID, + Ppid: sshd0PID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshd2PID, + Sid: sshd1PID, + Ppid: sshd1PID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshd2PID, + Pgid: bashPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + PIDs: types.PIDInfo{ + Tgid: lsPID, + Sid: bashPID, + Ppid: bashPID, + Pgid: lsPID, + }, + }) + + // systemd + systemd, err := db.GetProcess(1) + require.Nil(t, err) + requireParentUnset(t, systemd) + requireEntryLeaderUnset(t, systemd) + + requireProcess(t, db, 1, systemdPath) + requireSessionLeader(t, db, 1, 1) + + // sshd0 + requireProcess(t, db, sshd0PID, sshdPath) + requireParent(t, db, sshd0PID, 1) + requireSessionLeader(t, db, sshd0PID, sshd0PID) + requireEntryLeader(t, db, sshd0PID, sshd0PID, Init) + + // sshd1 + requireProcess(t, db, sshd1PID, sshdPath) + requireParent(t, db, sshd1PID, sshd0PID) + requireSessionLeader(t, db, sshd1PID, sshd1PID) + requireEntryLeader(t, db, sshd1PID, sshd0PID, Init) + + // sshd2 + requireProcess(t, db, sshd2PID, sshdPath) + requireParent(t, db, sshd2PID, sshd1PID) + requireSessionLeader(t, db, sshd2PID, sshd1PID) + requireEntryLeader(t, db, sshd2PID, sshd0PID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshd2PID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) + + // ls + requireProcess(t, db, lsPID, lsPath) + requireParent(t, db, lsPID, bashPID) + requireSessionLeader(t, db, lsPID, bashPID) + requireEntryLeader(t, db, lsPID, bashPID, Sshd) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// +// containerd-shim-runc-v2 will reparent itself to init just prior to +// executing the containerized process. +func TestInitContainerdContainerdShim(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + containerdPID := uint32(100) + containerdShimPID := uint32(1000) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + PIDs: types.PIDInfo{ + Tgid: containerdPID, + Sid: containerdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + PIDs: types.PIDInfo{ + Tgid: containerdShimPID, + Sid: containerdPID, + Ppid: containerdPID, + }, + }) + + // containerd + requireProcess(t, db, containerdPID, containerdPath) + requireParent(t, db, containerdPID, 1) + requireSessionLeader(t, db, containerdPID, containerdPID) + requireEntryLeader(t, db, containerdPID, containerdPID, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPID, containerdShimPath) + requireParent(t, db, containerdShimPID, containerdPID) + requireSessionLeader(t, db, containerdShimPID, containerdPID) + requireEntryLeader(t, db, containerdShimPID, containerdPID, Init) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// | +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// \___ bash (1001, 1001, "container", 1000) +// +// Note that containerd originally forks and exec's +// containerd-shim-runc-v2, which then forks such that it is reparented to +// init. +func TestInitContainerdShimBashContainerdShimIsReparentedToInit(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + containerdPID := uint32(100) + containerdShimPID := uint32(1000) + bashPID := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + PIDs: types.PIDInfo{ + Tgid: containerdPID, + Sid: containerdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + PIDs: types.PIDInfo{ + Tgid: containerdShimPID, + Sid: containerdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: containerdShimPID, + }, + }) + + // containerd + requireProcess(t, db, containerdPID, containerdPath) + requireParent(t, db, containerdPID, 1) + requireSessionLeader(t, db, containerdPID, containerdPID) + requireEntryLeader(t, db, containerdPID, containerdPID, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPID, containerdShimPath) + requireParent(t, db, containerdShimPID, 1) + requireSessionLeader(t, db, containerdShimPID, containerdPID) + requireEntryLeader(t, db, containerdShimPID, containerdPID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, containerdShimPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Container) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd +// +// \___ containerd (100, 100, "init", 100) +// | +// \___ containerd-shim-runc-v2 (1000, 100, "init", 100) +// \___ pause (1001, 1001, "container", 1001) +// +// The pause binary is a Kubernetes internal binary that is exec'd in a +// container by the container runtime. It is responsible for holding +// open the pod sandbox while other containers start and stop +func TestInitContainerdShimPauseContainerdShimIsReparentedToInit(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + containerdPID := uint32(100) + containerdShimPID := uint32(1000) + pausePID := uint32(1001) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdPath, + PIDs: types.PIDInfo{ + Tgid: containerdPID, + Sid: containerdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: containerdShimPath, + PIDs: types.PIDInfo{ + Tgid: containerdShimPID, + Sid: containerdPID, + Ppid: 1, + }, + }) + + pausePath := "/usr/bin/pause" + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: pausePath, + PIDs: types.PIDInfo{ + Tgid: pausePID, + Sid: pausePID, + Ppid: containerdShimPID, + }, + }) + + // containerd + requireProcess(t, db, containerdPID, containerdPath) + requireParent(t, db, containerdPID, 1) + requireSessionLeader(t, db, containerdPID, containerdPID) + requireEntryLeader(t, db, containerdPID, containerdPID, Init) + + // containerd-shim-runc-v2 + requireProcess(t, db, containerdShimPID, containerdShimPath) + requireParent(t, db, containerdShimPID, 1) + requireSessionLeader(t, db, containerdShimPID, containerdPID) + requireEntryLeader(t, db, containerdShimPID, containerdPID, Init) + + // pause + requireProcess(t, db, pausePID, pausePath) + requireParent(t, db, pausePID, containerdShimPID) + requireSessionLeader(t, db, pausePID, pausePID) + requireEntryLeader(t, db, pausePID, pausePID, Container) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// | +// \___ grep (1002, 1000, "sshd", 1000) /* ppid/sid data is missing */ +// +// Grep does not have ppid or sid set, only pgid. Entry evaluation code +// should fallback to grabbing entry leader data from ls, the process group +// leader. +func TestInitSshdBashLsAndGrepGrepOnlyHasGroupLeader(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshdPID := uint32(100) + bashPID := uint32(1000) + lsPID := uint32(1001) + grepPID := uint32(1002) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshdPID, + Sid: sshdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshdPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + PIDs: types.PIDInfo{ + Tgid: lsPID, + Sid: bashPID, + Ppid: bashPID, + Pgid: lsPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: grepPath, + PIDs: types.PIDInfo{ + Tgid: grepPID, + Pgid: lsPID, + }, + }) + + // sshd + requireProcess(t, db, sshdPID, sshdPath) + requireParent(t, db, sshdPID, 1) + requireSessionLeader(t, db, sshdPID, sshdPID) + requireEntryLeader(t, db, sshdPID, sshdPID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshdPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) + + // ls + requireProcess(t, db, lsPID, lsPath) + requireParent(t, db, lsPID, bashPID) + requireSessionLeader(t, db, lsPID, bashPID) + requireEntryLeader(t, db, lsPID, bashPID, Sshd) + + // grep + grep, err := db.GetProcess(grepPID) + require.Nil(t, err) + requireParentUnset(t, grep) + + requireProcess(t, db, grepPID, grepPath) + requireEntryLeader(t, db, grepPID, bashPID, Sshd) +} + +// / (pid, sid, entry meta, entry leader) +// +// systemd (1, 1, none, none) +// +// \___ sshd (100, 100, "init", 100) +// \___ bash (1000, 1000, "sshd", 1000) +// \___ ls (1001, 1000, "sshd", 1000) +// | +// \___ grep (1002, 1000, "sshd", 1000) /* ppid/pgid data is missing */ +// +// Grep does not have ppid or pgid set, ppid. Entry evaluation code should +// fallback to grabbing entry leader data from sshd, the session leader. +func TestInitSshdBashLsAndGrepGrepOnlyHasSessionLeader(t *testing.T) { + reader := procfs.NewMockReader() + populateProcfsWithInit(reader) + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + sshdPID := uint32(100) + bashPID := uint32(1000) + lsPID := uint32(1001) + grepPID := uint32(1002) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: sshdPath, + PIDs: types.PIDInfo{ + Tgid: sshdPID, + Sid: sshdPID, + Ppid: 1, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: bashPath, + PIDs: types.PIDInfo{ + Tgid: bashPID, + Sid: bashPID, + Ppid: sshdPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: lsPath, + PIDs: types.PIDInfo{ + Tgid: lsPID, + Sid: bashPID, + Ppid: bashPID, + Pgid: lsPID, + }, + }) + + insertForkAndExec(t, db, types.ProcessExecEvent{ + Filename: grepPath, + PIDs: types.PIDInfo{ + Tgid: grepPID, + Sid: bashPID, + }, + }) + + // sshd + requireProcess(t, db, sshdPID, sshdPath) + requireParent(t, db, sshdPID, 1) + requireSessionLeader(t, db, sshdPID, sshdPID) + requireEntryLeader(t, db, sshdPID, sshdPID, Init) + + // bash + requireProcess(t, db, bashPID, bashPath) + requireParent(t, db, bashPID, sshdPID) + requireSessionLeader(t, db, bashPID, bashPID) + requireEntryLeader(t, db, bashPID, bashPID, Sshd) + + // ls + requireProcess(t, db, lsPID, lsPath) + requireParent(t, db, lsPID, bashPID) + requireSessionLeader(t, db, lsPID, bashPID) + requireEntryLeader(t, db, lsPID, bashPID, Sshd) + + // grep + grep, err := db.GetProcess(grepPID) + require.Nil(t, err) + requireParentUnset(t, grep) + + requireProcess(t, db, grepPID, grepPath) + requireSessionLeader(t, db, grepPID, bashPID) + requireEntryLeader(t, db, grepPID, bashPID, Sshd) +} + +// / (pid, sid, entry meta, entry leader) +// +// grep (1001, 1000, "unknown", 1001) +// +// No parent, session leader, or process group leader exists to draw +// on to get an entry leader for grep, fallback to assigning it an +// entry meta type of "unknown" and making it an entry leader. +func TestGrepInIsolation(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + db.ScrapeProcfs() + + grepPID := uint32(1001) + + db.InsertExec(types.ProcessExecEvent{ + Filename: grepPath, + PIDs: types.PIDInfo{ + Tgid: grepPID, + Ppid: 1000, + Sid: grepPID, + }, + }) + + process, err := db.GetProcess(grepPID) + require.Nil(t, err) + requireParentUnset(t, process) + + requireProcess(t, db, grepPID, grepPath) + requireSessionLeader(t, db, grepPID, grepPID) + requireEntryLeader(t, db, grepPID, grepPID, EntryUnknown) +} + +// / (pid, sid, entry meta, entry leader) +// +// kthreadd (2, 0, , ) +// +// \___ rcu_gp (3, 0, , ) +// +// Kernel threads should never have an entry meta type or entry leader set. +func TestKernelThreads(t *testing.T) { + reader := procfs.NewMockReader() + db, err := NewDB(reader, *logger) + require.Nil(t, err) + + kthreaddPID := uint32(2) + rcuGpPID := uint32(3) + + kthreaddPath := "kthreadd" + rcuGpPath := "rcu_gp" + + db.InsertExec(types.ProcessExecEvent{ + Filename: kthreaddPath, + PIDs: types.PIDInfo{ + Tgid: kthreaddPID, + Ppid: 1, + Sid: 0, + }, + }) + + db.InsertExec(types.ProcessExecEvent{ + Filename: rcuGpPath, + PIDs: types.PIDInfo{ + Tgid: rcuGpPID, + Ppid: kthreaddPID, + Sid: 0, + }, + }) + + // kthreadd + kthreadd, err := db.GetProcess(kthreaddPID) + require.Nil(t, err) + requireParentUnset(t, kthreadd) + requireSessionLeaderUnset(t, kthreadd) + requireEntryLeaderUnset(t, kthreadd) + + requireProcess(t, db, kthreaddPID, kthreaddPath) + + // rcu_gp + rcuGp, err := db.GetProcess(rcuGpPID) + require.Nil(t, err) + requireSessionLeaderUnset(t, rcuGp) + requireEntryLeaderUnset(t, rcuGp) + + requireProcess(t, db, rcuGpPID, rcuGpPath) + requireParent(t, db, rcuGpPID, kthreaddPID) +} diff --git a/x-pack/auditbeat/processors/sessionmd/procfs/mock.go b/x-pack/auditbeat/processors/sessionmd/procfs/mock.go new file mode 100644 index 000000000000..1689873044ec --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/procfs/mock.go @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package procfs + +import ( + "fmt" +) + +type MockReader struct { + entries map[uint32]ProcessInfo +} + +func NewMockReader() *MockReader { + return &MockReader{ + entries: make(map[uint32]ProcessInfo), + } +} + +func (r *MockReader) AddEntry(pid uint32, entry ProcessInfo) { + r.entries[pid] = entry +} + +func (r *MockReader) GetProcess(pid uint32) (ProcessInfo, error) { + entry, ok := r.entries[pid] + if !ok { + return ProcessInfo{}, fmt.Errorf("not found") + } + return entry, nil +} + +func (r *MockReader) GetAllProcesses() ([]ProcessInfo, error) { + ret := make([]ProcessInfo, 0, len(r.entries)) + + for _, entry := range r.entries { + ret = append(ret, entry) + } + return ret, nil +} diff --git a/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go new file mode 100644 index 000000000000..f3a7d41d1d71 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/procfs/procfs.go @@ -0,0 +1,252 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package procfs + +import ( + "fmt" + "strconv" + "strings" + + "github.com/prometheus/procfs" + "golang.org/x/sys/unix" + + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/timeutils" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/elastic-agent-libs/logp" +) + +func MajorTTY(ttyNr uint32) uint16 { + return uint16((ttyNr >> 8) & 0xf) +} + +func MinorTTY(ttyNr uint32) uint16 { + return uint16(((ttyNr & 0xfff00000) >> 20) | (ttyNr & 0xff)) +} + +// this interface exists so that we can inject a mock procfs reader for deterministic testing +type Reader interface { + GetProcess(pid uint32) (ProcessInfo, error) + GetAllProcesses() ([]ProcessInfo, error) +} + +type ProcfsReader struct { + logger logp.Logger +} + +func NewProcfsReader(logger logp.Logger) ProcfsReader { + return ProcfsReader{ + logger: logger, + } +} + +type Stat procfs.ProcStat + +type ProcessInfo struct { + PIDs types.PIDInfo + Creds types.CredInfo + CTTY types.TTYDev + Argv []string + Cwd string + Env map[string]string + Filename string + CGroupPath string +} + +func credsFromProc(proc procfs.Proc) (types.CredInfo, error) { + status, err := proc.NewStatus() + if err != nil { + return types.CredInfo{}, err + } + + ruid, err := strconv.Atoi(status.UIDs[0]) + if err != nil { + return types.CredInfo{}, err + } + + euid, err := strconv.Atoi(status.UIDs[1]) + if err != nil { + return types.CredInfo{}, err + } + + suid, err := strconv.Atoi(status.UIDs[2]) + if err != nil { + return types.CredInfo{}, err + } + + rgid, err := strconv.Atoi(status.GIDs[0]) + if err != nil { + return types.CredInfo{}, err + } + + egid, err := strconv.Atoi(status.GIDs[1]) + if err != nil { + return types.CredInfo{}, err + } + + sgid, err := strconv.Atoi(status.GIDs[2]) + if err != nil { + return types.CredInfo{}, err + } + + // procfs library doesn't grab CapEff or CapPrm, make the direct syscall + hdr := unix.CapUserHeader{ + Version: unix.LINUX_CAPABILITY_VERSION_3, + Pid: int32(proc.PID), + } + var data [2]unix.CapUserData + err = unix.Capget(&hdr, &data[0]) + if err != nil { + return types.CredInfo{}, err + } + permitted := uint64(data[1].Permitted) << 32 + permitted += uint64(data[0].Permitted) + effective := uint64(data[1].Effective) << 32 + effective += uint64(data[0].Effective) + + return types.CredInfo{ + Ruid: uint32(ruid), + Euid: uint32(euid), + Suid: uint32(suid), + Rgid: uint32(rgid), + Egid: uint32(egid), + Sgid: uint32(sgid), + CapPermitted: permitted, + CapEffective: effective, + }, nil +} + +func (r ProcfsReader) getProcessInfo(proc procfs.Proc) (ProcessInfo, error) { + pid := uint32(proc.PID) + // All other info can be best effort, but failing to get pid info and + // start time is needed to register the process in the database + stat, err := proc.Stat() + if err != nil { + return ProcessInfo{}, fmt.Errorf("failed to read /proc/%d/stat: %w", pid, err) + } + + argv, err := proc.CmdLine() + if err != nil { + argv = []string{} + } + + exe, err := proc.Executable() + if err != nil { + if len(argv) > 0 { + r.logger.Debugf("pid %d: got executable from cmdline: %s", pid, argv[0]) + exe = argv[0] + } else { + r.logger.Debugf("pid %d: failed to get executable path: %v", pid, err) + exe = "" + } + } + + environ, err := r.getEnviron(pid) + if err != nil { + environ = nil + } + + cwd, err := proc.Cwd() + if err != nil { + cwd = "" + } + + creds, err := credsFromProc(proc) + if err != nil { + creds = types.CredInfo{} + } + + cGroupPath := "" + cgroups, err := proc.Cgroups() + if err == nil { + out: + // Find the cgroup path from the PID controller. + // NOTE: This does not support the unified hierarchy from cgroup v2, as bpf also does not currently support it. + // When support is added for unified hierarchies, it should be added in bpf and userspace at the same time. + // (Currently all supported cgroup v2 systems (GKE) are working as they send backwards compatible v1 hierarchies as well) + for _, cgroup := range cgroups { + for _, controller := range cgroup.Controllers { + if controller == "pids" { + cGroupPath = cgroup.Path + break out + } + } + } + } + + startTimeNs := timeutils.TicksToNs(stat.Starttime) + return ProcessInfo{ + PIDs: types.PIDInfo{ + StartTimeNS: startTimeNs, + Tid: pid, + Tgid: pid, + Ppid: uint32(stat.PPID), + Pgid: uint32(stat.PGRP), + Sid: uint32(stat.Session), + }, + Creds: creds, + CTTY: types.TTYDev{ + Major: MajorTTY(uint32(stat.TTY)), + Minor: MinorTTY(uint32(stat.TTY)), + }, + Cwd: cwd, + Argv: argv, + Env: environ, + Filename: exe, + CGroupPath: cGroupPath, + }, nil +} + +func (r ProcfsReader) GetProcess(pid uint32) (ProcessInfo, error) { + proc, err := procfs.NewProc(int(pid)) + if err != nil { + return ProcessInfo{}, err + } + return r.getProcessInfo(proc) +} + +// returns empty slice on error +func (r ProcfsReader) GetAllProcesses() ([]ProcessInfo, error) { + procs, err := procfs.AllProcs() + if err != nil { + return nil, err + } + + ret := make([]ProcessInfo, 0) + for _, proc := range procs { + process_info, err := r.getProcessInfo(proc) + if err != nil { + r.logger.Warnf("failed to read process info for %v", proc.PID) + } + ret = append(ret, process_info) + } + + return ret, nil +} + +func (r ProcfsReader) getEnviron(pid uint32) (map[string]string, error) { + proc, err := procfs.NewProc(int(pid)) + if err != nil { + return nil, err + } + + flatEnviron, err := proc.Environ() + if err != nil { + return nil, err + } + + ret := make(map[string]string) + for _, entry := range flatEnviron { + index := strings.Index(entry, "=") + if index == -1 { + continue + } + + ret[entry[0:index]] = entry[index:] + } + + return ret, nil +} diff --git a/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go new file mode 100644 index 000000000000..2b9b540e037c --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/ebpf_provider/ebpf_provider.go @@ -0,0 +1,157 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package ebpf_provider + +import ( + "context" + "fmt" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/ebpf" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/processdb" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/provider" + "github.com/elastic/beats/v7/x-pack/auditbeat/processors/sessionmd/types" + "github.com/elastic/ebpfevents" + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + name = "add_session_metadata" + eventMask = ebpf.EventMask(ebpfevents.EventTypeProcessFork | ebpfevents.EventTypeProcessExec | ebpfevents.EventTypeProcessExit) +) + +type prvdr struct { + ctx context.Context + logger *logp.Logger + db *processdb.DB +} + +func NewProvider(ctx context.Context, logger *logp.Logger, db *processdb.DB) (provider.Provider, error) { + p := prvdr{ + ctx: ctx, + logger: logger, + db: db, + } + + w, err := ebpf.GetWatcher() + if err != nil { + return nil, fmt.Errorf("get ebpf watcher: %w", err) + } + + records := w.Subscribe(name, eventMask) + + go func(logger logp.Logger) { + for { + r := <-records + if r.Error != nil { + logger.Warnw("received error from the ebpf subscription", "error", err) + continue + } + if r.Event == nil { + continue + } + ev := r.Event + switch ev.Type { + case ebpfevents.EventTypeProcessFork: + body, ok := ev.Body.(*ebpfevents.ProcessFork) + if !ok { + logger.Errorf("unexpected event body, got %T", ev.Body) + continue + } + pe := types.ProcessForkEvent{ + ParentPIDs: types.PIDInfo{ + Tid: body.ParentPids.Tid, + Tgid: body.ParentPids.Tgid, + Ppid: body.ParentPids.Ppid, + Pgid: body.ParentPids.Pgid, + Sid: body.ParentPids.Sid, + StartTimeNS: body.ParentPids.StartTimeNs, + }, + ChildPIDs: types.PIDInfo{ + Tid: body.ChildPids.Tid, + Tgid: body.ChildPids.Tgid, + Ppid: body.ChildPids.Ppid, + Pgid: body.ChildPids.Pgid, + Sid: body.ChildPids.Sid, + StartTimeNS: body.ChildPids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + } + p.db.InsertFork(pe) + case ebpfevents.EventTypeProcessExec: + body, ok := ev.Body.(*ebpfevents.ProcessExec) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExecEvent{ + PIDs: types.PIDInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNS: body.Pids.StartTimeNs, + }, + Creds: types.CredInfo{ + Ruid: body.Creds.Ruid, + Rgid: body.Creds.Rgid, + Euid: body.Creds.Euid, + Egid: body.Creds.Egid, + Suid: body.Creds.Suid, + Sgid: body.Creds.Sgid, + CapPermitted: body.Creds.CapPermitted, + CapEffective: body.Creds.CapEffective, + }, + CTTY: types.TTYDev{ + Major: body.CTTY.Major, + Minor: body.CTTY.Minor, + }, + CWD: body.Cwd, + Argv: body.Argv, + Env: body.Env, + Filename: body.Filename, + } + p.db.InsertExec(pe) + case ebpfevents.EventTypeProcessExit: + body, ok := ev.Body.(*ebpfevents.ProcessExit) + if !ok { + logger.Errorf("unexpected event body") + continue + } + pe := types.ProcessExitEvent{ + PIDs: types.PIDInfo{ + Tid: body.Pids.Tid, + Tgid: body.Pids.Tgid, + Ppid: body.Pids.Ppid, + Pgid: body.Pids.Pgid, + Sid: body.Pids.Sid, + StartTimeNS: body.Pids.StartTimeNs, + }, + ExitCode: body.ExitCode, + } + p.db.InsertExit(pe) + } + } + }(*p.logger) + + return &p, nil +} + +func (s prvdr) UpdateDB(ev *beat.Event) error { + // no-op for ebpf, DB is updated from pushed ebpf events + return nil +} diff --git a/x-pack/auditbeat/processors/sessionmd/provider/provider.go b/x-pack/auditbeat/processors/sessionmd/provider/provider.go new file mode 100644 index 000000000000..e3fa1547806c --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/provider/provider.go @@ -0,0 +1,15 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package provider + +import ( + "github.com/elastic/beats/v7/libbeat/beat" +) + +type Provider interface { + UpdateDB(*beat.Event) error +} diff --git a/x-pack/auditbeat/processors/sessionmd/timeutils/time.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go new file mode 100644 index 000000000000..5c8dd7450df0 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/timeutils/time.go @@ -0,0 +1,77 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package timeutils + +import ( + "fmt" + "sync" + "time" + + "github.com/prometheus/procfs" + "github.com/tklauser/go-sysconf" +) + +var ( + getBootTimeOnce = sync.OnceValues(getBootTime) + getTicksPerSecondOnce = sync.OnceValues(getTicksPerSecond) +) + +func getBootTime() (time.Time, error) { + fs, err := procfs.NewDefaultFS() + if err != nil { + return time.Time{}, fmt.Errorf("could not get procfs: %w", err) + } + + stat, err := fs.Stat() + if err != nil { + return time.Time{}, fmt.Errorf("could not read /proc/stat: %w", err) + } + return time.Unix(int64(stat.BootTime), 0), nil +} + +func getTicksPerSecond() (uint64, error) { + tps, err := sysconf.Sysconf(sysconf.SC_CLK_TCK) + if err != nil { + return 0, fmt.Errorf("sysconf(SC_CLK_TCK) failed: %w", err) + } + return uint64(tps), nil +} + +func TicksToNs(ticks uint64) uint64 { + ticksPerSecond, err := getTicksPerSecondOnce() + if err != nil { + return 0 + } + return ticks * uint64(time.Second.Nanoseconds()) / ticksPerSecond +} + +func TimeFromNsSinceBoot(t time.Duration) *time.Time { + bootTime, err := getBootTimeOnce() + if err != nil { + return nil + } + timestamp := bootTime.Add(t) + return ×tamp +} + +// When generating an `entity_id` in ECS we need to reduce the precision of a +// process's start time to that of procfs. Process start times can come from either +// BPF (high precision) or procfs (lower precision). We must reduce them all to the +// lowest common denominator such that entity ID's generated are always consistent. +// +// - Timestamps we get from the kernel are in nanosecond precision. +// - Timestamps we get from procfs are typically 1/100th second precision. We +// get this precision from `sysconf()` +// - We store timestamps as nanoseconds, but reduce the precision to 1/100th +// second +func ReduceTimestampPrecision(timeNs uint64) time.Duration { + ticksPerSecond, err := getTicksPerSecondOnce() + if err != nil { + return 0 + } + return time.Duration(timeNs).Truncate(time.Second / time.Duration(ticksPerSecond)) +} diff --git a/x-pack/auditbeat/processors/sessionmd/timeutils/time_test.go b/x-pack/auditbeat/processors/sessionmd/timeutils/time_test.go new file mode 100644 index 000000000000..1aa5abdf469a --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/timeutils/time_test.go @@ -0,0 +1,24 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +//go:build linux + +package timeutils + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestReduceTimestampPrecision(t *testing.T) { + oneSecond := time.Second.Nanoseconds() + result1 := ReduceTimestampPrecision(uint64(oneSecond)) + require.Equal(t, time.Duration(oneSecond), result1) + + oneSecondWithDelay := oneSecond + 10 + result2 := ReduceTimestampPrecision(uint64(oneSecondWithDelay)) + require.Equal(t, time.Duration(oneSecond), result2) +} diff --git a/x-pack/auditbeat/processors/sessionmd/types/events.go b/x-pack/auditbeat/processors/sessionmd/types/events.go new file mode 100644 index 000000000000..5f8d67d763f1 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/types/events.go @@ -0,0 +1,94 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package types + +//go:generate stringer -linecomment=true -type=Type,HookPoint,Field -output=gen_types_string.go + +type Type uint64 + +const ( + ProcessFork Type = iota + ProcessExec + ProcessExit + ProcessSetsid +) + +type ( + Field uint32 +) + +const ( + CWD Field = iota + 1 + Argv + Env + Filename +) + +type PIDInfo struct { + StartTimeNS uint64 + Tid uint32 + Tgid uint32 + Vpid uint32 + Ppid uint32 + Pgid uint32 + Sid uint32 +} + +type CredInfo struct { + Ruid uint32 + Rgid uint32 + Euid uint32 + Egid uint32 + Suid uint32 + Sgid uint32 + CapPermitted uint64 + CapEffective uint64 +} + +type TTYWinsize struct { + Rows uint16 + Cols uint16 +} + +type TTYTermios struct { + CIflag uint32 + COflag uint32 + CLflag uint32 + CCflag uint32 +} + +type TTYDev struct { + Minor uint16 + Major uint16 + Winsize TTYWinsize + Termios TTYTermios +} + +type ProcessForkEvent struct { + ParentPIDs PIDInfo + ChildPIDs PIDInfo + Creds CredInfo +} + +type ProcessExecEvent struct { + PIDs PIDInfo + Creds CredInfo + CTTY TTYDev + + // varlen fields + CWD string + Argv []string + Env map[string]string + Filename string +} + +type ProcessExitEvent struct { + PIDs PIDInfo + ExitCode int32 +} + +type ProcessSetsidEvent struct { + PIDs PIDInfo +} diff --git a/x-pack/auditbeat/processors/sessionmd/types/process.go b/x-pack/auditbeat/processors/sessionmd/types/process.go new file mode 100644 index 000000000000..e5a07d099876 --- /dev/null +++ b/x-pack/auditbeat/processors/sessionmd/types/process.go @@ -0,0 +1,456 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package types + +import ( + "time" + + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// These fields contain information about a process. +// These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. +type Process struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The time the process ended. + End *time.Time `json:"end,omitempty"` + + // The exit code of the process, if this is a termination event. + // The field should be absent if there is no exit code for the event (e.g. process start). + ExitCode *int64 `json:"exit_code,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + Vpid uint32 `json:"vpid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // An array of previous executions for the process, including the initial fork. Only executable and args are set. + Previous []struct { + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + } `json:"previous,omitempty"` + + Thread struct { + Capabilities struct { + Permitted []string `json:"permitted,omitempty"` + + Effective []string `json:"effective,omitempty"` + } `json:"capabilities,omitempty"` + } `json:"thread,omitempty"` + + // Information about the parent process. + Parent struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + Thread struct { + Capabilities struct { + Permitted []string `json:"permitted,omitempty"` + + Effective []string `json:"effective,omitempty"` + } `json:"capabilities,omitempty"` + } `json:"thread,omitempty"` + } `json:"parent,omitempty"` + + // Information about the process group leader. In some cases this may be the same as the top level process. + GroupLeader struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // This boolean is used to identify if a leader process is the same as the top level process. + // For example, if `process.group_leader.same_as_process = true`, it means the process event in question is the leader of its process group. Details under `process.*` like `pid` would be the same under `process.group_leader.*` The same applies for both `process.session_leader` and `process.entry_leader`. + // This field exists to the benefit of EQL and other rule engines since it's not possible to compare equality between two fields in a single document. e.g `process.entity_id` = `process.group_leader.entity_id` (top level process is the process group leader) OR `process.entity_id` = `process.entry_leader.entity_id` (top level process is the entry session leader) + // Instead these rules could be written like: `process.group_leader.same_as_process: true` OR `process.entry_leader.same_as_process: true` + // Note: This field is only set on `process.entry_leader`, `process.session_leader` and `process.group_leader`. + SameAsProcess *bool `json:"same_as_process,omitempty"` + } `json:"group_leader,omitempty"` + + // Often the same as entry_leader. When it differs, it represents a session started within another session. e.g. using tmux + SessionLeader struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // This boolean is used to identify if a leader process is the same as the top level process. + // For example, if `process.group_leader.same_as_process = true`, it means the process event in question is the leader of its process group. Details under `process.*` like `pid` would be the same under `process.group_leader.*` The same applies for both `process.session_leader` and `process.entry_leader`. + // This field exists to the benefit of EQL and other rule engines since it's not possible to compare equality between two fields in a single document. e.g `process.entity_id` = `process.group_leader.entity_id` (top level process is the process group leader) OR `process.entity_id` = `process.entry_leader.entity_id` (top level process is the entry session leader) + // Instead these rules could be written like: `process.group_leader.same_as_process: true` OR `process.entry_leader.same_as_process: true` + // Note: This field is only set on `process.entry_leader`, `process.session_leader` and `process.group_leader`. + SameAsProcess *bool `json:"same_as_process,omitempty"` + } `json:"session_leader,omitempty"` + + // First process from terminal or remote access via SSH, SSM, etc OR a service directly started by the init process. + EntryLeader struct { + // Unique identifier for the process. + // The implementation of this is specified by the data source, but some examples of what could be used here are a process-generated UUID, Sysmon Process GUIDs, or a hash of some uniquely identifying components of a process. + // Constructing a globally unique identifier is a common practice to mitigate PID reuse as well as to identify a specific process over time, across multiple monitored hosts. + EntityID string `json:"entity_id,omitempty"` + + // Absolute path to the process executable. + Executable string `json:"executable,omitempty"` + + // Whether the process is connected to an interactive shell. + // Process interactivity is inferred from the processes file descriptors. If the character device for the controlling tty is the same as stdin and stderr for the process, the process is considered interactive. + // Note: A non-interactive process can belong to an interactive session and is simply one that does not have open file descriptors reading the controlling TTY on FD 0 (stdin) or writing to the controlling TTY on FD 2 (stderr). A backgrounded process is still considered interactive if stdin and stderr are connected to the controlling TTY. + Interactive *bool `json:"interactive,omitempty"` + + // Process name. + // Sometimes called program name or similar. + Name string `json:"name,omitempty"` + + // The time the process started. + Start *time.Time `json:"start,omitempty"` + + // The working directory of the process. + WorkingDirectory string `json:"working_directory,omitempty"` + + EntryMeta struct { + // The entry type for the entry session leader. Values include: init(e.g systemd), sshd, ssm, kubelet, teleport, terminal, console + // Note: This field is only set on process.session_leader. + Type string `json:"type,omitempty"` + } `json:"entry_meta,omitempty"` + + // The effective user (euid). + User struct { + // Unique identifier of the user. + ID string `json:"id,omitempty"` + + // Short name or login of the user. + Name string `json:"name,omitempty"` + } `json:"user,omitempty"` + + // The effective group (egid). + Group struct { + // Unique identifier for the group on the system/platform. + ID string `json:"id,omitempty"` + + // Name of the group. + Name string `json:"name,omitempty"` + } `json:"group,omitempty"` + + // Process id. + PID uint32 `json:"pid,omitempty"` + + // Array of process arguments, starting with the absolute path to the executable. + // May be filtered to protect sensitive information. + Args []string `json:"args,omitempty"` + + // This boolean is used to identify if a leader process is the same as the top level process. + // For example, if `process.group_leader.same_as_process = true`, it means the process event in question is the leader of its process group. Details under `process.*` like `pid` would be the same under `process.group_leader.*` The same applies for both `process.session_leader` and `process.entry_leader`. + // This field exists to the benefit of EQL and other rule engines since it's not possible to compare equality between two fields in a single document. e.g `process.entity_id` = `process.group_leader.entity_id` (top level process is the process group leader) OR `process.entity_id` = `process.entry_leader.entity_id` (top level process is the entry session leader) + // Instead these rules could be written like: `process.group_leader.same_as_process: true` OR `process.entry_leader.same_as_process: true` + // Note: This field is only set on `process.entry_leader`, `process.session_leader` and `process.group_leader`. + SameAsProcess *bool `json:"same_as_process,omitempty"` + } `json:"entry_leader,omitempty"` +} + +func (p *Process) ToMap() mapstr.M { + process := mapstr.M{ + "entity_id": p.EntityID, + "executable": p.Executable, + "name": p.Name, + "exit_code": p.ExitCode, + "interactive": p.Interactive, + "working_directory": p.WorkingDirectory, + "user": mapstr.M{ + "id": p.User.ID, + "name": p.User.Name, + }, + "group": mapstr.M{ + "id": p.Group.ID, + "name": p.Group.Name, + }, + "pid": p.PID, + "vpid": p.Vpid, + "args": p.Args, + "thread": mapstr.M{ + "capabilities": mapstr.M{ + "permitted": p.Thread.Capabilities.Permitted, + "effective": p.Thread.Capabilities.Effective, + }, + }, + "parent": mapstr.M{ + "entity_id": p.Parent.EntityID, + "executable": p.Parent.Executable, + "name": p.Parent.Name, + "interactive": p.Parent.Interactive, + "working_directory": p.Parent.WorkingDirectory, + "user": mapstr.M{ + "id": p.Parent.User.ID, + "name": p.Parent.User.Name, + }, + "group": mapstr.M{ + "id": p.Parent.Group.ID, + "name": p.Parent.Group.Name, + }, + "pid": p.Parent.PID, + "args": p.Parent.Args, + "thread": mapstr.M{ + "capabilities": mapstr.M{ + "permitted": p.Parent.Thread.Capabilities.Permitted, + "effective": p.Parent.Thread.Capabilities.Effective, + }, + }, + }, + "group_leader": mapstr.M{ + "entity_id": p.GroupLeader.EntityID, + "executable": p.GroupLeader.Executable, + "name": p.GroupLeader.Name, + "interactive": p.GroupLeader.Interactive, + "working_directory": p.GroupLeader.WorkingDirectory, + "user": mapstr.M{ + "id": p.GroupLeader.User.ID, + "name": p.GroupLeader.User.Name, + }, + "group": mapstr.M{ + "id": p.GroupLeader.Group.ID, + "name": p.GroupLeader.Group.Name, + }, + "pid": p.GroupLeader.PID, + "args": p.GroupLeader.Args, + "same_as_process": p.GroupLeader.SameAsProcess, + }, + "session_leader": mapstr.M{ + "entity_id": p.SessionLeader.EntityID, + "executable": p.SessionLeader.Executable, + "name": p.SessionLeader.Name, + "interactive": p.SessionLeader.Interactive, + "working_directory": p.SessionLeader.WorkingDirectory, + "user": mapstr.M{ + "id": p.SessionLeader.User.ID, + "name": p.SessionLeader.User.Name, + }, + "group": mapstr.M{ + "id": p.SessionLeader.Group.ID, + "name": p.SessionLeader.Group.Name, + }, + "pid": p.SessionLeader.PID, + "args": p.SessionLeader.Args, + "same_as_process": p.SessionLeader.SameAsProcess, + }, + "entry_leader": mapstr.M{ + "entity_id": p.EntryLeader.EntityID, + "executable": p.EntryLeader.Executable, + "name": p.EntryLeader.Name, + "interactive": p.EntryLeader.Interactive, + "working_directory": p.EntryLeader.WorkingDirectory, + "entry_meta": mapstr.M{ + "type": p.EntryLeader.EntryMeta.Type, + }, + "user": mapstr.M{ + "id": p.EntryLeader.User.ID, + "name": p.EntryLeader.User.Name, + }, + "group": mapstr.M{ + "id": p.EntryLeader.Group.ID, + "name": p.EntryLeader.Group.Name, + }, + "pid": p.EntryLeader.PID, + "args": p.EntryLeader.Args, + "same_as_process": p.EntryLeader.SameAsProcess, + }, + } + + // nil timestamps will cause a panic within the publisher, only add the mapping if it exists + if p.Start != nil { + process.Put("start", p.Start) + } + if p.Parent.Start != nil { + process.Put("parent.start", p.Parent.Start) + } + if p.GroupLeader.Start != nil { + process.Put("group_leader.start", p.GroupLeader.Start) + } + if p.SessionLeader.Start != nil { + process.Put("session_leader.start", p.SessionLeader.Start) + } + if p.EntryLeader.Start != nil { + process.Put("entry_leader.start", p.EntryLeader.Start) + } + + return process +} diff --git a/x-pack/filebeat/docs/inputs/input-salesforce.asciidoc b/x-pack/filebeat/docs/inputs/input-salesforce.asciidoc new file mode 100644 index 000000000000..3b85b8511b6f --- /dev/null +++ b/x-pack/filebeat/docs/inputs/input-salesforce.asciidoc @@ -0,0 +1,276 @@ +[role="xpack"] + +:type: salesforce + +[id="{beatname_lc}-input-{type}"] +=== Salesforce input + +++++ +Salesforce +++++ + +Use the `salesforce` input to monitor Salesforce events either via the https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_eventlogfile.htm[Salesforce EventLogFile (ELF) API] or the https://developer.salesforce.com/blogs/2020/05/introduction-to-real-time-event-monitoring[Salesforce Real-time event monitoring API]. Both use REST API (to execute SOQL queries in the Salesforce instance) under the hood to query the relevant objects to fetch the events. + +Here the `input` have cursor state(s) that will be provided to the next execution of event monitoring to fetch the events from the last cursor state. The cursor states can be used to control the behaviour of the program. + +This input supports: + +* Auth +** OAuth2 +*** User-Password flow +*** JWT Bearer flow +* Event Monitoring +** EventLogFile (ELF) using REST API +** REST API for objects (Used for Setup Audit Trail and for monitoring real-time events) + +Event Monitoring methods are highly configurable and can be used to monitor any supported object or event log file. The input can be configured to monitor multiple objects or event log files at the same time. + +Example configurations: + +["source","yaml",subs="attributes"] +---- +filebeat.inputs: + - type: salesforce + enabled: true + version: 56 + auth.oauth2: + user_password_flow: + enabled: true + client.id: client-id + client.secret: client-secret + token_url: https://instance-id.develop.my.salesforce.com + username: salesforce-instance@user.in + password: salesforce-instance-password + jwt_bearer_flow: + enabled: true + client.id: client-id + client.username: salesforce-instance@user.in + client.key_path: server_client.key + url: https://login.salesforce.com + url: https://instance-id.develop.my.salesforce.com + event_monitoring_method: + event_log_file: + enabled: true + interval: 1h + query: + default: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST" + value: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY CreatedDate ASC NULLS FIRST" + cursor: + field: "CreatedDate" + object: + enabled: true + interval: 5m + query: + default: "SELECT FIELDS(STANDARD) FROM LoginEvent" + value: "SELECT FIELDS(STANDARD) FROM LoginEvent WHERE EventDate > [[ .cursor.object.first_event_time ]]" + cursor: + field: "EventDate" +---- + + +==== Execution + +The `salesforce` input is a long-running program that retrieves events from a Salesforce instance and sends them to the specified output. The program executes in a loop, fetching events from the Salesforce instance at a preconfigured interval. Each event monitoring method can be configured to run separately and at different intervals. To prevent a sudden spike in memory usage, if multiple event monitoring methods are configured, they are scheduled to run one at a time. Even if the intervals overlap, only one method will be executed randomly, and the other will be executed after the first one completes. + +There are two methods to fetch the events from the Salesforce instance: + +- event_log_file: https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_eventlogfile.htm[EventLogFile] is a standard object in Salesforce and the event monitoring method uses the REST API under the hood to gather the Salesforce org's operational events from the object. There is a field EventType that helps distinguish between the types of operational events like — Login, Logout, etc. Uses Salesforce's query language SOQL to query the object. + +- object: This method is a general way of retrieving events from a Salesforce instance by using the REST API. It can be used for the SetupAuditTrail and for monitoring objects in real-time. In real-time event monitoring, subscribing to the events is a common practice, but the events are also stored in Salesforce org (if configured), specifically in big object tables that are preconfigured for each event type. To query the object, Salesforce's query language SOQL is used. + +==== Configuration options + +The `salesforce` input supports the following configuration options plus the +<<{beatname_lc}-input-{type}-common-options>> described later. + +[bool] +==== `enabled` + +Whether the input is enabled or not. Default: `false`. + +[integer] +==== `version` + +The version of the Salesforce API to use. Minimum supported version is 46. + +[object] +==== `auth` + +The authentication settings for the Salesforce instance. + +[object] +==== `auth.oauth2` + +The OAuth2 authentication options for the Salesforce instance. + +There are two OAuth2 authentication flows supported: + +* `user_password_flow`: User-Password flow +* `jwt_bearer_flow`: JWT Bearer flow + +[bool] +==== `auth.oauth2.user_password_flow.enabled` + +Whether to use the user-password flow for authentication. Default: `false`. + +NOTE: Only one authentication flow can be enabled at a time. + +[string] +==== `auth.oauth2.user_password_flow.client.id` + +The client ID for the user-password flow. + +[string] +==== `auth.oauth2.user_password_flow.client.secret` + +The client secret for the user-password flow. + +[string] +==== `auth.oauth2.user_password_flow.token_url` + +The token URL for the user-password flow. + +[string] +==== `auth.oauth2.user_password_flow.username` + +The username for the user-password flow. + +[string] +==== `auth.oauth2.user_password_flow.password` + +The password for the user-password flow. + +[bool] +==== `auth.oauth2.jwt_bearer_flow.enabled` + +Whether to use the JWT bearer flow for authentication. Default: `false`. + +NOTE: Only one authentication flow can be enabled at a time. + +[string] +==== `auth.oauth2.jwt_bearer_flow.client.id` + +The client ID for the JWT bearer flow. + +[string] +==== `auth.oauth2.jwt_bearer_flow.client.username` + +The username for the JWT bearer flow. + +[string] +==== `auth.oauth2.jwt_bearer_flow.client.key_path` + +The path to the private key file for the JWT bearer flow. The file must be PEM encoded PKCS1 or PKCS8 private key and must have the right permissions set to have read access for the user running the program. + +[string] +==== `auth.oauth2.jwt_bearer_flow.url` + +The URL for the JWT bearer flow. + +[string] +==== `url` + +The URL of the Salesforce instance. Required. + +[[resource-parameters]] +[float] +==== `resource.timeout` + +Duration before declaring that the HTTP client connection has timed out. Valid time units are `ns`, `us`, `ms`, `s`, `m`, `h`. Default: `30s`. + +[float] +==== `resource.retry.max_attempts` + +The maximum number of retries for the HTTP client. Default: `5`. + +[float] +==== `resource.retry.wait_min` + +The minimum time to wait before a retry is attempted. Default: `1s`. + +[float] +==== `resource.retry.wait_max` + +The maximum time to wait before a retry is attempted. Default: `60s`. + +[object] +==== `event_monitoring_method` + +The event monitoring method to use. There are two event monitoring methods supported: + +* `event_log_file`: EventLogFile (ELF) using REST API + +* `object`: Real-time event monitoring using REST API (objects) + +[object] +==== `event_monitoring_method.event_log_file` + +The event monitoring method to use — event_log_file. Uses the EventLogFile API to fetch the events from the Salesforce instance. + +[bool] +==== `event_monitoring_method.event_log_file.enabled` + +Whether to use the EventLogFile API for event monitoring. Default: `false`. + +[duration] +==== `event_monitoring_method.event_log_file.interval` + +The interval to collect the events from the Salesforce instance using the EventLogFile API. + +[string] +==== `event_monitoring_method.event_log_file.query.default` + +The default query to fetch the events from the Salesforce instance using the EventLogFile API. + +In case the cursor state is not available, the default query will be used to fetch the events from the Salesforce instance. The default query must be a valid SOQL query. If the SOQL query in `event_monitoring_method.event_log_file.query.value` is not valid, the default query will be used to fetch the events from the Salesforce instance. + +[string] +==== `event_monitoring_method.event_log_file.query.value` + +The SOQL query to fetch the events from the Salesforce instance using the EventLogFile API but it uses the cursor state to fetch the events from the Salesforce instance. The SOQL query must be a valid SOQL query. If the SOQL query is not valid, the default query will be used to fetch the events from the Salesforce instance. + +In case of restarts or subsequent executions, the cursor state will be used to fetch the events from the Salesforce instance. The cursor state is the last event time of the last event fetched from the Salesforce instance. The cursor state is taken from `event_monitoring_method.event_log_file.cursor.field` field for the last event fetched from the Salesforce instance. + +[string] +==== `event_monitoring_method.event_log_file.cursor.field` + +The field to use to fetch the cursor state from the last event fetched from the Salesforce instance. The field must be a valid field in the SOQL query specified in `event_monitoring_method.event_log_file.query.default` and `event_monitoring_method.event_log_file.query.value` i.e., part of the selected fields in the SOQL query. + +[object] +==== `event_monitoring_method.object` + +The event monitoring method to use — object. Uses REST API to fetch the events directly from the objects from the Salesforce instance. This method is used for Setup Audit Trail and for monitoring real-time events. + +[bool] +==== `event_monitoring_method.object.enabled` + +Whether to use the REST API for objects for event monitoring. Default: `false`. + +[duration] +==== `event_monitoring_method.object.interval` + +The interval to collect the events from the Salesforce instance using the REST API from objects. + +[string] +==== `event_monitoring_method.object.query.default` + +The default SOQL query to fetch the events from the Salesforce instance using the REST API from objects. + +In case the cursor state is not available, the default query will be used to fetch the events from the Salesforce instance. The default query must be a valid SOQL query. If the SOQL query in `event_monitoring_method.object.query.value` is not valid, the default query will be used to fetch the events from the Salesforce instance. + +[string] +==== `event_monitoring_method.object.query.value` + +The SOQL query to fetch the events from the Salesforce instance using the REST API from objects but it uses the cursor state to fetch the events from the Salesforce instance. The SOQL query must be a valid SOQL query. If the SOQL query is not valid, the default query will be used to fetch the events from the Salesforce instance. + +In case of restarts or subsequent executions, the cursor state will be used to fetch the events from the Salesforce instance. The cursor state is the last event time of the last event fetched from the Salesforce instance. The cursor state is taken from `event_monitoring_method.object.cursor.field` field for the last event fetched from the Salesforce instance. + +[string] +==== `event_monitoring_method.object.cursor.field` + +The field to use to fetch the cursor state from the last event fetched from the Salesforce instance. The field must be a valid field in the SOQL query specified in `event_monitoring_method.object.query.default` and `event_monitoring_method.object.query.value` i.e., part of the selected fields in the SOQL query. + +[id="{beatname_lc}-input-{type}-common-options"] +include::../../../../filebeat/docs/inputs/input-common-options.asciidoc[] + +:type!: diff --git a/x-pack/filebeat/input/default-inputs/inputs_other.go b/x-pack/filebeat/input/default-inputs/inputs_other.go index 91d5917f261c..ab682e4e0010 100644 --- a/x-pack/filebeat/input/default-inputs/inputs_other.go +++ b/x-pack/filebeat/input/default-inputs/inputs_other.go @@ -22,6 +22,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/lumberjack" "github.com/elastic/beats/v7/x-pack/filebeat/input/netflow" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" + "github.com/elastic/beats/v7/x-pack/filebeat/input/salesforce" "github.com/elastic/beats/v7/x-pack/filebeat/input/shipper" "github.com/elastic/beats/v7/x-pack/filebeat/input/websocket" "github.com/elastic/elastic-agent-libs/logp" @@ -40,6 +41,7 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 awss3.Plugin(store), awscloudwatch.Plugin(), lumberjack.Plugin(), + salesforce.Plugin(log, store), shipper.Plugin(log, store), websocket.Plugin(log, store), netflow.Plugin(log), diff --git a/x-pack/filebeat/input/salesforce/config.go b/x-pack/filebeat/input/salesforce/config.go new file mode 100644 index 000000000000..55ee1c81e835 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/config.go @@ -0,0 +1,131 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "fmt" + "time" + + "github.com/elastic/elastic-agent-libs/transport/httpcommon" +) + +type config struct { + Resource *resourceConfig `config:"resource"` + Auth *authConfig `config:"auth"` + EventMonitoringMethod *eventMonitoringMethod `config:"event_monitoring_method"` + URL string `config:"url" validate:"required"` + Version int `config:"version" validate:"required"` +} + +type resourceConfig struct { + Retry retryConfig `config:"retry"` + Transport httpcommon.HTTPTransportSettings `config:",inline"` +} + +type retryConfig struct { + MaxAttempts *int `config:"max_attempts"` + WaitMin *time.Duration `config:"wait_min"` + WaitMax *time.Duration `config:"wait_max"` +} + +func (c retryConfig) Validate() error { + switch { + case c.MaxAttempts != nil && *c.MaxAttempts <= 0: + return errors.New("max_attempts must be greater than zero") + case c.WaitMin != nil && *c.WaitMin <= 0: + return errors.New("wait_min must be greater than zero") + case c.WaitMax != nil && *c.WaitMax <= 0: + return errors.New("wait_max must be greater than zero") + } + return nil +} + +func (c retryConfig) getMaxAttempts() int { + if c.MaxAttempts == nil { + return 0 + } + return *c.MaxAttempts +} + +func (c retryConfig) getWaitMin() time.Duration { + if c.WaitMin == nil { + return 0 + } + return *c.WaitMin +} + +func (c retryConfig) getWaitMax() time.Duration { + if c.WaitMax == nil { + return 0 + } + return *c.WaitMax +} + +type eventMonitoringMethod struct { + EventLogFile EventMonitoringConfig `config:"event_log_file"` + Object EventMonitoringConfig `config:"object"` +} + +type EventMonitoringConfig struct { + Enabled *bool `config:"enabled"` + Query *QueryConfig `config:"query"` + Cursor *cursorConfig `config:"cursor"` + Interval time.Duration `config:"interval"` +} + +func (e *EventMonitoringConfig) isEnabled() bool { + return e != nil && (e.Enabled != nil && *e.Enabled) +} + +type cursorConfig struct { + Field string `config:"field"` +} + +// Validate validates the configuration. +func (c *config) Validate() error { + switch { + case !c.Auth.OAuth2.JWTBearerFlow.isEnabled() && !c.Auth.OAuth2.UserPasswordFlow.isEnabled(): + return errors.New("no auth provider enabled") + case c.Auth.OAuth2.JWTBearerFlow.isEnabled() && c.Auth.OAuth2.UserPasswordFlow.isEnabled(): + return errors.New("only one auth provider must be enabled") + case c.URL == "": + return errors.New("no instance url is configured") + case !c.EventMonitoringMethod.Object.isEnabled() && !c.EventMonitoringMethod.EventLogFile.isEnabled(): + return errors.New(`at least one of "event_monitoring_method.event_log_file.enabled" or "event_monitoring_method.object.enabled" must be set to true`) + case c.EventMonitoringMethod.EventLogFile.isEnabled() && c.EventMonitoringMethod.EventLogFile.Interval == 0: + return fmt.Errorf("not a valid interval %d", c.EventMonitoringMethod.EventLogFile.Interval) + case c.EventMonitoringMethod.Object.isEnabled() && c.EventMonitoringMethod.Object.Interval == 0: + return fmt.Errorf("not a valid interval %d", c.EventMonitoringMethod.Object.Interval) + + case c.Version < 46: + // - EventLogFile object is available in API version 32.0 or later + // - SetupAuditTrail object is available in API version 15.0 or later + // - Real-Time Event monitoring objects that were introduced as part of + // the beta release in API version 46.0 + // + // To keep things simple, only one version is entertained i.e., the + // minimum version supported by all objects for which we have support + // for. + // + // minimum_vesion_supported_by_all_objects([32.0, 15.0, 46.0]) = 46.0 + // + // (Objects like EventLogFile, SetupAuditTrail and Real-time monitoring + // objects are available in v46.0 and above) + + // References: + // https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_eventlogfile.htm + // https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_setupaudittrail.htm + // https://developer.salesforce.com/docs/atlas.en-us.platform_events.meta/platform_events/platform_events_objects_monitoring.htm + return errors.New("not a valid version i.e., 46.0 or above") + } + + return nil +} + +type QueryConfig struct { + Default *valueTpl `config:"default"` + Value *valueTpl `config:"value"` +} diff --git a/x-pack/filebeat/input/salesforce/config_auth.go b/x-pack/filebeat/input/salesforce/config_auth.go new file mode 100644 index 000000000000..6e0f9361def6 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/config_auth.go @@ -0,0 +1,92 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import "errors" + +type authConfig struct { + // See: https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_flows.htm&type=5 + // for more information about OAuth2 flows. + OAuth2 *OAuth2 `config:"oauth2"` +} + +type OAuth2 struct { + // See: https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_username_password_flow.htm&type=5 + UserPasswordFlow *UserPasswordFlow `config:"user_password_flow"` + // See: https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_jwt_flow.htm&type=5 + JWTBearerFlow *JWTBearerFlow `config:"jwt_bearer_flow"` +} + +type UserPasswordFlow struct { + Enabled *bool `config:"enabled"` + + ClientID string `config:"client.id"` + ClientSecret string `config:"client.secret"` + Password string `config:"password"` + TokenURL string `config:"token_url"` + Username string `config:"username"` +} + +type JWTBearerFlow struct { + Enabled *bool `config:"enabled"` + + URL string `config:"url"` + ClientID string `config:"client.id"` + ClientUsername string `config:"client.username"` + ClientKeyPath string `config:"client.key_path"` +} + +// isEnabled returns true if the `enable` field is set to true in the yaml. +func (o *UserPasswordFlow) isEnabled() bool { + return o != nil && (o.Enabled != nil && *o.Enabled) +} + +// Validate checks if User Passworld Flow config is valid. +func (o *UserPasswordFlow) Validate() error { + if !o.isEnabled() { + return nil + } + + switch { + case o.TokenURL == "": + return errors.New("token_url must be provided") + case o.ClientID == "": + return errors.New("client.id must be provided") + case o.ClientSecret == "": + return errors.New("client.secret must be provided") + case o.Username == "": + return errors.New("username must be provided") + case o.Password == "": + return errors.New("password must be provided") + + } + + return nil +} + +// isEnabled returns true if the `enable` field is set to true in the yaml. +func (o *JWTBearerFlow) isEnabled() bool { + return o != nil && (o.Enabled != nil && *o.Enabled) +} + +// Validate checks if JWT Bearer Flow config is valid. +func (o *JWTBearerFlow) Validate() error { + if !o.isEnabled() { + return nil + } + + switch { + case o.URL == "": + return errors.New("url must be provided") + case o.ClientID == "": + return errors.New("client.id must be provided") + case o.ClientUsername == "": + return errors.New("client.username must be provided") + case o.ClientKeyPath == "": + return errors.New("client.key_path must be provided") + } + + return nil +} diff --git a/x-pack/filebeat/input/salesforce/config_auth_test.go b/x-pack/filebeat/input/salesforce/config_auth_test.go new file mode 100644 index 000000000000..0b378f7b47f0 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/config_auth_test.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOAuth2Config(t *testing.T) { + tests := map[string]struct { + wantErr error + config UserPasswordFlow + }{ + "auth disabled I": {config: UserPasswordFlow{}, wantErr: nil}, + "auth disabled II": {config: UserPasswordFlow{Enabled: pointer(false)}, wantErr: nil}, + "tokenURL missing": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: ""}, wantErr: errors.New("token_url must be provided")}, + "clientID missing": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: "https://salesforce.com", ClientID: ""}, wantErr: errors.New("client.id must be provided")}, + "clientSecret missing": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: "https://salesforce.com", ClientID: "xyz", ClientSecret: ""}, wantErr: errors.New("client.secret must be provided")}, + "username missing": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: "https://salesforce.com", ClientID: "xyz", ClientSecret: "abc", Username: ""}, wantErr: errors.New("username must be provided")}, + "password missing": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: "https://salesforce.com", ClientID: "xyz", ClientSecret: "abc", Username: "user", Password: ""}, wantErr: errors.New("password must be provided")}, + "all present": {config: UserPasswordFlow{Enabled: pointer(true), TokenURL: "https://salesforce.com", ClientID: "xyz", ClientSecret: "abc", Username: "user", Password: "pass"}, wantErr: nil}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := tc.config.Validate() + assert.Equal(t, tc.wantErr, got) + }) + } +} + +func TestJWTConfig(t *testing.T) { + tests := map[string]struct { + wantErr error + config JWTBearerFlow + }{ + "auth disabled I": {config: JWTBearerFlow{}, wantErr: nil}, + "auth disabled II": {config: JWTBearerFlow{Enabled: pointer(false)}, wantErr: nil}, + "url missing": {config: JWTBearerFlow{Enabled: pointer(true), URL: ""}, wantErr: errors.New("url must be provided")}, + "clientID missing": {config: JWTBearerFlow{Enabled: pointer(true), URL: "https://salesforce.com", ClientID: ""}, wantErr: errors.New("client.id must be provided")}, + "clientUsername missing": {config: JWTBearerFlow{Enabled: pointer(true), URL: "https://salesforce.com", ClientID: "xyz", ClientUsername: ""}, wantErr: errors.New("client.username must be provided")}, + "clientKeyPath missing": {config: JWTBearerFlow{Enabled: pointer(true), URL: "https://salesforce.com", ClientID: "xyz", ClientUsername: "abc", ClientKeyPath: ""}, wantErr: errors.New("client.key_path must be provided")}, + "all present": {config: JWTBearerFlow{Enabled: pointer(true), URL: "https://salesforce.com", ClientID: "xyz", ClientUsername: "abc", ClientKeyPath: "def"}, wantErr: nil}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := tc.config.Validate() + assert.Equal(t, tc.wantErr, got) + }) + } +} diff --git a/x-pack/filebeat/input/salesforce/config_test.go b/x-pack/filebeat/input/salesforce/config_test.go new file mode 100644 index 000000000000..b620d9bf9a9e --- /dev/null +++ b/x-pack/filebeat/input/salesforce/config_test.go @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestValidate(t *testing.T) { + tests := map[string]struct { + wantErr error + inputCfg config + }{ + "no auth provider enabled (no password or jwt)": { + inputCfg: config{ + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{}, + JWTBearerFlow: &JWTBearerFlow{}, + }, + }, + }, + wantErr: errors.New("no auth provider enabled"), + }, + "only one auth provider is allowed (either password or jwt)": { + inputCfg: config{ + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + JWTBearerFlow: &JWTBearerFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: errors.New("only one auth provider must be enabled"), + }, + "no instance url is configured (empty url)": { + inputCfg: config{ + URL: "", + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: errors.New("no instance url is configured"), + }, + "no data collection method configured": { + inputCfg: config{ + EventMonitoringMethod: &eventMonitoringMethod{}, + URL: "https://some-dummy-subdomain.salesforce.com/services/oauth2/token", + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: errors.New(`at least one of "event_monitoring_method.event_log_file.enabled" or "event_monitoring_method.object.enabled" must be set to true`), + }, + "invalid elf interval (1h)": { + inputCfg: config{ + EventMonitoringMethod: &eventMonitoringMethod{ + EventLogFile: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Duration(0), + }, + }, + URL: "https://some-dummy-subdomain.salesforce.com/services/oauth2/token", + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: fmt.Errorf("not a valid interval %d", time.Duration(0)), + }, + "invalid object interval (1h)": { + inputCfg: config{ + EventMonitoringMethod: &eventMonitoringMethod{ + Object: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Duration(0), + }, + }, + URL: "https://some-dummy-subdomain.salesforce.com/services/oauth2/token", + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: fmt.Errorf("not a valid interval %d", time.Duration(0)), + }, + "invalid api version (v45)": { + inputCfg: config{ + Version: 45, + EventMonitoringMethod: &eventMonitoringMethod{ + Object: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Hour, + }, + }, + URL: "https://some-dummy-subdomain.salesforce.com/services/oauth2/token", + Auth: &authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{Enabled: pointer(true)}, + }, + }, + }, + wantErr: errors.New("not a valid version i.e., 46.0 or above"), + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := tc.inputCfg.Validate() + assert.Equal(t, tc.wantErr, got) + }) + } +} diff --git a/x-pack/filebeat/input/salesforce/helper.go b/x-pack/filebeat/input/salesforce/helper.go new file mode 100644 index 000000000000..8869ca9aa3f1 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/helper.go @@ -0,0 +1,33 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import "time" + +// timeNow wraps time.Now to mock time for tests. +var timeNow = time.Now + +// mockTimeNow mocks timeNow for tests. +func mockTimeNow(t time.Time) { + timeNow = func() time.Time { + return t + } +} + +// resetTimeNow resets timeNow to time.Now. +func resetTimeNow() { + timeNow = time.Now +} + +// pointer returns a pointer to the given value. +// +// For example: Assigning &true to value of type *bool is not possible but +// pointer(true) is assignable to the same value of type *bool as address operator +// can be applied to pointer(true) as the returned value is an addressable value. +// +// See: https://go.dev/ref/spec#Address_operators +func pointer[T any](d T) *T { + return &d +} diff --git a/x-pack/filebeat/input/salesforce/input.go b/x-pack/filebeat/input/salesforce/input.go new file mode 100644 index 000000000000..1e893625ed20 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/input.go @@ -0,0 +1,573 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/g8rswimmer/go-sfdc" + "github.com/g8rswimmer/go-sfdc/credentials" + "github.com/g8rswimmer/go-sfdc/session" + "github.com/g8rswimmer/go-sfdc/soql" + "github.com/golang-jwt/jwt" + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/zap" + "golang.org/x/exp/slices" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/feature" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/go-concert/ctxtool" +) + +const ( + inputName = "salesforce" + formatRFC3339Like = "2006-01-02T15:04:05.999Z" +) + +type salesforceInput struct { + ctx context.Context + publisher inputcursor.Publisher + cancel context.CancelCauseFunc + cursor *state + srcConfig *config + sfdcConfig *sfdc.Configuration + log *logp.Logger + clientSession *session.Session + soqlr *soql.Resource + config +} + +// // The Filebeat user-agent is provided to the program as useragent. +// var userAgent = useragent.UserAgent("Filebeat", version.GetDefaultVersion(), version.Commit(), version.BuildTime().String()) + +// Plugin returns the input plugin. +func Plugin(log *logp.Logger, store inputcursor.StateStore) v2.Plugin { + return v2.Plugin{ + Name: inputName, + Stability: feature.Stable, + Manager: NewInputManager(log, store), + } +} + +func (s *salesforceInput) Name() string { return inputName } + +func (s *salesforceInput) Test(_ inputcursor.Source, _ v2.TestContext) error { + return nil +} + +// Run starts the input and blocks until it ends completes. It will return on +// context cancellation or type invalidity errors, any other error will be retried. +func (s *salesforceInput) Run(env v2.Context, src inputcursor.Source, cursor inputcursor.Cursor, pub inputcursor.Publisher) (err error) { + st := &state{} + if !cursor.IsNew() { + if err = cursor.Unpack(&st); err != nil { + return err + } + } + + if err = s.Setup(env, src, st, pub); err != nil { + return err + } + + return s.run() +} + +// Setup sets up the input. It will create a new SOQL resource and all other +// necessary configurations. +func (s *salesforceInput) Setup(env v2.Context, src inputcursor.Source, cursor *state, pub inputcursor.Publisher) (err error) { + cfg := src.(*source).cfg + + ctx := ctxtool.FromCanceller(env.Cancelation) + childCtx, cancel := context.WithCancelCause(ctx) + + s.srcConfig = &cfg + s.ctx = childCtx + s.cancel = cancel + s.publisher = pub + s.cursor = cursor + s.log = env.Logger.With("input_url", cfg.URL) + s.sfdcConfig, err = s.getSFDCConfig(&cfg) + if err != nil { + return fmt.Errorf("error with configuration: %w", err) + } + + s.soqlr, err = s.SetupSFClientConnection() // create a new SOQL resource + if err != nil { + return fmt.Errorf("error setting up connection to Salesforce: %w", err) + } + + return nil +} + +// run is the main loop of the input. It will run until the context is cancelled +// and based on the configuration, it will run the different methods -- EventLogFile +// or Object to collect events at defined intervals. +func (s *salesforceInput) run() error { + if s.srcConfig.EventMonitoringMethod.EventLogFile.isEnabled() { + err := s.RunEventLogFile() + if err != nil { + s.log.Errorf("Problem running EventLogFile collection: %s", err) + } + } + + if s.srcConfig.EventMonitoringMethod.Object.isEnabled() { + err := s.RunObject() + if err != nil { + s.log.Errorf("Problem running Object collection: %s", err) + } + } + + eventLogFileTicker, objectMethodTicker := &time.Ticker{}, &time.Ticker{} + eventLogFileTicker.C, objectMethodTicker.C = nil, nil + + if s.srcConfig.EventMonitoringMethod.EventLogFile.isEnabled() { + eventLogFileTicker = time.NewTicker(s.srcConfig.EventMonitoringMethod.EventLogFile.Interval) + defer eventLogFileTicker.Stop() + } + + if s.srcConfig.EventMonitoringMethod.Object.isEnabled() { + objectMethodTicker = time.NewTicker(s.srcConfig.EventMonitoringMethod.Object.Interval) + defer objectMethodTicker.Stop() + } + + for { + // Always check for cancel first, to not accidentally trigger another + // run if the context is already cancelled, but we have already received + // another ticker making the channel ready. + select { + case <-s.ctx.Done(): + return s.isError(s.ctx.Err()) + default: + } + + select { + case <-s.ctx.Done(): + return s.isError(s.ctx.Err()) + case <-eventLogFileTicker.C: + if err := s.RunEventLogFile(); err != nil { + s.log.Errorf("Problem running EventLogFile collection: %s", err) + } + case <-objectMethodTicker.C: + if err := s.RunObject(); err != nil { + s.log.Errorf("Problem running Object collection: %s", err) + } + } + } +} + +func (s *salesforceInput) isError(err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + s.log.Infof("input stopped because context was cancelled with: %v", err) + return nil + } + + return err +} + +func (s *salesforceInput) SetupSFClientConnection() (*soql.Resource, error) { + if s.sfdcConfig == nil { + return nil, errors.New("internal error: salesforce configuration is not set properly") + } + + // Open creates a session using the configuration. + session, err := session.Open(*s.sfdcConfig) + if err != nil { + return nil, err + } + + // Set clientSession for re-use. + s.clientSession = session + + // Create a new SOQL resource using the session. + soqlr, err := soql.NewResource(session) + if err != nil { + return nil, fmt.Errorf("error setting up salesforce SOQL resource: %w", err) + } + return soqlr, nil +} + +// FormQueryWithCursor takes a queryConfig and a cursor and returns a querier. +func (s *salesforceInput) FormQueryWithCursor(queryConfig *QueryConfig, cursor mapstr.M) (*querier, error) { + qr, err := parseCursor(queryConfig, cursor, s.log) + if err != nil { + return nil, err + } + + s.log.Infof("Salesforce query: %s", qr) + + return &querier{Query: qr}, err +} + +// isZero checks if the given value v is the zero value for its type. +// It compares v to the zero value obtained by new(T). +func isZero[T comparable](v T) bool { + return v == *new(T) +} + +// RunObject runs the Object method of the Event Monitoring API to collect events. +func (s *salesforceInput) RunObject() error { + s.log.Debugf("scrape object(s) every %s", s.srcConfig.EventMonitoringMethod.Object.Interval) + + var cursor mapstr.M + if !(isZero(s.cursor.Object.FirstEventTime) && isZero(s.cursor.Object.LastEventTime)) { + object := make(mapstr.M) + if !isZero(s.cursor.Object.FirstEventTime) { + object.Put("first_event_time", s.cursor.Object.FirstEventTime) + } + if !isZero(s.cursor.Object.LastEventTime) { + object.Put("last_event_time", s.cursor.Object.LastEventTime) + } + cursor = mapstr.M{"object": object} + } + + query, err := s.FormQueryWithCursor(s.config.EventMonitoringMethod.Object.Query, cursor) + if err != nil { + return fmt.Errorf("error forming query based on cursor: %w", err) + } + + res, err := s.soqlr.Query(query, false) + if err != nil { + return err + } + + totalEvents := 0 + firstEvent := true + + for res.TotalSize() > 0 { + for _, rec := range res.Records() { + val := rec.Record().Fields() + + jsonStrEvent, err := json.Marshal(val) + if err != nil { + return err + } + + if timestamp, ok := val[s.config.EventMonitoringMethod.Object.Cursor.Field].(string); ok { + if firstEvent { + s.cursor.Object.FirstEventTime = timestamp + } + s.cursor.Object.LastEventTime = timestamp + } + + err = publishEvent(s.publisher, s.cursor, jsonStrEvent, "Object") + if err != nil { + return err + } + firstEvent = false + totalEvents++ + } + + if !res.MoreRecords() { // returns true if there are more records. + break + } + + res, err = res.Next() + if err != nil { + return err + } + } + s.log.Debugf("Total events: %d", totalEvents) + + return nil +} + +// RunEventLogFile runs the EventLogFile method of the Event Monitoring API to +// collect events. +func (s *salesforceInput) RunEventLogFile() error { + s.log.Debugf("scrape eventLogFile(s) every %s", s.srcConfig.EventMonitoringMethod.EventLogFile.Interval) + + var cursor mapstr.M + if !(isZero(s.cursor.Object.FirstEventTime) && isZero(s.cursor.Object.LastEventTime)) { + eventLogFile := make(mapstr.M) + if !isZero(s.cursor.Object.FirstEventTime) { + eventLogFile.Put("first_event_time", s.cursor.EventLogFile.FirstEventTime) + } + if !isZero(s.cursor.Object.LastEventTime) { + eventLogFile.Put("last_event_time", s.cursor.EventLogFile.LastEventTime) + } + cursor = mapstr.M{"event_log_file": eventLogFile} + } + + query, err := s.FormQueryWithCursor(s.config.EventMonitoringMethod.EventLogFile.Query, cursor) + if err != nil { + return fmt.Errorf("error forming query based on cursor: %w", err) + } + + res, err := s.soqlr.Query(query, false) + if err != nil { + return err + } + + // NOTE: This is a failsafe check because the HTTP client is always set. + // This check allows unit tests to verify correct behavior when the HTTP + // client is nil. + if s.sfdcConfig.Client == nil { + return errors.New("internal error: salesforce configuration is not set properly") + } + + totalEvents, firstEvent := 0, true + for res.TotalSize() > 0 { + for _, rec := range res.Records() { + req, err := http.NewRequestWithContext(s.ctx, http.MethodGet, s.config.URL+rec.Record().Fields()["LogFile"].(string), nil) + if err != nil { + return err + } + + s.clientSession.AuthorizationHeader(req) + + // NOTE: If we ever see a production issue relaated to this, then only + // we should consider adding the header: "X-PrettyPrint:1" + // + // // NOTE: X-PrettyPrint:1 is for formatted response and ideally we do + // // not need it. But see: + // // https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/dome_event_log_file_download.htm?q=X-PrettyPrint%3A1 + // req.Header.Add("X-PrettyPrint", "1") + + resp, err := s.sfdcConfig.Client.Do(req) + if err != nil { + return err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + resp.Body.Close() + return err + } + resp.Body.Close() + + recs, err := decodeAsCSV(body) + if err != nil { + return err + } + + if timestamp, ok := rec.Record().Fields()[s.config.EventMonitoringMethod.EventLogFile.Cursor.Field].(string); ok { + if firstEvent { + s.cursor.EventLogFile.FirstEventTime = timestamp + } + s.cursor.EventLogFile.LastEventTime = timestamp + } + + for _, val := range recs { + jsonStrEvent, err := json.Marshal(val) + if err != nil { + return err + } + + err = publishEvent(s.publisher, s.cursor, jsonStrEvent, "EventLogFile") + if err != nil { + return err + } + totalEvents++ + } + firstEvent = false + } + + if !res.MoreRecords() { + break + } + + res, err = res.Next() + if err != nil { + return err + } + } + s.log.Debugf("Total events: %d", totalEvents) + + return nil +} + +// getSFDCConfig returns a new Salesforce configuration based on the configuration. +func (s *salesforceInput) getSFDCConfig(cfg *config) (*sfdc.Configuration, error) { + var ( + creds *credentials.Credentials + err error + ) + + if cfg.Auth == nil { + return nil, errors.New("no auth provider enabled") + } + + switch { + case cfg.Auth.OAuth2.JWTBearerFlow != nil && cfg.Auth.OAuth2.JWTBearerFlow.isEnabled(): + pemBytes, err := os.ReadFile(cfg.Auth.OAuth2.JWTBearerFlow.ClientKeyPath) + if err != nil { + return nil, fmt.Errorf("problem with client key path for JWT auth: %w", err) + } + + signKey, err := jwt.ParseRSAPrivateKeyFromPEM(pemBytes) + if err != nil { + return nil, fmt.Errorf("problem with client key for JWT auth: %w", err) + } + + passCreds := credentials.JwtCredentials{ + URL: cfg.Auth.OAuth2.JWTBearerFlow.URL, + ClientId: cfg.Auth.OAuth2.JWTBearerFlow.ClientID, + ClientUsername: cfg.Auth.OAuth2.JWTBearerFlow.ClientUsername, + ClientKey: signKey, + } + + creds, err = credentials.NewJWTCredentials(passCreds) + if err != nil { + return nil, fmt.Errorf("error creating jwt credentials: %w", err) + } + + case cfg.Auth.OAuth2.UserPasswordFlow != nil && cfg.Auth.OAuth2.UserPasswordFlow.isEnabled(): + passCreds := credentials.PasswordCredentials{ + URL: cfg.Auth.OAuth2.UserPasswordFlow.TokenURL, + Username: cfg.Auth.OAuth2.UserPasswordFlow.Username, + Password: cfg.Auth.OAuth2.UserPasswordFlow.Password, + ClientID: cfg.Auth.OAuth2.UserPasswordFlow.ClientID, + ClientSecret: cfg.Auth.OAuth2.UserPasswordFlow.ClientSecret, + } + + creds, err = credentials.NewPasswordCredentials(passCreds) + if err != nil { + return nil, fmt.Errorf("error creating password credentials: %w", err) + } + + } + + client, err := newClient(*cfg, s.log) + if err != nil { + return nil, fmt.Errorf("problem with client: %w", err) + } + + return &sfdc.Configuration{ + Credentials: creds, + Client: client, + Version: cfg.Version, + }, nil +} + +// retryLog is a shim for the retryablehttp.Client.Logger. +type retryLog struct{ log *logp.Logger } + +func newRetryLog(log *logp.Logger) *retryLog { + return &retryLog{log: log.Named("retryablehttp").WithOptions(zap.AddCallerSkip(1))} +} + +func (l *retryLog) Error(msg string, kv ...interface{}) { l.log.Errorw(msg, kv...) } +func (l *retryLog) Info(msg string, kv ...interface{}) { l.log.Infow(msg, kv...) } +func (l *retryLog) Debug(msg string, kv ...interface{}) { l.log.Debugw(msg, kv...) } +func (l *retryLog) Warn(msg string, kv ...interface{}) { l.log.Warnw(msg, kv...) } + +// retryErrorHandler returns a retryablehttp.ErrorHandler that will log retry resignation +// but return the last retry attempt's response and a nil error to allow the retryablehttp.Client +// evaluate the response status itself. Any error passed to the retryablehttp.ErrorHandler +// is returned unaltered. +func retryErrorHandler(max int, log *logp.Logger) retryablehttp.ErrorHandler { + return func(resp *http.Response, err error, numTries int) (*http.Response, error) { + log.Warnw("giving up retries", "method", resp.Request.Method, "url", resp.Request.URL, "retries", max+1) + return resp, err + } +} + +func newClient(cfg config, log *logp.Logger) (*http.Client, error) { + c, err := cfg.Resource.Transport.Client() + if err != nil { + return nil, err + } + + if maxAttempts := cfg.Resource.Retry.getMaxAttempts(); maxAttempts > 1 { + c = (&retryablehttp.Client{ + HTTPClient: c, + Logger: newRetryLog(log), + RetryWaitMin: cfg.Resource.Retry.getWaitMin(), + RetryWaitMax: cfg.Resource.Retry.getWaitMax(), + RetryMax: maxAttempts, + CheckRetry: retryablehttp.DefaultRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, + ErrorHandler: retryErrorHandler(maxAttempts, log), + }).StandardClient() + + // BUG: retryablehttp ignores the timeout previously set. So, setting it + // again. + c.Timeout = cfg.Resource.Transport.Timeout + } + + return c, nil +} + +// publishEvent publishes an event using the configured publisher pub. +func publishEvent(pub inputcursor.Publisher, cursor *state, jsonStrEvent []byte, dataCollectionMethod string) error { + event := beat.Event{ + Timestamp: timeNow(), + Fields: mapstr.M{ + "message": string(jsonStrEvent), + "event": mapstr.M{ + "provider": dataCollectionMethod, + }, + }, + } + + return pub.Publish(event, cursor) +} + +type textContextError struct { + error + body []byte +} + +// decodeAsCSV decodes p as a headed CSV document into dst. +func decodeAsCSV(p []byte) ([]map[string]string, error) { + r := csv.NewReader(bytes.NewReader(p)) + + // To share the backing array for performance. + r.ReuseRecord = true + + // Header row is always expected, otherwise we can't map values to keys in + // the event. + header, err := r.Read() + if err != nil { + if err == io.EOF { //nolint:errorlint // csv.Reader never wraps io.EOF. + return nil, nil + } + return nil, err + } + + // As buffer reuse is enabled, copying header is important. + header = slices.Clone(header) + + var results []map[string]string //nolint:prealloc // not sure about the size to prealloc with + + // NOTE: + // + // Read sets `r.FieldsPerRecord` to the number of fields in the first record, + // so that future records must have the same field count. + // So, if len(header) != len(event), the Read will return an error and hence + // we need not put an explicit check. + event, err := r.Read() + for ; err == nil; event, err = r.Read() { + if err != nil { + continue + } + o := make(map[string]string, len(header)) + for i, h := range header { + o[h] = event[i] + } + results = append(results, o) + } + + if err != nil { + if err != io.EOF { //nolint:errorlint // csv.Reader never wraps io.EOF. + return nil, textContextError{error: err, body: p} + } + } + + return results, nil +} diff --git a/x-pack/filebeat/input/salesforce/input_manager.go b/x-pack/filebeat/input/salesforce/input_manager.go new file mode 100644 index 000000000000..1fb0ae42e916 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/input_manager.go @@ -0,0 +1,89 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "fmt" + "time" + + "github.com/elastic/go-concert/unison" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/transport/httpcommon" +) + +// compile-time check if querier implements InputManager +var _ v2.InputManager = InputManager{} + +// InputManager wraps one stateless input manager +// and one cursor input manager. It will create one or the other +// based on the config that is passed. +type InputManager struct { + cursor *inputcursor.InputManager +} + +// NewInputManager creates a new input manager. +func NewInputManager(log *logp.Logger, store inputcursor.StateStore) InputManager { + return InputManager{ + cursor: &inputcursor.InputManager{ + Logger: log, + StateStore: store, + Type: inputName, + Configure: cursorConfigure, + }, + } +} + +func defaultConfig() config { + apiVersion := 58 + maxAttempts := 5 + waitMin := time.Second + waitMax := time.Minute + transport := httpcommon.DefaultHTTPTransportSettings() + transport.Timeout = 30 * time.Second + + return config{ + Version: apiVersion, + Resource: &resourceConfig{ + Transport: transport, + Retry: retryConfig{ + MaxAttempts: &maxAttempts, + WaitMin: &waitMin, + WaitMax: &waitMax, + }, + }, + } +} + +// cursorConfigure configures the cursor input manager. +func cursorConfigure(cfg *conf.C) ([]inputcursor.Source, inputcursor.Input, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, nil, fmt.Errorf("reading config: %w", err) + } + sources := []inputcursor.Source{&source{cfg: config}} + return sources, &salesforceInput{config: config}, nil +} + +type source struct{ cfg config } + +func (s *source) Name() string { return s.cfg.URL } + +// Init initializes both wrapped input managers. +func (m InputManager) Init(grp unison.Group, mode v2.Mode) error { + return m.cursor.Init(grp, mode) +} + +// Create creates a cursor input manager. +func (m InputManager) Create(cfg *conf.C) (v2.Input, error) { + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + return m.cursor.Create(cfg) +} diff --git a/x-pack/filebeat/input/salesforce/input_manager_test.go b/x-pack/filebeat/input/salesforce/input_manager_test.go new file mode 100644 index 000000000000..90647e9d302b --- /dev/null +++ b/x-pack/filebeat/input/salesforce/input_manager_test.go @@ -0,0 +1,82 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + v2 "github.com/elastic/beats/v7/filebeat/input/v2" + cursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/beats/v7/libbeat/statestore/storetest" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/go-concert/unison" +) + +func makeTestStore(data map[string]interface{}) *statestore.Store { + memstore := &storetest.MapStore{Table: data} + reg := statestore.NewRegistry(&storetest.MemoryStore{ + Stores: map[string]*storetest.MapStore{ + "test": memstore, + }, + }) + store, err := reg.Get("test") + if err != nil { + panic("failed to create test store") + } + return store +} + +type stateStore struct{} + +func (stateStore) Access() (*statestore.Store, error) { + return makeTestStore(map[string]interface{}{"hello": "world"}), nil +} +func (stateStore) CleanupInterval() time.Duration { return time.Duration(0) } + +// compile-time check if stateStore implements cursor.StateStore +var _ cursor.StateStore = stateStore{} + +func TestInputManager(t *testing.T) { + inputManager := NewInputManager(logp.NewLogger("salesforce_test"), stateStore{}) + + var inputTaskGroup unison.TaskGroup + defer inputTaskGroup.Stop() //nolint:errcheck // ignore error in test + + err := inputManager.Init(&inputTaskGroup, v2.ModeRun) + assert.NoError(t, err) + + config, err := conf.NewConfigFrom(map[string]interface{}{ + "url": "https://salesforce.com", + "version": 46, + "auth": &authConfig{ + OAuth2: &OAuth2{JWTBearerFlow: &JWTBearerFlow{ + Enabled: pointer(true), + URL: "https://salesforce.com", + ClientID: "xyz", + ClientUsername: "xyz", + ClientKeyPath: "xyz", + }}, + }, + "event_monitoring_method": &eventMonitoringMethod{ + Object: EventMonitoringConfig{Enabled: pointer(true), Interval: 4}, + }, + }) + assert.NoError(t, err) + + _, err = inputManager.Create(config) + assert.NoError(t, err) +} + +func TestSource(t *testing.T) { + want := "https://salesforce.com" + src := source{cfg: config{URL: want}} + got := src.Name() + assert.Equal(t, want, got) +} diff --git a/x-pack/filebeat/input/salesforce/input_test.go b/x-pack/filebeat/input/salesforce/input_test.go new file mode 100644 index 000000000000..a579036e4d88 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/input_test.go @@ -0,0 +1,803 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "sync" + "testing" + "time" + + "github.com/g8rswimmer/go-sfdc" + "github.com/g8rswimmer/go-sfdc/soql" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" + + inputcursor "github.com/elastic/beats/v7/filebeat/input/v2/input-cursor" + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" + conf "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/transport/httpcommon" +) + +const ( + PaginationFlow = "PaginationFlow" + NoPaginationFlow = "NoPaginationFlow" + IntervalFlow = "IntervalFlow" + BadReponseFlow = "BadReponseFlow" + + defaultLoginObjectQuery = "SELECT FIELDS(STANDARD) FROM LoginEvent" + valueLoginObjectQuery = "SELECT FIELDS(STANDARD) FROM LoginEvent WHERE EventDate > [[ .cursor.object.first_event_time ]]" + defaultLoginObjectQueryWithCursor = "SELECT FIELDS(STANDARD) FROM LoginEvent WHERE EventDate > 2023-12-06T05:44:24.973+0000" + + defaultLoginEventLogFileQuery = "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' ORDER BY CreatedDate ASC NULLS FIRST" + valueLoginEventLogFileQuery = "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event_time ]] ORDER BY CreatedDate ASC NULLS FIRST" + + invalidDefaultLoginEventObjectQuery = "SELECT FIELDS(STANDARD) FROM LoginEvnt" + invalidDefaultLoginEventLogFileQuery = "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' ORDER BY ASC NULLS FIRST" + + invalidValueLoginObjectQuery = "SELECT FIELDS(STANDARD) FROM LoginEvent WHERE EventDate > [[ .cursor.object.first_event ]]" + invalidValueLoginEventLogFileQuery = "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.last_event ]] ORDER BY CreatedDate ASC NULLS FIRST" + + oneEventLogfileFirstResponseJSON = `{ "totalSize": 1, "done": true, "records": [ { "attributes": { "type": "EventLogFile", "url": "/services/data/v58.0/sobjects/EventLogFile/0AT5j00002LqQTxGAN" }, "Id": "0AT5j00002LqQTxGAN", "CreatedDate": "2023-12-19T21:04:35.000+0000", "LogDate": "2023-12-18T00:00:00.000+0000", "LogFile": "/services/data/v58.0/sobjects/EventLogFile/0AT5j00002LqQTxGAN/LogFile" } ] }` + oneEventLogfileSecondResponseCSV = `"EVENT_TYPE","TIMESTAMP","REQUEST_ID","ORGANIZATION_ID","USER_ID","RUN_TIME","CPU_TIME","URI","SESSION_KEY","LOGIN_KEY","USER_TYPE","REQUEST_STATUS","DB_TOTAL_TIME","LOGIN_TYPE","BROWSER_TYPE","API_TYPE","API_VERSION","USER_NAME","TLS_PROTOCOL","CIPHER_SUITE","AUTHENTICATION_METHOD_REFERENCE","LOGIN_SUB_TYPE","TIMESTAMP_DERIVED","USER_ID_DERIVED","CLIENT_IP","URI_ID_DERIVED","LOGIN_STATUS","SOURCE_IP" +"Login","20231218054831.655","4u6LyuMrDvb_G-l1cJIQk-","00D5j00000DgAYG","0055j00000AT6I1","1219","127","/services/oauth2/token","","bY5Wfv8t/Ith7WVE","Standard","","1051271151","i","Go-http-client/1.1","","9998.0","salesforceinstance@devtest.in","TLSv1.2","ECDHE-RSA-AES256-GCM-SHA384","","","2023-12-18T05:48:31.655Z","0055j00000AT6I1AAL","Salesforce.com IP","","LOGIN_NO_ERROR","103.108.207.58" +` + + expectedELFEvent = `{"API_TYPE":"","API_VERSION":"9998.0","AUTHENTICATION_METHOD_REFERENCE":"","BROWSER_TYPE":"Go-http-client/1.1","CIPHER_SUITE":"ECDHE-RSA-AES256-GCM-SHA384","CLIENT_IP":"Salesforce.com IP","CPU_TIME":"127","DB_TOTAL_TIME":"1051271151","EVENT_TYPE":"Login","LOGIN_KEY":"bY5Wfv8t/Ith7WVE","LOGIN_STATUS":"LOGIN_NO_ERROR","LOGIN_SUB_TYPE":"","LOGIN_TYPE":"i","ORGANIZATION_ID":"00D5j00000DgAYG","REQUEST_ID":"4u6LyuMrDvb_G-l1cJIQk-","REQUEST_STATUS":"","RUN_TIME":"1219","SESSION_KEY":"","SOURCE_IP":"103.108.207.58","TIMESTAMP":"20231218054831.655","TIMESTAMP_DERIVED":"2023-12-18T05:48:31.655Z","TLS_PROTOCOL":"TLSv1.2","URI":"/services/oauth2/token","URI_ID_DERIVED":"","USER_ID":"0055j00000AT6I1","USER_ID_DERIVED":"0055j00000AT6I1AAL","USER_NAME":"salesforceinstance@devtest.in","USER_TYPE":"Standard"}` + + oneObjectEvents = `{ "totalSize": 1, "done": true, "records": [ { "attributes": { "type": "LoginEvent", "url": "/services/data/v58.0/sobjects/LoginEvent/000000000000000AAA" }, "AdditionalInfo": "{}", "ApiType": "N/A", "ApiVersion": "N/A", "Application": "salesforce_test", "Browser": "Unknown", "CipherSuite": "ECDHE-RSA-AES256-GCM-SHA384", "City": "Mumbai", "ClientVersion": "N/A", "Country": "India", "CountryIso": "IN", "CreatedDate": "2023-12-06T05:44:34.942+0000", "EvaluationTime": 0, "EventDate": "2023-12-06T05:44:24.973+0000", "EventIdentifier": "00044326-ed4a-421a-a0a8-e62ea626f3af", "HttpMethod": "POST", "Id": "000000000000000AAA", "LoginGeoId": "04F5j00003NvV1cEAF", "LoginHistoryId": "0Ya5j00003k2scQCAQ", "LoginKey": "pgOVoLbV96U9o08W", "LoginLatitude": 19.0748, "LoginLongitude": 72.8856, "LoginType": "Remote Access 2.0", "LoginUrl": "login.salesforce.com", "Platform": "Unknown", "PostalCode": "400070", "SessionLevel": "STANDARD", "SourceIp": "134.238.252.19", "Status": "Success", "Subdivision": "Maharashtra", "TlsProtocol": "TLS 1.2", "UserId": "0055j00000AT6I1AAL", "UserType": "Standard", "Username": "salesforceinstance@devtest.in" } ] }` + oneObjectEventsPageOne = `{ "totalSize": 1, "done": true, "nextRecordsUrl": "/nextRecords/LoginEvents/ABCABCDABCDE", "records": [ { "attributes": { "type": "LoginEvent", "url": "/services/data/v58.0/sobjects/LoginEvent/000000000000000AAA" }, "AdditionalInfo": "{}", "ApiType": "N/A", "ApiVersion": "N/A", "Application": "salesforce_test", "Browser": "Unknown", "CipherSuite": "ECDHE-RSA-AES256-GCM-SHA384", "City": "Mumbai", "ClientVersion": "N/A", "Country": "India", "CountryIso": "IN", "CreatedDate": "2023-12-06T05:44:34.942+0000", "EvaluationTime": 0, "EventDate": "2023-12-06T05:44:24.973+0000", "EventIdentifier": "00044326-ed4a-421a-a0a8-e62ea626f3af", "HttpMethod": "POST", "Id": "000000000000000AAA", "LoginGeoId": "04F5j00003NvV1cEAF", "LoginHistoryId": "0Ya5j00003k2scQCAQ", "LoginKey": "pgOVoLbV96U9o08W", "LoginLatitude": 19.0748, "LoginLongitude": 72.8856, "LoginType": "Remote Access 2.0", "LoginUrl": "login.salesforce.com", "Platform": "Unknown", "PostalCode": "400070", "SessionLevel": "STANDARD", "SourceIp": "134.238.252.19", "Status": "Success", "Subdivision": "Maharashtra", "TlsProtocol": "TLS 1.2", "UserId": "0055j00000AT6I1AAL", "UserType": "Standard", "Username": "salesforceinstance@devtest.in" } ] }` + oneObjectEventsPageTwo = `{ "totalSize": 1, "done": true, "records": [ { "attributes": { "type": "LoginEvent", "url": "/services/data/v58.0/sobjects/LoginEvent/000000000000000AAA" }, "AdditionalInfo": "{}", "ApiType": "N/A", "ApiVersion": "N/A", "Application": "salesforce_test", "Browser": "Unknown", "CipherSuite": "ECDHE-RSA-AES256-GCM-SHA384", "City": "Mumbai", "ClientVersion": "N/A", "Country": "India", "CountryIso": "IN", "CreatedDate": "2023-12-06T05:44:34.942+0000", "EvaluationTime": 0, "EventDate": "2023-12-06T05:44:24.973+0000", "EventIdentifier": "00044326-ed4a-421a-a0a8-e62ea626f3af", "HttpMethod": "POST", "Id": "000000000000000AAA", "LoginGeoId": "04F5j00003NvV1cEAF", "LoginHistoryId": "0Ya5j00003k2scQCAQ", "LoginKey": "pgOVoLbV96U9o08W", "LoginLatitude": 19.0748, "LoginLongitude": 72.8856, "LoginType": "Remote Access 2.0", "LoginUrl": "login.salesforce.com", "Platform": "Unknown", "PostalCode": "400070", "SessionLevel": "STANDARD", "SourceIp": "134.238.252.19", "Status": "Success", "Subdivision": "Maharashtra", "TlsProtocol": "TLS 1.2", "UserId": "0055j00000AT6I1AAL", "UserType": "Standard", "Username": "salesforceinstance@devtest.in" } ] }` + + expectedObjectEvent = `{"AdditionalInfo":"{}","ApiType":"N/A","ApiVersion":"N/A","Application":"salesforce_test","Browser":"Unknown","CipherSuite":"ECDHE-RSA-AES256-GCM-SHA384","City":"Mumbai","ClientVersion":"N/A","Country":"India","CountryIso":"IN","CreatedDate":"2023-12-06T05:44:34.942+0000","EvaluationTime":0,"EventDate":"2023-12-06T05:44:24.973+0000","EventIdentifier":"00044326-ed4a-421a-a0a8-e62ea626f3af","HttpMethod":"POST","Id":"000000000000000AAA","LoginGeoId":"04F5j00003NvV1cEAF","LoginHistoryId":"0Ya5j00003k2scQCAQ","LoginKey":"pgOVoLbV96U9o08W","LoginLatitude":19.0748,"LoginLongitude":72.8856,"LoginType":"Remote Access 2.0","LoginUrl":"login.salesforce.com","Platform":"Unknown","PostalCode":"400070","SessionLevel":"STANDARD","SourceIp":"134.238.252.19","Status":"Success","Subdivision":"Maharashtra","TlsProtocol":"TLS 1.2","UserId":"0055j00000AT6I1AAL","UserType":"Standard","Username":"salesforceinstance@devtest.in"}` +) + +func TestFormQueryWithCursor(t *testing.T) { + logp.TestingSetup() + + mockTimeNow(time.Date(2023, time.May, 18, 12, 0, 0, 0, time.UTC)) + t.Cleanup(resetTimeNow) + + tests := map[string]struct { + wantErr error + cursor mapstr.M + defaultSOQLTemplate string + valueSOQLTemplate string + wantQuery string + initialInterval time.Duration + }{ + "valid soql templates with nil cursor": { // expect default query with LogDate > initialInterval + initialInterval: 60 * 24 * time.Hour, // 60 * 24h = 1440h = 60 days = 2 months + defaultSOQLTemplate: `SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND Logdate > [[ (formatTime (now.Add (parseDuration "-1440h")) "RFC3339") ]] ORDER BY CreatedDate ASC NULLS FIRST`, + valueSOQLTemplate: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.logdate ]] ORDER BY CreatedDate ASC NULLS FIRST", + wantQuery: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND Logdate > 2023-03-19T12:00:00Z ORDER BY CreatedDate ASC NULLS FIRST", + cursor: nil, + }, + "valid soql templates with non-empty .cursor.object.logdate": { // expect value SOQL query with .cursor.object.logdate set + initialInterval: 60 * 24 * time.Hour, // 60 * 24h = 1440h = 60 days = 2 months + defaultSOQLTemplate: `SELECT Id,CreatedDate,LogDate,LogFile FROM LoginEvent WHERE EventDate > [[ (formatTime (now.Add (parseDuration "-1440h")) "RFC3339") ]]`, + valueSOQLTemplate: "SELECT Id,CreatedDate,LogDate,LogFile FROM LoginEvent WHERE CreatedDate > [[ .cursor.object.logdate ]]", + wantQuery: "SELECT Id,CreatedDate,LogDate,LogFile FROM LoginEvent WHERE CreatedDate > 2023-05-18T12:00:00Z", + cursor: mapstr.M{"object": mapstr.M{"logdate": timeNow().Format(formatRFC3339Like)}}, + }, + "valid soql templates with non-empty .cursor.event_log_file.logdate": { // expect value SOQL query with .cursor.event_log_file.logdate set + initialInterval: 60 * 24 * time.Hour, // 60 * 24h = 1440h = 60 days = 2 months + defaultSOQLTemplate: `SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND Logdate > [[ (formatTime (now.Add (parseDuration "-1440h")) "RFC3339") ]] ORDER BY CreatedDate ASC NULLS FIRST`, + valueSOQLTemplate: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.logdate ]] ORDER BY CreatedDate ASC NULLS FIRST", + wantQuery: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > 2023-05-18T12:00:00Z ORDER BY CreatedDate ASC NULLS FIRST", + cursor: mapstr.M{"event_log_file": mapstr.M{"logdate": timeNow().Format(formatRFC3339Like)}}, + }, + "invalid soql templates wrong cursor name .cursor.event_log_file.logdate1": { // expect value SOQL query with .cursor.event_log_file.logdate set + initialInterval: 60 * 24 * time.Hour, // 60 * 24h = 1440h = 60 days = 2 months + defaultSOQLTemplate: `SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND Logdate > [[ (formatTime (now.Add (parseDuration "-1440h")) "RFC3339") ]] ORDER BY CreatedDate ASC NULLS FIRST`, + valueSOQLTemplate: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > [[ .cursor.event_log_file.logdate1 ]] ORDER BY CreatedDate ASC NULLS FIRST", + wantQuery: "SELECT Id,CreatedDate,LogDate,LogFile FROM EventLogFile WHERE EventType = 'Login' AND CreatedDate > 2023-05-18T12:00:00Z ORDER BY CreatedDate ASC NULLS FIRST", + cursor: mapstr.M{"event_log_file": mapstr.M{"logdate": timeNow().Format(formatRFC3339Like)}}, + wantErr: errors.New(`template: :1:110: executing "" at <.cursor.event_log_file.logdate1>: map has no entry for key "logdate1"`), + }, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + v1, v2 := &valueTpl{}, &valueTpl{} + + err := v1.Unpack(tc.defaultSOQLTemplate) + assert.NoError(t, err) + + err = v2.Unpack(tc.valueSOQLTemplate) + assert.NoError(t, err) + + queryConfig := &QueryConfig{ + Default: v1, + Value: v2, + } + + sfInput := &salesforceInput{ + config: config{}, + log: logp.NewLogger("salesforce_test"), + } + + querier, err := sfInput.FormQueryWithCursor(queryConfig, tc.cursor) + if fmt.Sprint(tc.wantErr) != fmt.Sprint(err) { + t.Errorf("got error %v, want error %v", err, tc.wantErr) + } + if tc.wantErr != nil { + return + } + + assert.EqualValues(t, tc.wantQuery, querier.Query) + }) + } +} + +var ( + defaultUserPasswordFlowMap = map[string]interface{}{ + "user_password_flow": map[string]interface{}{ + "enabled": true, + "client.id": "clientid", + "client.secret": "clientsecret", + "token_url": "https://instance_id.develop.my.salesforce.com/services/oauth2/token", + "username": "username", + "password": "password", + }, + } + wrongUserPasswordFlowMap = map[string]interface{}{ + "user_password_flow": map[string]interface{}{ + "enabled": true, + "client.id": "clientid-wrong", + "client.secret": "clientsecret-wrong", + "token_url": "https://instance_id.develop.my.salesforce.com/services/oauth2/token", + "username": "username-wrong", + "password": "password-wrong", + }, + } + + defaultObjectMonitoringMethodConfigMap = map[string]interface{}{ + "interval": "5s", + "enabled": true, + "query": map[string]interface{}{ + "default": defaultLoginObjectQuery, + "value": valueLoginObjectQuery, + }, + "cursor": map[string]interface{}{ + "field": "EventDate", + }, + } + defaultEventLogFileMonitoringMethodMap = map[string]interface{}{ + "interval": "5s", + "enabled": true, + "query": map[string]interface{}{ + "default": defaultLoginEventLogFileQuery, + "value": valueLoginEventLogFileQuery, + }, + "cursor": map[string]interface{}{ + "field": "CreatedDate", + }, + } + + invalidObjectMonitoringMethodMap = map[string]interface{}{ + "interval": "5m", + "enabled": true, + "query": map[string]interface{}{ + "default": invalidDefaultLoginEventObjectQuery, + "value": valueLoginEventLogFileQuery, + }, + "cursor": map[string]interface{}{ + "field": "CreatedDate", + }, + } + invalidEventLogFileMonitoringMethodMap = map[string]interface{}{ + "interval": "5m", + "enabled": true, + "query": map[string]interface{}{ + "default": invalidDefaultLoginEventLogFileQuery, + "value": invalidValueLoginEventLogFileQuery, + }, + "cursor": map[string]interface{}{ + "field": "CreatedDate", + }, + } +) + +func TestInput(t *testing.T) { + logp.TestingSetup() + + tests := []struct { + setupServer func(testing.TB, http.HandlerFunc, map[string]interface{}) + baseConfig map[string]interface{} + handler http.HandlerFunc + persistentCursor *state + name string + expected []string + timeout time.Duration + wantErr bool + AuthFail bool + }{ + // Object + { + name: "Positive/event_monitoring_method_object_with_default_query_only", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "object": defaultObjectMonitoringMethodConfigMap, + }, + }, + handler: defaultHandler(NoPaginationFlow, false, "", oneObjectEvents), + expected: []string{expectedObjectEvent}, + }, + { + name: "Negative/event_monitoring_method_object_with_error_in_data_collection", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "object": invalidObjectMonitoringMethodMap, + }, + }, + handler: defaultHandler(NoPaginationFlow, false, "", `{"error": "invalid_query"}`), + wantErr: true, + }, + { + name: "Positive/event_monitoring_method_object_with_interval_5s", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "object": defaultObjectMonitoringMethodConfigMap, + }, + }, + handler: defaultHandler(IntervalFlow, false, "", oneObjectEventsPageTwo), + expected: []string{expectedObjectEvent, expectedObjectEvent}, + timeout: 20 * time.Second, + }, + { + name: "Positive/event_monitoring_method_object_with_Pagination", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "object": defaultObjectMonitoringMethodConfigMap, + }, + }, + handler: defaultHandler(PaginationFlow, false, oneObjectEventsPageOne, oneObjectEventsPageTwo), + expected: []string{expectedObjectEvent, expectedObjectEvent}, + }, + + // EventLogFile + { + name: "Positive/event_monitoring_method_elf_with_default_query_only", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "event_log_file": defaultEventLogFileMonitoringMethodMap, + }, + }, + handler: defaultHandler(NoPaginationFlow, false, oneEventLogfileFirstResponseJSON, oneEventLogfileSecondResponseCSV), + expected: []string{expectedELFEvent}, + }, + { + name: "Negative/event_monitoring_method_elf_with_error_in_auth", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": wrongUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "event_log_file": defaultEventLogFileMonitoringMethodMap, + }, + }, + handler: defaultHandler(NoPaginationFlow, false, "", `{"error": "invalid_client_id"}`), + wantErr: true, + AuthFail: true, + }, + { + name: "Negative/event_monitoring_method_elf_with_error_in_data_collection", + setupServer: newTestServer(httptest.NewServer), + baseConfig: map[string]interface{}{ + "version": 56, + "auth.oauth2": defaultUserPasswordFlowMap, + "event_monitoring_method": map[string]interface{}{ + "event_log_file": invalidEventLogFileMonitoringMethodMap, + }, + }, + handler: defaultHandler(NoPaginationFlow, false, "", `{"error": "invalid_query"}`), + wantErr: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.setupServer(t, tc.handler, tc.baseConfig) + + cfg := defaultConfig() + err := conf.MustNewConfigFrom(tc.baseConfig).Unpack(&cfg) + assert.NoError(t, err) + timeout := 5 * time.Second + if tc.timeout != 0 { + timeout = tc.timeout + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + var client publisher + client.done = func() { + if len(client.published) >= len(tc.expected) { + cancel() + } + } + + salesforceInput := salesforceInput{config: cfg} + assert.Equal(t, "salesforce", salesforceInput.Name()) + + ctx, cancelClause := context.WithCancelCause(ctx) + + salesforceInput.cursor = &state{} + if tc.persistentCursor != nil { + salesforceInput.cursor = tc.persistentCursor + } + salesforceInput.ctx = ctx + salesforceInput.cancel = cancelClause + salesforceInput.srcConfig = &cfg + salesforceInput.publisher = &client + salesforceInput.log = logp.L().With("input_url", "salesforce") + + salesforceInput.sfdcConfig, err = salesforceInput.getSFDCConfig(&cfg) + assert.NoError(t, err) + + salesforceInput.soqlr, err = salesforceInput.SetupSFClientConnection() + if err != nil && !tc.wantErr { + t.Errorf("unexpected error from running input: %v", err) + } + if tc.wantErr && tc.AuthFail { + return + } + + err = salesforceInput.run() + if err != nil && !tc.wantErr { + t.Errorf("unexpected error from running input: %v", err) + } + if tc.wantErr { + return + } + + if len(client.published) < len(tc.expected) { + t.Errorf("unexpected number of published events: got:%d want at least:%d", len(client.published), len(tc.expected)) + tc.expected = tc.expected[:len(client.published)] + } + + client.published = client.published[:len(tc.expected)] + for i, got := range client.published { + if !reflect.DeepEqual(got.Fields["message"], tc.expected[i]) { + t.Errorf("unexpected result for event %d: got:- want:+\n%s", i, cmp.Diff(got.Fields, tc.expected[i])) + } + } + }) + } +} + +func defaultHandler(flow string, withoutQuery bool, msg1, msg2 string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("content-type", "application/json") + switch { + case flow == PaginationFlow && r.FormValue("q") == defaultLoginObjectQuery: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(msg1)) + case r.RequestURI == "/nextRecords/LoginEvents/ABCABCDABCDE": + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(msg2)) + case r.RequestURI == "/services/oauth2/token" && r.Method == http.MethodPost && r.FormValue("client_id") == "clientid": + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"access_token":"abcd","instance_url":"http://` + r.Host + `","token_type":"Bearer","id_token":"abcd","refresh_token":"abcd"}`)) + case r.FormValue("client_id") == "clientid-wrong": + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(msg2)) + case r.FormValue("q") == defaultLoginEventLogFileQuery: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(msg1)) + case r.FormValue("q") == defaultLoginObjectQuery, r.FormValue("q") == defaultLoginObjectQueryWithCursor, r.RequestURI == "/services/data/v58.0/sobjects/EventLogFile/0AT5j00002LqQTxGAN/LogFile": + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(msg2)) + case r.FormValue("q") == invalidDefaultLoginEventLogFileQuery, r.FormValue("q") == invalidDefaultLoginEventObjectQuery: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(msg2)) + case flow == BadReponseFlow && (withoutQuery && r.FormValue("q") == ""): + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":"internal server error"}`)) + } + } +} + +func newTestServer(newServer func(http.Handler) *httptest.Server) func(testing.TB, http.HandlerFunc, map[string]interface{}) { + return func(t testing.TB, h http.HandlerFunc, config map[string]interface{}) { + server := newServer(h) + config["url"] = server.URL + config["auth.oauth2"].(map[string]interface{})["user_password_flow"].(map[string]interface{})["token_url"] = server.URL + t.Cleanup(server.Close) + } +} + +var _ inputcursor.Publisher = (*publisher)(nil) + +type publisher struct { + done func() + published []beat.Event + cursors []map[string]interface{} + mu sync.Mutex +} + +func (p *publisher) Publish(e beat.Event, cursor interface{}) error { + p.mu.Lock() + defer p.mu.Unlock() + + p.published = append(p.published, e) + if cursor != nil { + var cv map[string]interface{} + err := typeconv.Convert(&cv, cursor) + if err != nil { + return err + } + + p.cursors = append(p.cursors, cv) + } + p.done() + + return nil +} + +func TestDecodeAsCSV(t *testing.T) { + sampleELF := `"EVENT_TYPE","TIMESTAMP","REQUEST_ID","ORGANIZATION_ID","USER_ID","RUN_TIME","CPU_TIME","URI","SESSION_KEY","LOGIN_KEY","USER_TYPE","REQUEST_STATUS","DB_TOTAL_TIME","LOGIN_TYPE","BROWSER_TYPE","API_TYPE","API_VERSION","USER_NAME","TLS_PROTOCOL","CIPHER_SUITE","AUTHENTICATION_METHOD_REFERENCE","LOGIN_SUB_TYPE","TIMESTAMP_DERIVED","USER_ID_DERIVED","CLIENT_IP","URI_ID_DERIVED","LOGIN_STATUS","SOURCE_IP" +"Login","20231218054831.655","4u6LyuMrDvb_G-l1cJIQk-","00D5j00000DgAYG","0055j00000AT6I1","1219","127","/services/oauth2/token","","bY5Wfv8t/Ith7WVE","Standard","","1051271151","i","Go-http-client/1.1","","9998.0","salesforceinstance@devtest.in","TLSv1.2","ECDHE-RSA-AES256-GCM-SHA384","","","2023-12-18T05:48:31.655Z","0055j00000AT6I1AAL","Salesforce.com IP","","LOGIN_NO_ERROR","103.108.207.58" +"Login","20231218054832.003","4u6LyuHSDv8LLVl1cJOqGV","00D5j00000DgAYG","0055j00000AT6I1","1277","104","/services/oauth2/token","","u60el7VqW8CSSKcW","Standard","","674857427","i","Go-http-client/1.1","","9998.0","salesforceinstance@devtest.in","TLSv1.2","ECDHE-RSA-AES256-GCM-SHA384","","","2023-12-18T05:48:32.003Z","0055j00000AT6I1AAL","103.108.207.58","","LOGIN_NO_ERROR","103.108.207.58"` + + mp, err := decodeAsCSV([]byte(sampleELF)) + assert.NoError(t, err) + + wantNumOfEvents := 2 + gotNumOfEvents := len(mp) + assert.Equal(t, wantNumOfEvents, gotNumOfEvents) + + wantEventFields := map[string]string{ + "LOGIN_TYPE": "i", + "API_VERSION": "9998.0", + "TIMESTAMP_DERIVED": "2023-12-18T05:48:31.655Z", + "TIMESTAMP": "20231218054831.655", + "USER_NAME": "salesforceinstance@devtest.in", + "SOURCE_IP": "103.108.207.58", + "CPU_TIME": "127", + "REQUEST_STATUS": "", + "DB_TOTAL_TIME": "1051271151", + "TLS_PROTOCOL": "TLSv1.2", + "AUTHENTICATION_METHOD_REFERENCE": "", + "REQUEST_ID": "4u6LyuMrDvb_G-l1cJIQk-", + "USER_ID": "0055j00000AT6I1", + "RUN_TIME": "1219", + "CIPHER_SUITE": "ECDHE-RSA-AES256-GCM-SHA384", + "CLIENT_IP": "Salesforce.com IP", + "EVENT_TYPE": "Login", + "LOGIN_SUB_TYPE": "", + "USER_ID_DERIVED": "0055j00000AT6I1AAL", + "URI_ID_DERIVED": "", + "ORGANIZATION_ID": "00D5j00000DgAYG", + "URI": "/services/oauth2/token", + "LOGIN_KEY": "bY5Wfv8t/Ith7WVE", + "USER_TYPE": "Standard", + "API_TYPE": "", + "SESSION_KEY": "", + "BROWSER_TYPE": "Go-http-client/1.1", + "LOGIN_STATUS": "LOGIN_NO_ERROR", + } + + assert.Equal(t, wantEventFields, mp[0]) +} + +func TestSalesforceInputRunWithMethod(t *testing.T) { + var ( + defaultUserPassAuthConfig = authConfig{ + OAuth2: &OAuth2{ + UserPasswordFlow: &UserPasswordFlow{ + Enabled: pointer(true), + TokenURL: "https://instance_id.develop.my.salesforce.com/services/oauth2/token", + ClientID: "clientid", + ClientSecret: "clientsecret", + Username: "username", + Password: "password", + }, + }, + } + objectEventMonitotingConfig = eventMonitoringMethod{ + Object: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Second * 5, + Query: &QueryConfig{ + Default: getValueTpl(defaultLoginObjectQuery), + Value: getValueTpl(valueLoginObjectQuery), + }, + Cursor: &cursorConfig{Field: "EventDate"}, + }, + } + objectEventMonitoringWithWrongQuery = eventMonitoringMethod{ + Object: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Second * 5, + Query: &QueryConfig{ + Default: getValueTpl(invalidDefaultLoginEventObjectQuery), + Value: getValueTpl(invalidValueLoginObjectQuery), + }, + Cursor: &cursorConfig{Field: "EventDate"}, + }, + } + + elfEventMonitotingConfig = eventMonitoringMethod{ + EventLogFile: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Second * 5, + Query: &QueryConfig{ + Default: getValueTpl(defaultLoginEventLogFileQuery), + Value: getValueTpl(valueLoginEventLogFileQuery), + }, + Cursor: &cursorConfig{Field: "EventDate"}, + }, + } + elfEventMonitotingWithWrongQuery = eventMonitoringMethod{ + EventLogFile: EventMonitoringConfig{ + Enabled: pointer(true), + Interval: time.Second * 5, + Query: &QueryConfig{ + Default: getValueTpl(invalidDefaultLoginEventLogFileQuery), + Value: getValueTpl(invalidValueLoginEventLogFileQuery), + }, + Cursor: &cursorConfig{Field: "EventDate"}, + }, + } + ) + + type fields struct { + ctx context.Context + publisher inputcursor.Publisher + cancel context.CancelCauseFunc + cursor *state + srcConfig *config + sfdcConfig *sfdc.Configuration + soqlr *soql.Resource + config config + } + + defaultResource := resourceConfig{ + Retry: retryConfig{ + MaxAttempts: pointer(5), + WaitMin: pointer(time.Minute), + WaitMax: pointer(time.Minute), + }, + Transport: httpcommon.DefaultHTTPTransportSettings(), + } + + tests := []struct { + fields fields + setupServer func(testing.TB, http.HandlerFunc, *config) + handler http.HandlerFunc + method string + name string + expected []string + wantErr bool + AuthFail bool + ClientConnectionFail bool + }{ + // Object + { + name: "Positive/object_get_one_event", + method: "Object", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, "", oneObjectEvents), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &objectEventMonitotingConfig, + Resource: &defaultResource, + }, + cursor: &state{}, + }, + expected: []string{expectedObjectEvent}, + }, + { + name: "Negative/object_error_from_wrong_default_query", + method: "Object", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, "", oneObjectEvents), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &objectEventMonitoringWithWrongQuery, + Resource: &defaultResource, + }, + cursor: &state{}, + }, + wantErr: true, + }, + { + name: "Negative/object_error_from_wrong_value_query", + method: "Object", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, "", oneObjectEvents), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &objectEventMonitoringWithWrongQuery, + Resource: &defaultResource, + }, + cursor: &state{ + Object: dateTimeCursor{ + FirstEventTime: "2020-01-01T00:00:00Z", + LastEventTime: "2020-01-01T00:00:00Z", + }, + }, + }, + wantErr: true, + }, + + // EventLogFile + { + name: "Positive/elf_get_one_event", + method: "ELF", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, oneEventLogfileFirstResponseJSON, oneEventLogfileSecondResponseCSV), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &elfEventMonitotingConfig, + Resource: &defaultResource, + }, + cursor: &state{}, + }, + expected: []string{expectedELFEvent}, + }, + { + name: "Negative/elf_error_from_wrong_default_query", + method: "ELF", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, oneEventLogfileFirstResponseJSON, oneEventLogfileSecondResponseCSV), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &elfEventMonitotingWithWrongQuery, + Resource: &defaultResource, + }, + cursor: &state{}, + }, + wantErr: true, + }, + { + name: "Negative/elf_error_from_wrong_value_query", + method: "ELF", + setupServer: newTestServerBasedOnConfig(httptest.NewServer), + handler: defaultHandler(NoPaginationFlow, false, oneEventLogfileFirstResponseJSON, oneEventLogfileSecondResponseCSV), + fields: fields{ + config: config{ + Version: 56, + Auth: &defaultUserPassAuthConfig, + EventMonitoringMethod: &elfEventMonitotingWithWrongQuery, + Resource: &defaultResource, + }, + cursor: &state{ + EventLogFile: dateTimeCursor{ + FirstEventTime: "2020-01-01T00:00:00Z", + LastEventTime: "2020-01-01T00:00:00Z", + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + config := tt.fields.config + + t.Run(tt.name, func(t *testing.T) { + tt.setupServer(t, tt.handler, &config) + + s := &salesforceInput{ + config: config, + ctx: tt.fields.ctx, + cancel: tt.fields.cancel, + publisher: tt.fields.publisher, + cursor: tt.fields.cursor, + srcConfig: tt.fields.srcConfig, + sfdcConfig: tt.fields.sfdcConfig, + log: logp.NewLogger("salesforceInput"), + soqlr: tt.fields.soqlr, + } + + ctx, cancel := context.WithCancelCause(context.Background()) + s.ctx = ctx + s.cancel = cancel + + var client publisher + client.done = func() { + if len(client.published) >= len(tt.expected) { + cancel(nil) + } + } + s.publisher = &client + s.srcConfig = &s.config + + var err error + s.sfdcConfig, err = s.getSFDCConfig(&s.config) + if err != nil && !tt.wantErr { + t.Errorf("unexpected error from running input: %v", err) + } + if tt.wantErr && tt.AuthFail { + return + } + + s.soqlr, err = s.SetupSFClientConnection() + if err != nil && !tt.wantErr { + t.Errorf("unexpected error from running input: %v", err) + } + if tt.wantErr && tt.ClientConnectionFail { + return + } + + if tt.method == "Object" { + if err := s.RunObject(); (err != nil) != tt.wantErr { + t.Errorf("salesforceInput.RunObject() error = %v, wantErr %v", err, tt.wantErr) + } + } else { + if err := s.RunEventLogFile(); (err != nil) != tt.wantErr { + t.Errorf("salesforceInput.RunEventLogFile() error = %v, wantErr %v", err, tt.wantErr) + } + } + + if len(client.published) < len(tt.expected) { + t.Errorf("unexpected number of published events: got:%d want at least:%d", len(client.published), len(tt.expected)) + tt.expected = tt.expected[:len(client.published)] + } + + client.published = client.published[:len(tt.expected)] + for i, got := range client.published { + if !reflect.DeepEqual(got.Fields["message"], tt.expected[i]) { + t.Errorf("unexpected result for event %d: got:- want:+\n%s", i, cmp.Diff(got.Fields, tt.expected[i])) + } + } + }) + } +} + +func getValueTpl(in string) *valueTpl { + vp := &valueTpl{} + vp.Unpack(in) //nolint:errcheck // ignore error in test + + return vp +} + +func newTestServerBasedOnConfig(newServer func(http.Handler) *httptest.Server) func(testing.TB, http.HandlerFunc, *config) { + return func(t testing.TB, h http.HandlerFunc, config *config) { + server := newServer(h) + config.URL = server.URL + config.Auth.OAuth2.UserPasswordFlow.TokenURL = server.URL + t.Cleanup(server.Close) + } +} + +func TestPlugin(t *testing.T) { + _ = Plugin(logp.NewLogger("salesforce_test"), stateStore{}) +} diff --git a/x-pack/filebeat/input/salesforce/soql.go b/x-pack/filebeat/input/salesforce/soql.go new file mode 100644 index 000000000000..44987644bdc6 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/soql.go @@ -0,0 +1,27 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "strings" + + "github.com/g8rswimmer/go-sfdc/soql" +) + +// compile-time check if querier implements soql.QueryFormatter +var _ soql.QueryFormatter = (*querier)(nil) + +type querier struct { + Query string +} + +// Format returns the query string. +func (q querier) Format() (string, error) { + if strings.TrimSpace(q.Query) == "" { + return "", errors.New("query is empty") + } + return q.Query, nil +} diff --git a/x-pack/filebeat/input/salesforce/soql_test.go b/x-pack/filebeat/input/salesforce/soql_test.go new file mode 100644 index 000000000000..137a59a91bda --- /dev/null +++ b/x-pack/filebeat/input/salesforce/soql_test.go @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFormat(t *testing.T) { + tests := map[string]struct { + wantErr error + input string + wantStr string + }{ + "empty query": {input: "", wantStr: "", wantErr: errors.New("query is empty")}, + "valid query": {input: "SELECT FIELDS(STANDARD) FROM LoginEvent", wantStr: "SELECT FIELDS(STANDARD) FROM LoginEvent", wantErr: nil}, + "invalid query": {input: "SELECT ", wantStr: "SELECT ", wantErr: nil}, + } + + var q querier + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + q.Query = tc.input + got, gotErr := q.Format() + if !assert.Equal(t, tc.wantErr, gotErr) { + t.FailNow() + } + if !assert.EqualValues(t, tc.wantStr, got) { + t.FailNow() + } + }) + } +} diff --git a/x-pack/filebeat/input/salesforce/state.go b/x-pack/filebeat/input/salesforce/state.go new file mode 100644 index 000000000000..2d8a96b68bd2 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/state.go @@ -0,0 +1,49 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// state is the state of the salesforce module. It is used to watermark the state +// to avoid pulling duplicate data from Salesforce. The state is persisted separately +// for EventLogFile and Object. +type state struct { + Object dateTimeCursor `json:"object,omitempty"` + EventLogFile dateTimeCursor `json:"event_log_file,omitempty"` +} + +// dateTimeCursor maintains two distinct states for the event collection iteration. +// The initial state represents the time of the first event, while the subsequent state denotes the time of the last event. +// In certain SOQL queries for specific objects, sorting by all fields may not be feasible, and there may be no specific order. +// This design allows users to exert maximum control over the iteration process. +// For instance, the LoginEvent object only supports sorting based on EventIdentifier and EventDate. +// Furthermore, if we desire to sort based on EventDate, it only supports descending order sorting. +// In this case by using first_event_time we can get latest event EventDate to query next set of events. +// Reference to LoginEvent: https://developer.salesforce.com/docs/atlas.en-us.platform_events.meta/platform_events/sforce_api_objects_loginevent.htm +type dateTimeCursor struct { + FirstEventTime string `struct:"first_event_time,omitempty"` + LastEventTime string `struct:"last_event_time,omitempty"` +} + +// parseCursor parses the cursor from the configuration and executes the +// template. If cursor is nil, the default templated query is used else +// the value templated query is used. See QueryConfig struct for more. +func parseCursor(cfg *QueryConfig, cursor mapstr.M, log *logp.Logger) (string, error) { + ctxTmpl := mapstr.M{"cursor": nil} + + if cursor != nil { + ctxTmpl["cursor"] = cursor + qr, err := cfg.Value.Execute(ctxTmpl, nil, log) + if err != nil { + return "", err + } + return qr, nil + } + + return cfg.Default.Execute(ctxTmpl, nil, log) +} diff --git a/x-pack/filebeat/input/salesforce/value_tpl.go b/x-pack/filebeat/input/salesforce/value_tpl.go new file mode 100644 index 000000000000..8a05ecc06867 --- /dev/null +++ b/x-pack/filebeat/input/salesforce/value_tpl.go @@ -0,0 +1,132 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package salesforce + +import ( + "errors" + "strings" + "text/template" + "time" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type valueTpl struct { + *template.Template +} + +var ( + errEmptyTemplateResult = errors.New("template result is empty") + errExecuteTemplate = errors.New("template execution failed") +) + +// Execute executes the template with the given data. If the template execution +// fails, then the defaultVal is used if it is not nil. Execute will return +// variable substituted query with nil error. +func (t *valueTpl) Execute(data any, defaultVal *valueTpl, log *logp.Logger) (val string, err error) { + fallback := func(err error) (string, error) { + if defaultVal != nil { + log.Debugf("template execution error: %s", err) + log.Info("fallback to default template") + return defaultVal.Execute(mapstr.M{}, nil, log) + } + return "", err + } + + defer func() { + if r := recover(); r != nil { + val, err = fallback(errExecuteTemplate) + } + if err != nil { + log.Debugf("template execution failed %s", err) + } + }() + + buf := new(strings.Builder) + + err = t.Template.Execute(buf, data) + if err != nil { + return fallback(err) + } + + val = buf.String() + if val == "" { + return fallback(errEmptyTemplateResult) + } + + return val, nil +} + +// Unpack parses the given string as a template. +func (t *valueTpl) Unpack(in string) error { + // Custom delimiters to prevent issues when using template values as part of + // other Go templates. + const ( + leftDelim = "[[" + rightDelim = "]]" + ) + + tpl, err := template.New(""). + Option("missingkey=error"). + Funcs(template.FuncMap{ + "now": timeNow, + "parseDuration": parseDuration, + "parseTime": parseTime, + "formatTime": formatTime, + }). + Delims(leftDelim, rightDelim). + Parse(in) + if err != nil { + return err + } + + *t = valueTpl{Template: tpl} + + return nil +} + +// parseDuration parses a duration string and returns the time.Duration value. +func parseDuration(s string) time.Duration { + d, _ := time.ParseDuration(s) + return d +} + +// predefinedLayouts contains some predefined layouts that are commonly used. +var predefinedLayouts = map[string]string{ + "ANSIC": time.ANSIC, + "UnixDate": time.UnixDate, + "RubyDate": time.RubyDate, + "RFC822": time.RFC822, + "RFC822Z": time.RFC822Z, + "RFC850": time.RFC850, + "RFC1123": time.RFC1123, + "RFC1123Z": time.RFC1123Z, + "RFC3339": time.RFC3339, // 2006-01-02T15:04:05Z07:00 + "CustomRFC3339Like": formatRFC3339Like, // 2006-01-02T15:04:05.999Z + "RFC3339Nano": time.RFC3339Nano, + "Kitchen": time.Kitchen, +} + +// parseTime parses a time string using the given layout. There are also some +// predefined layouts that can be used; see predefinedLayouts for more. +func parseTime(ts, layout string) time.Time { + if found := predefinedLayouts[layout]; found != "" { + layout = found + } + + t, _ := time.Parse(layout, ts) + return t +} + +// formatTime formats a time using the given layout. There are also some +// predefined layouts that can be used; see predefinedLayouts for more. +func formatTime(t time.Time, layout string) string { + if found := predefinedLayouts[layout]; found != "" { + layout = found + } + + return t.Format(layout) +}