diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml new file mode 100644 index 000000000000..34321b61161b --- /dev/null +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml new file mode 100644 index 000000000000..34321b61161b --- /dev/null +++ b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml new file mode 100644 index 000000000000..34321b61161b --- /dev/null +++ b/.buildkite/heartbeat/heartbeat-pipeline.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +steps: + - label: "Example test" + command: echo "Hello!" diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2cfac0a00b25..43d8974f3bf6 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -47,6 +47,54 @@ "skip_target_branches": [ ], "skip_ci_on_only_changed": [ ], "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "auditbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test auditbeat$", + "always_trigger_comment_regex": "^/test auditbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ "^auditbeat/.*", ".buildkite/auditbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "heartbeat", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test heartbeat$", + "always_trigger_comment_regex": "^/test heartbeat$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ "^heartbeat/.*", ".buildkite/heartbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"] + }, + { + "enabled": true, + "pipelineSlug": "deploy-k8s", + "allow_org_users": true, + "allowed_repo_permissions": ["admin", "write"], + "allowed_list": [ ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^/test deploy/kubernetes$", + "always_trigger_comment_regex": "^/test deploy/kubernetes$", + "skip_ci_labels": [ ], + "skip_target_branches": [ ], + "skip_ci_on_only_changed": [ ], + "always_require_ci_on_changed": [ "^deploy/kubernetes/.*", ".buildkite/deploy/kubernetes/.*", "^libbeat/docs/version.asciidoc"] } ] } diff --git a/.go-version b/.go-version index 4bb1a22f8ec5..3b9e4a0c187a 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.20.11 +1.20.12 diff --git a/.golangci.yml b/.golangci.yml index b4c16d3291da..3cc6336695e3 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -114,7 +114,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -132,19 +132,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" gosec: excludes: diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 98253379f221..dcb2179d3539 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,97 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.11.4]] +=== Beats version 8.11.4 +https://github.com/elastic/beats/compare/v8.11.3\...v8.11.4[View commits] + +==== Bugfixes + +*Heartbeat* + +- Added fix for formatting the logs from stateloader properly. {pull}37369[37369] +- Remove duplicated syscall from ARM seccomp profile. {pull}37440[37440] + +*Metricbeat* + +- Nest the `region` and `availability_zone` ECS fields within the cloud field. {pull}37015[37015] +- Fix CPU and memory metrics collection from privileged process on Windows. {issue}17314[17314]{pull}37027[37027] +- Add memory hard limit from container metadata and remove usage percentage in AWS Fargate. {pull}37194[37194] +- Ignore parser errors from unsupported metrics types on Prometheus client and continue parsing until EOF is reached. {pull}37383[37383] +- Fix the reference time rounding on Azure Metrics. {issue}37204[37204] {pull}37365[37365] + +==== Added + +*Packetbeat* + +- Bump Windows Npcap version to v1.78. {issue}37300[37300] {pull}37370[37370] + + +[[release-notes-8.11.3]] +=== Beats version 8.11.3 +https://github.com/elastic/beats/compare/v8.11.2\...v8.11.3[View commits] + +The 8.11.3 patch release contains a fix for a potential security vulnerability. Please see our link:https://discuss.elastic.co/c/announcements/security-announcements/31[security advisory for more details]. + +==== Breaking changes + +*Affecting all Beats* + +- `queue.mem.events` is changing from `4096` to `3200`. +- `queue.mem.flush.min_events` is changing from `2048` to `1600`. +- `queue.mem.flush.timeout` is changing from `1s` to `10s`. +- `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. +- `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. + + +[[release-notes-8.11.2]] +=== Beats version 8.11.2 +https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] + +==== Breaking changes + +*Affecting all Beats* + +- Avoid logging fields values when handling Elasticsearch output errors except at the debug log level. The debug log level must now be used to see detailed errors, for example mapping errors and their cause. {pull}37229[37229] + +==== Bugfixes + +*Affecting all Beats* +- Fix memqueue producer blocking indefinitely even after being cancelled. {issue}22813[22813] {pull}37077[37077] + +*Auditbeat* + +- Fix documentation regarding socket type selection. {issue}37174[37174] {pull}37175[37175] +- Fix guess trigger for system/socket credentials on newer kernels. {issue}36905[36905] {pull}37136[37136] + +*Filebeat* + +- Do not error when Okta API returns no data. {pull}37092[37092] +- Fix request body close behaviour in HTTP_Endpoint when handling GZIP compressed content. {pull}37091[37091] +- Make CEL input `now` global evaluate to a time in UTC. {pull}37159[37159] + +*Heartbeat* + +- Fix mapping errors with invalid URLs. {pull}37255[37255] + +*Metricbeat* + +- Fix memory leak on Windows {issue}37142[37142] {pull}37171[37171] +- Enhance Azure Metrics metricset with refined grouping logic and resolved duplication issues for TSDB compatibility. {pull}36823[36823] +- Fix unintended skip in metric collection on Azure Monitor. {issue}37204[37204] {pull}37203[37203] +- Fix the "api-version query parameter (?api-version=) is required for all requests" error in Azure Billing. {pull}37158[37158] + +*Winlogbeat* + +- Fix dashboards under Kibana 8.x. {issue}37080[37080] {pull}37085[37085] + +==== Added + +*Affecting all Beats* + +- Upgrade to Go 1.20.11. {pull}37123[37123] + + [[release-notes-8.11.1]] === Beats version 8.11.1 https://github.com/elastic/beats/compare/v8.11.0\...v8.11.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 53b183671495..ace3eeec5dcf 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -99,6 +99,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Fix mapping errors with invalid urls. {pull}37255[37255] +- Added fix for formatting the logs from stateloader properly. {pull}37369[37369] *Metricbeat* @@ -125,6 +126,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix unintended skip in metric collection on Azure Monitor {issue}37204[37204] {pull}37203[37203] - Fix the "api-version query parameter (?api-version=) is required for all requests" error in Azure Billing. {pull}37158[37158] - Add memory hard limit from container metadata and remove usage percentage in AWS Fargate. {pull}37194[37194] +- Fix the reference time rounding on Azure Metrics {issue}37204[37204] {pull}37365[37365] *Osquerybeat* @@ -159,8 +161,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor -- Upgrade to Go 1.20.11. {pull}37123[37123] +- Upgrade to Go 1.20.12. {pull}37350[37350] - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] +- Upgrade elastic-agent-system-metrics to v0.9.1. See https://github.com/elastic/elastic-agent-system-metrics/releases/tag/v0.9.1. {pull}37353[pull] +- Upgrade to elastic-agent-libs v0.7.3 and golang.org/x/crypto v0.17.0. {pull}37544[37544] *Auditbeat* @@ -331,6 +335,15 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d + + + + + + + + + diff --git a/NOTICE.txt b/NOTICE.txt index b7e6c5325f0a..de0dea6dca01 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Elastic Beats -Copyright 2014-2023 Elasticsearch BV +Copyright 2014-2024 Elasticsearch BV This product includes software developed by The Apache Software Foundation (http://www.apache.org/). @@ -4812,11 +4812,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/aws/aws-lambda-go -Version: v1.13.3 +Version: v1.44.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/aws/aws-lambda-go@v1.13.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/aws/aws-lambda-go@v1.44.0/LICENSE: Apache License @@ -12268,11 +12268,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.5 +Version: v0.6.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: Apache License Version 2.0, January 2004 @@ -12712,11 +12712,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.7.2 +Version: v0.7.3 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.7.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.7.3/LICENSE: Apache License Version 2.0, January 2004 @@ -13026,11 +13026,11 @@ these terms. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-system-metrics -Version: v0.9.0 +Version: v0.9.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-system-metrics@v0.9.1/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -24701,11 +24701,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.14.0 +Version: v0.17.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.17.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24923,11 +24923,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24960,11 +24960,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.13.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51287,11 +51287,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 4f7612cd90e3..742041d66af8 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/catalog-info.yaml b/catalog-info.yaml index 3ccb648d0979..92757fd4c134 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -152,3 +152,138 @@ spec: access_level: MANAGE_BUILD_AND_READ everyone: access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-auditbeat + description: "Auditbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/auditbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: auditbeat + description: "Auditbeat pipeline" + spec: + # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + pipeline_file: ".buildkite/auditbeat/auditbeat-pipeline.yml" + # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-heartbeat + description: "Heartbeat pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/heartbeat + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: heartbeat + description: "Heartbeat pipeline" + spec: + # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + pipeline_file: ".buildkite/heartbeat/heartbeat-pipeline.yml" + # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-deploy-k8s + description: "Deploy K8S pipeline" + links: + - title: Pipeline + url: https://buildkite.com/elastic/deploy-k8s + +spec: + type: buildkite-pipeline + owner: group:ingest-fp + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: deploy-k8s + description: "Deploy K8S pipeline" + spec: + # branch_configuration: "main 7.* 8.* v7.* v8.*" TODO: temporarily commented to build PRs from forks + pipeline_file: ".buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml" + # maximum_timeout_in_minutes: 120 TODO: uncomment when pipeline is ready + provider_settings: + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_tags: true + filter_enabled: true + filter_condition: >- + build.pull_request.id == null || (build.creator.name == 'elasticmachine' && build.pull_request.id != null) + repository: elastic/beats + cancel_intermediate_builds: true + cancel_intermediate_builds_branch_filter: "!main !7.* !8.*" + skip_intermediate_builds: true + skip_intermediate_builds_branch_filter: "!main !7.* !8.*" + # env: + # ELASTIC_PR_COMMENTS_ENABLED: "true" TODO: uncomment when pipeline is ready + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index 61e036848474..c0c8768861cc 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index df065094897e..aa48a8d58d7e 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index 241c186998d1..854cf1ac32be 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/mage/gotool/go.go b/dev-tools/mage/gotool/go.go index bb7066a4f3e5..e507282c15ce 100644 --- a/dev-tools/mage/gotool/go.go +++ b/dev-tools/mage/gotool/go.go @@ -58,12 +58,19 @@ var Test goTest = runGoTest // GetModuleName returns the name of the module. func GetModuleName() (string, error) { - lines, err := getLines(callGo(nil, "list", "-m")) + lines, err := getLines(callGo( + // Disabling the Go workspace prevents 'go list' from listing all + // modules within the workspace. + map[string]string{"GOWORK": "off"}, + "list", + "-m")) if err != nil { return "", err } + if len(lines) != 1 { - return "", fmt.Errorf("unexpected number of lines") + return "", fmt.Errorf("expected 'go list -m' to return 1 line, got %d", + len(lines)) } return lines[0], nil } diff --git a/filebeat/input/filestream/copytruncate_prospector.go b/filebeat/input/filestream/copytruncate_prospector.go index 10884cb9b94e..5b1c6bdd4277 100644 --- a/filebeat/input/filestream/copytruncate_prospector.go +++ b/filebeat/input/filestream/copytruncate_prospector.go @@ -28,6 +28,7 @@ import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" ) @@ -330,7 +331,7 @@ func (p *copyTruncateFileProspector) onRotatedFile( return } descCopy := fe.Descriptor - descCopy.Info = fi + descCopy.Info = file.ExtendFileInfo(fi) originalSrc := p.identifier.GetSource(loginp.FSEvent{NewPath: originalPath, Descriptor: descCopy}) p.rotatedFiles.addOriginalFile(originalPath, originalSrc) p.rotatedFiles.addRotatedFile(originalPath, fe.NewPath, src) diff --git a/filebeat/input/filestream/environment_test.go b/filebeat/input/filestream/environment_test.go index 1713a964c527..7c3c8ccd4d3b 100644 --- a/filebeat/input/filestream/environment_test.go +++ b/filebeat/input/filestream/environment_test.go @@ -37,6 +37,7 @@ import ( v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/beats/v7/libbeat/statestore" "github.com/elastic/beats/v7/libbeat/statestore/storetest" @@ -372,7 +373,13 @@ func (e *inputTestingEnvironment) getRegistryState(key string) (registryEntry, e func getIDFromPath(filepath, inputID string, fi os.FileInfo) string { identifier, _ := newINodeDeviceIdentifier(nil) - src := identifier.GetSource(loginp.FSEvent{Descriptor: loginp.FileDescriptor{Info: fi}, Op: loginp.OpCreate, NewPath: filepath}) + src := identifier.GetSource(loginp.FSEvent{ + Descriptor: loginp.FileDescriptor{ + Info: file.ExtendFileInfo(fi), + }, + Op: loginp.OpCreate, + NewPath: filepath, + }) return "filestream::" + inputID + "::" + src.Name() } diff --git a/filebeat/input/filestream/fswatch.go b/filebeat/input/filestream/fswatch.go index a32b4409ef2d..454a5b428b05 100644 --- a/filebeat/input/filestream/fswatch.go +++ b/filebeat/input/filestream/fswatch.go @@ -32,6 +32,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input/file" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + commonfile "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/match" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -406,7 +407,7 @@ type ingestTarget struct { filename string originalFilename string symlink bool - info os.FileInfo + info commonfile.ExtendedFileInfo } func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err error) { @@ -421,10 +422,11 @@ func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err err it.filename = filename it.originalFilename = filename - it.info, err = os.Lstat(it.filename) // to determine if it's a symlink + info, err := os.Lstat(it.filename) // to determine if it's a symlink if err != nil { return it, fmt.Errorf("failed to lstat %q: %w", it.filename, err) } + it.info = commonfile.ExtendFileInfo(info) if it.info.IsDir() { return it, fmt.Errorf("file %q is a directory", it.filename) @@ -438,10 +440,11 @@ func (s *fileScanner) getIngestTarget(filename string) (it ingestTarget, err err } // now we know it's a symlink, we stat with link resolution - it.info, err = os.Stat(it.filename) + info, err := os.Stat(it.filename) if err != nil { return it, fmt.Errorf("failed to stat the symlink %q: %w", it.filename, err) } + it.info = commonfile.ExtendFileInfo(info) it.originalFilename, err = filepath.EvalSymlinks(it.filename) if err != nil { diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go index f9f58734360c..6c9d88b858e2 100644 --- a/filebeat/input/filestream/fswatch_test.go +++ b/filebeat/input/filestream/fswatch_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/require" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -68,7 +69,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 5}, // 5 bytes written + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 5}), // 5 bytes written }, } requireEqualEvents(t, expEvent, e) @@ -91,7 +92,7 @@ scanner: Op: loginp.OpWrite, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 10}, // +5 bytes appended + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 10}), // +5 bytes appended }, } requireEqualEvents(t, expEvent, e) @@ -113,7 +114,7 @@ scanner: Op: loginp.OpRename, Descriptor: loginp.FileDescriptor{ Filename: newFilename, - Info: testFileInfo{name: newBasename, size: 10}, + Info: file.ExtendFileInfo(&testFileInfo{name: newBasename, size: 10}), }, } requireEqualEvents(t, expEvent, e) @@ -133,7 +134,7 @@ scanner: Op: loginp.OpTruncate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -153,7 +154,7 @@ scanner: Op: loginp.OpTruncate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -172,7 +173,7 @@ scanner: Op: loginp.OpDelete, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 2}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 2}), }, } requireEqualEvents(t, expEvent, e) @@ -210,7 +211,7 @@ scanner: Descriptor: loginp.FileDescriptor{ Filename: filename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -241,7 +242,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -298,7 +299,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: filename, - Info: testFileInfo{name: basename, size: 5}, // +5 bytes appended + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 5}), // +5 bytes appended }, } requireEqualEvents(t, expEvent, e) @@ -332,7 +333,7 @@ scanner: Descriptor: loginp.FileDescriptor{ Filename: filename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{name: basename, size: 1024}, + Info: file.ExtendFileInfo(&testFileInfo{name: basename, size: 1024}), }, } requireEqualEvents(t, expEvent, e) @@ -384,7 +385,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: firstFilename, - Info: testFileInfo{name: firstBasename, size: 5}, // "line\n" + Info: file.ExtendFileInfo(&testFileInfo{name: firstBasename, size: 5}), // "line\n" }, }, { @@ -392,7 +393,7 @@ scanner: Op: loginp.OpCreate, Descriptor: loginp.FileDescriptor{ Filename: secondFilename, - Info: testFileInfo{name: secondBasename, size: 5}, // "line\n" + Info: file.ExtendFileInfo(&testFileInfo{name: secondBasename, size: 5}), // "line\n" }, }, } @@ -494,38 +495,38 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -543,31 +544,31 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -586,24 +587,24 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -617,17 +618,17 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ normalFilename: { Filename: normalFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, undersizedFilename: { Filename: undersizedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[undersizedFilename], name: undersizedBasename, - }, + }), }, }, }, @@ -646,10 +647,10 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -663,10 +664,10 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, }, }, @@ -680,17 +681,17 @@ scanner: expDesc: map[string]loginp.FileDescriptor{ excludedIncludedFilename: { Filename: excludedIncludedFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -709,34 +710,34 @@ scanner: normalFilename: { Filename: normalFilename, Fingerprint: "2edc986847e209b4016e141a6dc8716d3207350f416969382d431539bf292e4a", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, excludedFilename: { Filename: excludedFilename, Fingerprint: "bd151321c3bbdb44185414a1b56b5649a00206dd4792e7230db8904e43987336", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, Fingerprint: "bfdb99a65297062658c26dfcea816d76065df2a2da2594bfd9b96e9e405da1c2", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, Fingerprint: "c4058942bffcea08810a072d5966dfa5c06eb79b902bf0011890dd8d22e1a5f8", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, @@ -755,35 +756,35 @@ scanner: normalFilename: { Filename: normalFilename, Fingerprint: "ffe054fe7ae0cb6dc65c3af9b61d5209f439851db43d0ba5997337df154668eb", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[normalFilename], name: normalBasename, - }, + }), }, // undersizedFilename got excluded because of the matching fingerprint excludedFilename: { Filename: excludedFilename, Fingerprint: "9c225a1e6a7df9c869499e923565b93937e88382bb9188145f117195cd41dcd1", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedFilename], name: excludedBasename, - }, + }), }, excludedIncludedFilename: { Filename: excludedIncludedFilename, Fingerprint: "7985b2b9750bdd3c76903db408aff3859204d6334279eaf516ecaeb618a218d5", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[excludedIncludedFilename], name: excludedIncludedBasename, - }, + }), }, travelerSymlinkFilename: { Filename: travelerSymlinkFilename, Fingerprint: "da437600754a8eed6c194b7241b078679551c06c7dc89685a9a71be7829ad7e5", - Info: testFileInfo{ + Info: file.ExtendFileInfo(&testFileInfo{ size: sizes[travelerFilename], name: travelerSymlinkBasename, - }, + }), }, }, }, diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go index 0cfeb031d633..a0cd7903e7ac 100644 --- a/filebeat/input/filestream/identifier.go +++ b/filebeat/input/filestream/identifier.go @@ -21,7 +21,6 @@ import ( "fmt" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" ) @@ -114,7 +113,7 @@ func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource { oldPath: e.OldPath, truncated: e.Op == loginp.OpTruncate, archived: e.Op == loginp.OpArchived, - fileID: i.name + identitySep + file.GetOSState(e.Descriptor.Info).String(), + fileID: i.name + identitySep + e.Descriptor.Info.GetOSState().String(), identifierGenerator: i.name, } } diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go index af6a56100862..05a768d2babf 100644 --- a/filebeat/input/filestream/identifier_inode_deviceid.go +++ b/filebeat/input/filestream/identifier_inode_deviceid.go @@ -27,7 +27,6 @@ import ( "time" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -93,7 +92,7 @@ func (i *inodeMarkerIdentifier) markerContents() string { } func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource { - osstate := file.GetOSState(e.Descriptor.Info) + osstate := e.Descriptor.Info.GetOSState() return fileSource{ desc: e.Descriptor, newPath: e.NewPath, diff --git a/filebeat/input/filestream/identifier_test.go b/filebeat/input/filestream/identifier_test.go index ca67ba375d60..1fcd4d73efa2 100644 --- a/filebeat/input/filestream/identifier_test.go +++ b/filebeat/input/filestream/identifier_test.go @@ -53,7 +53,7 @@ func TestFileIdentifier(t *testing.T) { src := identifier.GetSource(loginp.FSEvent{ NewPath: tmpFile.Name(), - Descriptor: loginp.FileDescriptor{Info: fi}, + Descriptor: loginp.FileDescriptor{Info: file.ExtendFileInfo(fi)}, }) assert.Equal(t, identifier.Name()+"::"+file.GetOSState(fi).String(), src.Name()) @@ -77,7 +77,7 @@ func TestFileIdentifier(t *testing.T) { src := identifier.GetSource(loginp.FSEvent{ NewPath: tmpFile.Name(), - Descriptor: loginp.FileDescriptor{Info: fi}, + Descriptor: loginp.FileDescriptor{Info: file.ExtendFileInfo(fi)}, }) assert.Equal(t, identifier.Name()+"::"+file.GetOSState(fi).String()+"-my-suffix", src.Name()) diff --git a/filebeat/input/filestream/internal/input-logfile/fswatch.go b/filebeat/input/filestream/internal/input-logfile/fswatch.go index dc00519c437b..1f91515036fa 100644 --- a/filebeat/input/filestream/internal/input-logfile/fswatch.go +++ b/filebeat/input/filestream/internal/input-logfile/fswatch.go @@ -18,11 +18,9 @@ package input_logfile import ( - "os" - "github.com/elastic/go-concert/unison" - file_helper "github.com/elastic/beats/v7/libbeat/common/file" + "github.com/elastic/beats/v7/libbeat/common/file" ) const ( @@ -63,7 +61,7 @@ type FileDescriptor struct { // the filename from the `Info`. Filename string // Info is the result of file stat - Info os.FileInfo + Info file.ExtendedFileInfo // Fingerprint is a computed hash of the file header Fingerprint string } @@ -75,7 +73,7 @@ func (fd FileDescriptor) FileID() string { if fd.Fingerprint != "" { return fd.Fingerprint } - return file_helper.GetOSState(fd.Info).String() + return fd.Info.GetOSState().String() } // SameFile returns true if descriptors point to the same file. diff --git a/filebeat/input/filestream/logger.go b/filebeat/input/filestream/logger.go index 7b644fd0d877..62cb416cdb26 100644 --- a/filebeat/input/filestream/logger.go +++ b/filebeat/input/filestream/logger.go @@ -19,7 +19,6 @@ package filestream import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" - "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/logp" ) @@ -31,8 +30,11 @@ func loggerWithEvent(logger *logp.Logger, event loginp.FSEvent, src loginp.Sourc if event.Descriptor.Fingerprint != "" { log = log.With("fingerprint", event.Descriptor.Fingerprint) } - if event.Descriptor.Info != nil && event.Descriptor.Info.Sys() != nil { - log = log.With("os_id", file.GetOSState(event.Descriptor.Info)) + if event.Descriptor.Info != nil { + osID := event.Descriptor.Info.GetOSState().String() + if osID != "" { + log = log.With("os_id", osID) + } } if event.NewPath != "" { log = log.With("new_path", event.NewPath) diff --git a/filebeat/input/filestream/prospector_test.go b/filebeat/input/filestream/prospector_test.go index 834784c81da7..552b4218c784 100644 --- a/filebeat/input/filestream/prospector_test.go +++ b/filebeat/input/filestream/prospector_test.go @@ -32,6 +32,7 @@ import ( loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/common/transform/typeconv" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/go-concert/unison" @@ -139,7 +140,7 @@ func TestProspector_InitUpdateIdentifiers(t *testing.T) { }, }, filesOnDisk: map[string]loginp.FileDescriptor{ - tmpFileName: {Info: fi}, + tmpFileName: {Info: file.ExtendFileInfo(fi)}, }, expectedUpdatedKeys: map[string]string{"not_path::key1": "path::" + tmpFileName}, }, @@ -205,12 +206,12 @@ func TestProspectorNewAndUpdatedFiles(t *testing.T) { { Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), }, { Op: loginp.OpWrite, NewPath: "/path/to/other/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), }, }, ignoreOlder: 10 * time.Second, @@ -223,12 +224,12 @@ func TestProspectorNewAndUpdatedFiles(t *testing.T) { { Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), }, { Op: loginp.OpWrite, NewPath: "/path/to/other/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/other/file", 5, minuteAgo, nil}), }, }, ignoreOlder: 5 * time.Minute, @@ -268,12 +269,12 @@ func TestProspectorHarvesterUpdateIgnoredFiles(t *testing.T) { eventCreate := loginp.FSEvent{ Op: loginp.OpCreate, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 5, minuteAgo, nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 5, minuteAgo, nil}), } eventUpdated := loginp.FSEvent{ Op: loginp.OpWrite, NewPath: "/path/to/file", - Descriptor: createTestFileDescriptorWithInfo(testFileInfo{"/path/to/file", 10, time.Now(), nil}), + Descriptor: createTestFileDescriptorWithInfo(&testFileInfo{"/path/to/file", 10, time.Now(), nil}), } expectedEvents := []harvesterEvent{ harvesterStart("path::/path/to/file"), @@ -730,20 +731,20 @@ type testFileInfo struct { sys interface{} } -func (t testFileInfo) Name() string { return t.name } -func (t testFileInfo) Size() int64 { return t.size } -func (t testFileInfo) Mode() os.FileMode { return 0 } -func (t testFileInfo) ModTime() time.Time { return t.time } -func (t testFileInfo) IsDir() bool { return false } -func (t testFileInfo) Sys() interface{} { return t.sys } +func (t *testFileInfo) Name() string { return t.name } +func (t *testFileInfo) Size() int64 { return t.size } +func (t *testFileInfo) Mode() os.FileMode { return 0 } +func (t *testFileInfo) ModTime() time.Time { return t.time } +func (t *testFileInfo) IsDir() bool { return false } +func (t *testFileInfo) Sys() interface{} { return t.sys } func createTestFileDescriptor() loginp.FileDescriptor { - return createTestFileDescriptorWithInfo(testFileInfo{}) + return createTestFileDescriptorWithInfo(&testFileInfo{}) } func createTestFileDescriptorWithInfo(fi fs.FileInfo) loginp.FileDescriptor { return loginp.FileDescriptor{ - Info: fi, + Info: file.ExtendFileInfo(fi), Fingerprint: "fingerprint", Filename: "filename", } diff --git a/go.mod b/go.mod index b827bb365cc5..0af433f1a79b 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/andrewkroh/sys v0.0.0-20151128191922-287798fe3e43 github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-lambda-go v1.13.3 + github.com/aws/aws-lambda-go v1.44.0 github.com/aws/aws-sdk-go-v2 v1.18.0 github.com/aws/aws-sdk-go-v2/config v1.17.7 github.com/aws/aws-sdk-go-v2/credentials v1.12.20 @@ -152,14 +152,14 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.17.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mod v0.10.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.10.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/sys v0.15.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.9.1 google.golang.org/api v0.126.0 @@ -201,10 +201,10 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.5 - github.com/elastic/elastic-agent-libs v0.7.2 + github.com/elastic/elastic-agent-autodiscover v0.6.6 + github.com/elastic/elastic-agent-libs v0.7.3 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 - github.com/elastic/elastic-agent-system-metrics v0.9.0 + github.com/elastic/elastic-agent-system-metrics v0.9.1 github.com/elastic/go-elasticsearch/v8 v8.11.1 github.com/elastic/mito v1.7.0 github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640 @@ -367,7 +367,7 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect @@ -413,7 +413,6 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 - ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index a0c7177c553f..c76f126b6e0c 100644 --- a/go.sum +++ b/go.sum @@ -274,8 +274,9 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-lambda-go v1.44.0 h1:Xp9PANXKsSJ23IhE4ths592uWTCEewswPhSH9qpAuQQ= +github.com/aws/aws-lambda-go v1.44.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= @@ -652,16 +653,16 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U= -github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= +github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA= github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0= -github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= -github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc= +github.com/elastic/elastic-agent-libs v0.7.3 h1:tc6JDXYR+2XFMHJVv+7+M0OwAbZPxm3caLJEd943dlE= +github.com/elastic/elastic-agent-libs v0.7.3/go.mod h1:9hlSaDPm0XTrUWrZjwvckgov1pDHnsGyybzAjNe/1wA= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3/go.mod h1:rWarFM7qYxJKsi9WcV6ONcFjH/NA3niDNpTxO+8/GVI= -github.com/elastic/elastic-agent-system-metrics v0.9.0 h1:sA3pSk+awH4g45M8tjcwd9U6uVQvNzUpKS8ajS3DuUk= -github.com/elastic/elastic-agent-system-metrics v0.9.0/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= +github.com/elastic/elastic-agent-system-metrics v0.9.1 h1:r0ofKHgPpl+W09ie7tzGcCDC0d4NZbQUv37rSgHf4FM= +github.com/elastic/elastic-agent-system-metrics v0.9.1/go.mod h1:9C1UEfj0P687HAzZepHszN6zXA+2tN2Lx3Osvq1zby8= github.com/elastic/elastic-transport-go/v8 v8.3.0 h1:DJGxovyQLXGr62e9nDMPSxRyWION0Bh6d9eCFBriiHo= github.com/elastic/elastic-transport-go/v8 v8.3.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 h1:cWPqxlPtir4RoQVCpGSRXmLqjEHpJKbR60rxh1nQZY4= @@ -2030,8 +2031,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2344,8 +2345,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2354,8 +2355,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2370,8 +2371,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index d90185df60bd..e3e4a8bf989c 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/heartbeat/beater/heartbeat.go b/heartbeat/beater/heartbeat.go index 8e4d39480f2f..e4e5a5129a8c 100644 --- a/heartbeat/beater/heartbeat.go +++ b/heartbeat/beater/heartbeat.go @@ -162,7 +162,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { logp.L().Info("heartbeat is running! Hit CTRL-C to stop it.") groups, _ := syscall.Getgroups() - logp.L().Info("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups) + logp.L().Infof("Effective user/group ids: %d/%d, with groups: %v", syscall.Geteuid(), syscall.Getegid(), groups) waitMonitors := monitors.NewSignalWait() @@ -226,7 +226,7 @@ func (bt *Heartbeat) Run(b *beat.Beat) error { waitPublished.AddChan(bt.done) waitPublished.Add(monitors.WithLog(pipelineWrapper.Wait, "shutdown: finished publishing events.")) if bt.config.PublishTimeout > 0 { - logp.Info("shutdown: output timer started. Waiting for max %v.", bt.config.PublishTimeout) + logp.L().Infof("shutdown: output timer started. Waiting for max %v.", bt.config.PublishTimeout) waitPublished.Add(monitors.WithLog(monitors.WaitDuration(bt.config.PublishTimeout), "shutdown: timed out waiting for pipeline to publish events.")) } @@ -241,7 +241,7 @@ func (bt *Heartbeat) RunStaticMonitors(b *beat.Beat, pipeline beat.Pipeline) (st created, err := bt.monitorFactory.Create(pipeline, cfg) if err != nil { if errors.Is(err, monitors.ErrMonitorDisabled) { - logp.L().Info("skipping disabled monitor: %s", err) + logp.L().Infof("skipping disabled monitor: %s", err) continue // don't stop loading monitors just because they're disabled } diff --git a/heartbeat/monitors/active/icmp/stdloop.go b/heartbeat/monitors/active/icmp/stdloop.go index b49d243ec4c6..f67ae402bc70 100644 --- a/heartbeat/monitors/active/icmp/stdloop.go +++ b/heartbeat/monitors/active/icmp/stdloop.go @@ -165,7 +165,7 @@ func (l *stdICMPLoop) runICMPRecv(conn *icmp.PacketConn, proto int) { bytes := make([]byte, 512) err := conn.SetReadDeadline(time.Now().Add(time.Second)) if err != nil { - logp.L().Error("could not set read deadline for ICMP: %w", err) + logp.L().Errorf("could not set read deadline for ICMP: %w", err) return } _, addr, err := conn.ReadFrom(bytes) diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go index 6a16c7d3f303..29e7713145ca 100644 --- a/heartbeat/monitors/monitor.go +++ b/heartbeat/monitors/monitor.go @@ -256,7 +256,7 @@ func (m *Monitor) Stop() { if m.close != nil { err := m.close() if err != nil { - logp.L().Error("error closing monitor %s: %w", m.String(), err) + logp.L().Errorf("error closing monitor %s: %w", m.String(), err) } } diff --git a/heartbeat/monitors/task.go b/heartbeat/monitors/task.go index ee0839fe14e5..a655e1d15467 100644 --- a/heartbeat/monitors/task.go +++ b/heartbeat/monitors/task.go @@ -80,13 +80,13 @@ func (t *configuredJob) Start(pubClient beat.Client) { t.pubClient = pubClient if err != nil { - logp.L().Info("could not start monitor: %v", err) + logp.L().Infof("could not start monitor: %v", err) return } t.cancelFn, err = t.monitor.addTask(t.config.Schedule, t.monitor.stdFields.ID, t.makeSchedulerTaskFunc(), t.config.Type) if err != nil { - logp.L().Info("could not start monitor: %v", err) + logp.L().Infof("could not start monitor: %v", err) } } @@ -107,7 +107,7 @@ func runPublishJob(job jobs.Job, pubClient beat.Client) []scheduler.TaskFunc { conts, err := job(event) if err != nil { - logp.L().Info("Job failed with: %s", err) + logp.L().Infof("Job failed with: %s", err) } hasContinuations := len(conts) > 0 diff --git a/heartbeat/monitors/wrappers/monitorstate/tracker.go b/heartbeat/monitors/wrappers/monitorstate/tracker.go index 03909d55aa83..e350294e46e8 100644 --- a/heartbeat/monitors/wrappers/monitorstate/tracker.go +++ b/heartbeat/monitors/wrappers/monitorstate/tracker.go @@ -104,7 +104,7 @@ func (t *Tracker) GetCurrentState(sf stdfields.StdMonitorFields) (state *State) time.Sleep(sleepFor) } if err != nil { - logp.L().Warn("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", tries, sf.ID) + logp.L().Warnf("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", tries, sf.ID) } if loadedState != nil { diff --git a/libbeat/common/file/file_info.go b/libbeat/common/file/file_info.go new file mode 100644 index 000000000000..1364cf8b193d --- /dev/null +++ b/libbeat/common/file/file_info.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file + +import ( + "os" +) + +type ExtendedFileInfo interface { + os.FileInfo + GetOSState() StateOS +} + +type extendedFileInfo struct { + os.FileInfo + osSpecific *StateOS +} + +// GetOSState returns the platform specific StateOS. +// The data is fetched once and cached. +func (f *extendedFileInfo) GetOSState() StateOS { + if f == nil || f.FileInfo == nil { + return StateOS{} + } + + if f.osSpecific != nil { + return *f.osSpecific + } + + osSpecific := GetOSState(f.FileInfo) + f.osSpecific = &osSpecific + return osSpecific +} + +// ExtendFileInfo wraps the standard FileInfo with an extended version. +func ExtendFileInfo(fi os.FileInfo) ExtendedFileInfo { + return &extendedFileInfo{FileInfo: fi} +} diff --git a/libbeat/common/file/file_other.go b/libbeat/common/file/file_other.go index 600e225bc189..3ba429ced201 100644 --- a/libbeat/common/file/file_other.go +++ b/libbeat/common/file/file_other.go @@ -32,7 +32,14 @@ type StateOS struct { // GetOSState returns the FileStateOS for non windows systems func GetOSState(info os.FileInfo) StateOS { - stat := info.Sys().(*syscall.Stat_t) + sys := info.Sys() + if sys == nil { + return StateOS{} + } + stat, ok := sys.(*syscall.Stat_t) + if !ok { + return StateOS{} + } // Convert inode and dev to uint64 to be cross platform compatible fileState := StateOS{ diff --git a/libbeat/common/file/file_windows.go b/libbeat/common/file/file_windows.go index 1b8a9da49dea..3c541c942d2c 100644 --- a/libbeat/common/file/file_windows.go +++ b/libbeat/common/file/file_windows.go @@ -42,6 +42,9 @@ var ( // GetOSState returns the platform specific StateOS func GetOSState(info os.FileInfo) StateOS { + if info == nil { + return StateOS{} + } // os.SameFile must be called to populate the id fields. Otherwise in case for example // os.Stat(file) is used to get the fileInfo, the ids are empty. // https://github.com/elastic/beats/filebeat/pull/53 @@ -56,14 +59,22 @@ func GetOSState(info os.FileInfo) StateOS { // Uint should already return uint64, but making sure this is the case // The required fields can be found here: https://github.com/golang/go/blob/master/src/os/types_windows.go#L78 fileState := StateOS{ - IdxHi: uint64(fileStat.FieldByName("idxhi").Uint()), - IdxLo: uint64(fileStat.FieldByName("idxlo").Uint()), - Vol: uint64(fileStat.FieldByName("vol").Uint()), + IdxHi: getFieldValue(fileStat, "idxhi"), + IdxLo: getFieldValue(fileStat, "idxlo"), + Vol: getFieldValue(fileStat, "vol"), } return fileState } +func getFieldValue(val reflect.Value, name string) uint64 { + fieldValue := val.FieldByName(name) + if !fieldValue.IsValid() { + return 0 + } + return uint64(fieldValue.Uint()) +} + // IsSame file checks if the files are identical func (fs StateOS) IsSame(state StateOS) bool { return fs.IdxHi == state.IdxHi && fs.IdxLo == state.IdxLo && fs.Vol == state.Vol diff --git a/libbeat/docs/queueconfig.asciidoc b/libbeat/docs/queueconfig.asciidoc index f4e2d62c6eae..08ece0f752f5 100644 --- a/libbeat/docs/queueconfig.asciidoc +++ b/libbeat/docs/queueconfig.asciidoc @@ -61,6 +61,7 @@ queue.mem: You can specify the following options in the `queue.mem` section of the +{beatname_lc}.yml+ config file: [float] +[[queue-mem-events-option]] ===== `events` Number of events the queue can store. This value should be evenly divisible by `flush.min_events` to @@ -69,6 +70,7 @@ avoid sending partial batches to the output. The default value is 3200 events. [float] +[[queue-mem-flush-min-events-option]] ===== `flush.min_events` Minimum number of events required for publishing. If this value is set to 0 or 1, events are @@ -80,6 +82,7 @@ sent by the output. The default value is 1600. [float] +[[queue-mem-flush-timeout-option]] ===== `flush.timeout` Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events are available to the diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 3e5bfdc177db..8034a904ff31 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,9 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> +* <> +* <> * <> * <> * <> diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index f3a5c492bfa8..d47d8cf89cc0 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.12.0 :doc-branch: main -:go-version: 1.20.11 +:go-version: 1.20.12 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc index 1b84948b2779..046c45a34dc1 100644 --- a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc +++ b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc @@ -98,6 +98,7 @@ output.elasticsearch: In the previous example, the Elasticsearch nodes are available at `https://10.45.3.2:9220/elasticsearch` and `https://10.45.3.1:9230/elasticsearch`. +[[compression-level-option]] ===== `compression_level` The gzip compression level. Setting this value to `0` disables compression. @@ -114,6 +115,7 @@ Configure escaping of HTML in strings. Set to `true` to enable escaping. The default value is `false`. +[[worker-option]] ===== `worker` The number of workers per configured host publishing events to Elasticsearch. This @@ -659,6 +661,7 @@ The default is 3. endif::[] +[[bulk-max-size-option]] ===== `bulk_max_size` The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 1600. @@ -691,6 +694,7 @@ default is `1s`. The maximum number of seconds to wait before attempting to connect to Elasticsearch after a network error. The default is `60s`. +[[idle-connection-timeout-option]] ===== `idle_connection_timeout` The maximum amount of time an idle connection will remain idle before closing itself. @@ -701,7 +705,7 @@ The default is 3s. The http request timeout in seconds for the Elasticsearch request. The default is 90. -==== `allow_older_versions` +===== `allow_older_versions` By default, {beatname_uc} expects the Elasticsearch instance to be on the same or newer version to provide optimal experience. We suggest you connect to the same version to make sure all features {beatname_uc} is using are @@ -759,6 +763,75 @@ output.elasticsearch: index: "my-dead-letter-index" ------------------------------------------------------------------------------ +===== `preset` + +The performance preset to apply to the output configuration. + +["source","yaml"] +------------------------------------------------------------------------------ +output.elasticsearch: + hosts: ["http://localhost:9200"] + preset: balanced +------------------------------------------------------------------------------ + +Performance presets apply a set of configuration overrides based on a desired performance goal. If set, a performance preset will override other configuration flags to match the recommended settings for that preset. Valid options are: +* `balanced`: good starting point for general efficiency +* `throughput`: good for high data volumes, may increase cpu and memory requirements +* `scale`: reduces ambient resource use in large low-throughput deployments +* `latency`: minimize the time for fresh data to become visible in Elasticsearch +* `custom`: apply user configuration directly with no overrides + +The default if unspecified is `custom`. + +Presets represent current recommendations based on the intended goal; their effect may change between versions to better suit those goals. Currently the presets have the following effects: + +[cols="2,1,1,1,1"] +|=== +|preset |balanced |throughput |scale |latency + +|<> +|1600 +|1600 +|1600 +|50 + +|<> +|1 +|4 +|1 +|1 + +|<> +|3200 +|12800 +|3200 +|4100 + +|<> +|1600 +|1600 +|1600 +|2050 + +|<> +|`10s` +|`5s` +|`20s` +|`1s` + +|<> +|1 +|1 +|1 +|1 + +|<> +|`3s` +|`15s` +|`1s` +|`60s` +|=== + [[es-apis]] ==== Elasticsearch APIs {beatname_uc} will use the `_bulk` API from {es}, the events are sent diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 0998a275ea4f..7c74c82268d1 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -50,15 +50,13 @@ type Enabled struct { type PluginConfig []map[string]config.C -func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig { - return kubeAnnotatorConfig{ - SyncPeriod: 10 * time.Minute, - CleanupTimeout: 60 * time.Second, - DefaultMatchers: Enabled{true}, - DefaultIndexers: Enabled{true}, - Scope: "node", - AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), - } +func (k *kubeAnnotatorConfig) InitDefaults() { + k.SyncPeriod = 10 * time.Minute + k.CleanupTimeout = 60 * time.Second + k.DefaultMatchers = Enabled{true} + k.DefaultIndexers = Enabled{true} + k.Scope = "node" + k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig() } func (k *kubeAnnotatorConfig) Validate() error { @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error { err := matcherCfg.Unpack(&logsPathMatcher) if err != nil { - return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err) + return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err) } if logsPathMatcher.LogsPath == "" { return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well") diff --git a/libbeat/processors/add_kubernetes_metadata/config_test.go b/libbeat/processors/add_kubernetes_metadata/config_test.go index e94089f388a9..3857eb148fa6 100644 --- a/libbeat/processors/add_kubernetes_metadata/config_test.go +++ b/libbeat/processors/add_kubernetes_metadata/config_test.go @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) { for _, test := range tests { cfg := config.MustNewConfigFrom(test.cfg) - c := defaultKubernetesAnnotatorConfig() + var c kubeAnnotatorConfig err := cfg.Unpack(&c) if test.error { @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) { for _, test := range tests { cfg, _ := config.NewConfigFrom(test.matcherConfig) - c := defaultKubernetesAnnotatorConfig() - c.DefaultMatchers = Enabled{false} + var c kubeAnnotatorConfig - err := cfg.Unpack(&c) + _ = cfg.Unpack(&c) + c.DefaultMatchers = Enabled{false} c.Matchers = PluginConfig{ { test.matcherName: *cfg, }, } - err = c.Validate() + err := c.Validate() if test.error { require.NotNil(t, err) } else { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index a8667aef0a8a..954a59ab3f12 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) { - config := defaultKubernetesAnnotatorConfig() - + var config kubeAnnotatorConfig err := cfg.Unpack(&config) if err != nil { return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err) diff --git a/libbeat/processors/script/javascript/session.go b/libbeat/processors/script/javascript/session.go index 337d2eae7997..5b08e7d6052f 100644 --- a/libbeat/processors/script/javascript/session.go +++ b/libbeat/processors/script/javascript/session.go @@ -210,7 +210,6 @@ func (s *session) runProcessFunc(b *beat.Event) (out *beat.Event, err error) { if r := recover(); r != nil { s.log.Errorw("The javascript processor caused an unexpected panic "+ "while processing an event. Recovering, but please report this.", - "event", mapstr.M{"original": b.Fields.String()}, "panic", r, zap.Stack("stack")) if !s.evt.IsCancelled() { diff --git a/libbeat/reader/readfile/fs_metafields_other.go b/libbeat/reader/readfile/fs_metafields_other.go index cc764c4bbcc7..1c1bcb8a24b8 100644 --- a/libbeat/reader/readfile/fs_metafields_other.go +++ b/libbeat/reader/readfile/fs_metafields_other.go @@ -21,7 +21,6 @@ package readfile import ( "fmt" - "os" "strconv" "github.com/elastic/beats/v7/libbeat/common/file" @@ -33,8 +32,8 @@ const ( inodeKey = "log.file.inode" ) -func setFileSystemMetadata(fi os.FileInfo, fields mapstr.M) error { - osstate := file.GetOSState(fi) +func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M) error { + osstate := fi.GetOSState() _, err := fields.Put(deviceIDKey, strconv.FormatUint(osstate.Device, 10)) if err != nil { return fmt.Errorf("failed to set %q: %w", deviceIDKey, err) diff --git a/libbeat/reader/readfile/fs_metafields_windows.go b/libbeat/reader/readfile/fs_metafields_windows.go index 97bfd5c72de1..0a928ba10474 100644 --- a/libbeat/reader/readfile/fs_metafields_windows.go +++ b/libbeat/reader/readfile/fs_metafields_windows.go @@ -19,7 +19,6 @@ package readfile import ( "fmt" - "os" "strconv" "github.com/elastic/beats/v7/libbeat/common/file" @@ -32,8 +31,8 @@ const ( volKey = "log.file.vol" ) -func setFileSystemMetadata(fi os.FileInfo, fields mapstr.M) error { - osstate := file.GetOSState(fi) +func setFileSystemMetadata(fi file.ExtendedFileInfo, fields mapstr.M) error { + osstate := fi.GetOSState() _, err := fields.Put(idxhiKey, strconv.FormatUint(osstate.IdxHi, 10)) if err != nil { return fmt.Errorf("failed to set %q: %w", idxhiKey, err) diff --git a/libbeat/reader/readfile/metafields.go b/libbeat/reader/readfile/metafields.go index c4c41e980f67..be74a9e07a57 100644 --- a/libbeat/reader/readfile/metafields.go +++ b/libbeat/reader/readfile/metafields.go @@ -19,8 +19,8 @@ package readfile import ( "fmt" - "os" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/reader" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -30,14 +30,14 @@ import ( type FileMetaReader struct { reader reader.Reader path string - fi os.FileInfo + fi file.ExtendedFileInfo fingerprint string offset int64 } // New creates a new Encode reader from input reader by applying // the given codec. -func NewFilemeta(r reader.Reader, path string, fi os.FileInfo, fingerprint string, offset int64) reader.Reader { +func NewFilemeta(r reader.Reader, path string, fi file.ExtendedFileInfo, fingerprint string, offset int64) reader.Reader { return &FileMetaReader{r, path, fi, fingerprint, offset} } diff --git a/libbeat/reader/readfile/metafields_other_test.go b/libbeat/reader/readfile/metafields_other_test.go index b9d25b854204..351d175156f3 100644 --- a/libbeat/reader/readfile/metafields_other_test.go +++ b/libbeat/reader/readfile/metafields_other_test.go @@ -20,23 +20,23 @@ package readfile import ( - "os" "syscall" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/mapstr" ) -func createTestFileInfo() os.FileInfo { - return testFileInfo{ +func createTestFileInfo() file.ExtendedFileInfo { + return file.ExtendFileInfo(testFileInfo{ name: "filename", size: 42, time: time.Now(), sys: &syscall.Stat_t{Dev: 17, Ino: 999}, - } + }) } func checkFields(t *testing.T, expected, actual mapstr.M) { diff --git a/libbeat/reader/readfile/metafields_windows_test.go b/libbeat/reader/readfile/metafields_windows_test.go index dce0b8d2161a..fb4cf2a5160a 100644 --- a/libbeat/reader/readfile/metafields_windows_test.go +++ b/libbeat/reader/readfile/metafields_windows_test.go @@ -18,12 +18,12 @@ package readfile import ( - "os" "testing" "time" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -34,8 +34,8 @@ type winTestInfo struct { vol uint32 } -func createTestFileInfo() os.FileInfo { - return &winTestInfo{ +func createTestFileInfo() file.ExtendedFileInfo { + return file.ExtendFileInfo(&winTestInfo{ testFileInfo: testFileInfo{ name: "filename", size: 42, @@ -44,7 +44,7 @@ func createTestFileInfo() os.FileInfo { idxhi: 100, idxlo: 200, vol: 300, - } + }) } func checkFields(t *testing.T, expected, actual mapstr.M) { diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 7f3b50109d56..634db34245b7 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt update \ diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 6b0bbc023f3c..9ff079faa3be 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -232,11 +232,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -276,13 +283,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3588aaec9769..d6b8b9e9475d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -504,11 +504,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -548,13 +555,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json new file mode 100644 index 000000000000..3cabff1ad069 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "uuid_from_es_output" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json new file mode 100644 index 000000000000..557b4163d359 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json new file mode 100644 index 000000000000..1464e65b6402 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "uuid_from_monitoring_config" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json new file mode 100644 index 000000000000..c204e26d2491 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go index b555c84bd402..5d8231046043 100644 --- a/metricbeat/module/beat/state/data.go +++ b/metricbeat/module/beat/state/data.go @@ -77,22 +77,19 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) return fmt.Errorf("failure parsing Beat's State API response: %w", err) } - event.MetricSetFields, _ = schema.Apply(data) - clusterUUID := getMonitoringClusterUUID(data) if clusterUUID == "" { if isOutputES(data) { clusterUUID = getClusterUUID(data) - if clusterUUID != "" { - event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) - if event.MetricSetFields != nil { - event.MetricSetFields.Put("cluster.uuid", clusterUUID) - } + if clusterUUID == "" { + return nil } } } + event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) + event.MetricSetFields, _ = schema.Apply(data) if event.MetricSetFields != nil { diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go index 6123b7539b7e..4d94a3f24da9 100644 --- a/metricbeat/module/beat/state/data_test.go +++ b/metricbeat/module/beat/state/data_test.go @@ -53,3 +53,92 @@ func TestEventMapping(t *testing.T) { require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestUuidFromEsOutput(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_es_output", uuid) +} + +func TestNoEventIfEsOutputButNoUuidYet(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output_pre_connect.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.Equal(t, 0, len(reporter.GetEvents())) + require.Equal(t, 0, len(reporter.GetErrors())) +} + +func TestUuidFromMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_monitoring_config", uuid) +} + +func TestNoUuidInMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_no_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "", uuid) +} diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 74c1fdb0bca5..bf7a9fc931ec 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 COPY test/main.go main.go diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index dcd59309119d..23f5ce8dea62 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -18,11 +18,18 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -62,13 +69,21 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 44ef19c97862..1c56e57b167f 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -16,15 +16,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 26728fccdaea..60b3360ab891 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // GetPodMetaGen requires cfg of type Config + // commonMetaConfig stores the metadata configuration of the resource itself commonMetaConfig := metadata.Config{} if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { logp.Err("Error initializing Kubernetes metadata enricher: %s", err) @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher( // update func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) - id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all + id := join(accessor.GetNamespace(), accessor.GetName()) switch r := r.(type) { case *kubernetes.Pod: @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher( return &nilEnricher{} } + // commonMetaConfig stores the metadata configuration of the resource itself + commonMetaConfig := metadata.Config{} + if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { + logp.Err("Error initializing Kubernetes metadata enricher: %s", err) + return &nilEnricher{} + } + cfg, _ := conf.NewConfigFrom(&commonMetaConfig) + // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher( } } - commonMetaConfig := metadata.Config{} - if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { - logp.Err("Error initializing Kubernetes metadata enricher: %s", err) - return &nilEnricher{} - } - cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata) enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { SyncPeriod: time.Minute * 10, AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), } + if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.New("error unpacking configs") } diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index 0340387144c0..44fdc695748e 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.20.11 AS build-env +FROM golang:1.20.12 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 05e0eaf3c74c..993137b89f3a 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.20.11 +FROM golang:1.20.12 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 23bd210a8357..12bbeee26ca5 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -19,15 +19,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 2040495b5510..fe13ef47b61f 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3ec6770467f0..b5811ed5fe8b 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-k08v7bga-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.12.0-k08v7bga-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.12.0-k08v7bga-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 907a08003b42..256cb7f997bf 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -181,12 +181,12 @@ "event.code": 609002, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T17:51:17.000-02:00", + "event.end": "2024-05-05T17:51:17.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%FTD-7-609002: Teardown local-host net:192.168.2.2 duration 0:00:00", "event.severity": 7, - "event.start": "2023-05-05T19:51:17.000Z", + "event.start": "2024-05-05T19:51:17.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -701,12 +701,12 @@ "event.code": 609002, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T18:24:31.000-02:00", + "event.end": "2024-05-05T18:24:31.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-7-609002: Teardown local-host identity:10.10.10.10 duration 0:00:00", "event.severity": 7, - "event.start": "2023-05-05T20:24:31.000Z", + "event.start": "2024-05-05T20:24:31.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -849,13 +849,13 @@ "event.code": 302014, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T18:29:32.000-02:00", + "event.end": "2024-05-05T18:29:32.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302014: Teardown TCP connection 2960892904 for out111:10.10.10.10/443 to fw111:192.168.2.2/55225 duration 0:00:00 bytes 0 TCP Reset-I", "event.reason": "TCP Reset-I", "event.severity": 6, - "event.start": "2023-05-05T20:29:32.000Z", + "event.start": "2024-05-05T20:29:32.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -966,12 +966,12 @@ "event.code": 305012, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T18:29:32.000-02:00", + "event.end": "2024-05-05T18:29:32.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-305012: Teardown dynamic UDP translation from fw111:10.10.10.10/54230 to out111:192.168.2.2/54230 duration 0:00:00", "event.severity": 6, - "event.start": "2023-05-05T20:29:32.000Z", + "event.start": "2024-05-05T20:29:32.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -1175,12 +1175,12 @@ "event.code": 302016, "event.dataset": "cisco.asa", "event.duration": 124000000000, - "event.end": "2023-05-05T18:40:50.000-02:00", + "event.end": "2024-05-05T18:40:50.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-2-302016: Teardown UDP connection 1671727 for intfacename:10.10.10.10/161 to net:192.186.2.2/53356 duration 0:02:04 bytes 64585", "event.severity": 2, - "event.start": "2023-05-05T20:38:46.000Z", + "event.start": "2024-05-05T20:38:46.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -1812,13 +1812,13 @@ "event.code": 302023, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T19:02:58.000-02:00", + "event.end": "2024-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for fw111:10.10.10.10/39210 to net:192.168.2.2/10051 duration 0:00:00 forwarded bytes 0 Cluster flow with CLU closed on owner", "event.reason": "Cluster flow with CLU closed on owner", "event.severity": 6, - "event.start": "2023-05-05T21:02:58.000Z", + "event.start": "2024-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -1868,13 +1868,13 @@ "event.code": 302023, "event.dataset": "cisco.asa", "event.duration": 0, - "event.end": "2023-05-05T19:02:58.000-02:00", + "event.end": "2024-05-05T19:02:58.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302023: Teardown stub TCP connection for net:10.10.10.10/10051 to unknown:192.168.2.2/39222 duration 0:00:00 forwarded bytes 0 Forwarding or redirect flow removed to create director or backup flow", "event.reason": "Forwarding or redirect flow removed to create director or backup flow", "event.severity": 6, - "event.start": "2023-05-05T21:02:58.000Z", + "event.start": "2024-05-05T21:02:58.000Z", "event.timezone": "-02:00", "event.type": [ "info" @@ -2687,13 +2687,13 @@ "event.code": 302304, "event.dataset": "cisco.asa", "event.duration": 3602000000000, - "event.end": "2023-04-27T04:12:23.000-02:00", + "event.end": "2024-04-27T04:12:23.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-302304: Teardown TCP state-bypass connection 2751765169 from server.deflan:81.2.69.143/54242 to server.deflan:67.43.156.12/9101 duration 1:00:02 bytes 245 Connection timeout", "event.reason": "Connection timeout", "event.severity": 6, - "event.start": "2023-04-27T05:12:21.000Z", + "event.start": "2024-04-27T05:12:21.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -3227,13 +3227,13 @@ "event.code": 113019, "event.dataset": "cisco.asa", "event.duration": 1936000000000, - "event.end": "2023-04-27T02:03:03.000-02:00", + "event.end": "2024-04-27T02:03:03.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-4-113019: Group = 81.2.69.143, Username = 81.2.69.143, IP = 81.2.69.143, Session disconnected. Session Type: LAN-to-LAN, Duration: 0h:32m:16s, Bytes xmt: 297103, Bytes rcv: 1216163, Reason: User Requested", "event.reason": "User Requested", "event.severity": 4, - "event.start": "2023-04-27T03:30:47.000Z", + "event.start": "2024-04-27T03:30:47.000Z", "event.timezone": "-02:00", "event.type": [ "info" diff --git a/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json index 63f46eabbba2..d7c455136e2f 100644 --- a/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/non-canonical.log-expected.json @@ -361,12 +361,12 @@ "event.code": 305012, "event.dataset": "cisco.asa", "event.duration": 41000000000, - "event.end": "2023-07-15T13:38:47.000-02:00", + "event.end": "2024-07-15T13:38:47.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/62409 to outside:81.2.69.142/62409 duration 0:00:41", "event.severity": 6, - "event.start": "2023-07-15T15:38:06.000Z", + "event.start": "2024-07-15T15:38:06.000Z", "event.timezone": "-02:00", "event.type": [ "connection", @@ -423,12 +423,12 @@ "event.code": 305012, "event.dataset": "cisco.asa", "event.duration": 30000000000, - "event.end": "2023-07-15T13:37:33.000-02:00", + "event.end": "2024-07-15T13:37:33.000-02:00", "event.kind": "event", "event.module": "cisco", "event.original": "%ASA-6-305012: Teardown dynamic UDP translation from SERVERS:exp-wait/56421 to outside:81.2.69.142/56421 duration 0:00:30", "event.severity": 6, - "event.start": "2023-07-15T15:37:03.000Z", + "event.start": "2024-07-15T15:37:03.000Z", "event.timezone": "-02:00", "event.type": [ "connection", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/288_auto_clear_users_history_start.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/288_auto_clear_users_history_start.log-expected.json index 129ad664676d..fb3cfbbb9cb5 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/288_auto_clear_users_history_start.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/288_auto_clear_users_history_start.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T03:00:20.000-02:00", + "@timestamp": "2024-03-08T03:00:20.000-02:00", "cyberarkpas.audit.action": "Auto Clear Users History start", "cyberarkpas.audit.desc": "Auto Clear Users History start", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/289_auto_clear_users_history_end.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/289_auto_clear_users_history_end.log-expected.json index de251078346b..9ad5b886c6ca 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/289_auto_clear_users_history_end.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/289_auto_clear_users_history_end.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T03:00:20.000-02:00", + "@timestamp": "2024-03-08T03:00:20.000-02:00", "cyberarkpas.audit.action": "Auto Clear Users History end", "cyberarkpas.audit.desc": "Auto Clear Users History end", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/310_monitor_dr_replication_start.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/310_monitor_dr_replication_start.log-expected.json index 41b565a5a4a0..9d813f639d65 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/310_monitor_dr_replication_start.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/310_monitor_dr_replication_start.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T02:48:07.000-02:00", + "@timestamp": "2024-03-08T02:48:07.000-02:00", "cyberarkpas.audit.action": "Monitor DR Replication start", "cyberarkpas.audit.desc": "Monitor DR Replication start", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/311_monitor_dr_replication_end.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/311_monitor_dr_replication_end.log-expected.json index 13cd9bf1248f..ee767935d3b0 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/311_monitor_dr_replication_end.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/311_monitor_dr_replication_end.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T02:48:07.000-02:00", + "@timestamp": "2024-03-08T02:48:07.000-02:00", "cyberarkpas.audit.action": "Monitor DR Replication end", "cyberarkpas.audit.desc": "Monitor DR Replication end", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/357_monitor_fw_rules_start.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/357_monitor_fw_rules_start.log-expected.json index 22738846d864..2943356268b9 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/357_monitor_fw_rules_start.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/357_monitor_fw_rules_start.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T02:32:56.000-02:00", + "@timestamp": "2024-03-08T02:32:56.000-02:00", "cyberarkpas.audit.action": "Monitor FW rules start", "cyberarkpas.audit.desc": "Monitor FW rules start", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/358_monitor_fw_rules_end.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/358_monitor_fw_rules_end.log-expected.json index 6518fbedab7f..bed2becb5d42 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/358_monitor_fw_rules_end.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/358_monitor_fw_rules_end.log-expected.json @@ -38,7 +38,7 @@ ] }, { - "@timestamp": "2023-03-08T02:32:56.000-02:00", + "@timestamp": "2024-03-08T02:32:56.000-02:00", "cyberarkpas.audit.action": "Monitor FW Rules end", "cyberarkpas.audit.desc": "Monitor FW Rules end", "cyberarkpas.audit.issuer": "Batch", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/479_security_warning_the_signature_hash_algorithm_of_the_vault_certificate_is_sha1.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/479_security_warning_the_signature_hash_algorithm_of_the_vault_certificate_is_sha1.log-expected.json index eafc4237e717..bb66629fa39b 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/479_security_warning_the_signature_hash_algorithm_of_the_vault_certificate_is_sha1.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/479_security_warning_the_signature_hash_algorithm_of_the_vault_certificate_is_sha1.log-expected.json @@ -39,7 +39,7 @@ ] }, { - "@timestamp": "2023-03-08T07:46:54.000-02:00", + "@timestamp": "2024-03-08T07:46:54.000-02:00", "cyberarkpas.audit.action": "Security warning - The Signature Hash Algorithm of the Vault certificate is SHA1.", "cyberarkpas.audit.desc": "Security warning - The Signature Hash Algorithm of the Vault certificate is SHA1.", "cyberarkpas.audit.issuer": "Builtin", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/59_clear_safe_history.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/59_clear_safe_history.log-expected.json index d3e8d85a46f5..ef8f8d42bb26 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/59_clear_safe_history.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/59_clear_safe_history.log-expected.json @@ -39,7 +39,7 @@ ] }, { - "@timestamp": "2023-03-08T03:10:31.000-02:00", + "@timestamp": "2024-03-08T03:10:31.000-02:00", "cyberarkpas.audit.action": "Clear Safe History", "cyberarkpas.audit.desc": "Clear Safe History", "cyberarkpas.audit.issuer": "PasswordManager", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/88_set_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/88_set_password.log-expected.json index 728e84742ea2..65ec1710d275 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/88_set_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/88_set_password.log-expected.json @@ -76,7 +76,7 @@ ] }, { - "@timestamp": "2023-03-08T02:54:46.000-02:00", + "@timestamp": "2024-03-08T02:54:46.000-02:00", "cyberarkpas.audit.action": "Set Password", "cyberarkpas.audit.desc": "Set Password", "cyberarkpas.audit.issuer": "PVWAGWUser", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/legacysyslog.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/legacysyslog.log-expected.json index c4e72e65c5f0..439a5355e95b 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/legacysyslog.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/legacysyslog.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2023-03-08T03:41:01.000-02:00", + "@timestamp": "2024-03-08T03:41:01.000-02:00", "cyberarkpas.audit.action": "Retrieve File", "cyberarkpas.audit.desc": "Retrieve File", "cyberarkpas.audit.file": "Root\\Policies\\Policy-BusinessWebsite.ini", diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index bd68544accce..662b27d669db 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/x-pack/heartbeat/monitors/browser/source/project.go b/x-pack/heartbeat/monitors/browser/source/project.go index af6bd96dfdc6..7c11de311345 100644 --- a/x-pack/heartbeat/monitors/browser/source/project.go +++ b/x-pack/heartbeat/monitors/browser/source/project.go @@ -168,8 +168,8 @@ func (p *ProjectSource) Close() error { func runSimpleCommand(cmd *exec.Cmd, dir string) error { cmd.Dir = dir - logp.L().Info("Running %s in %s", cmd, dir) + logp.L().Infof("Running %s in %s", cmd, dir) output, err := cmd.CombinedOutput() - logp.L().Info("Ran %s (%d) got '%s': (%s) as (%d/%d)", cmd, cmd.ProcessState.ExitCode(), string(output), err, syscall.Getuid(), syscall.Geteuid()) + logp.L().Infof("Ran %s (%d) got '%s': (%s) as (%d/%d)", cmd, cmd.ProcessState.ExitCode(), string(output), err, syscall.Getuid(), syscall.Geteuid()) return err } diff --git a/x-pack/heartbeat/monitors/browser/sourcejob.go b/x-pack/heartbeat/monitors/browser/sourcejob.go index c62c50b3bb17..697e51abf51a 100644 --- a/x-pack/heartbeat/monitors/browser/sourcejob.go +++ b/x-pack/heartbeat/monitors/browser/sourcejob.go @@ -125,7 +125,7 @@ func (sj *SourceJob) extraArgs(uiOrigin bool) []string { s, err := json.Marshal(sj.browserCfg.PlaywrightOpts) if err != nil { // This should never happen, if it was parsed as a config it should be serializable - logp.L().Warn("could not serialize playwright options '%v': %w", sj.browserCfg.PlaywrightOpts, err) + logp.L().Warnf("could not serialize playwright options '%v': %w", sj.browserCfg.PlaywrightOpts, err) } else { extraArgs = append(extraArgs, "--playwright-options", string(s)) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go index fbfb71526cc5..32f127de98c3 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go @@ -219,7 +219,7 @@ func runCmd( break } if err != nil { - logp.L().Warn("error decoding json for test json results: %w", err) + logp.L().Warnf("error decoding json for test json results: %w", err) } mpx.writeSynthEvent(&se) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 8555ae448a71..ddd928b216d6 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -96,7 +96,7 @@ func (se SynthEvent) ToMap() (m mapstr.M) { u, e := url.Parse(se.URL) if e != nil { _, _ = m.Put("url", mapstr.M{"full": se.URL}) - logp.L().Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) + logp.L().Warnf("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) } else { _, _ = m.Put("url", wraputil.URLFields(u)) } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 46e951ccb56e..a22db4f7f8cf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -901,11 +901,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -945,13 +952,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/x-pack/metricbeat/module/azure/azure.go b/x-pack/metricbeat/module/azure/azure.go index 7812feed838c..dd7f121b2697 100644 --- a/x-pack/metricbeat/module/azure/azure.go +++ b/x-pack/metricbeat/module/azure/azure.go @@ -96,9 +96,14 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { // depending on metric time grain (check `MetricRegistry` // for more information). // - // We truncate the reference time to the second to avoid millisecond - // variations in the collection period causing skipped collections. - referenceTime := time.Now().UTC().Truncate(time.Second) + // We round the reference time to the nearest second to avoid + // millisecond variations in the collection period causing + // skipped collections. + // + // See "Round outer limits" and "Round inner limits" tests in + // the metric_registry_test.go for more information. + //referenceTime := time.Now().UTC().Round(time.Second) + referenceTime := time.Now().UTC() // Initialize cloud resources and monitor metrics // information. diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index ce9a6cb824fc..3b22a5713cd3 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -16,110 +16,6 @@ import ( "github.com/elastic/elastic-agent-libs/logp" ) -// NewMetricRegistry instantiates a new metric registry. -func NewMetricRegistry(logger *logp.Logger) *MetricRegistry { - return &MetricRegistry{ - logger: logger, - collectionsInfo: make(map[string]MetricCollectionInfo), - } -} - -// MetricRegistry keeps track of the last time a metric was collected and -// the time grain used. -// -// This is used to avoid collecting the same metric values over and over again -// when the time grain is larger than the collection interval. -type MetricRegistry struct { - logger *logp.Logger - collectionsInfo map[string]MetricCollectionInfo -} - -// Update updates the metric registry with the latest timestamp and -// time grain for the given metric. -func (m *MetricRegistry) Update(metric Metric, info MetricCollectionInfo) { - m.collectionsInfo[m.buildMetricKey(metric)] = info -} - -// NeedsUpdate returns true if the metric needs to be collected again -// for the given `referenceTime`. -func (m *MetricRegistry) NeedsUpdate(referenceTime time.Time, metric Metric) bool { - // Build a key to store the metric in the registry. - // The key is a combination of the namespace, - // resource ID and metric names. - metricKey := m.buildMetricKey(metric) - - // Get the now time in UTC, only to be used for logging. - // It's interesting to see when the registry evaluate each - // metric in relation to the reference time. - now := time.Now().UTC() - - if collection, exists := m.collectionsInfo[metricKey]; exists { - // Turn the time grain into a duration (for example, PT5M -> 5 minutes). - timeGrainDuration := convertTimeGrainToDuration(collection.timeGrain) - - // Calculate the start time of the time grain in relation to - // the reference time. - timeGrainStartTime := referenceTime.Add(-timeGrainDuration) - - // If the last collection time is after the start time of the time grain, - // it means that we already have a value for the given time grain. - // - // In this case, the metricset does not need to collect the metric - // values again. - if collection.timestamp.After(timeGrainStartTime) { - m.logger.Debugw( - "MetricRegistry: Metric does not need an update", - "needs_update", false, - "reference_time", referenceTime, - "now", now, - "time_grain_start_time", timeGrainStartTime, - "last_collection_at", collection.timestamp, - ) - - return false - } - - // The last collection time is before the start time of the time grain, - // it means that the metricset needs to collect the metric values again. - m.logger.Debugw( - "MetricRegistry: Metric needs an update", - "needs_update", true, - "reference_time", referenceTime, - "now", now, - "time_grain_start_time", timeGrainStartTime, - "last_collection_at", collection.timestamp, - ) - - return true - } - - // If the metric is not in the registry, it means that it has never - // been collected before. - // - // In this case, we need to collect the metric. - m.logger.Debugw( - "MetricRegistry: Metric needs an update", - "needs_update", true, - "reference_time", referenceTime, - "now", now, - ) - - return true -} - -// buildMetricKey builds a key for the metric registry. -// -// The key is a combination of the namespace, resource ID and metric names. -func (m *MetricRegistry) buildMetricKey(metric Metric) string { - keyComponents := []string{ - metric.Namespace, - metric.ResourceId, - } - keyComponents = append(keyComponents, metric.Names...) - - return strings.Join(keyComponents, ",") -} - // MetricCollectionInfo contains information about the last time // a metric was collected and the time grain used. type MetricCollectionInfo struct { diff --git a/x-pack/metricbeat/module/azure/client_utils.go b/x-pack/metricbeat/module/azure/client_utils.go index 986125ba6b68..114ccd95baf2 100644 --- a/x-pack/metricbeat/module/azure/client_utils.go +++ b/x-pack/metricbeat/module/azure/client_utils.go @@ -135,14 +135,14 @@ func compareMetricValues(metVal *float64, metricVal *float64) bool { return false } -// convertTimeGrainToDuration converts the Azure time grain options to the equivalent +// asDuration converts the Azure time grain options to the equivalent // `time.Duration` value. // // For example, converts "PT1M" to `time.Minute`. // // See https://docs.microsoft.com/en-us/azure/azure-monitor/platform/metrics-supported#time-grain // for more information. -func convertTimeGrainToDuration(timeGrain string) time.Duration { +func asDuration(timeGrain string) time.Duration { var duration time.Duration switch timeGrain { case "PT1M": diff --git a/x-pack/metricbeat/module/azure/metric_registry.go b/x-pack/metricbeat/module/azure/metric_registry.go new file mode 100644 index 000000000000..cdaa9496b5d6 --- /dev/null +++ b/x-pack/metricbeat/module/azure/metric_registry.go @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "strings" + "time" + + "github.com/elastic/elastic-agent-libs/logp" +) + +// NewMetricRegistry instantiates a new metric registry. +func NewMetricRegistry(logger *logp.Logger) *MetricRegistry { + return &MetricRegistry{ + logger: logger, + collectionsInfo: make(map[string]MetricCollectionInfo), + jitter: 1 * time.Second, + } +} + +// MetricRegistry keeps track of the last time a metric was collected and +// the time grain used. +// +// This is used to avoid collecting the same metric values over and over again +// when the time grain is larger than the collection interval. +type MetricRegistry struct { + logger *logp.Logger + collectionsInfo map[string]MetricCollectionInfo + // The collection period can be jittered by a second. + // We introduce a small jitter to avoid skipping collections + // when the collection period is close (usually < 1s) to the + // time grain start time. + jitter time.Duration +} + +// Update updates the metric registry with the latest timestamp and +// time grain for the given metric. +func (m *MetricRegistry) Update(metric Metric, info MetricCollectionInfo) { + m.collectionsInfo[m.buildMetricKey(metric)] = info +} + +// NeedsUpdate returns true if the metric needs to be collected again +// for the given `referenceTime`. +func (m *MetricRegistry) NeedsUpdate(referenceTime time.Time, metric Metric) bool { + // Build a key to store the metric in the registry. + // The key is a combination of the namespace, + // resource ID and metric names. + metricKey := m.buildMetricKey(metric) + + if lastCollection, exists := m.collectionsInfo[metricKey]; exists { + // Turn the time grain into a duration (for example, PT5M -> 5 minutes). + timeGrainDuration := asDuration(lastCollection.timeGrain) + + // Adjust the last collection time by adding a small jitter to avoid + // skipping collections when the collection period is close (usually < 1s). + timeSinceLastCollection := time.Since(lastCollection.timestamp) + m.jitter + + if timeSinceLastCollection < timeGrainDuration { + m.logger.Debugw( + "MetricRegistry: Metric does not need an update", + "needs_update", false, + "reference_time", referenceTime, + "last_collection_time", lastCollection.timestamp, + "time_since_last_collection_seconds", timeSinceLastCollection.Seconds(), + "time_grain", lastCollection.timeGrain, + "time_grain_duration_seconds", timeGrainDuration.Seconds(), + "resource_id", metric.ResourceId, + "namespace", metric.Namespace, + "aggregation", metric.Aggregations, + "names", strings.Join(metric.Names, ","), + ) + + return false + } + + // The last collection time is before the start time of the time grain, + // it means that the metricset needs to collect the metric values again. + m.logger.Debugw( + "MetricRegistry: Metric needs an update", + "needs_update", true, + "reference_time", referenceTime, + "last_collection_time", lastCollection.timestamp, + "time_since_last_collection_seconds", timeSinceLastCollection.Seconds(), + "time_grain", lastCollection.timeGrain, + "time_grain_duration_seconds", timeGrainDuration.Seconds(), + "resource_id", metric.ResourceId, + "namespace", metric.Namespace, + "aggregation", metric.Aggregations, + "names", strings.Join(metric.Names, ","), + ) + + return true + } + + // If the metric is not in the registry, it means that it has never + // been collected before. + // + // In this case, we need to collect the metric. + m.logger.Debugw( + "MetricRegistry: Metric needs an update (no collection info in the metric registry)", + "needs_update", true, + "reference_time", referenceTime, + "resource_id", metric.ResourceId, + "namespace", metric.Namespace, + "aggregation", metric.Aggregations, + "names", strings.Join(metric.Names, ","), + ) + + return true +} + +// buildMetricKey builds a key for the metric registry. +// +// The key is a combination of the namespace, resource ID and metric names. +func (m *MetricRegistry) buildMetricKey(metric Metric) string { + keyComponents := []string{ + metric.Namespace, + metric.ResourceId, + } + keyComponents = append(keyComponents, metric.Names...) + + return strings.Join(keyComponents, ",") +} diff --git a/x-pack/metricbeat/module/azure/metric_registry_test.go b/x-pack/metricbeat/module/azure/metric_registry_test.go new file mode 100644 index 000000000000..a0ecdc84b85d --- /dev/null +++ b/x-pack/metricbeat/module/azure/metric_registry_test.go @@ -0,0 +1,93 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package azure + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestNewMetricRegistry(t *testing.T) { + logger := logp.NewLogger("test azure monitor") + + t.Run("Collect metrics with a regular 5 minutes period", func(t *testing.T) { + metricRegistry := NewMetricRegistry(logger) + + // Create a lastCollectionAt parsing the string 2023-12-08T16:37:50.000Z into a time.Time + lastCollectionAt, _ := time.Parse(time.RFC3339, "2023-12-08T16:37:50.000Z") + + // Create a referenceTime parsing 2023-12-08T16:42:50.000Z into a time.Time + referenceTime, _ := time.Parse(time.RFC3339, "2023-12-08T16:42:50.000Z") + + metric := Metric{ + ResourceId: "test", + Namespace: "test", + } + metricCollectionInfo := MetricCollectionInfo{ + timeGrain: "PT5M", + timestamp: lastCollectionAt, + } + + metricRegistry.Update(metric, metricCollectionInfo) + + needsUpdate := metricRegistry.NeedsUpdate(referenceTime, metric) + + assert.True(t, needsUpdate, "metric should need update") + }) + + t.Run("Collect metrics using a period 3 seconds longer than previous", func(t *testing.T) { + metricRegistry := NewMetricRegistry(logger) + + // Create a lastCollectionAt parsing the string 2023-12-08T16:37:50.000Z into a time.Time + lastCollectionAt, _ := time.Parse(time.RFC3339, "2023-12-08T16:37:50.000Z") + + // Create a referenceTime parsing 2023-12-08T16:42:50.000Z into a time.Time + referenceTime, _ := time.Parse(time.RFC3339, "2023-12-08T16:42:53.000Z") + + metric := Metric{ + ResourceId: "test", + Namespace: "test", + } + metricCollectionInfo := MetricCollectionInfo{ + timeGrain: "PT5M", + timestamp: lastCollectionAt, + } + + metricRegistry.Update(metric, metricCollectionInfo) + + needsUpdate := metricRegistry.NeedsUpdate(referenceTime, metric) + + assert.True(t, needsUpdate, "metric should need update") + }) + + t.Run("Collect metrics using a period (1 second) shorter than previous", func(t *testing.T) { + metricRegistry := NewMetricRegistry(logger) + + // Create a referenceTime parsing 2023-12-08T16:42:50.000Z into a time.Time + referenceTime, _ := time.Parse(time.RFC3339, "2023-12-08T10:58:33.000Z") + + // Create a lastCollectionAt parsing the string 2023-12-08T16:37:50.000Z into a time.Time + lastCollectionAt, _ := time.Parse(time.RFC3339, "2023-12-08T10:53:34.000Z") + + metric := Metric{ + ResourceId: "test", + Namespace: "test", + } + metricCollectionInfo := MetricCollectionInfo{ + timeGrain: "PT5M", + timestamp: lastCollectionAt, + } + + metricRegistry.Update(metric, metricCollectionInfo) + + needsUpdate := metricRegistry.NeedsUpdate(referenceTime, metric) + + assert.True(t, needsUpdate, "metric should not need update") + }) +} diff --git a/x-pack/metricbeat/module/azure/resources.go b/x-pack/metricbeat/module/azure/resources.go index 0a723c82bd5a..6a633663cb1c 100644 --- a/x-pack/metricbeat/module/azure/resources.go +++ b/x-pack/metricbeat/module/azure/resources.go @@ -38,7 +38,7 @@ type Metric struct { Values []MetricValue TimeGrain string ResourceId string - // ResourceSubId is used for the metric values api as namespaces can apply to sub resrouces ex. storage account: container, blob, vm scaleset: vms + // ResourceSubId is used for the metric values api as namespaces can apply to sub resources ex. storage account: container, blob, vm scaleset: vms ResourceSubId string } diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 20604392f603..92ee1d834571 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.20.11 AS build-env +FROM golang:1.20.12 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build .