diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee4de95884df..61fa4b04bcdc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -24,24 +24,24 @@ CHANGELOG* /deploy/ @elastic/elastic-agent-data-plane /deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring /dev-tools/ @elastic/elastic-agent-data-plane +/dev-tools/kubernetes @elastic/obs-ds-hosted-services /docs/ @elastic/elastic-agent-data-plane /filebeat @elastic/elastic-agent-data-plane /filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /filebeat/input/syslog/ @elastic/security-external-integrations /filebeat/input/winlog/ @elastic/security-external-integrations -/filebeat/module/ @elastic/integrations /filebeat/module/apache @elastic/obs-infraobs-integrations /filebeat/module/auditd @elastic/security-external-integrations /filebeat/module/elasticsearch/ @elastic/infra-monitoring-ui /filebeat/module/haproxy @elastic/obs-infraobs-integrations -/filebeat/module/icinga @elastic/integrations +/filebeat/module/icinga # TODO: find right team /filebeat/module/iis @elastic/obs-infraobs-integrations /filebeat/module/kafka @elastic/obs-infraobs-integrations /filebeat/module/kibana/ @elastic/infra-monitoring-ui /filebeat/module/logstash/ @elastic/infra-monitoring-ui /filebeat/module/mongodb @elastic/obs-infraobs-integrations /filebeat/module/mysql @elastic/security-external-integrations -/filebeat/module/nats @elastic/integrations +/filebeat/module/nats @elastic/obs-infraobs-integrations /filebeat/module/nginx @elastic/obs-infraobs-integrations /filebeat/module/osquery @elastic/security-external-integrations /filebeat/module/pensando @elastic/security-external-integrations @@ -49,8 +49,8 @@ CHANGELOG* /filebeat/module/redis @elastic/obs-infraobs-integrations /filebeat/module/santa @elastic/security-external-integrations /filebeat/module/system @elastic/elastic-agent-data-plane -/filebeat/module/traefik @elastic/integrations -/heartbeat/ @elastic/hosted-services +/filebeat/module/traefik # TODO: find right team +/heartbeat/ @elastic/obs-ds-hosted-services /journalbeat @elastic/elastic-agent-data-plane /libbeat/ @elastic/elastic-agent-data-plane /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs @@ -68,7 +68,6 @@ CHANGELOG* /metricbeat/ @elastic/elastic-agent-data-plane /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /metricbeat/helper/kubernetes @elastic/obs-cloudnative-monitoring -/metricbeat/module/ @elastic/integrations /metricbeat/module/apache @elastic/obs-infraobs-integrations /metricbeat/module/beat/ @elastic/infra-monitoring-ui /metricbeat/module/ceph @elastic/obs-infraobs-integrations @@ -122,7 +121,6 @@ CHANGELOG* /x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations -/x-pack/filebeat/module/ @elastic/integrations /x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations /x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring /x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring @@ -174,10 +172,9 @@ CHANGELOG* /x-pack/filebeat/module/zscaler @elastic/security-external-integrations /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-external-integrations /x-pack/filebeat/processors/decode_cef/ @elastic/security-external-integrations -/x-pack/heartbeat/ @elastic/hosted-services +/x-pack/heartbeat/ @elastic/obs-ds-hosted-services /x-pack/metricbeat/ @elastic/elastic-agent-data-plane /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. -/x-pack/metricbeat/module/ @elastic/integrations /x-pack/metricbeat/module/activemq @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/airflow @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/cloudfoundry @elastic/obs-infraobs-integrations diff --git a/.go-version b/.go-version index 4bb1a22f8ec5..3b9e4a0c187a 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.20.11 +1.20.12 diff --git a/.golangci.yml b/.golangci.yml index b4c16d3291da..3cc6336695e3 100755 --- a/.golangci.yml +++ b/.golangci.yml @@ -114,7 +114,7 @@ linters-settings: gosimple: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 @@ -132,19 +132,19 @@ linters-settings: staticcheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" checks: ["all"] stylecheck: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" # Disabled: # ST1005: error strings should not be capitalized checks: ["all", "-ST1005"] unused: # Select the Go version to target. The default is '1.13'. - go: "1.20.11" + go: "1.20.12" gosec: excludes: diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ada44dfd7b0a..1f1f3e359353 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -32,6 +32,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Winlogbeat* - Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193] +- Add "keystore.path" configuration settings to $workdir\data\{{.BeatName}}.keystore. Issue {issue}12315[12315] {pull}37237[37237] *Functionbeat* @@ -53,6 +54,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640] - Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820] - Support build of projects outside of beats directory {pull}36126[36126] +- Support Elastic Agent control protocol chunking support {pull}37343[37343] *Auditbeat* @@ -148,7 +150,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - elasticsearch output now supports `idle_connection_timeout`. {issue}35616[35615] {pull}36843[36843] - Upgrade golang/x/net to v0.17.0. Updates the publicsuffix table used by the registered_domain processor. {pull}36969[36969] Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor -- Upgrade to Go 1.20.11. {pull}37123[37123] +- Upgrade to Go 1.20.12. {pull}37350[37350] - The Elasticsearch output can now configure performance presets with the `preset` configuration field. {pull}37259[37259] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 0070ee8ff110..192819bb6e72 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,5 +1,5 @@ Elastic Beats -Copyright 2014-2023 Elasticsearch BV +Copyright 2014-2024 Elasticsearch BV This product includes software developed by The Apache Software Foundation (http://www.apache.org/). @@ -12268,11 +12268,11 @@ SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.6.5 +Version: v0.6.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.5/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.6.6/LICENSE: Apache License Version 2.0, January 2004 @@ -12479,11 +12479,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.5.0 +Version: v7.6.0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.5.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.6.0/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -17323,44 +17323,6 @@ Contents of probable licence file $GOMODCACHE/github.com/golang/mock@v1.6.0/LICE limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/golang/protobuf -Version: v1.5.3 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.3/LICENSE: - -Copyright 2010 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - -------------------------------------------------------------------------------- Dependency : github.com/golang/snappy Version: v0.0.4 @@ -24701,11 +24663,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.14.0 +Version: v0.16.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.16.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24775,11 +24737,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/mod -Version: v0.10.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24812,11 +24774,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.17.0 +Version: v0.19.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.19.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24886,11 +24848,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sync -Version: v0.3.0 +Version: v0.5.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.3.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.5.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24923,11 +24885,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24960,11 +24922,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.13.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25034,11 +24996,48 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/tools -Version: v0.9.1 +Version: v0.16.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.16.0/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : golang.org/x/tools/go/vcs +Version: v0.1.0-deprecated Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.9.1/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/tools/go/vcs@v0.1.0-deprecated/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -37721,6 +37720,44 @@ third-party archives. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/golang/protobuf +Version: v1.5.3 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.3/LICENSE: + +Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + -------------------------------------------------------------------------------- Dependency : github.com/google/gofuzz Version: v1.2.0 @@ -51250,11 +51287,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/exp -Version: v0.0.0-20220921023135-46d9e7742f1e +Version: v0.0.0-20231127185646-65229373498e Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20220921023135-46d9e7742f1e/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51287,11 +51324,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 4f7612cd90e3..742041d66af8 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/deploy/kubernetes/filebeat-kubernetes.yaml b/deploy/kubernetes/filebeat-kubernetes.yaml index c9015c0e1473..6c365ced4cb9 100644 --- a/deploy/kubernetes/filebeat-kubernetes.yaml +++ b/deploy/kubernetes/filebeat-kubernetes.yaml @@ -113,6 +113,7 @@ data: filebeat.yml: |- filebeat.inputs: - type: filestream + id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} paths: - /var/log/containers/*.log parsers: diff --git a/deploy/kubernetes/filebeat/filebeat-configmap.yaml b/deploy/kubernetes/filebeat/filebeat-configmap.yaml index f2614e8c035b..8c2fb6603a48 100644 --- a/deploy/kubernetes/filebeat/filebeat-configmap.yaml +++ b/deploy/kubernetes/filebeat/filebeat-configmap.yaml @@ -9,6 +9,7 @@ data: filebeat.yml: |- filebeat.inputs: - type: filestream + id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} paths: - /var/log/containers/*.log parsers: diff --git a/dev-tools/kubernetes/Tiltfile b/dev-tools/kubernetes/Tiltfile index 0a373ceb05fa..5ef4217b849a 100644 --- a/dev-tools/kubernetes/Tiltfile +++ b/dev-tools/kubernetes/Tiltfile @@ -137,7 +137,7 @@ def k8s_expose( # `beat`: `metricbeat` to test Metricbeat, `filebeat` to test Filebeat # `mode`: `debug` to start a remote debugger that you can connect to from your IDE with hot reloading enabled, `run` to just run Metricbeat without a debugger but still with hot reloading enabled # `arch`: `amd64` to build go binary for amd64 architecture, `arm64` to build go binary for arm64 (aka M1 Apple chip) architecture -# `k8s_env`: `kind` to run against a Kind cluster with no docker repo, `gcp` to use a docker repo on GCP +# `k8s_env`: `kind` to run against a Kind cluster with no docker repo, `gcp` to use a docker repo on GCP, `aws` to use a docker repo on AWS # `k8s_cluster`: `single` to use a single node k8s cluster, `multi` to use a k8s with more than 1 node. # if running on a multi-node cluster we expect to have at least 2 workers and a control plane node. One of the workers (eg. worker1) # should have a taint and a label (for node affinity) to make sure that only the debugger runs on that node. You need to run the following commands: @@ -203,10 +203,20 @@ def beat( k8s_expose(beat=beat, mode=mode, k8s_cluster=k8s_cluster) +# Note: Select only one of the following examples or modify one with the parameters you want to use + +# Run metricbeat in run mode against a single node k8s cluster with a docker repo on AWS +# beat(beat="metricbeat", +# mode="run", +# arch="amd64", +# k8s_env="aws", +# k8s_cluster="single", +# ) + +# Run on Mac M1 against a single node k8s cluster beat(beat="metricbeat", - # mode="debug", - mode="run", - arch="amd64", - k8s_env="aws", + mode="debug", + arch="arm64", + k8s_env="kind", k8s_cluster="single", ) diff --git a/dev-tools/kubernetes/filebeat/Dockerfile.debug b/dev-tools/kubernetes/filebeat/Dockerfile.debug index 61e036848474..c0c8768861cc 100644 --- a/dev-tools/kubernetes/filebeat/Dockerfile.debug +++ b/dev-tools/kubernetes/filebeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/filebeat/manifest.debug.yaml b/dev-tools/kubernetes/filebeat/manifest.debug.yaml index 36600e5bf5cf..36fc03bc559c 100644 --- a/dev-tools/kubernetes/filebeat/manifest.debug.yaml +++ b/dev-tools/kubernetes/filebeat/manifest.debug.yaml @@ -1,3 +1,106 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap @@ -9,9 +112,17 @@ metadata: data: filebeat.yml: |- filebeat.inputs: - - type: container + - type: filestream + id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} paths: - /var/log/containers/*.log + parsers: + - container: ~ + prospector: + scanner: + fingerprint.enabled: true + symlinks: true + file_identity.fingerprint: ~ processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -20,15 +131,23 @@ data: logs_path: "/var/log/containers/" # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: - #filebeat.autodiscover: + # filebeat.autodiscover: # providers: # - type: kubernetes # node: ${NODE_NAME} # hints.enabled: true # hints.default_config: - # type: container + # type: filestream + # id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} # paths: - # - /var/log/containers/*${data.kubernetes.container.id}.log + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # parsers: + # - container: ~ + # prospector: + # scanner: + # fingerprint.enabled: true + # symlinks: true + # file_identity.fingerprint: ~ processors: - add_cloud_metadata: @@ -71,7 +190,6 @@ spec: args: [ "-c", "/etc/filebeat.yml", "-e", - "-system.hostfs=/hostfs", ] ports: - containerPort: 56268 @@ -136,107 +254,3 @@ spec: path: /var/lib/filebeat-data type: DirectoryOrCreate --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: filebeat -subjects: -- kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat - namespace: kube-system -subjects: - - kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: Role - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: Role - name: filebeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: filebeat - labels: - k8s-app: filebeat -rules: -- apiGroups: [""] # "" indicates the core API group - resources: - - namespaces - - pods - - nodes - verbs: - - get - - watch - - list -- apiGroups: ["apps"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: - - jobs - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat - # should be the namespace where filebeat is running - namespace: kube-system - labels: - k8s-app: filebeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat-kubeadm-config - namespace: kube-system - labels: - k8s-app: filebeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: filebeat - namespace: kube-system - labels: - k8s-app: filebeat ---- diff --git a/dev-tools/kubernetes/filebeat/manifest.run.yaml b/dev-tools/kubernetes/filebeat/manifest.run.yaml index 70e4612aee1e..2263bdd77e67 100644 --- a/dev-tools/kubernetes/filebeat/manifest.run.yaml +++ b/dev-tools/kubernetes/filebeat/manifest.run.yaml @@ -1,3 +1,106 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: filebeat + labels: + k8s-app: filebeat +rules: +- apiGroups: [""] # "" indicates the core API group + resources: + - namespaces + - pods + - nodes + verbs: + - get + - watch + - list +- apiGroups: ["apps"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat + # should be the namespace where filebeat is running + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: filebeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: filebeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: filebeat +subjects: +- kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: filebeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: filebeat + namespace: kube-system +roleRef: + kind: Role + name: filebeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap @@ -9,9 +112,17 @@ metadata: data: filebeat.yml: |- filebeat.inputs: - - type: container + - type: filestream + id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} paths: - /var/log/containers/*.log + parsers: + - container: ~ + prospector: + scanner: + fingerprint.enabled: true + symlinks: true + file_identity.fingerprint: ~ processors: - add_kubernetes_metadata: host: ${NODE_NAME} @@ -20,15 +131,23 @@ data: logs_path: "/var/log/containers/" # To enable hints based autodiscover, remove `filebeat.inputs` configuration and uncomment this: - #filebeat.autodiscover: + # filebeat.autodiscover: # providers: # - type: kubernetes # node: ${NODE_NAME} # hints.enabled: true # hints.default_config: - # type: container + # type: filestream + # id: kubernetes-container-logs-${data.kubernetes.pod.name}-${data.kubernetes.container.id} # paths: - # - /var/log/containers/*${data.kubernetes.container.id}.log + # - /var/log/containers/*-${data.kubernetes.container.id}.log + # parsers: + # - container: ~ + # prospector: + # scanner: + # fingerprint.enabled: true + # symlinks: true + # file_identity.fingerprint: ~ processors: - add_cloud_metadata: @@ -71,7 +190,6 @@ spec: args: [ "-c", "/etc/filebeat.yml", "-e", - "-system.hostfs=/hostfs", ] env: - name: ELASTICSEARCH_HOST @@ -131,107 +249,3 @@ spec: path: /var/lib/filebeat-data type: DirectoryOrCreate --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: filebeat -subjects: -- kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat - namespace: kube-system -subjects: - - kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: Role - name: filebeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: filebeat-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: filebeat - namespace: kube-system -roleRef: - kind: Role - name: filebeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: filebeat - labels: - k8s-app: filebeat -rules: -- apiGroups: [""] # "" indicates the core API group - resources: - - namespaces - - pods - - nodes - verbs: - - get - - watch - - list -- apiGroups: ["apps"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: - - jobs - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat - # should be the namespace where filebeat is running - namespace: kube-system - labels: - k8s-app: filebeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: filebeat-kubeadm-config - namespace: kube-system - labels: - k8s-app: filebeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: filebeat - namespace: kube-system - labels: - k8s-app: filebeat ---- diff --git a/dev-tools/kubernetes/heartbeat/Dockerfile.debug b/dev-tools/kubernetes/heartbeat/Dockerfile.debug index df065094897e..aa48a8d58d7e 100644 --- a/dev-tools/kubernetes/heartbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/heartbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/Dockerfile.debug b/dev-tools/kubernetes/metricbeat/Dockerfile.debug index 241c186998d1..854cf1ac32be 100644 --- a/dev-tools/kubernetes/metricbeat/Dockerfile.debug +++ b/dev-tools/kubernetes/metricbeat/Dockerfile.debug @@ -1,4 +1,4 @@ -FROM golang:1.20.11 as builder +FROM golang:1.20.12 as builder ENV PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/go/bin:/usr/local/go/bin diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml index 12f51a2a500d..6dd492804c8b 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.multi.yaml @@ -1,3 +1,134 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap @@ -30,6 +161,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset @@ -41,6 +173,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -337,131 +472,3 @@ spec: path: /var/lib/metricbeat-data type: DirectoryOrCreate --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - services - - persistentvolumes - - persistentvolumeclaims - verbs: ["get", "list", "watch"] -# Enable this rule only if planing to use Kubernetes keystore -#- apiGroups: [""] -# resources: -# - secrets -# verbs: ["get"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - - daemonsets - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: - - jobs - - cronjobs - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get -- nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat - # should be the namespace where metricbeat is running - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat ---- diff --git a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml index 7e7d6e8f2ad1..398d7fa85606 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.debug.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.debug.yaml @@ -1,3 +1,134 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap @@ -30,6 +161,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset @@ -41,6 +173,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -232,131 +367,3 @@ spec: path: /var/lib/metricbeat-data type: DirectoryOrCreate --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - services - - persistentvolumes - - persistentvolumeclaims - verbs: ["get", "list", "watch"] -# Enable this rule only if planing to use Kubernetes keystore -#- apiGroups: [""] -# resources: -# - secrets -# verbs: ["get"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - - daemonsets - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: - - jobs - - cronjobs - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get -- nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat - # should be the namespace where metricbeat is running - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat ---- diff --git a/dev-tools/kubernetes/metricbeat/manifest.run.yaml b/dev-tools/kubernetes/metricbeat/manifest.run.yaml index 883b44862489..21c9727d45ef 100644 --- a/dev-tools/kubernetes/metricbeat/manifest.run.yaml +++ b/dev-tools/kubernetes/metricbeat/manifest.run.yaml @@ -1,3 +1,134 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metricbeat + namespace: kube-system + labels: + k8s-app: metricbeat +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metricbeat + labels: + k8s-app: metricbeat +rules: +- apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - services + - persistentvolumes + - persistentvolumeclaims + verbs: ["get", "list", "watch"] +# Enable this rule only if planing to use Kubernetes keystore +#- apiGroups: [""] +# resources: +# - secrets +# verbs: ["get"] +- apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] +- apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + - daemonsets + verbs: ["get", "list", "watch"] +- apiGroups: ["batch"] + resources: + - jobs + - cronjobs + verbs: ["get", "list", "watch"] +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["get", "list", "watch"] +- apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get +- nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat + # should be the namespace where metricbeat is running + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: ["get", "create", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system + labels: + k8s-app: metricbeat +rules: + - apiGroups: [""] + resources: + - configmaps + resourceNames: + - kubeadm-config + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metricbeat +subjects: +- kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: ClusterRole + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metricbeat-kubeadm-config + namespace: kube-system +subjects: + - kind: ServiceAccount + name: metricbeat + namespace: kube-system +roleRef: + kind: Role + name: metricbeat-kubeadm-config + apiGroup: rbac.authorization.k8s.io --- apiVersion: v1 kind: ConfigMap @@ -30,6 +161,7 @@ data: period: 10s add_metadata: true metricsets: + - state_namespace - state_node - state_deployment - state_daemonset @@ -41,6 +173,9 @@ data: - state_resourcequota - state_statefulset - state_service + - state_persistentvolume + - state_persistentvolumeclaim + - state_storageclass # If `https` is used to access `kube-state-metrics`, uncomment following settings: # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token # ssl.certificate_authorities: @@ -227,131 +362,3 @@ spec: path: /var/lib/metricbeat-data type: DirectoryOrCreate --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: metricbeat -subjects: -- kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: ClusterRole - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system -subjects: - - kind: ServiceAccount - name: metricbeat - namespace: kube-system -roleRef: - kind: Role - name: metricbeat-kubeadm-config - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: metricbeat - labels: - k8s-app: metricbeat -rules: -- apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - services - - persistentvolumes - - persistentvolumeclaims - verbs: ["get", "list", "watch"] -# Enable this rule only if planing to use Kubernetes keystore -#- apiGroups: [""] -# resources: -# - secrets -# verbs: ["get"] -- apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - - daemonsets - verbs: ["get", "list", "watch"] -- apiGroups: ["batch"] - resources: - - jobs - - cronjobs - verbs: ["get", "list", "watch"] -- apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get -- nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat - # should be the namespace where metricbeat is running - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: ["get", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: metricbeat-kubeadm-config - namespace: kube-system - labels: - k8s-app: metricbeat -rules: - - apiGroups: [""] - resources: - - configmaps - resourceNames: - - kubeadm-config - verbs: ["get"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: metricbeat - namespace: kube-system - labels: - k8s-app: metricbeat ---- diff --git a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl index 840f8dd97d3e..279862dcd1de 100644 --- a/dev-tools/packaging/templates/windows/install-service.ps1.tmpl +++ b/dev-tools/packaging/templates/windows/install-service.ps1.tmpl @@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path # Create the new service. New-Service -name {{.BeatName}} ` -displayName {{.BeatName | title}} ` - -binaryPathName "`"$workdir\{{.BeatName}}.exe`" --environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" --path.home `"$workdir`" --path.data `"$env:PROGRAMDATA\{{.BeatName}}`" --path.logs `"$env:PROGRAMDATA\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true" + -binaryPathName "`"$workdir\{{.BeatName}}.exe`" --environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" --path.home `"$workdir`" --path.data `"$env:PROGRAMDATA\{{.BeatName}}`" --path.logs `"$env:PROGRAMDATA\{{.BeatName}}\logs`" -E keystore.path=`"$workdir\data\{{.BeatName}}.keystore`" -E logging.files.redirect_stderr=true" # Attempt to set the service to delayed start using sc config. Try { diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index b935161c1269..6cedd1bd9d16 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -59,7 +59,6 @@ type fileMeta struct { type filestream struct { readerConfig readerConfig encodingFactory encoding.EncodingFactory - encoding encoding.Encoding closerConfig closerConfig parsers parser.Config } @@ -175,7 +174,7 @@ func initState(log *logp.Logger, c loginp.Cursor, s fileSource) state { } func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSource, offset int64) (reader.Reader, error) { - f, err := inp.openFile(log, fs.newPath, offset) + f, encoding, err := inp.openFile(log, fs.newPath, offset) if err != nil { return nil, err } @@ -216,7 +215,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo var r reader.Reader r, err = readfile.NewEncodeReader(dbgReader, readfile.Config{ - Codec: inp.encoding, + Codec: encoding, BufferSize: inp.readerConfig.BufferSize, Terminator: inp.readerConfig.LineTerminator, MaxBytes: encReaderMaxBytes, @@ -241,33 +240,33 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo // or the file cannot be opened because for example of failing read permissions, an error // is returned and the harvester is closed. The file will be picked up again the next time // the file system is scanned -func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*os.File, error) { +func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*os.File, encoding.Encoding, error) { fi, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("failed to stat source file %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to stat source file %s: %w", path, err) } // it must be checked if the file is not a named pipe before we try to open it // if it is a named pipe os.OpenFile fails, so there is no need to try opening it. if fi.Mode()&os.ModeNamedPipe != 0 { - return nil, fmt.Errorf("failed to open file %s, named pipes are not supported", fi.Name()) + return nil, nil, fmt.Errorf("failed to open file %s, named pipes are not supported", fi.Name()) } ok := false f, err := file.ReadOpen(path) if err != nil { - return nil, fmt.Errorf("failed opening %s: %w", path, err) + return nil, nil, fmt.Errorf("failed opening %s: %w", path, err) } defer cleanup.IfNot(&ok, cleanup.IgnoreError(f.Close)) fi, err = f.Stat() if err != nil { - return nil, fmt.Errorf("failed to stat source file %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to stat source file %s: %w", path, err) } err = checkFileBeforeOpening(fi) if err != nil { - return nil, err + return nil, nil, err } if fi.Size() < offset { @@ -276,20 +275,20 @@ func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*o } err = inp.initFileOffset(f, offset) if err != nil { - return nil, err + return nil, nil, err } - inp.encoding, err = inp.encodingFactory(f) + encoding, err := inp.encodingFactory(f) if err != nil { f.Close() if errors.Is(err, transform.ErrShortSrc) { - return nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) + return nil, nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) } - return nil, fmt.Errorf("initialising encoding for '%v' failed: %w", f, err) + return nil, nil, fmt.Errorf("initialising encoding for '%v' failed: %w", f, err) } ok = true - return f, nil + return f, encoding, nil } func checkFileBeforeOpening(fi os.FileInfo) error { diff --git a/filebeat/input/filestream/input_test.go b/filebeat/input/filestream/input_test.go index 55b8d2e7fc66..a1d9729c5aad 100644 --- a/filebeat/input/filestream/input_test.go +++ b/filebeat/input/filestream/input_test.go @@ -21,10 +21,11 @@ import ( "context" "fmt" "os" + "path/filepath" + "sync/atomic" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" @@ -38,25 +39,26 @@ import ( func BenchmarkFilestream(b *testing.B) { logp.TestingSetup(logp.ToDiscardOutput()) - lineCount := 10000 - filename := generateFile(b, lineCount) - b.ResetTimer() + b.Run("single file", func(b *testing.B) { + lineCount := 10000 + filename := generateFile(b, b.TempDir(), lineCount) + b.ResetTimer() - b.Run("filestream default throughput", func(b *testing.B) { - cfg := ` + b.Run("inode throughput", func(b *testing.B) { + cfg := ` type: filestream prospector.scanner.check_interval: 1s paths: - ` + filename + ` ` - for i := 0; i < b.N; i++ { - runFilestreamBenchmark(b, fmt.Sprintf("default-benchmark-%d", i), cfg, lineCount) - } - }) + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("one-file-inode-benchmark-%d", i), cfg, lineCount) + } + }) - b.Run("filestream fingerprint throughput", func(b *testing.B) { - cfg := ` + b.Run("fingerprint throughput", func(b *testing.B) { + cfg := ` type: filestream prospector.scanner: fingerprint.enabled: true @@ -65,9 +67,51 @@ file_identity.fingerprint: ~ paths: - ` + filename + ` ` - for i := 0; i < b.N; i++ { - runFilestreamBenchmark(b, fmt.Sprintf("fp-benchmark-%d", i), cfg, lineCount) + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("one-file-fp-benchmark-%d", i), cfg, lineCount) + } + }) + }) + + b.Run("many files", func(b *testing.B) { + lineCount := 1000 + fileCount := 100 + dir := b.TempDir() + + for i := 0; i < fileCount; i++ { + _ = generateFile(b, dir, lineCount) } + + ingestPath := filepath.Join(dir, "*") + expEvents := lineCount * fileCount + b.ResetTimer() + + b.Run("inode throughput", func(b *testing.B) { + cfg := ` +type: filestream +prospector.scanner.check_interval: 1s +paths: + - ` + ingestPath + ` +` + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("many-files-inode-benchmark-%d", i), cfg, expEvents) + } + }) + + b.Run("fingerprint throughput", func(b *testing.B) { + cfg := ` +type: filestream +prospector.scanner: + fingerprint.enabled: true + check_interval: 1s +file_identity.fingerprint: ~ +paths: + - ` + ingestPath + ` +` + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("many-files-fp-benchmark-%d", i), cfg, expEvents) + } + }) }) } @@ -76,13 +120,13 @@ paths: // `cfg` must be a valid YAML string containing valid filestream configuration // `expEventCount` is an expected amount of produced events func runFilestreamBenchmark(b *testing.B, testID string, cfg string, expEventCount int) { + b.Helper() // we don't include initialization in the benchmark time b.StopTimer() - runner := createFilestreamTestRunner(b, testID, cfg, expEventCount) + runner := createFilestreamTestRunner(context.Background(), b, testID, cfg, int64(expEventCount), false) // this is where the benchmark actually starts b.StartTimer() - events := runner(b) - require.Len(b, events, expEventCount) + _ = runner(b) } // createFilestreamTestRunner can be used for both benchmarks and regular tests to run a filestream input @@ -90,9 +134,11 @@ func runFilestreamBenchmark(b *testing.B, testID string, cfg string, expEventCou // `testID` must be unique for each test run // `cfg` must be a valid YAML string containing valid filestream configuration // `eventLimit` is an amount of produced events after which the filestream will shutdown +// `collectEvents` if `true` the runner will return a list of all events produced by the filestream input. +// Events should not be collected in benchmarks due to high extra costs of using the channel. // // returns a runner function that returns produced events. -func createFilestreamTestRunner(b testing.TB, testID string, cfg string, eventLimit int) func(t testing.TB) []beat.Event { +func createFilestreamTestRunner(ctx context.Context, b testing.TB, testID string, cfg string, eventLimit int64, collectEvents bool) func(t testing.TB) []beat.Event { logger := logp.L() c, err := conf.NewConfigWithYAML([]byte(cfg), cfg) require.NoError(b, err) @@ -101,41 +147,43 @@ func createFilestreamTestRunner(b testing.TB, testID string, cfg string, eventLi input, err := p.Manager.Create(c) require.NoError(b, err) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) context := v2.Context{ Logger: logger, ID: testID, Cancelation: ctx, } - events := make([]beat.Event, 0, eventLimit) - connector, eventsDone := newTestPipeline(eventLimit, &events) - done := make(chan struct{}) + connector, events := newTestPipeline(eventLimit, collectEvents) + var out []beat.Event + if collectEvents { + out = make([]beat.Event, 0, eventLimit) + } + go func() { + // even if `collectEvents` is false we need to range the channel + // and wait until it's closed indicating that the input finished its job + for event := range events { + out = append(out, event) + } + cancel() + }() return func(t testing.TB) []beat.Event { - go func() { - err := input.Run(context, connector) - assert.NoError(b, err) - close(done) - }() + err := input.Run(context, connector) + require.NoError(b, err) - <-eventsDone - cancel() - <-done // for more stable results we should wait until the full shutdown - return events + return out } } -func generateFile(t testing.TB, lineCount int) string { +func generateFile(t testing.TB, dir string, lineCount int) string { t.Helper() - dir := t.TempDir() - file, err := os.CreateTemp(dir, "lines.log") + file, err := os.CreateTemp(dir, "*") require.NoError(t, err) - + filename := file.Name() for i := 0; i < lineCount; i++ { - fmt.Fprintf(file, "rather mediocre log line message - %d\n", i) + fmt.Fprintf(file, "rather mediocre log line message in %s - %d\n", filename, i) } - filename := file.Name() err = file.Close() require.NoError(t, err) return filename @@ -161,15 +209,15 @@ func (s *testStore) CleanupInterval() time.Duration { return time.Second } -func newTestPipeline(eventLimit int, out *[]beat.Event) (pc beat.PipelineConnector, done <-chan struct{}) { - ch := make(chan struct{}) - return &testPipeline{limit: eventLimit, done: ch, out: out}, ch +func newTestPipeline(eventLimit int64, collectEvents bool) (pc beat.PipelineConnector, out <-chan beat.Event) { + ch := make(chan beat.Event, eventLimit) + return &testPipeline{limit: eventLimit, out: ch, collect: collectEvents}, ch } type testPipeline struct { - done chan struct{} - limit int - out *[]beat.Event + limit int64 + out chan beat.Event + collect bool } func (p *testPipeline) ConnectWith(beat.ClientConfig) (beat.Client, error) { @@ -184,13 +232,15 @@ type testClient struct { } func (c *testClient) Publish(event beat.Event) { - c.testPipeline.limit-- - if c.testPipeline.limit < 0 { + newLimit := atomic.AddInt64(&c.testPipeline.limit, -1) + if newLimit < 0 { return } - *c.testPipeline.out = append(*c.testPipeline.out, event) - if c.testPipeline.limit == 0 { - close(c.testPipeline.done) + if c.testPipeline.collect { + c.testPipeline.out <- event + } + if newLimit == 0 { + close(c.testPipeline.out) } } diff --git a/go.mod b/go.mod index 77aa9c7861ce..e4943105c6a4 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.5.0 + github.com/elastic/elastic-agent-client/v7 v7.6.0 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.4.0 github.com/elastic/go-licenser v0.4.1 @@ -95,7 +95,6 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 github.com/gomodule/redigo v1.8.3 github.com/google/flatbuffers v23.3.3+incompatible @@ -152,16 +151,16 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.16.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/mod v0.10.0 - golang.org/x/net v0.17.0 + golang.org/x/mod v0.14.0 + golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.10.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.15.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.9.1 + golang.org/x/tools v0.16.0 google.golang.org/api v0.126.0 google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/grpc v1.58.3 @@ -201,7 +200,7 @@ require ( github.com/aws/smithy-go v1.13.5 github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20220623125934-28468a6701b5 github.com/elastic/bayeux v1.0.5 - github.com/elastic/elastic-agent-autodiscover v0.6.5 + github.com/elastic/elastic-agent-autodiscover v0.6.6 github.com/elastic/elastic-agent-libs v0.7.2 github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 github.com/elastic/elastic-agent-system-metrics v0.9.1 @@ -224,6 +223,7 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.4.7 go.elastic.co/apm/v2 v2.4.7 go.mongodb.org/mongo-driver v1.5.1 + golang.org/x/tools/go/vcs v0.1.0-deprecated google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -290,6 +290,7 @@ require ( github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect @@ -366,8 +367,8 @@ require ( go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect - golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect @@ -413,7 +414,6 @@ replace ( github.com/snowflakedb/gosnowflake => github.com/snowflakedb/gosnowflake v1.6.19 github.com/tonistiigi/fifo => github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c k8s.io/kubernetes v1.13.0 => k8s.io/kubernetes v1.24.15 - ) // Exclude this version because the version has an invalid checksum. diff --git a/go.sum b/go.sum index e836a3d92670..c83841bdcb59 100644 --- a/go.sum +++ b/go.sum @@ -652,10 +652,10 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqrj3lotWinO9+jFmeDXIC4gvIQs= github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= -github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U= -github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= -github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA= -github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0= +github.com/elastic/elastic-agent-autodiscover v0.6.6 h1:P1y0dDpbhJc7Uw/xe85irPEad4Vljygc+y4iSxtqW7A= +github.com/elastic/elastic-agent-autodiscover v0.6.6/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= +github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= +github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= @@ -2030,8 +2030,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2047,8 +2047,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e h1:Ctm9yurWsg7aWwIpH9Bnap/IdSVxixymIb3MhiMEQQA= -golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2089,8 +2089,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2173,8 +2173,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2207,8 +2207,8 @@ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2344,8 +2344,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2354,8 +2354,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2370,8 +2370,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2475,8 +2475,10 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4= +golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index d90185df60bd..e3e4a8bf989c 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/libbeat/docs/metrics-in-logs.asciidoc b/libbeat/docs/metrics-in-logs.asciidoc index c499e7462f4d..97aac4f3a302 100644 --- a/libbeat/docs/metrics-in-logs.asciidoc +++ b/libbeat/docs/metrics-in-logs.asciidoc @@ -170,6 +170,7 @@ endif::[] | `.output.events.total` | Integer | Number of events currently being processed by the output. | If this number grows over time, it may indicate that the output destination (e.g. {ls} pipeline or {es} cluster) is not able to accept events at the same or faster rate than what {beatname_uc} is sending to it. | `.output.events.acked` | Integer | Number of events acknowledged by the output destination. | Generally, we want this number to be the same as `.output.events.total` as this indicates that the output destination has reliably received all the events sent to it. | `.output.events.failed` | Integer | Number of events that {beatname_uc} tried to send to the output destination, but the destination failed to receive them. | Generally, we want this field to be absent or its value to be zero. When the value is greater than zero, it's useful to check {beatname_uc}'s logs right before this log entry's `@timestamp` to see if there are any connectivity issues with the output destination. Note that failed events are not lost or dropped; they will be sent back to the publisher pipeline for retrying later. +| `.output.write.latency` | Object | Reports statistics on the time to send an event to the connected output, in milliseconds. This can be used to diagnose delays and performance issues caused by I/O or output configuration. This metric is available for the Elasticsearch, file, redis, and logstash outputs. |=== ifeval::["{beatname_lc}"=="filebeat"] diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 6d2c70722207..ccb198c4a924 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 8.13.0 :doc-branch: main -:go-version: 1.20.11 +:go-version: 1.20.12 :release-state: unreleased :python: 3.7 :docker: 1.12 diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 4996dba887e2..8aeef2c623e7 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -221,7 +221,7 @@ func (client *Client) Publish(ctx context.Context, batch publisher.Batch) error func (client *Client) publishEvents(ctx context.Context, data []publisher.Event) ([]publisher.Event, error) { span, ctx := apm.StartSpan(ctx, "publishEvents", "output") defer span.End() - begin := time.Now() + st := client.observer if st != nil { @@ -246,8 +246,10 @@ func (client *Client) publishEvents(ctx context.Context, data []publisher.Event) return nil, nil } + begin := time.Now() params := map[string]string{"filter_path": "errors,items.*.error,items.*.status"} status, result, sendErr := client.conn.Bulk(ctx, "", "", params, bulkItems) + timeSinceSend := time.Since(begin) if sendErr != nil { if status == http.StatusRequestEntityTooLarge { @@ -265,7 +267,7 @@ func (client *Client) publishEvents(ctx context.Context, data []publisher.Event) client.log.Debugf("PublishEvents: %d events have been published to elasticsearch in %v.", pubCount, - time.Since(begin)) + timeSinceSend) // check response for transient errors var failedEvents []publisher.Event @@ -289,6 +291,8 @@ func (client *Client) publishEvents(ctx context.Context, data []publisher.Event) st.Dropped(dropped) st.Duplicate(duplicates) st.ErrTooMany(stats.tooMany) + st.ReportLatency(timeSinceSend) + } if failed > 0 { diff --git a/libbeat/outputs/fileout/file.go b/libbeat/outputs/fileout/file.go index d12a11b25c3c..4ddc5955d6ef 100644 --- a/libbeat/outputs/fileout/file.go +++ b/libbeat/outputs/fileout/file.go @@ -21,6 +21,7 @@ import ( "context" "os" "path/filepath" + "time" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/outputs" @@ -119,6 +120,7 @@ func (out *fileOutput) Publish(_ context.Context, batch publisher.Batch) error { st.NewBatch(len(events)) dropped := 0 + for i := range events { event := &events[i] @@ -135,6 +137,7 @@ func (out *fileOutput) Publish(_ context.Context, batch publisher.Batch) error { continue } + begin := time.Now() if _, err = out.rotator.Write(append(serializedEvent, '\n')); err != nil { st.WriteError(err) @@ -149,9 +152,12 @@ func (out *fileOutput) Publish(_ context.Context, batch publisher.Batch) error { } st.WriteBytes(len(serializedEvent) + 1) + took := time.Since(begin) + st.ReportLatency(took) } st.Dropped(dropped) + st.Acked(len(events) - dropped) return nil diff --git a/libbeat/outputs/logstash/sync.go b/libbeat/outputs/logstash/sync.go index ad4293eb9f79..2a49324c46f9 100644 --- a/libbeat/outputs/logstash/sync.go +++ b/libbeat/outputs/logstash/sync.go @@ -114,6 +114,7 @@ func (c *syncClient) Publish(_ context.Context, batch publisher.Batch) error { } for len(events) > 0 { + // check if we need to reconnect if c.ticker != nil { select { @@ -136,12 +137,14 @@ func (c *syncClient) Publish(_ context.Context, batch publisher.Batch) error { err error ) + begin := time.Now() if c.win == nil { n, err = c.sendEvents(events) } else { n, err = c.publishWindowed(events) } - + took := time.Since(begin) + st.ReportLatency(took) c.log.Debugf("%v events out of %v events sent to logstash host %s. Continue sending", n, len(events), c.Host()) @@ -163,6 +166,7 @@ func (c *syncClient) Publish(_ context.Context, batch publisher.Batch) error { return err } + } batch.ACK() diff --git a/libbeat/outputs/metrics.go b/libbeat/outputs/metrics.go index 77374df3e61e..5502c4e4ae06 100644 --- a/libbeat/outputs/metrics.go +++ b/libbeat/outputs/metrics.go @@ -17,7 +17,14 @@ package outputs -import "github.com/elastic/elastic-agent-libs/monitoring" +import ( + "time" + + "github.com/rcrowley/go-metrics" + + "github.com/elastic/elastic-agent-libs/monitoring" + "github.com/elastic/elastic-agent-libs/monitoring/adapter" +) // Stats implements the Observer interface, for collecting metrics on common // outputs events. @@ -46,13 +53,15 @@ type Stats struct { readBytes *monitoring.Uint // total amount of bytes read readErrors *monitoring.Uint // total number of errors while waiting for response on output + + sendLatencyMillis metrics.Sample } // NewStats creates a new Stats instance using a backing monitoring registry. // This function will create and register a number of metrics with the registry passed. // The registry must not be null. func NewStats(reg *monitoring.Registry) *Stats { - return &Stats{ + obj := &Stats{ batches: monitoring.NewUint(reg, "events.batches"), events: monitoring.NewUint(reg, "events.total"), acked: monitoring.NewUint(reg, "events.acked"), @@ -69,7 +78,11 @@ func NewStats(reg *monitoring.Registry) *Stats { readBytes: monitoring.NewUint(reg, "read.bytes"), readErrors: monitoring.NewUint(reg, "read.errors"), + + sendLatencyMillis: metrics.NewUniformSample(1024), } + _ = adapter.NewGoMetrics(reg, "write.latency", adapter.Accept).Register("histogram", metrics.NewHistogram(obj.sendLatencyMillis)) + return obj } // NewBatch updates active batch and event metrics. @@ -81,6 +94,10 @@ func (s *Stats) NewBatch(n int) { } } +func (s *Stats) ReportLatency(time time.Duration) { + s.sendLatencyMillis.Update(time.Milliseconds()) +} + // Acked updates active and acked event metrics. func (s *Stats) Acked(n int) { if s != nil { diff --git a/libbeat/outputs/observer.go b/libbeat/outputs/observer.go index 9d7a3aec4a0c..3a330e4a43ac 100644 --- a/libbeat/outputs/observer.go +++ b/libbeat/outputs/observer.go @@ -17,21 +17,24 @@ package outputs +import "time" + // Observer provides an interface used by outputs to report common events on // documents/events being published and I/O workload. type Observer interface { - NewBatch(int) // report new batch being processed with number of events - Acked(int) // report number of acked events - Failed(int) // report number of failed events - Dropped(int) // report number of dropped events - Duplicate(int) // report number of events detected as duplicates (e.g. on resends) - Cancelled(int) // report number of cancelled events - Split() // report a batch was split for being too large to ingest - WriteError(error) // report an I/O error on write - WriteBytes(int) // report number of bytes being written - ReadError(error) // report an I/O error on read - ReadBytes(int) // report number of bytes being read - ErrTooMany(int) // report too many requests response + NewBatch(int) // report new batch being processed with number of events + ReportLatency(time.Duration) // report the duration a send to the output takes + Acked(int) // report number of acked events + Failed(int) // report number of failed events + Dropped(int) // report number of dropped events + Duplicate(int) // report number of events detected as duplicates (e.g. on resends) + Cancelled(int) // report number of cancelled events + Split() // report a batch was split for being too large to ingest + WriteError(error) // report an I/O error on write + WriteBytes(int) // report number of bytes being written + ReadError(error) // report an I/O error on read + ReadBytes(int) // report number of bytes being read + ErrTooMany(int) // report too many requests response } type emptyObserver struct{} @@ -43,15 +46,16 @@ func NewNilObserver() Observer { return nilObserver } -func (*emptyObserver) NewBatch(int) {} -func (*emptyObserver) Acked(int) {} -func (*emptyObserver) Duplicate(int) {} -func (*emptyObserver) Failed(int) {} -func (*emptyObserver) Dropped(int) {} -func (*emptyObserver) Cancelled(int) {} -func (*emptyObserver) Split() {} -func (*emptyObserver) WriteError(error) {} -func (*emptyObserver) WriteBytes(int) {} -func (*emptyObserver) ReadError(error) {} -func (*emptyObserver) ReadBytes(int) {} -func (*emptyObserver) ErrTooMany(int) {} +func (*emptyObserver) NewBatch(int) {} +func (*emptyObserver) ReportLatency(_ time.Duration) {} +func (*emptyObserver) Acked(int) {} +func (*emptyObserver) Duplicate(int) {} +func (*emptyObserver) Failed(int) {} +func (*emptyObserver) Dropped(int) {} +func (*emptyObserver) Cancelled(int) {} +func (*emptyObserver) Split() {} +func (*emptyObserver) WriteError(error) {} +func (*emptyObserver) WriteBytes(int) {} +func (*emptyObserver) ReadError(error) {} +func (*emptyObserver) ReadBytes(int) {} +func (*emptyObserver) ErrTooMany(int) {} diff --git a/libbeat/outputs/redis/client.go b/libbeat/outputs/redis/client.go index 5165d894f654..5a299749aac8 100644 --- a/libbeat/outputs/redis/client.go +++ b/libbeat/outputs/redis/client.go @@ -233,8 +233,11 @@ func (c *client) publishEventsBulk(conn redis.Conn, command string) publishFn { return nil, nil } + start := time.Now() // RPUSH returns total length of list -> fail and retry all on error _, err := conn.Do(command, args...) + took := time.Since(start) + c.observer.ReportLatency(took) if err != nil { c.log.Errorf("Failed to %v to redis list with: %+v", command, err) return okEvents, err @@ -283,7 +286,7 @@ func (c *client) publishEventsPipeline(conn redis.Conn, command string) publishF for i := range serialized { _, err := conn.Receive() if err != nil { - if _, ok := err.(redis.Error); ok { + if _, ok := err.(redis.Error); ok { //nolint:errorlint //this line checks against a type, not an instance of an error c.log.Errorf("Failed to %v event to list with %+v", command, err) failed = append(failed, data[i]) diff --git a/libbeat/outputs/shipper/shipper_test.go b/libbeat/outputs/shipper/shipper_test.go index ef6b628ba8c9..e26d44635aff 100644 --- a/libbeat/outputs/shipper/shipper_test.go +++ b/libbeat/outputs/shipper/shipper_test.go @@ -637,15 +637,16 @@ type TestObserver struct { errTooMany int } -func (to *TestObserver) NewBatch(batch int) { to.batch += batch } -func (to *TestObserver) Acked(acked int) { to.acked += acked } -func (to *TestObserver) Duplicate(duplicate int) { to.duplicate += duplicate } -func (to *TestObserver) Failed(failed int) { to.failed += failed } -func (to *TestObserver) Dropped(dropped int) { to.dropped += dropped } -func (to *TestObserver) Cancelled(cancelled int) { to.cancelled += cancelled } -func (to *TestObserver) Split() { to.split++ } -func (to *TestObserver) WriteError(we error) { to.writeError = we } -func (to *TestObserver) WriteBytes(wb int) { to.writeBytes += wb } -func (to *TestObserver) ReadError(re error) { to.readError = re } -func (to *TestObserver) ReadBytes(rb int) { to.readBytes += rb } -func (to *TestObserver) ErrTooMany(err int) { to.errTooMany = +err } +func (to *TestObserver) NewBatch(batch int) { to.batch += batch } +func (to *TestObserver) Acked(acked int) { to.acked += acked } +func (to *TestObserver) ReportLatency(_ time.Duration) {} +func (to *TestObserver) Duplicate(duplicate int) { to.duplicate += duplicate } +func (to *TestObserver) Failed(failed int) { to.failed += failed } +func (to *TestObserver) Dropped(dropped int) { to.dropped += dropped } +func (to *TestObserver) Cancelled(cancelled int) { to.cancelled += cancelled } +func (to *TestObserver) Split() { to.split++ } +func (to *TestObserver) WriteError(we error) { to.writeError = we } +func (to *TestObserver) WriteBytes(wb int) { to.writeBytes += wb } +func (to *TestObserver) ReadError(re error) { to.readError = re } +func (to *TestObserver) ReadBytes(rb int) { to.readBytes += rb } +func (to *TestObserver) ErrTooMany(err int) { to.errTooMany = +err } diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index 0998a275ea4f..7c74c82268d1 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -50,15 +50,13 @@ type Enabled struct { type PluginConfig []map[string]config.C -func defaultKubernetesAnnotatorConfig() kubeAnnotatorConfig { - return kubeAnnotatorConfig{ - SyncPeriod: 10 * time.Minute, - CleanupTimeout: 60 * time.Second, - DefaultMatchers: Enabled{true}, - DefaultIndexers: Enabled{true}, - Scope: "node", - AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), - } +func (k *kubeAnnotatorConfig) InitDefaults() { + k.SyncPeriod = 10 * time.Minute + k.CleanupTimeout = 60 * time.Second + k.DefaultMatchers = Enabled{true} + k.DefaultIndexers = Enabled{true} + k.Scope = "node" + k.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig() } func (k *kubeAnnotatorConfig) Validate() error { @@ -83,7 +81,7 @@ func (k *kubeAnnotatorConfig) Validate() error { err := matcherCfg.Unpack(&logsPathMatcher) if err != nil { - return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %s", err) + return fmt.Errorf("fail to unpack the `logs_path` matcher configuration: %w", err) } if logsPathMatcher.LogsPath == "" { return fmt.Errorf("invalid logs_path matcher configuration: when resource_type is defined, logs_path must be set as well") diff --git a/libbeat/processors/add_kubernetes_metadata/config_test.go b/libbeat/processors/add_kubernetes_metadata/config_test.go index e94089f388a9..3857eb148fa6 100644 --- a/libbeat/processors/add_kubernetes_metadata/config_test.go +++ b/libbeat/processors/add_kubernetes_metadata/config_test.go @@ -50,7 +50,7 @@ func TestConfigValidate(t *testing.T) { for _, test := range tests { cfg := config.MustNewConfigFrom(test.cfg) - c := defaultKubernetesAnnotatorConfig() + var c kubeAnnotatorConfig err := cfg.Unpack(&c) if test.error { @@ -116,16 +116,16 @@ func TestConfigValidate_LogsPatchMatcher(t *testing.T) { for _, test := range tests { cfg, _ := config.NewConfigFrom(test.matcherConfig) - c := defaultKubernetesAnnotatorConfig() - c.DefaultMatchers = Enabled{false} + var c kubeAnnotatorConfig - err := cfg.Unpack(&c) + _ = cfg.Unpack(&c) + c.DefaultMatchers = Enabled{false} c.Matchers = PluginConfig{ { test.matcherName: *cfg, }, } - err = c.Validate() + err := c.Validate() if test.error { require.NotNil(t, err) } else { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index a8667aef0a8a..954a59ab3f12 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -123,8 +123,7 @@ func New(cfg *config.C) (beat.Processor, error) { } func newProcessorConfig(cfg *config.C, register *Register) (kubeAnnotatorConfig, error) { - config := defaultKubernetesAnnotatorConfig() - + var config kubeAnnotatorConfig err := cfg.Unpack(&config) if err != nil { return config, fmt.Errorf("fail to unpack the kubernetes configuration: %w", err) diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 7f3b50109d56..634db34245b7 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt update \ diff --git a/metricbeat/docs/modules/kubernetes.asciidoc b/metricbeat/docs/modules/kubernetes.asciidoc index 6b0bbc023f3c..9ff079faa3be 100644 --- a/metricbeat/docs/modules/kubernetes.asciidoc +++ b/metricbeat/docs/modules/kubernetes.asciidoc @@ -232,11 +232,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -276,13 +283,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 3588aaec9769..d6b8b9e9475d 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -504,11 +504,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -548,13 +555,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index 74c1fdb0bca5..bf7a9fc931ec 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 COPY test/main.go main.go diff --git a/metricbeat/module/kubernetes/_meta/config.reference.yml b/metricbeat/module/kubernetes/_meta/config.reference.yml index dcd59309119d..23f5ce8dea62 100644 --- a/metricbeat/module/kubernetes/_meta/config.reference.yml +++ b/metricbeat/module/kubernetes/_meta/config.reference.yml @@ -18,11 +18,18 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -62,13 +69,21 @@ # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/_meta/config.yml b/metricbeat/module/kubernetes/_meta/config.yml index 44ef19c97862..1c56e57b167f 100644 --- a/metricbeat/module/kubernetes/_meta/config.yml +++ b/metricbeat/module/kubernetes/_meta/config.yml @@ -16,15 +16,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 26728fccdaea..60b3360ab891 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -161,7 +161,7 @@ func NewResourceMetadataEnricher( return &nilEnricher{} } - // GetPodMetaGen requires cfg of type Config + // commonMetaConfig stores the metadata configuration of the resource itself commonMetaConfig := metadata.Config{} if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { logp.Err("Error initializing Kubernetes metadata enricher: %s", err) @@ -206,7 +206,7 @@ func NewResourceMetadataEnricher( // update func(m map[string]mapstr.M, r kubernetes.Resource) { accessor, _ := meta.Accessor(r) - id := join(accessor.GetNamespace(), accessor.GetName()) //nolint:all + id := join(accessor.GetNamespace(), accessor.GetName()) switch r := r.(type) { case *kubernetes.Pod: @@ -308,6 +308,14 @@ func NewContainerMetadataEnricher( return &nilEnricher{} } + // commonMetaConfig stores the metadata configuration of the resource itself + commonMetaConfig := metadata.Config{} + if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { + logp.Err("Error initializing Kubernetes metadata enricher: %s", err) + return &nilEnricher{} + } + cfg, _ := conf.NewConfigFrom(&commonMetaConfig) + // Resource is Pod so we need to create watchers for Replicasets and Jobs that it might belongs to // in order to be able to retrieve 2nd layer Owner metadata like in case of: // Deployment -> Replicaset -> Pod @@ -331,13 +339,6 @@ func NewContainerMetadataEnricher( } } - commonMetaConfig := metadata.Config{} - if err := base.Module().UnpackConfig(&commonMetaConfig); err != nil { - logp.Err("Error initializing Kubernetes metadata enricher: %s", err) - return &nilEnricher{} - } - cfg, _ := conf.NewConfigFrom(&commonMetaConfig) - metaGen := metadata.GetPodMetaGen(cfg, watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, config.AddResourceMetadata) enricher := buildMetadataEnricher(watcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, @@ -508,6 +509,7 @@ func GetConfig(base mb.BaseMetricSet) (*kubernetesConfig, error) { SyncPeriod: time.Minute * 10, AddResourceMetadata: metadata.GetDefaultResourceMetadataConfig(), } + if err := base.Module().UnpackConfig(&config); err != nil { return nil, errors.New("error unpacking configs") } diff --git a/metricbeat/module/nats/_meta/Dockerfile b/metricbeat/module/nats/_meta/Dockerfile index 0340387144c0..44fdc695748e 100644 --- a/metricbeat/module/nats/_meta/Dockerfile +++ b/metricbeat/module/nats/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG NATS_VERSION=2.0.4 FROM nats:$NATS_VERSION # build stage -FROM golang:1.20.11 AS build-env +FROM golang:1.20.12 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/nats.go.git /nats-go RUN cd /nats-go/examples/nats-bench && git checkout tags/v1.10.0 && go build . diff --git a/metricbeat/module/vsphere/_meta/Dockerfile b/metricbeat/module/vsphere/_meta/Dockerfile index 05e0eaf3c74c..993137b89f3a 100644 --- a/metricbeat/module/vsphere/_meta/Dockerfile +++ b/metricbeat/module/vsphere/_meta/Dockerfile @@ -1,5 +1,5 @@ ARG VSPHERE_GOLANG_VERSION -FROM golang:1.20.11 +FROM golang:1.20.12 RUN apt-get install curl git RUN go install github.com/vmware/govmomi/vcsim@v0.30.4 diff --git a/metricbeat/modules.d/kubernetes.yml.disabled b/metricbeat/modules.d/kubernetes.yml.disabled index 23bd210a8357..12bbeee26ca5 100644 --- a/metricbeat/modules.d/kubernetes.yml.disabled +++ b/metricbeat/modules.d/kubernetes.yml.disabled @@ -19,15 +19,21 @@ # Enriching parameters: #add_metadata: true + # If kube_config is not set, KUBECONFIG environment variable will be checked + # and if not present it will fall back to InCluster + #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] #labels.dedot: true #annotations.dedot: true + # When used outside the cluster: #node: node_name - # If kube_config is not set, KUBECONFIG environment variable will be checked - # and if not present it will fall back to InCluster - #kube_config: ~/.kube/config + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/packetbeat/Dockerfile b/packetbeat/Dockerfile index 2040495b5510..fe13ef47b61f 100644 --- a/packetbeat/Dockerfile +++ b/packetbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3ec6770467f0..525657c20fd1 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-vqaxdghw-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-vqaxdghw-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-vqaxdghw-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" diff --git a/x-pack/functionbeat/Dockerfile b/x-pack/functionbeat/Dockerfile index bd68544accce..662b27d669db 100644 --- a/x-pack/functionbeat/Dockerfile +++ b/x-pack/functionbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.11 +FROM golang:1.20.12 RUN \ apt-get update \ diff --git a/x-pack/functionbeat/provider/aws/aws/kinesis_test.go b/x-pack/functionbeat/provider/aws/aws/kinesis_test.go index 242ed63d4a86..20d92ce36060 100644 --- a/x-pack/functionbeat/provider/aws/aws/kinesis_test.go +++ b/x-pack/functionbeat/provider/aws/aws/kinesis_test.go @@ -14,8 +14,9 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/awslabs/kinesis-aggregation/go/v2/deaggregator" aggRecProto "github.com/awslabs/kinesis-aggregation/go/v2/records" - "github.com/golang/protobuf/proto" //nolint:staticcheck // SA1019 dependency uses deprecated package "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/runtime/protoimpl" "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" conf "github.com/elastic/elastic-agent-libs/config" @@ -133,14 +134,14 @@ func generateAggregatedKinesisEvent(validRec bool) events.KinesisEvent { partKeyTable = append(partKeyTable, "0") aggRec.PartitionKeyTable = partKeyTable - data, _ := proto.Marshal(aggRec) + data, _ := proto.Marshal(protoimpl.X.ProtoMessageV2Of(aggRec)) md5Hash := md5.Sum(data) aggRecBytes = append(aggRecBytes, data...) aggRecBytes = append(aggRecBytes, md5Hash[:]...) return events.KinesisEvent{ Records: []events.KinesisEventRecord{ - events.KinesisEventRecord{ + { AwsRegion: "east-1", EventID: "1234", EventName: "connect", diff --git a/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go b/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go index e29bba6cde32..ae0dcb12eee4 100644 --- a/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go +++ b/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go @@ -14,12 +14,13 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/kinesis/types" + "google.golang.org/protobuf/runtime/protoimpl" "github.com/aws/aws-lambda-go/events" "github.com/awslabs/kinesis-aggregation/go/v2/deaggregator" aggRecProto "github.com/awslabs/kinesis-aggregation/go/v2/records" - "github.com/golang/protobuf/proto" //nolint:staticcheck // SA1019 dependency uses deprecated package "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/mapstr" @@ -33,7 +34,7 @@ func TestCloudwatch(t *testing.T) { SubscriptionFilters: []string{"MyFilter"}, MessageType: "DATA_MESSAGE", LogEvents: []events.CloudwatchLogsLogEvent{ - events.CloudwatchLogsLogEvent{ + { ID: "1234567890123456789", Timestamp: 1566908691193, Message: "my interesting message", @@ -393,7 +394,7 @@ func generateKinesisAggregateRecord(numRecords int, valid bool) []byte { } aggRec.PartitionKeyTable = partKeyTable - data, _ := proto.Marshal(aggRec) + data, _ := proto.Marshal(protoimpl.X.ProtoMessageV2Of(aggRec)) md5Hash := md5.Sum(data) aggRecBytes = append(aggRecBytes, data...) aggRecBytes = append(aggRecBytes, md5Hash[:]...) diff --git a/x-pack/libbeat/management/managerV2.go b/x-pack/libbeat/management/managerV2.go index fd38051efb3a..235325c0cbfc 100644 --- a/x-pack/libbeat/management/managerV2.go +++ b/x-pack/libbeat/management/managerV2.go @@ -35,8 +35,6 @@ import ( "github.com/elastic/beats/v7/libbeat/version" ) -var errStoppingOnOutputChange = errors.New("stopping Beat on output change") - // diagnosticHandler is a wrapper type that's a bit of a hack, the compiler won't let us send the raw unit struct, // since there's a type disagreement with the `client.DiagnosticHook` argument, and due to licensing issues we can't import the agent client types into the reloader type diagnosticHandler struct { @@ -173,7 +171,7 @@ func NewV2AgentManager(config *conf.C, registry *reload.Registry) (lbmanagement. client.VersionInfo{ Name: "beat-v2-client-for-testing", Version: version.GetDefaultVersion(), - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) } else { // Normal Elastic-Agent-Client initialisation agentClient, _, err = client.NewV2FromReader(os.Stdin, client.VersionInfo{ @@ -557,7 +555,7 @@ func (cm *BeatV2Manager) reload(units map[unitKey]*client.Unit) { for _, unit := range units { errs := unitErrors[unit.ID()] if len(errs) != 0 { - unit.UpdateState(client.UnitStateFailed, errors.Join(errs...).Error(), nil) + _ = unit.UpdateState(client.UnitStateFailed, errors.Join(errs...).Error(), nil) } } }() diff --git a/x-pack/libbeat/management/managerV2_test.go b/x-pack/libbeat/management/managerV2_test.go index 9fe238605b49..ea67fdd89f40 100644 --- a/x-pack/libbeat/management/managerV2_test.go +++ b/x-pack/libbeat/management/managerV2_test.go @@ -209,7 +209,7 @@ func TestManagerV2(t *testing.T) { Meta: map[string]string{ "key": "value", }, - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient(&Config{ Enabled: true, @@ -321,7 +321,7 @@ func TestOutputError(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ @@ -488,7 +488,7 @@ func TestErrorPerUnit(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ diff --git a/x-pack/libbeat/management/simple_input_config_test.go b/x-pack/libbeat/management/simple_input_config_test.go index ef88781a8622..7822d347112b 100644 --- a/x-pack/libbeat/management/simple_input_config_test.go +++ b/x-pack/libbeat/management/simple_input_config_test.go @@ -133,7 +133,7 @@ func TestSimpleInputConfig(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ diff --git a/x-pack/libbeat/management/tests/mock_server.go b/x-pack/libbeat/management/tests/mock_server.go index 84805c1f5cc8..8671b1242339 100644 --- a/x-pack/libbeat/management/tests/mock_server.go +++ b/x-pack/libbeat/management/tests/mock_server.go @@ -103,7 +103,7 @@ func NewMockServer(t *testing.T, canStop func(string) bool, inputConfig *proto.U Meta: map[string]string{ "key": "value", }, - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) return MockV2Handler{Srv: srv, Client: client} } diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 46e951ccb56e..a22db4f7f8cf 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -901,11 +901,18 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: @@ -945,13 +952,21 @@ metricbeat.modules: # Enriching parameters: add_metadata: true - # When used outside the cluster: - #node: node_name # If kube_config is not set, KUBECONFIG environment variable will be checked # and if not present it will fall back to InCluster #kube_config: ~/.kube/config + #include_labels: [] + #exclude_labels: [] + #include_annotations: [] + #labels.dedot: true + #annotations.dedot: true + + # When used outside the cluster: + #node: node_name + # Set the namespace to watch for resources #namespace: staging + # To configure additionally node and namespace metadata `add_resource_metadata` can be defined. # By default all labels will be included while annotations are not added by default. # add_resource_metadata: diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index cb678607c242..b6cd6e228f85 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -8,9 +8,9 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" "gotest.tools/assert" "github.com/elastic/elastic-agent-libs/mapstr" @@ -43,10 +43,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -55,10 +55,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go index 8cfae6675928..17213b8e1b89 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go @@ -8,10 +8,10 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" ) var fake = &monitoring.TimeSeries{ @@ -41,10 +41,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -53,10 +53,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index a39f320ef447..e7e7f081e526 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -12,12 +12,11 @@ import ( "sync" "time" - "github.com/golang/protobuf/ptypes/duration" - monitoring "cloud.google.com/go/monitoring/apiv3/v2" "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/api/iterator" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -256,7 +255,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) string { } // Returns a GCP TimeInterval based on the ingestDelay and samplePeriod from ListMetricDescriptor -func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod *duration.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) { +func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod *durationpb.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) { var startTime, endTime, currentTime time.Time var needsAggregation bool currentTime = time.Now().UTC() @@ -280,10 +279,10 @@ func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duratio } interval := &monitoringpb.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: startTime.Unix(), }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: endTime.Unix(), }, } diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go index 2b058de2beed..658568b66ca5 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/golang/protobuf/ptypes/duration" "github.com/stretchr/testify/assert" "go.uber.org/zap/zapcore" + "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -21,7 +21,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { title string ingestDelay time.Duration samplePeriod time.Duration - collectionPeriod *duration.Duration + collectionPeriod *durationpb.Duration inputAligner string expectedAligner string }{ @@ -29,7 +29,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod equals to samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(60), }, "", @@ -39,7 +39,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod larger than samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(300), }, "ALIGN_MEAN", @@ -49,7 +49,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod smaller than samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(30), }, "ALIGN_MAX", @@ -59,7 +59,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod equals to samplePeriod with given aligner", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(60), }, "ALIGN_MEAN", diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index ffe5cd788a1f..49cdce5d73a8 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -13,10 +13,10 @@ import ( monitoring "cloud.google.com/go/monitoring/apiv3/v2" "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/duration" "google.golang.org/api/iterator" "google.golang.org/api/option" "google.golang.org/genproto/googleapis/api/metric" + "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" @@ -107,7 +107,7 @@ type config struct { CredentialsJSON string `config:"credentials_json"` opt []option.ClientOption - period *duration.Duration + period *durationpb.Duration } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -139,7 +139,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return m, fmt.Errorf("no credentials_file_path or credentials_json specified") } - m.config.period = &duration.Duration{ + m.config.period = &durationpb.Duration{ Seconds: int64(m.Module().Config().Period.Seconds()), } diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go index 9fc790bf9e51..48ac45661ed9 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go @@ -8,10 +8,10 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" ) var fake = &monitoring.TimeSeries{ @@ -41,10 +41,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -53,10 +53,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, diff --git a/x-pack/metricbeat/module/stan/_meta/Dockerfile b/x-pack/metricbeat/module/stan/_meta/Dockerfile index 20604392f603..92ee1d834571 100644 --- a/x-pack/metricbeat/module/stan/_meta/Dockerfile +++ b/x-pack/metricbeat/module/stan/_meta/Dockerfile @@ -2,7 +2,7 @@ ARG STAN_VERSION=0.15.1 FROM nats-streaming:$STAN_VERSION # build stage -FROM golang:1.20.11 AS build-env +FROM golang:1.20.12 AS build-env RUN apt-get install git mercurial gcc RUN git clone https://github.com/nats-io/stan.go.git /stan-go RUN cd /stan-go/examples/stan-bench && git checkout tags/v0.5.2 && go build .