From b16c581fcd5715ccfd84f50de42b69a99b2c2157 Mon Sep 17 00:00:00 2001 From: kruskall <99559985+kruskall@users.noreply.github.com> Date: Thu, 20 Feb 2025 15:12:22 +0100 Subject: [PATCH 1/2] refactor: replace x/exp usage with iterators (#6933) * refactor: replace x/exp usage with iterators go 1.23 added iterators, let's use them * Update container_init_test.go * Update container_init_test.go --- NOTICE.txt | 74 +++++++++---------- go.mod | 2 +- internal/pkg/agent/cmd/container_init_test.go | 9 ++- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index ccefde4afeb..5f55d9b370a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -18238,43 +18238,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Dependency : golang.org/x/exp -Version: v0.0.0-20240719175910-8a7402abbf56 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20240719175910-8a7402abbf56/LICENSE: - -Copyright 2009 The Go Authors. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google LLC nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------- Dependency : golang.org/x/net Version: v0.34.0 @@ -104445,6 +104408,43 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : golang.org/x/exp +Version: v0.0.0-20240719175910-8a7402abbf56 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20240719175910-8a7402abbf56/LICENSE: + +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : golang.org/x/mod Version: v0.21.0 diff --git a/go.mod b/go.mod index ee1b986a506..636ed2b2b79 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,6 @@ require ( go.opentelemetry.io/collector/receiver/nopreceiver v0.119.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.32.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/net v0.34.0 golang.org/x/sync v0.10.0 golang.org/x/sys v0.29.0 @@ -618,6 +617,7 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/internal/pkg/agent/cmd/container_init_test.go b/internal/pkg/agent/cmd/container_init_test.go index 4113bfc69bb..2efbb2172a4 100644 --- a/internal/pkg/agent/cmd/container_init_test.go +++ b/internal/pkg/agent/cmd/container_init_test.go @@ -8,13 +8,14 @@ package cmd import ( "errors" + "maps" "os" "path/filepath" + "slices" "testing" "github.com/stretchr/testify/require" - "golang.org/x/exp/maps" "kernel.org/pub/linux/libs/security/libcap/cap" ) @@ -192,8 +193,8 @@ func Test_raiseEffectiveCapabilities(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - require.ElementsMatch(t, tt.expectedCaps, maps.Keys(tt.mockedProcCaps.effectiveCaps)) - require.ElementsMatch(t, tt.expectedCaps, maps.Keys(tt.mockedProcCaps.inheritableCaps)) + require.ElementsMatch(t, tt.expectedCaps, slices.Collect(maps.Keys(tt.mockedProcCaps.effectiveCaps))) + require.ElementsMatch(t, tt.expectedCaps, slices.Collect(maps.Keys(tt.mockedProcCaps.inheritableCaps))) } }) } @@ -291,7 +292,7 @@ func Test_raiseAmbientCapabilities(t *testing.T) { require.Error(t, err) } else { require.NoError(t, err) - require.ElementsMatch(t, tt.expectedAmbientCaps, maps.Keys(tt.mockedBoundCaps.ambientCaps)) + require.ElementsMatch(t, tt.expectedAmbientCaps, slices.Collect(maps.Keys(tt.mockedBoundCaps.ambientCaps))) } }) } From e1aa748f7c8d2d727b7407a5b6bd0f8ab32f8a1b Mon Sep 17 00:00:00 2001 From: Carson Ip Date: Thu, 20 Feb 2025 15:00:27 +0000 Subject: [PATCH 2/2] Never split metrics requests in batchprocessor in kube-stack helm values (#6928) Setting non-zero send_batch_max_size for metrics risks TSDB version_conflict_engine_exception as it causes metrics grouping in es exporter to not work properly. --- ...1-kube-stack-helm-chart-metrics-batch.yaml | 33 +++++++++++++++++++ .../edot-collector/kube-stack/values.yaml | 18 +++++++--- 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100644 changelog/fragments/1740054191-kube-stack-helm-chart-metrics-batch.yaml diff --git a/changelog/fragments/1740054191-kube-stack-helm-chart-metrics-batch.yaml b/changelog/fragments/1740054191-kube-stack-helm-chart-metrics-batch.yaml new file mode 100644 index 00000000000..f94a5ce9866 --- /dev/null +++ b/changelog/fragments/1740054191-kube-stack-helm-chart-metrics-batch.yaml @@ -0,0 +1,33 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: bug-fix + +# Change summary; a 80ish characters long description of the change. +summary: Fix TSDB version_conflict_engine_exception caused by incorrect kube-stack Helm values + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +description: Setting non-zero send_batch_max_size for metrics risks TSDB version_conflict_engine_exception as it causes metrics grouping in es exporter to not work properly. + + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: "elastic-agent" + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/deploy/helm/edot-collector/kube-stack/values.yaml b/deploy/helm/edot-collector/kube-stack/values.yaml index a7102efedd7..0a11f0e8a9e 100644 --- a/deploy/helm/edot-collector/kube-stack/values.yaml +++ b/deploy/helm/edot-collector/kube-stack/values.yaml @@ -210,7 +210,11 @@ collectors: insecure: true processors: # [Batch Processor](https://github.com/open-telemetry/opentelemetry-collector/tree/main/processor/batchprocessor) - batch: {} + batch: {} # inherit any values from helm chart + batch/metrics: + # explicitly set send_batch_max_size to 0, as splitting metrics requests may cause version_conflict_engine_exception in TSDB + send_batch_max_size: 0 + timeout: 1s # [Resource Detection Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor) resourcedetection/eks: detectors: [env, eks] # Detects resources from environment variables and EKS (Elastic Kubernetes Service). @@ -482,7 +486,7 @@ collectors: - kubeletstats - hostmetrics processors: - - batch + - batch/metrics - k8sattributes - resourcedetection/system - resourcedetection/eks @@ -498,7 +502,7 @@ collectors: receivers: - otlp processors: - - batch + - batch/metrics - resource/hostname exporters: - otlp/gateway @@ -834,8 +838,14 @@ collectors: send_batch_size: 1000 timeout: 1s send_batch_max_size: 1500 + batch/metrics: + # explicitly set send_batch_max_size to 0, as splitting metrics requests may cause version_conflict_engine_exception in TSDB + send_batch_max_size: 0 + timeout: 1s batch/aggs: send_batch_size: 16384 # 2x the default + # explicitly set send_batch_max_size to 0, as splitting metrics requests may cause version_conflict_engine_exception in TSDB + send_batch_max_size: 0 timeout: 10s # [Elastic Trace Processor](https://github.com/elastic/opentelemetry-collector-components/tree/main/processor/elastictraceprocessor) elastictrace: {} # The processor enriches traces with elastic specific requirements. @@ -888,7 +898,7 @@ collectors: - elasticinframetrics - attributes/dataset - resource/process - - batch + - batch/metrics exporters: [debug, elasticsearch/ecs] metrics/otel: receivers: [routing]