feat(meshmetric): filter out internal clusters (#9754)

* feat(meshmetric): filter out internal clusters Signed-off-by: slonka <[email protected]> * feat(meshmetric): filter out clusters with underscore for future proofing Signed-off-by: slonka <[email protected]> * feat(meshmetric): add all profile to e2e test Signed-off-by: slonka <[email protected]> * feat(meshmetric): one more test missing Signed-off-by: slonka <[email protected]> --------- Signed-off-by: slonka <[email protected]>
kumahq · Mar 29, 2024 · 9d93621 · 9d93621
1 parent 444cf2d
commit 9d93621
Show file tree

Hide file tree

Showing 7 changed files with 142 additions and 6 deletions.
diff --git a/app/kuma-dp/pkg/dataplane/metrics/profiles.go b/app/kuma-dp/pkg/dataplane/metrics/profiles.go
@@ -7,6 +7,7 @@ import (
 	io_prometheus_client "github.com/prometheus/client_model/go"
 
 	"github.com/kumahq/kuma/pkg/plugins/policies/meshmetric/api/v1alpha1"
+	"github.com/kumahq/kuma/pkg/xds/envoy/names"
 )
 
 type (
@@ -152,17 +153,51 @@ var basicProfile = []selectorFunction{
 	// end of dashboards
 }
 
+var basicProfileLabels = []selectorFunction{
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.PrefixSelectorType,
+		Match: names.GetInternalClusterNamePrefix(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.ExactSelectorType,
+		Match: names.GetAdsClusterName(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.ExactSelectorType,
+		Match: names.GetAccessLogSinkClusterName(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.ExactSelectorType,
+		Match: names.GetEnvoyAdminClusterName(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.ExactSelectorType,
+		Match: names.GetMetricsHijackerClusterName(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.PrefixSelectorType,
+		Match: names.GetOpenTelemetryClusterPrefix(),
+	}),
+	selectorToFilterFunction(v1alpha1.Selector{
+		Type:  v1alpha1.PrefixSelectorType,
+		Match: names.GetTracingClusterPrefix(),
+	}),
+}
+
 func ProfileMutatorGenerator(sidecar *v1alpha1.Sidecar) PrometheusMutator {
-	effectiveSelectors := []selectorFunction{alwaysSelect} // default is All
+	effectiveSelectors := basicProfile             // default is Basic
+	effectiveLabelsSelectors := basicProfileLabels // default is Basic
+	profile := v1alpha1.BasicProfileName
 	if sidecar != nil && sidecar.Profiles != nil && sidecar.Profiles.AppendProfiles != nil && len(*sidecar.Profiles.AppendProfiles) == 1 {
-		profile := (*sidecar.Profiles.AppendProfiles)[0].Name
+		profile = (*sidecar.Profiles.AppendProfiles)[0].Name
 		switch profile {
 		case v1alpha1.AllProfileName:
 			effectiveSelectors = []selectorFunction{alwaysSelect}
 		case v1alpha1.NoneProfileName:
 			effectiveSelectors = []selectorFunction{neverSelect}
 		case v1alpha1.BasicProfileName:
 			effectiveSelectors = basicProfile
+			effectiveLabelsSelectors = basicProfileLabels
 		}
 		logger.V(1).Info("selected profile", "name", profile)
 	}
@@ -198,6 +233,31 @@ func ProfileMutatorGenerator(sidecar *v1alpha1.Sidecar) PrometheusMutator {
 				}
 			}
 
+			// filter out internal clusters
+			// only activate this on basic profile so there is no overhead on other profiles
+			if profile == v1alpha1.BasicProfileName {
+				var metrics []*io_prometheus_client.Metric
+				for _, m := range metricFamily.Metric {
+					includeMetric := true
+					for _, l := range m.Label {
+						if metricFromInternalCluster(l, effectiveLabelsSelectors) {
+							includeMetric = false
+							break
+						}
+					}
+					if includeMetric {
+						metrics = append(metrics, m)
+					}
+				}
+
+				// if after cleaning there is no metrics remove this metric family
+				if len(metrics) == 0 {
+					include = false
+				} else {
+					metricFamily.Metric = metrics
+				}
+			}
+
 			if !include {
 				delete(in, key)
 			}
@@ -207,6 +267,15 @@ func ProfileMutatorGenerator(sidecar *v1alpha1.Sidecar) PrometheusMutator {
 	}
 }
 
+func metricFromInternalCluster(l *io_prometheus_client.LabelPair, effectiveLabelsSelectors []selectorFunction) bool {
+	for _, selector := range effectiveLabelsSelectors {
+		if l.GetName() == EnvoyClusterLabelName && selector(l.GetValue()) {
+			return true
+		}
+	}
+	return false
+}
+
 func selectorToFilterFunction(selector v1alpha1.Selector) selectorFunction {
 	switch selector.Type {
 	case v1alpha1.ContainsSelectorType:

diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/profiles/dashboards.in b/app/kuma-dp/pkg/dataplane/metrics/testdata/profiles/dashboards.in
@@ -1,5 +1,9 @@
 # TYPE envoy_cluster_external_upstream_rq_xx counter
 envoy_cluster_external_upstream_rq_xx{envoy_cluster_name="example"} 1
+envoy_cluster_external_upstream_rq_xx{envoy_cluster_name="ads_cluster"} 2
+
+# TYPE envoy_cluster_assignment_timeout_received counter
+envoy_cluster_assignment_timeout_received{envoy_cluster_name="ads_cluster"} 1
 
 # TYPE envoy_cluster_health_check_attempt counter
 envoy_cluster_health_check_attempt{envoy_cluster_name="example"} 1

diff --git a/app/kuma-dp/pkg/dataplane/metrics/testdata/profiles/exclude.yaml b/app/kuma-dp/pkg/dataplane/metrics/testdata/profiles/exclude.yaml
@@ -7,6 +7,8 @@ targetRef:
 default:
   sidecar:
     profiles:
+      appendProfiles:
+        - name: All
       exclude:
         - type: Regex
           match: "envoy_rbac.*"
diff --git a/pkg/xds/bootstrap/template_v3.go b/pkg/xds/bootstrap/template_v3.go
@@ -26,6 +26,7 @@ import (
 	core_xds "github.com/kumahq/kuma/pkg/core/xds"
 	util_proto "github.com/kumahq/kuma/pkg/util/proto"
 	clusters_v3 "github.com/kumahq/kuma/pkg/xds/envoy/clusters/v3"
+	"github.com/kumahq/kuma/pkg/xds/envoy/names"
 	"github.com/kumahq/kuma/pkg/xds/envoy/tls"
 )
 
@@ -37,8 +38,8 @@ func RegisterBootstrapCluster(c string) string {
 }
 
 var (
-	adsClusterName           = RegisterBootstrapCluster("ads_cluster")
-	accessLogSinkClusterName = RegisterBootstrapCluster("access_log_sink")
+	adsClusterName           = RegisterBootstrapCluster(names.GetAdsClusterName())
+	accessLogSinkClusterName = RegisterBootstrapCluster(names.GetAccessLogSinkClusterName())
 )
 
 func genConfig(parameters configParameters, proxyConfig xds.Proxy, enableReloadableTokens bool) (*envoy_bootstrap_v3.Bootstrap, error) {

diff --git a/pkg/xds/envoy/names/resource_names.go b/pkg/xds/envoy/names/resource_names.go
@@ -65,12 +65,28 @@ func GetMetricsHijackerClusterName() string {
 	return Join("kuma", "metrics", "hijacker")
 }
 
+func GetInternalClusterNamePrefix() string {
+	return "_"
+}
+
+func GetAdsClusterName() string {
+	return "ads_cluster"
+}
+
+func GetAccessLogSinkClusterName() string {
+	return "access_log_sink"
+}
+
 func GetOpenTelemetryListenerName(backendName string) string {
 	return Join("_kuma", "metrics", "opentelemetry", backendName)
 }
 
+func GetOpenTelemetryClusterPrefix() string {
+	return Join("_kuma", "metrics", "opentelemetry")
+}
+
 func GetOpenTelemetryClusterName(backendName string) string {
-	return Join("_kuma", "metrics", "opentelemetry", backendName)
+	return Join(GetOpenTelemetryClusterPrefix(), backendName)
 }
 
 func GetPrometheusListenerName() string {
@@ -81,8 +97,12 @@ func GetAdminListenerName() string {
 	return Join("kuma", "envoy", "admin")
 }
 
+func GetTracingClusterPrefix() string {
+	return Join("tracing")
+}
+
 func GetTracingClusterName(backendName string) string {
-	return Join("tracing", backendName)
+	return Join(GetTracingClusterPrefix(), backendName)
 }
 
 func GetDNSListenerName() string {

diff --git a/test/e2e_env/kubernetes/gateway/delegated/meshmetric.go b/test/e2e_env/kubernetes/gateway/delegated/meshmetric.go
@@ -31,6 +31,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: OpenTelemetry
         openTelemetry: 

diff --git a/test/e2e_env/kubernetes/meshmetric/meshmetric.go b/test/e2e_env/kubernetes/meshmetric/meshmetric.go
@@ -31,6 +31,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: Prometheus
         prometheus: 
@@ -90,6 +94,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: Prometheus
         prometheus: 
@@ -122,6 +130,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: Prometheus
         prometheus: 
@@ -148,6 +160,10 @@ spec:
     kind: MeshService
     name: %s
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: Prometheus
         prometheus: 
@@ -173,6 +189,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     applications:
       - name: "%s"
         path: "%s"
@@ -201,6 +221,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: OpenTelemetry
         openTelemetry: 
@@ -223,6 +247,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     sidecar:
       regex: .*upstream.*
       includeUnused: false
@@ -248,6 +276,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: OpenTelemetry
         openTelemetry: 
@@ -276,6 +308,10 @@ spec:
   targetRef:
     kind: Mesh
   default:
+    sidecar:
+      profiles:
+        appendProfiles:
+          - name: All
     backends:
       - type: OpenTelemetry
         openTelemetry: