From 9dbd88101caa91b849dcc34bc31bd3ba6e2ebbd9 Mon Sep 17 00:00:00 2001 From: Weifeng Wang Date: Wed, 22 Nov 2023 00:09:50 +0800 Subject: [PATCH] bump to grafana:10.2.2 grafana/agent:v0.38.0 Signed-off-by: Weifeng Wang --- Makefile | 2 +- .../compose-include/agent-collect-logs.yaml | 2 +- .../agent-collect-metrics.yaml | 2 +- .../agent-collect-profiles.yaml | 2 +- .../compose-include/agent-collect-traces.yaml | 2 +- .../logs/docker-compose.yaml | 2 +- .../metrics/docker-compose.yaml | 2 +- .../profiles/docker-compose.yaml | 2 +- .../traces/docker-compose.yaml | 2 +- .../all-in-one/docker-compose.yaml | 4 +- .../monolithic-mode/logs/docker-compose.yaml | 2 +- .../metrics/docker-compose.yaml | 2 +- .../profiles/docker-compose.yaml | 2 +- .../traces/docker-compose.yaml | 2 +- .../read-write-mode/logs/docker-compose.yaml | 2 +- .../metrics/docker-compose.yaml | 2 +- .../common/grafana-agent/k8s-all-in-one.yaml | 36 ++++++----- .../common/grafana-agent/kustomization.yaml | 4 +- kubernetes/common/grafana/k8s-all-in-one.yaml | 24 +++---- kubernetes/common/grafana/kustomization.yaml | 4 +- kubernetes/common/grafana/values.yaml | 19 ++++-- monitoring-mixins/k8s-all-in-one.yaml | 64 ------------------- 22 files changed, 67 insertions(+), 118 deletions(-) diff --git a/Makefile b/Makefile index 6fddcf3d..582b8834 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ CONFIG_FILES_IN_DOCKER = $(subst ./, /data/, $(CONFIG_FILES)) fmt: ## Uses Grafana Agent to fmt the river config @for c in $(CONFIG_FILES_IN_DOCKER); do \ echo "$$c"; \ - docker run -e AGENT_MODE=flow --rm --volume "$(shell pwd):/data" -u $(shell id -u) grafana/agent:v0.37.4 fmt -w $$c ; \ + docker run -e AGENT_MODE=flow --rm --volume "$(shell pwd):/data" -u $(shell id -u) grafana/agent:v0.38.0 fmt -w $$c ; \ done ##@ Docker compose diff --git a/docker-compose/common/compose-include/agent-collect-logs.yaml b/docker-compose/common/compose-include/agent-collect-logs.yaml index 27b0f4bc..6b210443 100644 --- a/docker-compose/common/compose-include/agent-collect-logs.yaml +++ b/docker-compose/common/compose-include/agent-collect-logs.yaml @@ -4,7 +4,7 @@ services: depends_on: gateway: condition: service_healthy - image: grafana/agent:v0.37.2 + image: grafana/agent:v0.38.0 volumes: - ../config/agent-flow:/etc/agent-config - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose/common/compose-include/agent-collect-metrics.yaml b/docker-compose/common/compose-include/agent-collect-metrics.yaml index b2e955c1..41760542 100644 --- a/docker-compose/common/compose-include/agent-collect-metrics.yaml +++ b/docker-compose/common/compose-include/agent-collect-metrics.yaml @@ -4,7 +4,7 @@ services: depends_on: gateway: condition: service_healthy - image: grafana/agent:v0.37.2 + image: grafana/agent:v0.38.0 volumes: - ../config/agent-flow:/etc/agent-config - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose/common/compose-include/agent-collect-profiles.yaml b/docker-compose/common/compose-include/agent-collect-profiles.yaml index 19c71ebf..8cc559f0 100644 --- a/docker-compose/common/compose-include/agent-collect-profiles.yaml +++ b/docker-compose/common/compose-include/agent-collect-profiles.yaml @@ -4,7 +4,7 @@ services: depends_on: gateway: condition: service_started - image: grafana/agent:v0.37.2 + image: grafana/agent:v0.38.0 volumes: - ../config/agent-flow:/etc/agent-config - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose/common/compose-include/agent-collect-traces.yaml b/docker-compose/common/compose-include/agent-collect-traces.yaml index 33427996..7b698f25 100644 --- a/docker-compose/common/compose-include/agent-collect-traces.yaml +++ b/docker-compose/common/compose-include/agent-collect-traces.yaml @@ -4,7 +4,7 @@ services: depends_on: gateway: condition: service_healthy - image: grafana/agent:v0.37.2 + image: grafana/agent:v0.38.0 volumes: - ../config/agent-flow:/etc/agent-config - /var/run/docker.sock:/var/run/docker.sock diff --git a/docker-compose/microservices-mode/logs/docker-compose.yaml b/docker-compose/microservices-mode/logs/docker-compose.yaml index fadeb1e9..7cf85027 100644 --- a/docker-compose/microservices-mode/logs/docker-compose.yaml +++ b/docker-compose/microservices-mode/logs/docker-compose.yaml @@ -139,7 +139,7 @@ services: - loki-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/microservices-mode/metrics/docker-compose.yaml b/docker-compose/microservices-mode/metrics/docker-compose.yaml index 487949dd..6ff391b7 100644 --- a/docker-compose/microservices-mode/metrics/docker-compose.yaml +++ b/docker-compose/microservices-mode/metrics/docker-compose.yaml @@ -205,7 +205,7 @@ services: - mimir-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/microservices-mode/profiles/docker-compose.yaml b/docker-compose/microservices-mode/profiles/docker-compose.yaml index fddb90f5..f73efbdd 100644 --- a/docker-compose/microservices-mode/profiles/docker-compose.yaml +++ b/docker-compose/microservices-mode/profiles/docker-compose.yaml @@ -174,7 +174,7 @@ services: - pyroscope-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/microservices-mode/traces/docker-compose.yaml b/docker-compose/microservices-mode/traces/docker-compose.yaml index 6120815a..90630f34 100644 --- a/docker-compose/microservices-mode/traces/docker-compose.yaml +++ b/docker-compose/microservices-mode/traces/docker-compose.yaml @@ -111,7 +111,7 @@ services: depends_on: agent: condition: service_started - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml index a0399200..1c21b8a6 100644 --- a/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml +++ b/docker-compose/monolithic-mode/all-in-one/docker-compose.yaml @@ -15,7 +15,7 @@ services: condition: service_healthy loki: condition: service_healthy - image: grafana/agent:v0.37.2 + image: grafana/agent:v0.38.0 volumes: - ../../common/config/agent-flow:/etc/agent-config - /var/run/docker.sock:/var/run/docker.sock @@ -152,7 +152,7 @@ services: # - "4040:4040" grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/monolithic-mode/logs/docker-compose.yaml b/docker-compose/monolithic-mode/logs/docker-compose.yaml index 1b7a2b95..86fd0c67 100644 --- a/docker-compose/monolithic-mode/logs/docker-compose.yaml +++ b/docker-compose/monolithic-mode/logs/docker-compose.yaml @@ -61,7 +61,7 @@ services: - loki-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/monolithic-mode/metrics/docker-compose.yaml b/docker-compose/monolithic-mode/metrics/docker-compose.yaml index d28c7020..9797e518 100644 --- a/docker-compose/monolithic-mode/metrics/docker-compose.yaml +++ b/docker-compose/monolithic-mode/metrics/docker-compose.yaml @@ -64,7 +64,7 @@ services: - mimir-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/monolithic-mode/profiles/docker-compose.yaml b/docker-compose/monolithic-mode/profiles/docker-compose.yaml index 9d5c95da..be3625c0 100644 --- a/docker-compose/monolithic-mode/profiles/docker-compose.yaml +++ b/docker-compose/monolithic-mode/profiles/docker-compose.yaml @@ -30,7 +30,7 @@ services: - gateway grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 hostname: grafana command: - --config=/etc/grafana-config/grafana.ini diff --git a/docker-compose/monolithic-mode/traces/docker-compose.yaml b/docker-compose/monolithic-mode/traces/docker-compose.yaml index 7988e7b8..e4f673fd 100644 --- a/docker-compose/monolithic-mode/traces/docker-compose.yaml +++ b/docker-compose/monolithic-mode/traces/docker-compose.yaml @@ -64,7 +64,7 @@ services: depends_on: minio: condition: service_started - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/read-write-mode/logs/docker-compose.yaml b/docker-compose/read-write-mode/logs/docker-compose.yaml index 04e39250..1c73ccc0 100644 --- a/docker-compose/read-write-mode/logs/docker-compose.yaml +++ b/docker-compose/read-write-mode/logs/docker-compose.yaml @@ -101,7 +101,7 @@ services: retries: 5 grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 command: - --config=/etc/grafana-config/grafana.ini volumes: diff --git a/docker-compose/read-write-mode/metrics/docker-compose.yaml b/docker-compose/read-write-mode/metrics/docker-compose.yaml index 189322f0..b444931d 100644 --- a/docker-compose/read-write-mode/metrics/docker-compose.yaml +++ b/docker-compose/read-write-mode/metrics/docker-compose.yaml @@ -94,7 +94,7 @@ services: - mimir-memberlist grafana: - image: grafana/grafana:10.2.0 + image: grafana/grafana:10.2.2 hostname: grafana command: - --config=/etc/grafana-config/grafana.ini diff --git a/kubernetes/common/grafana-agent/k8s-all-in-one.yaml b/kubernetes/common/grafana-agent/k8s-all-in-one.yaml index 84ffb320..b5060e55 100644 --- a/kubernetes/common/grafana-agent/k8s-all-in-one.yaml +++ b/kubernetes/common/grafana-agent/k8s-all-in-one.yaml @@ -5,8 +5,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent namespace: monitoring-system --- @@ -17,8 +17,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent rules: - apiGroups: @@ -103,8 +103,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent roleRef: apiGroup: rbac.authorization.k8s.io @@ -464,8 +464,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent namespace: monitoring-system spec: @@ -486,8 +486,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent-cluster namespace: monitoring-system spec: @@ -509,8 +509,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent namespace: monitoring-system spec: @@ -547,6 +547,8 @@ spec: env: - name: AGENT_MODE value: flow + - name: AGENT_DEPLOY_MODE + value: helm - name: HOSTNAME valueFrom: fieldRef: @@ -554,7 +556,7 @@ spec: envFrom: - secretRef: name: agent-env - image: docker.io/grafana/agent:v0.38.0-rc.0 + image: docker.io/grafana/agent:v0.38.0 imagePullPolicy: IfNotPresent name: grafana-agent ports: @@ -608,8 +610,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent namespace: monitoring-system spec: @@ -628,8 +630,8 @@ metadata: app.kubernetes.io/instance: grafana-agent app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana-agent - app.kubernetes.io/version: v0.37.4 - helm.sh/chart: grafana-agent-0.27.2 + app.kubernetes.io/version: v0.38.0 + helm.sh/chart: grafana-agent-0.28.0 name: grafana-agent namespace: monitoring-system spec: diff --git a/kubernetes/common/grafana-agent/kustomization.yaml b/kubernetes/common/grafana-agent/kustomization.yaml index 286929cc..c6e1769a 100644 --- a/kubernetes/common/grafana-agent/kustomization.yaml +++ b/kubernetes/common/grafana-agent/kustomization.yaml @@ -5,7 +5,7 @@ namespace: monitoring-system helmCharts: - name: grafana-agent - version: 0.27.2 + version: 0.28.0 repo: https://grafana.github.io/helm-charts releaseName: grafana-agent namespace: monitoring-system @@ -14,7 +14,7 @@ helmCharts: images: - name: docker.io/grafana/agent - newTag: v0.38.0-rc.0 + newTag: v0.38.0 configMapGenerator: - name: agent-config diff --git a/kubernetes/common/grafana/k8s-all-in-one.yaml b/kubernetes/common/grafana/k8s-all-in-one.yaml index c750c9a2..ae12cb85 100644 --- a/kubernetes/common/grafana/k8s-all-in-one.yaml +++ b/kubernetes/common/grafana/k8s-all-in-one.yaml @@ -11,7 +11,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system --- @@ -23,7 +23,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system rules: [] @@ -36,7 +36,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana-clusterrole rules: - apiGroups: @@ -57,7 +57,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system roleRef: @@ -77,7 +77,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana-clusterrolebinding roleRef: apiGroup: rbac.authorization.k8s.io @@ -156,7 +156,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana-config-dashboards namespace: monitoring-system --- @@ -3350,7 +3350,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system spec: @@ -3372,7 +3372,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system spec: @@ -3389,7 +3389,7 @@ spec: annotations: checksum/config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b checksum/dashboards-json-config: 01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b - checksum/sc-dashboard-provider-config: eeb32ac4580a7c57e39b245bcaa0038f191b3b2f37506d4dfb9c40f60fb79764 + checksum/sc-dashboard-provider-config: 8d71ea9c11146f8c439d477feab1447c1178f5fd966361f6e052af868d400663 kubectl.kubernetes.io/default-container: grafana logs.agent.grafana.com/scrape: "true" logs.agent.grafana.com/scrub-level: debug @@ -3508,7 +3508,7 @@ spec: - configMapRef: name: grafana-env-dkmb477tfh optional: true - image: docker.io/grafana/grafana:10.2.1 + image: docker.io/grafana/grafana:10.2.2 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 10 @@ -3581,7 +3581,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 team: team-infra name: grafana namespace: monitoring-system @@ -3610,7 +3610,7 @@ metadata: app.kubernetes.io/managed-by: Helm app.kubernetes.io/name: grafana app.kubernetes.io/version: 10.1.5 - helm.sh/chart: grafana-7.0.4 + helm.sh/chart: grafana-7.0.8 name: grafana namespace: monitoring-system spec: diff --git a/kubernetes/common/grafana/kustomization.yaml b/kubernetes/common/grafana/kustomization.yaml index 22722dae..12cf80ad 100644 --- a/kubernetes/common/grafana/kustomization.yaml +++ b/kubernetes/common/grafana/kustomization.yaml @@ -5,7 +5,7 @@ namespace: monitoring-system helmCharts: - name: grafana - version: 7.0.4 + version: 7.0.8 repo: https://grafana.github.io/helm-charts releaseName: grafana namespace: monitoring-system @@ -18,7 +18,7 @@ resources: images: - name: docker.io/grafana/grafana - newTag: 10.2.1 + newTag: 10.2.2 configMapGenerator: - name: grafana diff --git a/kubernetes/common/grafana/values.yaml b/kubernetes/common/grafana/values.yaml index ab259791..07502cc5 100644 --- a/kubernetes/common/grafana/values.yaml +++ b/kubernetes/common/grafana/values.yaml @@ -1241,14 +1241,25 @@ networkPolicy: ## created allowing grafana to connect to external data sources from kubernetes cluster. enabled: false ## + ## @param networkPolicy.egress.blockDNSResolution When enabled, DNS resolution will be blocked + ## for all pods in the grafana namespace. + blockDNSResolution: false + ## ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress ports: [] ## Add ports to the egress by specifying - port: ## E.X. - ## ports: - ## - port: 80 - ## - port: 443 - ## + ## - port: 80 + ## - port: 443 + ## + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + to: [] + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [grafana]} ## ## ## diff --git a/monitoring-mixins/k8s-all-in-one.yaml b/monitoring-mixins/k8s-all-in-one.yaml index 75b50388..0ed9720d 100644 --- a/monitoring-mixins/k8s-all-in-one.yaml +++ b/monitoring-mixins/k8s-all-in-one.yaml @@ -70890,70 +70890,6 @@ spec: --- apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule -metadata: - name: grafana-agent-alerts - namespace: monitoring-system -spec: - groups: - - name: clustering - rules: - - alert: ClusterNotConverging - annotations: - message: Cluster is not converging. - expr: stddev by (cluster, namespace) (sum without (state) (cluster_node_peers)) - != 0 - for: 5m - - alert: ClusterSplitBrain - annotations: - message: Cluster nodes have entered a split brain state. - expr: | - sum without (state) (cluster_node_peers) != - on (cluster, namespace) group_left - count by (cluster, namespace) (cluster_node_info) - for: 5m - - alert: ClusterLamportClockDrift - annotations: - message: Cluster nodes' lamport clocks are not converging. - expr: stddev by (cluster, namespace) (cluster_node_lamport_time) > 4 * sqrt(count - by (cluster, namespace) (cluster_node_info)) - for: 5m - - alert: ClusterNodeUnhealthy - annotations: - message: Cluster node is reporting a health score > 0. - expr: | - cluster_node_gossip_health_score > 0 - for: 5m - - alert: ClusterLamportClockStuck - annotations: - message: Cluster nodes's lamport clocks is not progressing. - expr: | - sum by (cluster, namespace, instance) (rate(cluster_node_lamport_time[2m])) == 0 - and on (cluster, namespace, instance) (cluster_node_peers > 1) - for: 5m - - alert: ClusterNodeNameConflict - annotations: - message: A node tried to join the cluster with a name conflicting with an - existing peer. - expr: sum by (cluster, namespace) (rate(cluster_node_gossip_received_events_total{event="node_conflict"}[2m])) - > 0 - for: 10m - - alert: ClusterNodeStuckTerminating - annotations: - message: Cluster node stuck in Terminating state. - expr: sum by (cluster, namespace, instance) (cluster_node_peers{state="terminating"}) - > 0 - for: 5m - - alert: ClusterConfigurationDrift - annotations: - message: Cluster nodes are not using the same configuration file. - expr: | - count without (sha256) ( - max by (cluster, namespace, sha256) (agent_config_hash and on(cluster, namespace) cluster_node_info) - ) > 1 - for: 5m ---- -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule metadata: name: loki-mixin-alerts namespace: monitoring-system