RedPanda Consumer Lag metric

redpanda/datadog_checks/redpanda/metrics.py

@@ -75,6 +75,7 @@
     'vectorized_kafka_rpc_requests_pending': 'kafka.rpc_requests_pending',
     'vectorized_kafka_rpc_sent_bytes': 'kafka.rpc_sent_bytes',
     'vectorized_kafka_rpc_service_errors': 'kafka.rpc_service_errors',
+    'vectorized_kafka_group_offset': 'kafka.group_offset',
 }
 
 VECTORIZED_LEADER = {

redpanda/metadata.csv

@@ -64,6 +64,7 @@ redpanda.kafka.rpc_requests_completed,count,,request,,kafka_rpc: Number of succe
 redpanda.kafka.rpc_requests_pending,gauge,,request,,kafka_rpc: Number of requests being processed by server,-1,redpanda,kafka rpc requests pending,
 redpanda.kafka.rpc_sent_bytes,count,,byte,,kafka_rpc: Number of bytes sent to clients,-1,redpanda,kafka rpc sent bytes,
 redpanda.kafka.rpc_service_errors,count,,error,,kafka_rpc: Number of service errors,-1,redpanda,kafka rpc service errors,
+redpanda.kafka.group_offset,gauge,,,,consumer lag offset,-1,redpanda,consumer lag offset,
 redpanda.leader.balancer_leader_transfer_error,count,,error,,Number of errors attempting to transfer leader,-1,redpanda,leader balancer leader transfer error,
 redpanda.leader.balancer_leader_transfer_no_improvement,count,,,,Number of times no balance improvement was found,-1,redpanda,leader balancer leader transfer no improvement,
 redpanda.leader.balancer_leader_transfer_succeeded,count,,success,,Number of successful leader transfers,-1,redpanda,leader balancer leader transfer succeeded,
@@ -165,4 +166,4 @@ redpanda.storage.log_partition_size,gauge,,byte,,Current size of partition in by
 redpanda.storage.log_read_bytes,count,,byte,,Total number of bytes read,-1,redpanda,storage log read bytes,
 redpanda.storage.log_readers_added,count,,read,,Number of readers added to cache,-1,redpanda,storage log readers added,
 redpanda.storage.log_readers_evicted,count,,read,,Number of readers evicted from cache,-1,redpanda,storage log readers evicted,
-redpanda.storage.log_written_bytes,count,,byte,,Total number of bytes written,-1,redpanda,storage log written bytes,
+redpanda.storage.log_written_bytes,count,,byte,,Total number of bytes written,-1,redpanda,storage log written bytes,

redpanda/tests/common.py

@@ -94,6 +94,7 @@
         'redpanda.kafka.rpc_requests_pending',
         'redpanda.kafka.rpc_sent_bytes.count',
         'redpanda.kafka.rpc_service_errors.count',
+        'redpanda.kafka.group_offset',
     ],
     'redpanda.leader': [
         'redpanda.leader.balancer_leader_transfer_error.count',

redpanda/tests/fixtures/redpanda_metrics.txt

@@ -354,6 +354,18 @@ vectorized_kafka_rpc_sent_bytes{shard="0"} 6164
 # HELP vectorized_kafka_rpc_service_errors kafka_rpc: Number of service errors
 # TYPE vectorized_kafka_rpc_service_errors counter
 vectorized_kafka_rpc_service_errors{shard="0"} 0
+# HELP vectorized_kafka_group_offset Group topic partition offset
+# TYPE vectorized_kafka_group_offset gauge
+vectorized_kafka_group_offset{group="123",partition="0",shard="0",topic="foo"} 132444.000000
+vectorized_kafka_group_offset{group="123",partition="1",shard="0",topic="foo"} 133757.000000
+vectorized_kafka_group_offset{group="123",partition="2",shard="0",topic="foo"} 136132.000000
+vectorized_kafka_group_offset{group="123",partition="3",shard="0",topic="foo"} 136116.000000
+vectorized_kafka_group_offset{group="123",partition="4",shard="0",topic="foo"} 127034.000000
+vectorized_kafka_group_offset{group="123",partition="5",shard="0",topic="foo"} 125198.000000
+vectorized_kafka_group_offset{group="123",partition="6",shard="0",topic="foo"} 121542.000000
+vectorized_kafka_group_offset{group="123",partition="7",shard="0",topic="foo"} 123378.000000
+vectorized_kafka_group_offset{group="123",partition="8",shard="0",topic="foo"} 119706.000000
+vectorized_kafka_group_offset{group="123",partition="9",shard="0",topic="foo"} 121526.000000
 # HELP vectorized_leader_balancer_leader_transfer_error Number of errors attempting to transfer leader
 # TYPE vectorized_leader_balancer_leader_transfer_error counter
 vectorized_leader_balancer_leader_transfer_error{shard="0"} 0
@@ -731,3 +743,4 @@ vectorized_storage_log_readers_evicted{namespace="redpanda",partition="0",shard=
 # TYPE vectorized_storage_log_written_bytes counter
 vectorized_storage_log_written_bytes{namespace="kafka_internal",partition="0",shard="0",topic="group"} 394
 vectorized_storage_log_written_bytes{namespace="redpanda",partition="0",shard="0",topic="controller"} 842
+

redpanda/tests/test_redpanda.py

@@ -97,5 +97,8 @@ def test_check(aggregator, dd_run_check):
     dd_run_check(check)
 
     for m in INSTANCE_DEFAULT_METRICS:
+        # skipping as its only exposed when consumer group is created
+        if m == "redpanda.kafka.group_offset":
+            continue
         aggregator.assert_metric(m)
     aggregator.assert_all_metrics_covered()

redpanda/tox.ini

@@ -3,7 +3,7 @@ minversion = 2.0
 skip_missing_interpreters = true
 basepython = py38
 envlist =
-    py{38}-{21.9}
+    py{38}-{21.11}
 
 [testenv]
 ensure_default_envdir = true
@@ -22,6 +22,6 @@ commands =
     pip install -r requirements.in
     pytest -v {posargs}
 setenv =
-	21.9: REDPANDA_VERSION=v21.9.6
+	21.11: REDPANDA_VERSION=v21.11.10
 	; OpenmetricsChecks sends generic tags
 	DDEV_SKIP_GENERIC_TAGS_CHECK=true