chore: improve prom cp connectivity tests resiliency

Signed-off-by: Sanskar Jaiswal <[email protected]>

spec/03-plugins/26-prometheus/04-status_api_spec.lua

@@ -532,6 +532,7 @@ end
 
 describe("CP/DP connectivity state #", function ()
   local status_client
+  local cp_running
 
   local function get_metrics()
     if not status_client then
@@ -553,15 +554,6 @@ describe("CP/DP connectivity state #", function ()
       name = "prometheus",
     }
 
-    assert(helpers.start_kong({
-      role = "control_plane",
-      prefix = "prom_cp",
-      cluster_cert = "spec/fixtures/kong_clustering.crt",
-      cluster_cert_key = "spec/fixtures/kong_clustering.key",
-      cluster_listen = "127.0.0.1:9005",
-      plugins = "bundled, prometheus",
-    }))
-
     assert(helpers.start_kong({
       role = "data_plane",
       database = "off",
@@ -585,14 +577,40 @@ describe("CP/DP connectivity state #", function ()
     end
 
     helpers.stop_kong("prom_dp")
+    if cp_running then
+      helpers.stop_kong("prom_cp")
+    end
   end)
 
-  it("exposes controlplane connectivity status", function ()
-    local body = get_metrics()
-    assert.matches('kong_control_plane_connected 1', body, nil, true)
+  it("exposes control plane connectivity status", function ()
+    assert.eventually(function()
+      local body = get_metrics()
+      assert.matches('kong_control_plane_connected 0', body, nil, true)
+    end).has_no_error("metric kong_control_plane_connected => 0")
+
+    assert(helpers.start_kong({
+      role = "control_plane",
+      prefix = "prom_cp",
+      cluster_cert = "spec/fixtures/kong_clustering.crt",
+      cluster_cert_key = "spec/fixtures/kong_clustering.key",
+      cluster_listen = "127.0.0.1:9005",
+      plugins = "bundled, prometheus",
+    }))
+    cp_running = true
+
+    -- it takes some time for the cp<->dp connection to get established and the
+    -- metric to reflect that, so set the timeout to 10 secs.
+    assert.with_timeout(10).eventually(function()
+      local body = get_metrics()
+      assert.matches('kong_control_plane_connected 1', body, nil, true)
+    end).has_no_error("metric kong_control_plane_connected => 1")
 
     helpers.stop_kong("prom_cp")
-    local body  = get_metrics()
-    assert.matches('kong_control_plane_connected 0', body, nil, true)
+    cp_running = false
+
+    assert.eventually(function()
+      local body = get_metrics()
+      assert.matches('kong_control_plane_connected 0', body, nil, true)
+    end).has_no_error("metric kong_control_plane_connected => 0")
   end)
 end)