issue-2969: add hive reconnect time counter (#2970)

* issue-2969: add hive reconnect time counter * update * update * update
ydb-platform · Feb 6, 2025 · 33359ef · 33359ef
1 parent 9c3f69c
commit 33359ef
Show file tree

Hide file tree

Showing 9 changed files with 155 additions and 35 deletions.
diff --git a/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp b/cloud/blockstore/libs/storage/init/disk_agent/actorsystem.cpp
@@ -52,15 +52,20 @@ class TStorageServicesInitializer final
         // HiveProxy
         //
 
-        auto hiveProxy = CreateHiveProxy({
-            .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
-            .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
-            .HiveLockExpireTimeout = Args.StorageConfig->GetHiveLockExpireTimeout(),
-            .LogComponent = TBlockStoreComponents::HIVE_PROXY,
-            .TabletBootInfoBackupFilePath = {},
-            .FallbackMode = false,
-            .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
-        });
+        auto hiveProxy = CreateHiveProxy(
+            {
+                .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
+                .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
+                .HiveLockExpireTimeout = Args.StorageConfig->GetHiveLockExpireTimeout(),
+                .LogComponent = TBlockStoreComponents::HIVE_PROXY,
+                .TabletBootInfoBackupFilePath = {},
+                .FallbackMode = false,
+                .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
+            },
+            appData
+                ->Counters
+                ->GetSubgroup("counters", "blockstore")
+                ->GetSubgroup("component", "service"));
 
         setup->LocalServices.emplace_back(
             MakeHiveProxyServiceId(),

diff --git a/cloud/blockstore/libs/storage/init/server/actorsystem.cpp b/cloud/blockstore/libs/storage/init/server/actorsystem.cpp
@@ -138,15 +138,20 @@ class TStorageServicesInitializer final
         // HiveProxy
         //
 
-        auto hiveProxy = CreateHiveProxy({
-            .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
-            .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
-            .HiveLockExpireTimeout = Args.StorageConfig->GetHiveLockExpireTimeout(),
-            .LogComponent = TBlockStoreComponents::HIVE_PROXY,
-            .TabletBootInfoBackupFilePath = Args.StorageConfig->GetTabletBootInfoBackupFilePath(),
-            .FallbackMode = Args.StorageConfig->GetHiveProxyFallbackMode(),
-            .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
-        });
+        auto hiveProxy = CreateHiveProxy(
+            {
+                .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
+                .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
+                .HiveLockExpireTimeout = Args.StorageConfig->GetHiveLockExpireTimeout(),
+                .LogComponent = TBlockStoreComponents::HIVE_PROXY,
+                .TabletBootInfoBackupFilePath = Args.StorageConfig->GetTabletBootInfoBackupFilePath(),
+                .FallbackMode = Args.StorageConfig->GetHiveProxyFallbackMode(),
+                .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
+            },
+            appData
+                ->Counters
+                ->GetSubgroup("counters", "blockstore")
+                ->GetSubgroup("component", "service"));
 
         setup->LocalServices.emplace_back(
             MakeHiveProxyServiceId(),

diff --git a/cloud/filestore/libs/storage/init/actorsystem.cpp b/cloud/filestore/libs/storage/init/actorsystem.cpp
@@ -124,16 +124,21 @@ class TStorageServicesInitializer final
         // HiveProxy
         //
 
-        auto hiveProxy = CreateHiveProxy({
-            .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
-            .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
-            // HiveLockExpireTimeout, used by NBS, doesn't matter
-            .HiveLockExpireTimeout = TDuration::Seconds(1),
-            .LogComponent = TFileStoreComponents::HIVE_PROXY,
-            .TabletBootInfoBackupFilePath = Args.StorageConfig->GetTabletBootInfoBackupFilePath(),
-            .FallbackMode = Args.StorageConfig->GetHiveProxyFallbackMode(),
-            .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
-        });
+        auto hiveProxy = CreateHiveProxy(
+            {
+                .PipeClientRetryCount = Args.StorageConfig->GetPipeClientRetryCount(),
+                .PipeClientMinRetryTime = Args.StorageConfig->GetPipeClientMinRetryTime(),
+                // HiveLockExpireTimeout, used by NBS, doesn't matter
+                .HiveLockExpireTimeout = TDuration::Seconds(1),
+                .LogComponent = TFileStoreComponents::HIVE_PROXY,
+                .TabletBootInfoBackupFilePath = Args.StorageConfig->GetTabletBootInfoBackupFilePath(),
+                .FallbackMode = Args.StorageConfig->GetHiveProxyFallbackMode(),
+                .TenantHiveTabletId = Args.StorageConfig->GetTenantHiveTabletId(),
+            },
+            appData
+                ->Counters
+                ->GetSubgroup("counters", "filestore")
+                ->GetSubgroup("component", "service"));
 
         setup->LocalServices.emplace_back(
             MakeHiveProxyServiceId(),

diff --git a/cloud/storage/core/libs/hive_proxy/hive_proxy.cpp b/cloud/storage/core/libs/hive_proxy/hive_proxy.cpp
@@ -10,12 +10,21 @@ using namespace NActors;
 ////////////////////////////////////////////////////////////////////////////////
 
 IActorPtr CreateHiveProxy(THiveProxyConfig config)
+{
+    return CreateHiveProxy(std::move(config), {});
+}
+
+IActorPtr CreateHiveProxy(
+    THiveProxyConfig config,
+    NMonitoring::TDynamicCounterPtr counters)
 {
     if (config.FallbackMode) {
         return std::make_unique<THiveProxyFallbackActor>(std::move(config));
     }
 
-    return std::make_unique<THiveProxyActor>(std::move(config));
+    return std::make_unique<THiveProxyActor>(
+        std::move(config),
+        std::move(counters));
 }
 
 }   // namespace NCloud::NStorage
diff --git a/cloud/storage/core/libs/hive_proxy/hive_proxy.h b/cloud/storage/core/libs/hive_proxy/hive_proxy.h
@@ -5,10 +5,15 @@
 #include <cloud/storage/core/libs/actors/public.h>
 #include <cloud/storage/core/libs/common/public.h>
 
+#include <library/cpp/monlib/dynamic_counters/counters.h>
+
 namespace NCloud::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
 NActors::IActorPtr CreateHiveProxy(THiveProxyConfig config);
+NActors::IActorPtr CreateHiveProxy(
+    THiveProxyConfig config,
+    NMonitoring::TDynamicCounterPtr counters);
 
 }   // namespace NCloud::NStorage
diff --git a/cloud/storage/core/libs/hive_proxy/hive_proxy_actor.cpp b/cloud/storage/core/libs/hive_proxy/hive_proxy_actor.cpp
@@ -34,12 +34,19 @@ std::unique_ptr<NTabletPipe::IClientCache> CreateTabletPipeClientCache(
 
 ////////////////////////////////////////////////////////////////////////////////
 
-THiveProxyActor::THiveProxyActor(THiveProxyConfig config)
+THiveProxyActor::THiveProxyActor(
+        THiveProxyConfig config,
+        NMonitoring::TDynamicCounterPtr counters)
     : ClientCache(CreateTabletPipeClientCache(config))
     , LockExpireTimeout(config.HiveLockExpireTimeout)
     , LogComponent(config.LogComponent)
     , TabletBootInfoBackupFilePath(config.TabletBootInfoBackupFilePath)
     , TenantHiveTabletId(config.TenantHiveTabletId)
+    , Counters(std::move(counters))
+{}
+
+THiveProxyActor::THiveProxyActor(THiveProxyConfig config)
+    : THiveProxyActor(std::move(config), {})
 {}
 
 void THiveProxyActor::Bootstrap(const TActorContext& ctx)
@@ -55,10 +62,24 @@ void THiveProxyActor::Bootstrap(const TActorContext& ctx)
         TabletBootInfoBackup = ctx.Register(
             cache.release(), TMailboxType::HTSwap, AppData()->IOPoolId);
     }
+    if (Counters) {
+        HiveReconnectTimeCounter = Counters->GetCounter("HiveReconnectTime", true);
+    }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
+void THiveProxyActor::SendRequest(
+    const TActorContext& ctx,
+    ui64 hive,
+    IEventBase* request)
+{
+    ClientCache->Send(ctx, hive, request);
+    if (HiveDisconnected) {
+        HiveReconnectStartCycles = GetCycleCount();
+    }
+}
+
 ui64 THiveProxyActor::GetHive(
     const TActorContext& ctx,
     ui64 tabletId,
@@ -98,15 +119,15 @@ void THiveProxyActor::SendLockRequest(
     hiveRequest->Record.SetMaxReconnectTimeout(
         LockExpireTimeout.MilliSeconds());
     hiveRequest->Record.SetReconnect(reconnect);
-    ClientCache->Send(ctx, hive, hiveRequest.release());
+    SendRequest(ctx, hive, hiveRequest.release());
 }
 
 void THiveProxyActor::SendUnlockRequest(
     const TActorContext& ctx, ui64 hive, ui64 tabletId)
 {
     auto hiveRequest =
         std::make_unique<TEvHive::TEvUnlockTabletExecution>(tabletId);
-    ClientCache->Send(ctx, hive, hiveRequest.release());
+    SendRequest(ctx, hive, hiveRequest.release());
 }
 
 void THiveProxyActor::SendGetTabletStorageInfoRequest(
@@ -115,7 +136,7 @@ void THiveProxyActor::SendGetTabletStorageInfoRequest(
 {
     auto hiveRequest =
         std::make_unique<TEvHive::TEvGetTabletStorageInfo>(tabletId);
-    ClientCache->Send(ctx, hive, hiveRequest.release());
+    SendRequest(ctx, hive, hiveRequest.release());
 }
 
 void THiveProxyActor::SendLockReply(
@@ -198,7 +219,7 @@ void THiveProxyActor::SendTabletMetrics(
         prTabletId.second.OnStatsSend();
     }
     if (record.TabletMetricsSize() > 0) {
-        ClientCache->Send(ctx, hive, event.Release());
+        SendRequest(ctx, hive, event.Release());
     }
 }
 
@@ -216,6 +237,14 @@ void THiveProxyActor::HandleConnect(
         auto error = MakeKikimrError(msg->Status, TStringBuilder()
             << "Connect to hive " << hive << " failed");
         HandleConnectionError(ctx, error, hive, true);
+    } else if (HiveReconnectStartCycles) {
+        if (HiveReconnectTimeCounter) {
+            HiveReconnectTimeCounter->Add(
+                CyclesToDuration(
+                    GetCycleCount() - HiveReconnectStartCycles).MicroSeconds());
+        }
+        HiveReconnectStartCycles = 0;
+        HiveDisconnected = false;
     }
 }
 
@@ -242,6 +271,8 @@ void THiveProxyActor::HandleConnectionError(
     Y_UNUSED(error);
     Y_UNUSED(connectFailed);
 
+    HiveDisconnected = true;
+
     LOG_ERROR_S(ctx, LogComponent,
         "Pipe to hive" << hive << " has been reset ");
 
@@ -307,6 +338,9 @@ void THiveProxyActor::HandleConnectionError(
 
         for (const auto& actorId: states->Actors) {
             auto clientId = ClientCache->Prepare(ctx, hive);
+            if (!HiveReconnectStartCycles) {
+                HiveReconnectStartCycles = GetCycleCount();
+            }
             NCloud::Send<TEvHiveProxyPrivate::TEvChangeTabletClient>(
                 ctx,
                 actorId,

diff --git a/cloud/storage/core/libs/hive_proxy/hive_proxy_actor.h b/cloud/storage/core/libs/hive_proxy/hive_proxy_actor.h
@@ -177,14 +177,28 @@ class THiveProxyActor final
 
     const ui64 TenantHiveTabletId;
 
+    const NMonitoring::TDynamicCounterPtr Counters;
+    NMonitoring::TDynamicCounters::TCounterPtr HiveReconnectTimeCounter;
+    ui64 HiveReconnectStartCycles = 0;
+    bool HiveDisconnected = true;
+
 public:
     explicit THiveProxyActor(THiveProxyConfig config);
 
+    THiveProxyActor(
+        THiveProxyConfig config,
+        NMonitoring::TDynamicCounterPtr counters);
+
     void Bootstrap(const NActors::TActorContext& ctx);
 
 private:
     STFUNC(StateWork);
 
+    void SendRequest(
+        const NActors::TActorContext& ctx,
+        ui64 hive,
+        NActors::IEventBase* request);
+
     ui64 GetHive(
         const NActors::TActorContext& ctx,
         ui64 tabletId,

diff --git a/cloud/storage/core/libs/hive_proxy/hive_proxy_ut.cpp b/cloud/storage/core/libs/hive_proxy/hive_proxy_ut.cpp
@@ -513,7 +513,9 @@ struct TTestEnv
             .TenantHiveTabletId = tenantHive,
         };
         HiveProxyActorId = Runtime.Register(
-            CreateHiveProxy(std::move(config)).release());
+            CreateHiveProxy(
+                std::move(config),
+                Runtime.GetAppData(0).Counters).release());
         Runtime.EnableScheduleForActor(HiveProxyActorId);
         Runtime.RegisterService(MakeHiveProxyServiceId(), HiveProxyActorId);
     }
@@ -1541,6 +1543,45 @@ Y_UNIT_TEST_SUITE(THiveProxyTest)
         UNIT_ASSERT_VALUES_EQUAL(1, hiveMessages);
         UNIT_ASSERT_VALUES_EQUAL(1, wakeups);
     }
+
+    Y_UNIT_TEST(ShouldReportHiveReconnectTime)
+    {
+        TTestBasicRuntime runtime;
+        TTestEnv env(runtime);
+
+        auto sender = runtime.AllocateEdgeActor();
+
+        auto counter = env.Runtime.GetAppData(0).Counters
+            ->GetCounter("HiveReconnectTime", true);
+
+        env.SendLockRequest(sender, FakeTablet2);
+        UNIT_ASSERT_VALUES_UNEQUAL(0, counter->Val());
+
+        auto oldVal = counter->Val();
+
+        int hiveLockRequests = 0;
+        runtime.SetObserverFunc([&](TAutoPtr<IEventHandle>& event) {
+                Y_UNUSED(runtime);
+                if (event->GetTypeRewrite() == TEvHive::EvLockTabletExecution) {
+                    ++hiveLockRequests;
+                }
+                return TTestActorRuntime::EEventAction::PROCESS;
+            });
+
+        env.EnableTabletResolverScheduling();
+        env.RebootHive();
+
+        while (!hiveLockRequests) {
+            // Pipe to hive may take a long time to connect
+            // Wait until hive receives the lock request
+            runtime.DispatchEvents(TDispatchOptions(), TDuration::Seconds(1));
+        }
+
+        runtime.SetObserverFunc(&TTestActorRuntime::DefaultObserverFunc);
+
+        // Rebooting hive should reconnect the lock
+        UNIT_ASSERT_GT(counter->Val(), oldVal);
+    }
 }
 
 }   // namespace NCloud::NStorage
diff --git a/cloud/storage/core/libs/hive_proxy/ya.make b/cloud/storage/core/libs/hive_proxy/ya.make
@@ -27,6 +27,8 @@ PEERDIR(
     contrib/ydb/core/tablet_flat
 
     contrib/ydb/library/actors/core
+
+    library/cpp/monlib/dynamic_counters
 )
 
 END()
-Original file line number
+Diff line change
@@ Expand Up / @@ -27,6 +27,8 @@ PEERDIR( @@
         contrib/ydb/core/tablet_flat
         contrib/ydb/library/actors/core
+        library/cpp/monlib/dynamic_counters
     )
     END()
@@ Expand Down @@