Write critical event when mismatch happened

cloud/blockstore/libs/diagnostics/critical_events.h

@@ -33,6 +33,8 @@ namespace NCloud::NBlockStore {
     xxx(MirroredDiskDeviceReplacementRateLimitExceeded)                        \
     xxx(MirroredDiskMinorityChecksumMismatch)                                  \
     xxx(MirroredDiskMajorityChecksumMismatch)                                  \
+    xxx(MirroredDiskChecksumMismatchUponRead)                                  \
+    xxx(MirroredDiskChecksumMismatchUponWrite)                                 \
     xxx(CounterUpdateRace)                                                     \
     xxx(EndpointStartingError)                                                 \
     xxx(ResyncFailed)                                                          \
@@ -64,7 +66,6 @@ namespace NCloud::NBlockStore {
     xxx(DiskRegistryPurgeHostError)                                            \
     xxx(DiskRegistryCleanupAgentConfigError)                                   \
     xxx(DiskRegistryOccupiedDeviceConfigurationHasChanged)                     \
-    xxx(MirroredDiskChecksumMismatchUponRead)                                  \
     xxx(DiskRegistryWrongMigratedDeviceOwnership)                              \
     xxx(DiskRegistryInitialAgentRejectionThresholdExceeded)                    \
 // BLOCKSTORE_CRITICAL_EVENTS

cloud/blockstore/libs/nbd/server_handler.cpp

@@ -951,6 +951,7 @@ IServerHandlerFactoryPtr CreateServerHandlerFactory(
 {
     auto deviceHandler = deviceHandlerFactory->CreateDeviceHandler(
         std::move(storage),
+        options.DiskId,
         options.ClientId,
         options.BlockSize,
         options.UnalignedRequestsDisabled,

cloud/blockstore/libs/service/aligned_device_handler.cpp

@@ -131,14 +131,16 @@ TBlocksInfo TBlocksInfo::MakeAligned() const
 
 TAlignedDeviceHandler::TAlignedDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         ui32 maxSubRequestSize,
         bool checkBufferModificationDuringWriting)
     : Storage(
-          checkBufferModificationDuringWriting
-              ? CreateChecksumStorageWrapper(std::move(storage))
-              : std::move(storage))
+          checkBufferModificationDuringWriting ? CreateChecksumStorageWrapper(
+                                                     std::move(storage),
+                                                     std::move(diskId))
+                                               : std::move(storage))
     , ClientId(std::move(clientId))
     , BlockSize(blockSize)
     , MaxBlockCount(maxSubRequestSize / BlockSize)

cloud/blockstore/libs/service/aligned_device_handler.h

@@ -55,6 +55,7 @@ class TAlignedDeviceHandler final
 public:
     TAlignedDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         ui32 maxSubRequestSize,

cloud/blockstore/libs/service/checksum_storage_wrapper.cpp

@@ -3,7 +3,9 @@
 #include "storage.h"
 
 #include <cloud/blockstore/libs/common/block_checksum.h>
+#include <cloud/blockstore/libs/common/block_range.h>
 #include <cloud/blockstore/libs/common/iovector.h>
+#include <cloud/blockstore/libs/diagnostics/critical_events.h>
 #include <cloud/blockstore/libs/service/context.h>
 #include <cloud/storage/core/libs/common/error.h>
 #include <cloud/storage/core/libs/common/sglist.h>
@@ -64,10 +66,12 @@ class TChecksumStorageWrapper final
     , public IStorage
 {
     const IStoragePtr Storage;
+    const TString DiskId;
 
 public:
-    explicit TChecksumStorageWrapper(IStoragePtr storage)
+    TChecksumStorageWrapper(IStoragePtr storage, TString diskId)
         : Storage(std::move(storage))
+        , DiskId(std::move(diskId))
     {}
 
     TFuture<NProto::TZeroBlocksResponse> ZeroBlocks(
@@ -165,6 +169,12 @@ TChecksumStorageWrapper::RetryWriteBlocksLocal(
     TCallContextPtr callContext,
     std::shared_ptr<NProto::TWriteBlocksLocalRequest> request)
 {
+    const auto range = TBlockRange64::WithLength(
+        request->GetStartIndex(),
+        request->BlocksCount);
+    ReportMirroredDiskChecksumMismatchUponWrite(
+        TStringBuilder() << "d:" << DiskId << ", r:" << range);
+
     auto guard = request->Sglist.Acquire();
     if (!guard) {
         return MakeFuture<NProto::TWriteBlocksLocalResponse>(
@@ -190,9 +200,11 @@ TChecksumStorageWrapper::RetryWriteBlocksLocal(
 
 ////////////////////////////////////////////////////////////////////////////////
 
-IStoragePtr CreateChecksumStorageWrapper(IStoragePtr storage)
+IStoragePtr CreateChecksumStorageWrapper(IStoragePtr storage, TString diskId)
 {
-    return std::make_shared<TChecksumStorageWrapper>(std::move(storage));
+    return std::make_shared<TChecksumStorageWrapper>(
+        std::move(storage),
+        std::move(diskId));
 }
 
 }   // namespace NCloud::NBlockStore

cloud/blockstore/libs/service/checksum_storage_wrapper.h

@@ -6,7 +6,7 @@ namespace NCloud::NBlockStore {
 
 ////////////////////////////////////////////////////////////////////////////////
 
-IStoragePtr CreateChecksumStorageWrapper(IStoragePtr storage);
+IStoragePtr CreateChecksumStorageWrapper(IStoragePtr storage, TString diskId);
 
 ////////////////////////////////////////////////////////////////////////////////
 

cloud/blockstore/libs/service/device_handler.cpp

@@ -30,6 +30,7 @@ struct TDefaultDeviceHandlerFactory final
 
     IDeviceHandlerPtr CreateDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         bool unalignedRequestsDisabled,
@@ -38,6 +39,7 @@ struct TDefaultDeviceHandlerFactory final
         if (unalignedRequestsDisabled) {
             return std::make_shared<TAlignedDeviceHandler>(
                 std::move(storage),
+                std::move(diskId),
                 std::move(clientId),
                 blockSize,
                 MaxSubRequestSize,
@@ -46,6 +48,7 @@ struct TDefaultDeviceHandlerFactory final
 
         return std::make_shared<TUnalignedDeviceHandler>(
             std::move(storage),
+            std::move(diskId),
             std::move(clientId),
             blockSize,
             MaxSubRequestSize,

cloud/blockstore/libs/service/device_handler.h

@@ -43,6 +43,7 @@ struct IDeviceHandlerFactory
 
     virtual IDeviceHandlerPtr CreateDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         bool unalignedRequestsDisabled,

cloud/blockstore/libs/service/device_handler_ut.cpp

@@ -4,10 +4,12 @@
 #include <cloud/blockstore/libs/service/context.h>
 #include <cloud/blockstore/libs/service/storage_test.h>
 #include <cloud/storage/core/libs/common/error.h>
-
 #include <cloud/storage/core/libs/common/sglist.h>
 #include <cloud/storage/core/libs/common/sglist_test.h>
+#include <cloud/storage/core/libs/diagnostics/critical_events.h>
+#include <cloud/storage/core/libs/diagnostics/monitoring.h>
 
+#include <library/cpp/monlib/dynamic_counters/counters.h>
 #include <library/cpp/testing/unittest/registar.h>
 
 #include <array>
@@ -20,6 +22,15 @@ namespace {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+auto SetupCriticalEvents()
+{
+    NMonitoring::TDynamicCountersPtr counters =
+        new NMonitoring::TDynamicCounters();
+    InitCriticalEventsCounter(counters);
+    return counters;
+}
+
+////////////////////////////////////////////////////////////////////////////////
 class TTestEnvironment
 {
 private:
@@ -126,6 +137,7 @@ class TTestEnvironment
         auto factory = CreateDeviceHandlerFactory(maxBlockCount * BlockSize);
         DeviceHandler = factory->CreateDeviceHandler(
             std::move(testStorage),
+            "disk1",
             "testClientId",
             BlockSize,
             unalignedRequestsDisabled,   // unalignedRequestsDisabled,
@@ -330,6 +342,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldSliceHugeZeroRequest)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
         const ui64 deviceBlocksCount = 8*1024;
@@ -340,6 +353,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         auto factory = CreateDeviceHandlerFactory(blocksCountLimit * blockSize);
         auto deviceHandler = factory->CreateDeviceHandler(
             storage,
+            diskId,
             clientId,
             blockSize,
             false,   // unalignedRequestsDisabled,
@@ -397,13 +411,15 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldHandleAlignedRequestsWhenUnalignedRequestsDisabled)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
 
         auto storage = std::make_shared<TTestStorage>();
 
         auto device = CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler(
             storage,
+            diskId,
             clientId,
             blockSize,
             true,   // unalignedRequestsDisabled,
@@ -485,13 +501,15 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldNotHandleUnalignedRequestsWhenUnalignedRequestsDisabled)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
 
         auto storage = std::make_shared<TTestStorage>();
 
         auto device = CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler(
             storage,
+            diskId,
             clientId,
             blockSize,
             true,   // unalignedRequestsDisabled,
@@ -594,6 +612,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     void DoShouldSliceHugeZeroRequest(bool requestUnaligned, bool unalignedRequestDisabled)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
         const ui64 deviceBlocksCount = 12;
@@ -607,6 +626,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         auto factory = CreateDeviceHandlerFactory(blocksCountLimit * blockSize);
         auto deviceHandler = factory->CreateDeviceHandler(
             storage,
+            diskId,
             clientId,
             blockSize,
             unalignedRequestDisabled,   // unalignedRequestsDisabled,
@@ -716,6 +736,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldReturnErrorForHugeUnalignedReadWriteRequests)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
 
@@ -724,6 +745,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         auto deviceHandler =
             CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler(
                 storage,
+                diskId,
                 clientId,
                 blockSize,
                 false,   // unalignedRequestsDisabled,
@@ -782,6 +804,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldReturnErrorForInvalidBufferSize)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
 
@@ -790,6 +813,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         auto deviceHandler =
             CreateDefaultDeviceHandlerFactory()->CreateDeviceHandler(
                 storage,
+                diskId,
                 clientId,
                 blockSize,
                 false,   // unalignedRequestsDisabled,
@@ -905,6 +929,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
 
     Y_UNIT_TEST(ShouldCopyBufferWhenClientModifyBuffer)
     {
+        const auto diskId = "disk1";
         const auto clientId = "testClientId";
         const ui32 blockSize = DefaultBlockSize;
         const ui64 deviceBlocksCount = 8*1024;
@@ -915,6 +940,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         auto factory = CreateDeviceHandlerFactory(blocksCountLimit * blockSize);
         auto deviceHandler = factory->CreateDeviceHandler(
             storage,
+            diskId,
             clientId,
             blockSize,
             false,   // unalignedRequestsDisabled,
@@ -945,6 +971,11 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
             return MakeFuture<NProto::TWriteBlocksLocalResponse>();
         };
 
+        auto counters = SetupCriticalEvents();
+        auto mirroredDiskChecksumMismatchUponWrite = counters->GetCounter(
+            "AppCriticalEvents/MirroredDiskChecksumMismatchUponWrite",
+            true);
+
         auto future = deviceHandler->Write(
             MakeIntrusive<TCallContext>(),
             0,
@@ -954,6 +985,7 @@ Y_UNIT_TEST_SUITE(TDeviceHandlerTest)
         const auto& response = future.GetValue(TDuration::Seconds(5));
         UNIT_ASSERT(!HasError(response));
         UNIT_ASSERT_VALUES_EQUAL(2, writeAttempts);
+        UNIT_ASSERT_VALUES_EQUAL(1, mirroredDiskChecksumMismatchUponWrite->Val());
     }
 }
 

cloud/blockstore/libs/service/unaligned_device_handler.cpp

@@ -468,13 +468,15 @@ TZeroRequest::TResponseFuture TZeroRequest::ModifyAndWrite()
 
 TUnalignedDeviceHandler::TUnalignedDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         ui32 maxSubRequestSize,
         ui32 maxUnalignedRequestSize,
         bool checkBufferModificationDuringWriting)
     : Backend(std::make_shared<TAlignedDeviceHandler>(
           std::move(storage),
+          std::move(diskId),
           std::move(clientId),
           blockSize,
           maxSubRequestSize,

cloud/blockstore/libs/service/unaligned_device_handler.h

@@ -43,6 +43,7 @@ class TUnalignedDeviceHandler final
 public:
     TUnalignedDeviceHandler(
         IStoragePtr storage,
+        TString diskId,
         TString clientId,
         ui32 blockSize,
         ui32 maxSubRequestSize,

cloud/blockstore/libs/service/ut/ya.make

@@ -8,4 +8,8 @@ SRCS(
     storage_ut.cpp
 )
 
+PEERDIR(
+    cloud/blockstore/libs/diagnostics
+)
+
 END()

cloud/blockstore/libs/vhost/server.cpp

@@ -419,6 +419,7 @@ class TExecutor final
     {
         auto deviceHandler = AppCtx.DeviceHandlerFactory->CreateDeviceHandler(
             std::move(storage),
+            options.DiskId,
             options.ClientId,
             options.BlockSize,
             options.UnalignedRequestsDisabled,