diff --git a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp index 6068c35c5bf..63de1db163b 100644 --- a/cloud/filestore/libs/storage/service/service_actor_writedata.cpp +++ b/cloud/filestore/libs/storage/service/service_actor_writedata.cpp @@ -50,6 +50,7 @@ class TWriteDataActor final: public TActorBootstrapped // in flight TMaybe InFlightRequest; TVector> InFlightBSRequests; + TVector StorageStatusFlags; const NCloud::NProto::EStorageMediaKind MediaKind; public: @@ -173,6 +174,7 @@ class TWriteDataActor final: public TActorBootstrapped RequestInfo->CallContext->RequestType = EFileStoreRequest::WriteBlob; InFlightBSRequests.reserve(RemainingBlobsToWrite); + StorageStatusFlags.resize(GenerateBlobIdsResponse.BlobsSize()); for (const auto& blob: GenerateBlobIdsResponse.GetBlobs()) { NKikimr::TLogoBlobID blobId = LogoBlobIDFromLogoBlobID(blob.GetBlobId()); @@ -249,11 +251,14 @@ class TWriteDataActor final: public TActorBootstrapped ui64 blobIdx = msg->Id.Cookie(); // It is implicitly expected that cookies are generated in increasing // order starting from 0. + // TODO: replace this TABLET_VERIFY with a critical event + WriteData + // fallback TABLET_VERIFY( blobIdx < InFlightBSRequests.size() && InFlightBSRequests[blobIdx] && !InFlightBSRequests[blobIdx]->IsCompleted()); InFlightBSRequests[blobIdx]->Complete(ctx.Now(), {}); + StorageStatusFlags[blobIdx] = msg->StatusFlags.Raw; --RemainingBlobsToWrite; if (RemainingBlobsToWrite == 0) { @@ -277,6 +282,11 @@ class TWriteDataActor final: public TActorBootstrapped request->Record.AddBlobIds()->Swap(blob.MutableBlobId()); } request->Record.SetCommitId(GenerateBlobIdsResponse.GetCommitId()); + request->Record.MutableStorageStatusFlags()->Reserve( + StorageStatusFlags.size()); + for (const auto flags: StorageStatusFlags) { + request->Record.AddStorageStatusFlags(flags); + } if (Range.Offset < BlobRange.Offset) { auto& unalignedHead = *request->Record.AddUnalignedDataRanges(); diff --git a/cloud/filestore/libs/storage/service/service_ut.cpp b/cloud/filestore/libs/storage/service/service_ut.cpp index a912e3b1146..9df0f2c4f9c 100644 --- a/cloud/filestore/libs/storage/service/service_ut.cpp +++ b/cloud/filestore/libs/storage/service/service_ut.cpp @@ -2608,6 +2608,70 @@ Y_UNIT_TEST_SUITE(TStorageServiceTest) // clang-format on } + Y_UNIT_TEST(ShouldSendBSGroupFlagsToTabletViaAddDataRequests) + { + TTestEnv env; + env.CreateSubDomain("nfs"); + + ui32 nodeIdx = env.CreateNode("nfs"); + + TServiceClient service(env.GetRuntime(), nodeIdx); + const TString fs = "test"; + service.CreateFileStore(fs, 1000); + + { + NProto::TStorageConfig newConfig; + newConfig.SetThreeStageWriteEnabled(true); + const auto response = + ExecuteChangeStorageConfig(std::move(newConfig), service); + env.GetRuntime().DispatchEvents({}, TDuration::Seconds(1)); + } + + auto headers = service.InitSession(fs, "client"); + ui64 nodeId = service + .CreateNode(headers, TCreateNodeArgs::File(RootNodeId, "file")) + ->Record.GetNode() + .GetId(); + ui64 handle = service + .CreateHandle(headers, fs, nodeId, "", TCreateHandleArgs::RDWR) + ->Record.GetHandle(); + + const auto yellowFlag = + NKikimrBlobStorage::EStatusFlags::StatusDiskSpaceYellowStop; + + NProtoPrivate::TAddDataRequest addData; + using TFlags = NKikimr::TStorageStatusFlags; + env.GetRuntime().SetEventFilter( + [&](auto& runtime, auto& event) + { + Y_UNUSED(runtime); + + switch (event->GetTypeRewrite()) { + case TEvBlobStorage::EvPutResult: { + auto* msg = + event->template Get(); + const_cast(msg->StatusFlags).Raw |= + ui32(yellowFlag); + break; + } + + case TEvIndexTablet::EvAddDataRequest: { + addData = event->template Get< + TEvIndexTablet::TEvAddDataRequest>()->Record; + break; + } + } + return false; + }); + + TString data = GenerateValidateData(256_KB); + service.WriteData(headers, fs, nodeId, handle, 0, data); + UNIT_ASSERT_VALUES_EQUAL(1, addData.BlobIdsSize()); + UNIT_ASSERT_VALUES_EQUAL(1, addData.StorageStatusFlagsSize()); + UNIT_ASSERT(NKikimr::TStorageStatusFlags( + addData.GetStorageStatusFlags(0)).Check(yellowFlag)); + } + void ConfigureFollowers( TServiceClient& service, const TString& fsId, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor.h b/cloud/filestore/libs/storage/tablet/tablet_actor.h index fea99188286..fcea252950a 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor.h +++ b/cloud/filestore/libs/storage/tablet/tablet_actor.h @@ -285,6 +285,11 @@ class TIndexTabletActor final void DefaultSignalTabletActive(const NActors::TActorContext& ctx) override; void OnActivateExecutor(const NActors::TActorContext& ctx) override; bool ReassignChannelsEnabled() const override; + void RegisterEvPutResult( + const NActors::TActorContext& ctx, + ui32 generation, + ui32 channel, + const NKikimr::TStorageStatusFlags flags); void ReassignDataChannelsIfNeeded(const NActors::TActorContext& ctx); bool OnRenderAppHtmlPage( NActors::NMon::TEvRemoteHttpInfo::TPtr ev, diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp index 7885777290f..68fae69945a 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_adddata.cpp @@ -438,6 +438,16 @@ void TIndexTabletActor::HandleAddData( blobIds.size(), unalignedMsg().c_str()); + const auto evPutResultCount = + Min(blobIds.size(), msg->Record.StorageStatusFlagsSize()); + for (ui32 i = 0; i < evPutResultCount; ++i) { + RegisterEvPutResult( + ctx, + blobIds[i].Generation(), + blobIds[i].Channel(), + msg->Record.GetStorageStatusFlags(i)); + } + AddTransaction(*requestInfo); ExecuteTx( diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp index 9cc3ad7bc71..99ad40d8920 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_writeblob.cpp @@ -360,6 +360,39 @@ void TIndexTabletActor::HandleWriteBlob( WorkerActors.insert(actorId); } +void TIndexTabletActor::RegisterEvPutResult( + const TActorContext& ctx, + ui32 generation, + ui32 channel, + const NKikimr::TStorageStatusFlags flags) +{ + const auto validFlag = NKikimrBlobStorage::EStatusFlags::StatusIsValid; + if (flags.Check(validFlag)) { + ui32 group = Info()->GroupFor(channel, generation); + + if (flags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) { + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s Yellow move flag received for channel %u and group %u", + LogTag.c_str(), + channel, + group); + + RegisterChannelToMove(channel); + } + if (flags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) { + LOG_WARN(ctx, TFileStoreComponents::TABLET, + "%s Yellow stop flag received for channel %u and group %u", + LogTag.c_str(), + channel, + group); + + RegisterUnwritableChannel(channel); + } + + ReassignDataChannelsIfNeeded(ctx); + } +} + void TIndexTabletActor::HandleWriteBlobCompleted( const TEvIndexTabletPrivate::TEvWriteBlobCompleted::TPtr& ev, const TActorContext& ctx) @@ -375,33 +408,12 @@ void TIndexTabletActor::HandleWriteBlobCompleted( Metrics.WriteBlob.Update(msg->Count, msg->Size, msg->Time); - const auto validFlag = NKikimrBlobStorage::EStatusFlags::StatusIsValid; for (const auto& result: msg->Results) { - if (result.StorageStatusFlags.Check(validFlag)) { - ui32 channel = result.BlobId.Channel(); - ui32 group = Info()->GroupFor(channel, result.BlobId.Generation()); - - if (result.StorageStatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceLightYellowMove)) { - LOG_WARN(ctx, TFileStoreComponents::TABLET, - "%s Yellow move flag received for channel %u and group %u", - LogTag.c_str(), - channel, - group); - - RegisterChannelToMove(channel); - } - if (result.StorageStatusFlags.Check(NKikimrBlobStorage::StatusDiskSpaceYellowStop)) { - LOG_WARN(ctx, TFileStoreComponents::TABLET, - "%s Yellow stop flag received for channel %u and group %u", - LogTag.c_str(), - channel, - group); - - RegisterUnwritableChannel(channel); - } - - ReassignDataChannelsIfNeeded(ctx); - } + RegisterEvPutResult( + ctx, + result.BlobId.Generation(), + result.BlobId.Channel(), + result.StorageStatusFlags); } if (FAILED(msg->GetStatus())) { diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index b6e6eb8aa7e..a97986a26a4 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -452,6 +452,9 @@ message TAddDataRequest // Unaligned data parts - supposed to contain unaligned head and tail. repeated TFreshDataRange UnalignedDataRanges = 9; + + // StorageStatusFlags for the written blobs. + repeated uint32 StorageStatusFlags = 10; } message TAddDataResponse