diff --git a/cloud/blockstore/apps/server/main.cpp b/cloud/blockstore/apps/server/main.cpp index 8733447938f..a20655d645f 100644 --- a/cloud/blockstore/apps/server/main.cpp +++ b/cloud/blockstore/apps/server/main.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -7,6 +8,8 @@ #include #include #include +#include +#include #include #include @@ -76,6 +79,22 @@ int main(int argc, char** argv) return NCloud::NBlockStore::CreateKmsClientStub(); }; + serverModuleFactories->RootKmsClientFactory = [] ( + const NProto::TRootKmsConfig& config, + NCloud::ILoggingServicePtr logging) + { + if (config.GetAddress()) { + return NCloud::NBlockStore::CreateRootKmsClient( + std::move(logging), + {.Address = config.GetAddress(), + .RootCertsFile = config.GetRootCertsFile(), + .CertChainFile = config.GetCertChainFile(), + .PrivateKeyFile = config.GetPrivateKeyFile()}); + } + + return NCloud::NBlockStore::CreateRootKmsClientStub(); + }; + serverModuleFactories->SpdkFactory = [] ( NSpdk::TSpdkEnvConfigPtr config) { diff --git a/cloud/blockstore/apps/server/ya.make b/cloud/blockstore/apps/server/ya.make index 032b3bb3523..9b0d0cca3b7 100644 --- a/cloud/blockstore/apps/server/ya.make +++ b/cloud/blockstore/apps/server/ya.make @@ -12,6 +12,7 @@ PEERDIR( cloud/blockstore/libs/kms/impl cloud/blockstore/libs/logbroker/iface cloud/blockstore/libs/rdma/impl + cloud/blockstore/libs/root_kms/impl cloud/blockstore/libs/service cloud/blockstore/libs/spdk/iface diff --git a/cloud/blockstore/config/root_kms.proto b/cloud/blockstore/config/root_kms.proto new file mode 100644 index 00000000000..dbbb2d278e4 --- /dev/null +++ b/cloud/blockstore/config/root_kms.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package NCloud.NBlockStore.NProto; + +option go_package = "github.com/ydb-platform/nbs/cloud/blockstore/config"; + +//////////////////////////////////////////////////////////////////////////////// + +message TRootKmsConfig +{ + // Address of the RootKMS server. + optional string Address = 1; + + // Key encryption key identifier. + optional string KeyId = 2; + + // mTLS. + optional string RootCertsFile = 3; + optional string CertChainFile = 4; + optional string PrivateKeyFile = 5; +} diff --git a/cloud/blockstore/config/ya.make b/cloud/blockstore/config/ya.make index d1945622754..99f7234e65a 100644 --- a/cloud/blockstore/config/ya.make +++ b/cloud/blockstore/config/ya.make @@ -14,6 +14,7 @@ SRCS( notify.proto plugin.proto rdma.proto + root_kms.proto server.proto spdk.proto storage.proto diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.cpp b/cloud/blockstore/libs/daemon/common/bootstrap.cpp index b4b7dda596c..c52bc578c17 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/common/bootstrap.cpp @@ -870,6 +870,7 @@ void TBootstrapBase::Start() START_KIKIMR_COMPONENT(IamTokenClient); START_KIKIMR_COMPONENT(ComputeClient); START_KIKIMR_COMPONENT(KmsClient); + START_KIKIMR_COMPONENT(RootKmsClient); START_KIKIMR_COMPONENT(YdbStorage); START_KIKIMR_COMPONENT(StatsUploader); START_COMMON_COMPONENT(Spdk); @@ -957,6 +958,7 @@ void TBootstrapBase::Stop() STOP_COMMON_COMPONENT(Spdk); STOP_KIKIMR_COMPONENT(StatsUploader); STOP_KIKIMR_COMPONENT(YdbStorage); + STOP_KIKIMR_COMPONENT(RootKmsClient); STOP_KIKIMR_COMPONENT(KmsClient); STOP_KIKIMR_COMPONENT(ComputeClient); STOP_KIKIMR_COMPONENT(IamTokenClient); diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.h b/cloud/blockstore/libs/daemon/common/bootstrap.h index 27fbec848c1..79bf75c2329 100644 --- a/cloud/blockstore/libs/daemon/common/bootstrap.h +++ b/cloud/blockstore/libs/daemon/common/bootstrap.h @@ -112,6 +112,7 @@ class TBootstrapBase virtual IStartable* GetIamTokenClient() = 0; virtual IStartable* GetComputeClient() = 0; virtual IStartable* GetKmsClient() = 0; + virtual IStartable* GetRootKmsClient() = 0; virtual void InitSpdk() = 0; virtual void InitRdmaClient() = 0; diff --git a/cloud/blockstore/libs/daemon/local/bootstrap.h b/cloud/blockstore/libs/daemon/local/bootstrap.h index 86f70729010..6c1c428a825 100644 --- a/cloud/blockstore/libs/daemon/local/bootstrap.h +++ b/cloud/blockstore/libs/daemon/local/bootstrap.h @@ -36,6 +36,7 @@ class TBootstrapLocal final IStartable* GetIamTokenClient() override { return nullptr; } IStartable* GetComputeClient() override { return nullptr; } IStartable* GetKmsClient() override { return nullptr; } + IStartable* GetRootKmsClient() override { return nullptr; } void InitSpdk() override; void InitRdmaClient() override; diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp index 68ee6dda56e..67384609e53 100644 --- a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp +++ b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include #include #include @@ -133,6 +135,7 @@ IStartable* TBootstrapYdb::GetCgroupStatsFetcher() { return CgroupStatsFetcher.g IStartable* TBootstrapYdb::GetIamTokenClient() { return IamTokenClient.get(); } IStartable* TBootstrapYdb::GetComputeClient() { return ComputeClient.get(); } IStartable* TBootstrapYdb::GetKmsClient() { return KmsClient.get(); } +IStartable* TBootstrapYdb::GetRootKmsClient() { return RootKmsClient.get(); } void TBootstrapYdb::InitConfigs() { @@ -151,6 +154,7 @@ void TBootstrapYdb::InitConfigs() Configs->InitNotifyConfig(); Configs->InitIamClientConfig(); Configs->InitKmsClientConfig(); + Configs->InitRootKmsConfig(); Configs->InitComputeClientConfig(); } @@ -363,6 +367,16 @@ void TBootstrapYdb::InitKikimrService() STORAGE_INFO("KmsKeyProvider initialized"); + RootKmsClient = ServerModuleFactories->RootKmsClientFactory( + Configs->RootKmsConfig, + logging); + + RootKmsKeyProvider = CreateRootKmsKeyProvider( + RootKmsClient, + Configs->RootKmsConfig.GetKeyId()); + + STORAGE_INFO("RootKmsKeyProvider initialized"); + auto discoveryConfig = Configs->DiscoveryConfig; if (discoveryConfig->GetConductorGroups() || discoveryConfig->GetInstanceListFile()) diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.h b/cloud/blockstore/libs/daemon/ydb/bootstrap.h index 9737f16b28e..b87ed13bbb6 100644 --- a/cloud/blockstore/libs/daemon/ydb/bootstrap.h +++ b/cloud/blockstore/libs/daemon/ydb/bootstrap.h @@ -1,12 +1,11 @@ #include "public.h" -#include - #include #include #include #include #include +#include #include #include @@ -15,6 +14,11 @@ #include +namespace NCloud::NBlockStore::NProto { + class TGrpcClientConfig; + class TRootKmsConfig; +} // namespace NCloud::NBlockStore::NProto + namespace NCloud::NBlockStore::NServer { //////////////////////////////////////////////////////////////////////////////// @@ -46,6 +50,10 @@ struct TServerModuleFactories NProto::TGrpcClientConfig config, ILoggingServicePtr logging)> KmsClientFactory; + std::function RootKmsClientFactory; + std::function SpdkFactory; std::function SpdkLogInitializer; public: @@ -90,7 +99,7 @@ struct TBootstrapYdb final std::shared_ptr moduleFactories, std::shared_ptr serverModuleFactories, IDeviceHandlerFactoryPtr deviceHandlerFactory); - ~TBootstrapYdb(); + ~TBootstrapYdb() override; TProgramShouldContinue& GetShouldContinue() override; @@ -110,6 +119,7 @@ struct TBootstrapYdb final IStartable* GetIamTokenClient() override; IStartable* GetComputeClient() override; IStartable* GetKmsClient() override; + IStartable* GetRootKmsClient() override; void InitSpdk() override; void InitRdmaClient() override; diff --git a/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp b/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp index 6aa114dc7ce..fbeae8efd12 100644 --- a/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp +++ b/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp @@ -37,7 +37,7 @@ using namespace NCloud::NBlockStore::NDiscovery; //////////////////////////////////////////////////////////////////////////////// TConfigInitializerYdb::TConfigInitializerYdb(TOptionsYdbPtr options) - : TConfigInitializerCommon(options) + : TConfigInitializerCommon(options) , NCloud::NStorage::TConfigInitializerYdbBase(options) , Options(options) {} @@ -146,6 +146,21 @@ void TConfigInitializerYdb::InitKmsClientConfig() KmsClientConfig = std::move(config); } +void TConfigInitializerYdb::InitRootKmsConfig() +{ + NProto::TRootKmsConfig config; + + if (Options->RootKmsConfig) { + ParseProtoTextFromFile(Options->RootKmsConfig, config); + } + + if (!config.GetRootCertsFile()) { + config.SetRootCertsFile(ServerConfig->GetRootCertsFile()); + } + + RootKmsConfig = std::move(config); +} + void TConfigInitializerYdb::InitComputeClientConfig() { NProto::TGrpcClientConfig config; @@ -340,6 +355,14 @@ void TConfigInitializerYdb::ApplyKmsClientConfig(const TString& text) KmsClientConfig = std::move(config); } +void TConfigInitializerYdb::ApplyRootKmsConfig(const TString& text) +{ + NProto::TRootKmsConfig config; + ParseProtoTextFromString(text, config); + + RootKmsConfig = std::move(config); +} + void TConfigInitializerYdb::ApplyComputeClientConfig(const TString& text) { NProto::TGrpcClientConfig config; @@ -384,6 +407,7 @@ void TConfigInitializerYdb::ApplyCustomCMSConfigs(const NKikimrConfig::TAppConfi { "YdbStatsConfig", &TSelf::ApplyYdbStatsConfig }, { "IamClientConfig", &TSelf::ApplyIamClientConfig }, { "KmsClientConfig", &TSelf::ApplyKmsClientConfig }, + { "RootKmsConfig", &TSelf::ApplyRootKmsConfig }, { "ComputeClientConfig", &TSelf::ApplyComputeClientConfig }, }; diff --git a/cloud/blockstore/libs/daemon/ydb/config_initializer.h b/cloud/blockstore/libs/daemon/ydb/config_initializer.h index 3edee9612b8..40a8fac040a 100644 --- a/cloud/blockstore/libs/daemon/ydb/config_initializer.h +++ b/cloud/blockstore/libs/daemon/ydb/config_initializer.h @@ -3,6 +3,7 @@ #include "public.h" #include +#include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -59,6 +61,7 @@ struct TConfigInitializerYdb final NIamClient::TIamClientConfigPtr IamClientConfig; NProto::TGrpcClientConfig KmsClientConfig; NProto::TGrpcClientConfig ComputeClientConfig; + NProto::TRootKmsConfig RootKmsConfig; TConfigInitializerYdb(TOptionsYdbPtr options); @@ -69,6 +72,7 @@ struct TConfigInitializerYdb final void InitStorageConfig(); void InitIamClientConfig(); void InitKmsClientConfig(); + void InitRootKmsConfig(); void InitComputeClientConfig(); bool GetUseNonreplicatedRdmaActor() const override; @@ -92,6 +96,7 @@ struct TConfigInitializerYdb final void ApplyYdbStatsConfig(const TString& text); void ApplyIamClientConfig(const TString& text); void ApplyKmsClientConfig(const TString& text); + void ApplyRootKmsConfig(const TString& text); void ApplyComputeClientConfig(const TString& text); }; diff --git a/cloud/blockstore/libs/daemon/ydb/options.cpp b/cloud/blockstore/libs/daemon/ydb/options.cpp index f4c9845feb1..f2898de93f8 100644 --- a/cloud/blockstore/libs/daemon/ydb/options.cpp +++ b/cloud/blockstore/libs/daemon/ydb/options.cpp @@ -37,6 +37,10 @@ TOptionsYdb::TOptionsYdb() .RequiredArgument("PATH") .StoreResult(&KmsConfig); + Opts.AddLongOption("root-kms-file") + .RequiredArgument("PATH") + .StoreResult(&RootKmsConfig); + Opts.AddLongOption("compute-file") .RequiredArgument("PATH") .StoreResult(&ComputeConfig); diff --git a/cloud/blockstore/libs/daemon/ydb/options.h b/cloud/blockstore/libs/daemon/ydb/options.h index ebd631c9840..488a8e5c4ad 100644 --- a/cloud/blockstore/libs/daemon/ydb/options.h +++ b/cloud/blockstore/libs/daemon/ydb/options.h @@ -21,6 +21,7 @@ struct TOptionsYdb final TString NotifyConfig; TString IamConfig; TString KmsConfig; + TString RootKmsConfig; TString ComputeConfig; TOptionsYdb(); diff --git a/cloud/blockstore/libs/daemon/ydb/ya.make b/cloud/blockstore/libs/daemon/ydb/ya.make index 565acd6e92a..874c974936a 100644 --- a/cloud/blockstore/libs/daemon/ydb/ya.make +++ b/cloud/blockstore/libs/daemon/ydb/ya.make @@ -18,6 +18,7 @@ PEERDIR( cloud/blockstore/libs/logbroker/iface cloud/blockstore/libs/notify cloud/blockstore/libs/nvme + cloud/blockstore/libs/root_kms/iface cloud/blockstore/libs/server cloud/blockstore/libs/service cloud/blockstore/libs/service_kikimr diff --git a/cloud/blockstore/libs/root_kms/impl/client.cpp b/cloud/blockstore/libs/root_kms/impl/client.cpp index b8b3b5eba3f..a4a914eeaa4 100644 --- a/cloud/blockstore/libs/root_kms/impl/client.cpp +++ b/cloud/blockstore/libs/root_kms/impl/client.cpp @@ -226,9 +226,9 @@ TRootKmsClient::~TRootKmsClient() void TRootKmsClient::Start() { grpc::SslCredentialsOptions sslOpts{ - .pem_root_certs = ReadFile(Params.RootCAPath), - .pem_private_key = ReadFile(Params.PrivateKeyPath), - .pem_cert_chain = ReadFile(Params.CertChainPath) + .pem_root_certs = ReadFile(Params.RootCertsFile), + .pem_private_key = ReadFile(Params.PrivateKeyFile), + .pem_cert_chain = ReadFile(Params.CertChainFile) }; STORAGE_INFO("Connect to " << Params.Address); diff --git a/cloud/blockstore/libs/root_kms/impl/client.h b/cloud/blockstore/libs/root_kms/impl/client.h index cd62dc0f643..cf2195b3887 100644 --- a/cloud/blockstore/libs/root_kms/impl/client.h +++ b/cloud/blockstore/libs/root_kms/impl/client.h @@ -11,9 +11,9 @@ namespace NCloud::NBlockStore { struct TCreateRootKmsClientParams { TString Address; - TString RootCAPath; - TString CertChainPath; - TString PrivateKeyPath; + TString RootCertsFile; + TString CertChainFile; + TString PrivateKeyFile; }; IRootKmsClientPtr CreateRootKmsClient( diff --git a/cloud/blockstore/libs/root_kms/impl/client_ut.cpp b/cloud/blockstore/libs/root_kms/impl/client_ut.cpp index aab6c0e3569..a7a5dfc6780 100644 --- a/cloud/blockstore/libs/root_kms/impl/client_ut.cpp +++ b/cloud/blockstore/libs/root_kms/impl/client_ut.cpp @@ -29,9 +29,9 @@ struct TFixture Client = CreateRootKmsClient( Logging, {.Address = "localhost:" + GetEnv("FAKE_ROOT_KMS_PORT"), - .RootCAPath = GetEnv("FAKE_ROOT_KMS_CA"), - .CertChainPath = GetEnv("FAKE_ROOT_KMS_CLIENT_CRT"), - .PrivateKeyPath = GetEnv("FAKE_ROOT_KMS_CLIENT_KEY")}); + .RootCertsFile = GetEnv("FAKE_ROOT_KMS_CA"), + .CertChainFile = GetEnv("FAKE_ROOT_KMS_CLIENT_CRT"), + .PrivateKeyFile = GetEnv("FAKE_ROOT_KMS_CLIENT_KEY")}); Client->Start(); }