issue-176: setup root kms client & provider (#2582)

ydb-platform · Nov 28, 2024 · 40e9574 · 40e9574
1 parent db47e2e
commit 40e9574
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 13 deletions.
diff --git a/cloud/blockstore/apps/server/main.cpp b/cloud/blockstore/apps/server/main.cpp
@@ -1,3 +1,4 @@
+#include <cloud/blockstore/config/root_kms.pb.h>
 #include <cloud/blockstore/libs/daemon/ydb/bootstrap.h>
 #include <cloud/blockstore/libs/kms/iface/compute_client.h>
 #include <cloud/blockstore/libs/kms/iface/kms_client.h>
@@ -7,6 +8,8 @@
 #include <cloud/blockstore/libs/rdma/impl/client.h>
 #include <cloud/blockstore/libs/rdma/impl/server.h>
 #include <cloud/blockstore/libs/rdma/impl/verbs.h>
+#include <cloud/blockstore/libs/root_kms/iface/client.h>
+#include <cloud/blockstore/libs/root_kms/impl/client.h>
 #include <cloud/blockstore/libs/service/device_handler.h>
 #include <cloud/blockstore/libs/spdk/iface/env_stub.h>
 
@@ -76,6 +79,22 @@ int main(int argc, char** argv)
         return NCloud::NBlockStore::CreateKmsClientStub();
     };
 
+    serverModuleFactories->RootKmsClientFactory = [] (
+        const NProto::TRootKmsConfig& config,
+        NCloud::ILoggingServicePtr logging)
+    {
+        if (config.GetAddress()) {
+            return NCloud::NBlockStore::CreateRootKmsClient(
+                std::move(logging),
+                {.Address = config.GetAddress(),
+                 .RootCertsFile = config.GetRootCertsFile(),
+                 .CertChainFile = config.GetCertChainFile(),
+                 .PrivateKeyFile = config.GetPrivateKeyFile()});
+        }
+
+        return NCloud::NBlockStore::CreateRootKmsClientStub();
+    };
+
     serverModuleFactories->SpdkFactory = [] (
         NSpdk::TSpdkEnvConfigPtr config)
     {

diff --git a/cloud/blockstore/apps/server/ya.make b/cloud/blockstore/apps/server/ya.make
@@ -12,6 +12,7 @@ PEERDIR(
     cloud/blockstore/libs/kms/impl
     cloud/blockstore/libs/logbroker/iface
     cloud/blockstore/libs/rdma/impl
+    cloud/blockstore/libs/root_kms/impl
     cloud/blockstore/libs/service
     cloud/blockstore/libs/spdk/iface
 

diff --git a/cloud/blockstore/config/root_kms.proto b/cloud/blockstore/config/root_kms.proto
@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package NCloud.NBlockStore.NProto;
+
+option go_package = "github.com/ydb-platform/nbs/cloud/blockstore/config";
+
+////////////////////////////////////////////////////////////////////////////////
+
+message TRootKmsConfig
+{
+    // Address of the RootKMS server.
+    optional string Address = 1;
+
+    // Key encryption key identifier.
+    optional string KeyId = 2;
+
+    // mTLS.
+    optional string RootCertsFile = 3;
+    optional string CertChainFile = 4;
+    optional string PrivateKeyFile = 5;
+}
diff --git a/cloud/blockstore/config/ya.make b/cloud/blockstore/config/ya.make
@@ -14,6 +14,7 @@ SRCS(
     notify.proto
     plugin.proto
     rdma.proto
+    root_kms.proto
     server.proto
     spdk.proto
     storage.proto

diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.cpp b/cloud/blockstore/libs/daemon/common/bootstrap.cpp
@@ -870,6 +870,7 @@ void TBootstrapBase::Start()
     START_KIKIMR_COMPONENT(IamTokenClient);
     START_KIKIMR_COMPONENT(ComputeClient);
     START_KIKIMR_COMPONENT(KmsClient);
+    START_KIKIMR_COMPONENT(RootKmsClient);
     START_KIKIMR_COMPONENT(YdbStorage);
     START_KIKIMR_COMPONENT(StatsUploader);
     START_COMMON_COMPONENT(Spdk);
@@ -957,6 +958,7 @@ void TBootstrapBase::Stop()
     STOP_COMMON_COMPONENT(Spdk);
     STOP_KIKIMR_COMPONENT(StatsUploader);
     STOP_KIKIMR_COMPONENT(YdbStorage);
+    STOP_KIKIMR_COMPONENT(RootKmsClient);
     STOP_KIKIMR_COMPONENT(KmsClient);
     STOP_KIKIMR_COMPONENT(ComputeClient);
     STOP_KIKIMR_COMPONENT(IamTokenClient);

diff --git a/cloud/blockstore/libs/daemon/common/bootstrap.h b/cloud/blockstore/libs/daemon/common/bootstrap.h
@@ -112,6 +112,7 @@ class TBootstrapBase
     virtual IStartable* GetIamTokenClient() = 0;
     virtual IStartable* GetComputeClient() = 0;
     virtual IStartable* GetKmsClient() = 0;
+    virtual IStartable* GetRootKmsClient() = 0;
 
     virtual void InitSpdk() = 0;
     virtual void InitRdmaClient() = 0;

diff --git a/cloud/blockstore/libs/daemon/local/bootstrap.h b/cloud/blockstore/libs/daemon/local/bootstrap.h
@@ -36,6 +36,7 @@ class TBootstrapLocal final
     IStartable* GetIamTokenClient() override     { return nullptr; }
     IStartable* GetComputeClient() override      { return nullptr; }
     IStartable* GetKmsClient() override          { return nullptr; }
+    IStartable* GetRootKmsClient() override      { return nullptr; }
 
     void InitSpdk() override;
     void InitRdmaClient() override;

diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp b/cloud/blockstore/libs/daemon/ydb/bootstrap.cpp
@@ -28,6 +28,8 @@
 #include <cloud/blockstore/libs/rdma/iface/client.h>
 #include <cloud/blockstore/libs/rdma/iface/config.h>
 #include <cloud/blockstore/libs/rdma/iface/server.h>
+#include <cloud/blockstore/libs/root_kms/iface/client.h>
+#include <cloud/blockstore/libs/root_kms/iface/key_provider.h>
 #include <cloud/blockstore/libs/server/config.h>
 #include <cloud/blockstore/libs/service/service_auth.h>
 #include <cloud/blockstore/libs/service_kikimr/auth_provider_kikimr.h>
@@ -133,6 +135,7 @@ IStartable* TBootstrapYdb::GetCgroupStatsFetcher() { return CgroupStatsFetcher.g
 IStartable* TBootstrapYdb::GetIamTokenClient()     { return IamTokenClient.get(); }
 IStartable* TBootstrapYdb::GetComputeClient()      { return ComputeClient.get(); }
 IStartable* TBootstrapYdb::GetKmsClient()          { return KmsClient.get(); }
+IStartable* TBootstrapYdb::GetRootKmsClient()      { return RootKmsClient.get(); }
 
 void TBootstrapYdb::InitConfigs()
 {
@@ -151,6 +154,7 @@ void TBootstrapYdb::InitConfigs()
     Configs->InitNotifyConfig();
     Configs->InitIamClientConfig();
     Configs->InitKmsClientConfig();
+    Configs->InitRootKmsConfig();
     Configs->InitComputeClientConfig();
 }
 
@@ -363,6 +367,16 @@ void TBootstrapYdb::InitKikimrService()
 
     STORAGE_INFO("KmsKeyProvider initialized");
 
+    RootKmsClient = ServerModuleFactories->RootKmsClientFactory(
+        Configs->RootKmsConfig,
+        logging);
+
+    RootKmsKeyProvider = CreateRootKmsKeyProvider(
+        RootKmsClient,
+        Configs->RootKmsConfig.GetKeyId());
+
+    STORAGE_INFO("RootKmsKeyProvider initialized");
+
     auto discoveryConfig = Configs->DiscoveryConfig;
     if (discoveryConfig->GetConductorGroups()
             || discoveryConfig->GetInstanceListFile())

diff --git a/cloud/blockstore/libs/daemon/ydb/bootstrap.h b/cloud/blockstore/libs/daemon/ydb/bootstrap.h
@@ -1,12 +1,11 @@
 #include "public.h"
 
-#include <cloud/blockstore/config/grpc_client.pb.h>
-
 #include <cloud/blockstore/libs/daemon/common/bootstrap.h>
 #include <cloud/blockstore/libs/kms/iface/public.h>
 #include <cloud/blockstore/libs/logbroker/iface/public.h>
 #include <cloud/blockstore/libs/notify/public.h>
 #include <cloud/blockstore/libs/rdma/iface/public.h>
+#include <cloud/blockstore/libs/root_kms/iface/public.h>
 #include <cloud/blockstore/libs/ydbstats/public.h>
 
 #include <cloud/storage/core/libs/actors/public.h>
@@ -15,6 +14,11 @@
 
 #include <contrib/ydb/core/driver_lib/run/factories.h>
 
+namespace NCloud::NBlockStore::NProto {
+    class TGrpcClientConfig;
+    class TRootKmsConfig;
+}   // namespace NCloud::NBlockStore::NProto
+
 namespace NCloud::NBlockStore::NServer {
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -46,6 +50,10 @@ struct TServerModuleFactories
         NProto::TGrpcClientConfig config,
         ILoggingServicePtr logging)> KmsClientFactory;
 
+    std::function<IRootKmsClientPtr(
+        const NProto::TRootKmsConfig& config,
+        ILoggingServicePtr logging)> RootKmsClientFactory;
+
     std::function<TSpdkParts(NSpdk::TSpdkEnvConfigPtr config)> SpdkFactory;
 
     std::function<NRdma::IServerPtr(
@@ -83,14 +91,15 @@ struct TBootstrapYdb final
     NIamClient::IIamTokenClientPtr IamTokenClient;
     IComputeClientPtr ComputeClient;
     IKmsClientPtr KmsClient;
+    IRootKmsClientPtr RootKmsClient;
     std::function<void(TLog& log)> SpdkLogInitializer;
 
 public:
     TBootstrapYdb(
         std::shared_ptr<NKikimr::TModuleFactories> moduleFactories,
         std::shared_ptr<TServerModuleFactories> serverModuleFactories,
         IDeviceHandlerFactoryPtr deviceHandlerFactory);
-    ~TBootstrapYdb();
+    ~TBootstrapYdb() override;
 
     TProgramShouldContinue& GetShouldContinue() override;
 
@@ -110,6 +119,7 @@ struct TBootstrapYdb final
     IStartable* GetIamTokenClient() override;
     IStartable* GetComputeClient() override;
     IStartable* GetKmsClient() override;
+    IStartable* GetRootKmsClient() override;
 
     void InitSpdk() override;
     void InitRdmaClient() override;

diff --git a/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp b/cloud/blockstore/libs/daemon/ydb/config_initializer.cpp
@@ -37,7 +37,7 @@ using namespace NCloud::NBlockStore::NDiscovery;
 ////////////////////////////////////////////////////////////////////////////////
 
 TConfigInitializerYdb::TConfigInitializerYdb(TOptionsYdbPtr options)
-    : TConfigInitializerCommon(options)
+        : TConfigInitializerCommon(options)
     , NCloud::NStorage::TConfigInitializerYdbBase(options)
     , Options(options)
 {}
@@ -146,6 +146,21 @@ void TConfigInitializerYdb::InitKmsClientConfig()
     KmsClientConfig = std::move(config);
 }
 
+void TConfigInitializerYdb::InitRootKmsConfig()
+{
+    NProto::TRootKmsConfig config;
+
+    if (Options->RootKmsConfig) {
+        ParseProtoTextFromFile(Options->RootKmsConfig, config);
+    }
+
+    if (!config.GetRootCertsFile()) {
+        config.SetRootCertsFile(ServerConfig->GetRootCertsFile());
+    }
+
+    RootKmsConfig = std::move(config);
+}
+
 void TConfigInitializerYdb::InitComputeClientConfig()
 {
     NProto::TGrpcClientConfig config;
@@ -340,6 +355,14 @@ void TConfigInitializerYdb::ApplyKmsClientConfig(const TString& text)
     KmsClientConfig = std::move(config);
 }
 
+void TConfigInitializerYdb::ApplyRootKmsConfig(const TString& text)
+{
+    NProto::TRootKmsConfig config;
+    ParseProtoTextFromString(text, config);
+
+    RootKmsConfig = std::move(config);
+}
+
 void TConfigInitializerYdb::ApplyComputeClientConfig(const TString& text)
 {
     NProto::TGrpcClientConfig config;
@@ -384,6 +407,7 @@ void TConfigInitializerYdb::ApplyCustomCMSConfigs(const NKikimrConfig::TAppConfi
         { "YdbStatsConfig",          &TSelf::ApplyYdbStatsConfig          },
         { "IamClientConfig",         &TSelf::ApplyIamClientConfig         },
         { "KmsClientConfig",         &TSelf::ApplyKmsClientConfig         },
+        { "RootKmsConfig",           &TSelf::ApplyRootKmsConfig           },
         { "ComputeClientConfig",     &TSelf::ApplyComputeClientConfig     },
     };
 

diff --git a/cloud/blockstore/libs/daemon/ydb/config_initializer.h b/cloud/blockstore/libs/daemon/ydb/config_initializer.h
@@ -3,6 +3,7 @@
 #include "public.h"
 
 #include <cloud/blockstore/config/grpc_client.pb.h>
+#include <cloud/blockstore/config/root_kms.pb.h>
 
 #include <cloud/blockstore/libs/client/config.h>
 #include <cloud/blockstore/libs/client/throttling.h>
@@ -15,6 +16,7 @@
 #include <cloud/blockstore/libs/kikimr/public.h>
 #include <cloud/blockstore/libs/logbroker/iface/public.h>
 #include <cloud/blockstore/libs/notify/public.h>
+#include <cloud/blockstore/libs/root_kms/iface/public.h>
 #include <cloud/blockstore/libs/server/public.h>
 #include <cloud/blockstore/libs/service/public.h>
 #include <cloud/blockstore/libs/spdk/iface/public.h>
@@ -59,6 +61,7 @@ struct TConfigInitializerYdb final
     NIamClient::TIamClientConfigPtr IamClientConfig;
     NProto::TGrpcClientConfig KmsClientConfig;
     NProto::TGrpcClientConfig ComputeClientConfig;
+    NProto::TRootKmsConfig RootKmsConfig;
 
     TConfigInitializerYdb(TOptionsYdbPtr options);
 
@@ -69,6 +72,7 @@ struct TConfigInitializerYdb final
     void InitStorageConfig();
     void InitIamClientConfig();
     void InitKmsClientConfig();
+    void InitRootKmsConfig();
     void InitComputeClientConfig();
 
     bool GetUseNonreplicatedRdmaActor() const override;
@@ -92,6 +96,7 @@ struct TConfigInitializerYdb final
     void ApplyYdbStatsConfig(const TString& text);
     void ApplyIamClientConfig(const TString& text);
     void ApplyKmsClientConfig(const TString& text);
+    void ApplyRootKmsConfig(const TString& text);
     void ApplyComputeClientConfig(const TString& text);
 };
 

diff --git a/cloud/blockstore/libs/daemon/ydb/options.cpp b/cloud/blockstore/libs/daemon/ydb/options.cpp
@@ -37,6 +37,10 @@ TOptionsYdb::TOptionsYdb()
         .RequiredArgument("PATH")
         .StoreResult(&KmsConfig);
 
+    Opts.AddLongOption("root-kms-file")
+        .RequiredArgument("PATH")
+        .StoreResult(&RootKmsConfig);
+
     Opts.AddLongOption("compute-file")
         .RequiredArgument("PATH")
         .StoreResult(&ComputeConfig);

diff --git a/cloud/blockstore/libs/daemon/ydb/options.h b/cloud/blockstore/libs/daemon/ydb/options.h
@@ -21,6 +21,7 @@ struct TOptionsYdb final
     TString NotifyConfig;
     TString IamConfig;
     TString KmsConfig;
+    TString RootKmsConfig;
     TString ComputeConfig;
 
     TOptionsYdb();

diff --git a/cloud/blockstore/libs/daemon/ydb/ya.make b/cloud/blockstore/libs/daemon/ydb/ya.make
@@ -18,6 +18,7 @@ PEERDIR(
     cloud/blockstore/libs/logbroker/iface
     cloud/blockstore/libs/notify
     cloud/blockstore/libs/nvme
+    cloud/blockstore/libs/root_kms/iface
     cloud/blockstore/libs/server
     cloud/blockstore/libs/service
     cloud/blockstore/libs/service_kikimr

diff --git a/cloud/blockstore/libs/root_kms/impl/client.cpp b/cloud/blockstore/libs/root_kms/impl/client.cpp
@@ -226,9 +226,9 @@ TRootKmsClient::~TRootKmsClient()
 void TRootKmsClient::Start()
 {
     grpc::SslCredentialsOptions sslOpts{
-        .pem_root_certs = ReadFile(Params.RootCAPath),
-        .pem_private_key = ReadFile(Params.PrivateKeyPath),
-        .pem_cert_chain = ReadFile(Params.CertChainPath)
+        .pem_root_certs = ReadFile(Params.RootCertsFile),
+        .pem_private_key = ReadFile(Params.PrivateKeyFile),
+        .pem_cert_chain = ReadFile(Params.CertChainFile)
     };
 
     STORAGE_INFO("Connect to " << Params.Address);

diff --git a/cloud/blockstore/libs/root_kms/impl/client.h b/cloud/blockstore/libs/root_kms/impl/client.h
@@ -11,9 +11,9 @@ namespace NCloud::NBlockStore {
 struct TCreateRootKmsClientParams
 {
     TString Address;
-    TString RootCAPath;
-    TString CertChainPath;
-    TString PrivateKeyPath;
+    TString RootCertsFile;
+    TString CertChainFile;
+    TString PrivateKeyFile;
 };
 
 IRootKmsClientPtr CreateRootKmsClient(

diff --git a/cloud/blockstore/libs/root_kms/impl/client_ut.cpp b/cloud/blockstore/libs/root_kms/impl/client_ut.cpp
@@ -29,9 +29,9 @@ struct TFixture
         Client = CreateRootKmsClient(
             Logging,
             {.Address = "localhost:" + GetEnv("FAKE_ROOT_KMS_PORT"),
-             .RootCAPath = GetEnv("FAKE_ROOT_KMS_CA"),
-             .CertChainPath = GetEnv("FAKE_ROOT_KMS_CLIENT_CRT"),
-             .PrivateKeyPath = GetEnv("FAKE_ROOT_KMS_CLIENT_KEY")});
+             .RootCertsFile = GetEnv("FAKE_ROOT_KMS_CA"),
+             .CertChainFile = GetEnv("FAKE_ROOT_KMS_CLIENT_CRT"),
+             .PrivateKeyFile = GetEnv("FAKE_ROOT_KMS_CLIENT_KEY")});
         Client->Start();
     }