From e2309c4831a79b58f675588bd375d16c4bdeb075 Mon Sep 17 00:00:00 2001 From: Yukang-Lian Date: Wed, 12 Feb 2025 12:14:14 +0800 Subject: [PATCH] 4 --- cloud/src/recycler/recycler.cpp | 4 +- cloud/test/recycler_test.cpp | 134 ++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index ed45c48e08ea74..e6b6181331620c 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -535,8 +535,6 @@ int InstanceRecycler::init_storage_vault_accessors() { LOG(WARNING) << "malformed storage vault, unable to deserialize key=" << hex(k); return -1; } - TEST_SYNC_POINT_CALLBACK("InstanceRecycler::init_storage_vault_accessors.mock_vault", - &accessor_map_, &vault); if (!config::valid_vault_name_set.empty()) { if (auto it = std::find(config::valid_vault_name_set.begin(), config::valid_vault_name_set.end(), vault.name()); @@ -554,6 +552,8 @@ int InstanceRecycler::init_storage_vault_accessors() { continue; } } + TEST_SYNC_POINT_CALLBACK("InstanceRecycler::init_storage_vault_accessors.mock_vault", + &accessor_map_, &vault); if (vault.has_hdfs_info()) { auto accessor = std::make_shared(vault.hdfs_info()); int ret = accessor->init(); diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index b6143c9545eb95..c6eea752c3dcb7 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -3616,6 +3616,140 @@ TEST(RecyclerTest, init_all_vault_accessors_failed_test) { EXPECT_EQ(recycler.init(), -2); } +TEST(RecyclerTest, valid_vault_name_set_test) { + auto* sp = SyncPoint::get_instance(); + std::unique_ptr> defer((int*)0x01, [&sp](int*) { + sp->clear_all_call_backs(); + sp->clear_trace(); + sp->disable_processing(); + }); + + auto txn_kv = std::make_shared(); + EXPECT_EQ(txn_kv->init(), 0); + std::unique_ptr txn; + ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string key; + std::string val; + + InstanceKeyInfo key_info {"test_instance"}; + instance_key(key_info, &key); + InstanceInfoPB instance; + instance.set_instance_id("GetObjStoreInfoTestInstance"); + { + ObjectStoreInfoPB obj_info; + StorageVaultPB vault; + obj_info.set_id("id"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + obj_info.set_provider(ObjectStoreInfoPB_Provider_COS); + vault.mutable_obj_info()->MergeFrom(obj_info); + vault.set_name("s3_1"); + vault.set_id("s3_1"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + txn->put(storage_vault_key({instance.instance_id(), "1"}), vault.SerializeAsString()); + } + + { + ObjectStoreInfoPB obj_info; + StorageVaultPB vault; + obj_info.set_id("id"); + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + obj_info.set_provider(ObjectStoreInfoPB_Provider_COS); + vault.mutable_obj_info()->MergeFrom(obj_info); + vault.set_name("s3_2"); + vault.set_id("s3_2"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString()); + } + + { + HdfsBuildConf hdfs_build_conf; + StorageVaultPB vault; + hdfs_build_conf.set_fs_name("fs_name"); + hdfs_build_conf.set_user("root"); + HdfsVaultInfo hdfs_info; + hdfs_info.set_prefix("root_path"); + hdfs_info.mutable_build_conf()->MergeFrom(hdfs_build_conf); + vault.mutable_hdfs_info()->MergeFrom(hdfs_info); + vault.set_name("hdfs_1"); + vault.set_id("hdfs_1"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "3"}), vault.SerializeAsString()); + } + + { + HdfsBuildConf hdfs_build_conf; + StorageVaultPB vault; + hdfs_build_conf.set_fs_name("fs_name"); + hdfs_build_conf.set_user("root"); + HdfsVaultInfo hdfs_info; + hdfs_info.set_prefix("root_path"); + hdfs_info.mutable_build_conf()->MergeFrom(hdfs_build_conf); + vault.mutable_hdfs_info()->MergeFrom(hdfs_info); + vault.set_name("hdfs_2"); + vault.set_id("hdfs_2"); + instance.add_storage_vault_names(vault.name()); + instance.add_resource_ids(vault.id()); + instance.set_instance_id("GetObjStoreInfoTestInstance"); + txn->put(storage_vault_key({instance.instance_id(), "4"}), vault.SerializeAsString()); + } + + auto accessor = std::make_shared(); + EXPECT_EQ(accessor->put_file("data/0/test.csv", ""), 0); + sp->set_call_back("HdfsAccessor::init.hdfs_init_failed", [](auto&& args) { + auto* ret = try_any_cast_ret(args); + ret->first = 0; + ret->second = true; + }); + sp->set_call_back( + "InstanceRecycler::init_storage_vault_accessors.mock_vault", [&accessor](auto&& args) { + auto* map = try_any_cast< + std::unordered_map>*>( + args[0]); + auto* vault = try_any_cast(args[1]); + map->emplace(vault->id(), accessor); + }); + sp->enable_processing(); + + val = instance.SerializeAsString(); + txn->put(key, val); + EXPECT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + EXPECT_EQ(accessor->exists("data/0/test.csv"), 0); + + { + config::valid_vault_name_set = {}; + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + EXPECT_EQ(recycler.init(), 0); + EXPECT_EQ(recycler.accessor_map_.size(), 4); + } + + { + config::valid_vault_name_set = {"s3_1", "s3_2", "hdfs_1", "hdfs_2"}; + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + EXPECT_EQ(recycler.init(), 0); + EXPECT_EQ(recycler.accessor_map_.size(), 4); + } + + { + config::valid_vault_name_set = {"s3_1", "hdfs_1"}; + InstanceRecycler recycler(txn_kv, instance, thread_group, + std::make_shared(txn_kv)); + EXPECT_EQ(recycler.init(), 0); + EXPECT_EQ(recycler.accessor_map_.size(), 2); + EXPECT_EQ(recycler.accessor_map_.at("s3_1")->exists("data/0/test.csv"), 0); + EXPECT_EQ(recycler.accessor_map_.at("hdfs_1")->exists("data/0/test.csv"), 0); + } +} + TEST(RecyclerTest, delete_tmp_rowset_data_with_idx_v1) { auto* sp = SyncPoint::get_instance(); std::unique_ptr> defer((int*)0x01, [&sp](int*) {