Allow multi data path on one disk (baidu#683)

(baidu#682) Use a flag to control wether allowing multi data path on one disk. This is mainly meant to make unit-test more convenient. Do not recommend for production environment.
lylei · Dec 12, 2016 · 1aca063 · 1aca063
1 parent 730e30f
commit 1aca063
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 30 deletions.
diff --git a/sandbox/deploy.sh b/sandbox/deploy.sh
@@ -28,6 +28,7 @@ echo '--bfs_web_kick_enable=true' >> bfs.flag
 echo "--ha_strategy=$strategy" >> bfs.flag
 echo '--nameserver_nodes=127.0.0.1:8827,127.0.0.1:8828,127.0.0.1:8829' >> bfs.flag
 echo '--sdk_wirte_mode=fanout' >> bfs.flag
+echo '--chunkserver_multi_path_on_one_disk=true' >> bfs.flag
 
 for((i=0;i<$ns_num;i++));
 do

diff --git a/src/chunkserver/block_manager.cc b/src/chunkserver/block_manager.cc
@@ -26,6 +26,7 @@
 DECLARE_int32(chunkserver_file_cache_size);
 DECLARE_int32(chunkserver_use_root_partition);
 DECLARE_int32(chunkserver_io_thread_num);
+DECLARE_bool(chunkserver_multi_path_on_one_disk);
 
 namespace baidu {
 namespace bfs {
@@ -65,54 +66,67 @@ int64_t BlockManager::DiskQuota() const{
 }
 
 void BlockManager::CheckStorePath(const std::string& store_path) {
-    int64_t disk_quota = 0;
-    common::SplitString(store_path, ",", &store_path_list_);
-    std::map<std::string, std::string> fs_map;
     std::string fsid_str;
     struct statfs fs_info;
-    int stat_ret = statfs("/home", &fs_info);
-    if (stat_ret != 0 && statfs("/", &fs_info) != 0) {
+    std::string home_fs;
+    if (statfs("/home", &fs_info) == 0) {
+        home_fs.assign((const char*)&fs_info.f_fsid, sizeof(fs_info.f_fsid));
+    } else if (statfs("/", &fs_info) == 0) {
+        LOG(WARNING, "statfs(\"/home\") fail: %s", strerror(errno));
+        home_fs.assign((const char*)&fs_info.f_fsid, sizeof(fs_info.f_fsid));
+    } else {
         LOG(FATAL, "statfs(\"/\") fail: %s", strerror(errno));
-    } else if (FLAGS_chunkserver_use_root_partition == 0) {
-        fsid_str.assign((const char*)&fs_info.f_fsid, sizeof(fs_info.f_fsid));
-        fs_map[fsid_str] = "Root";
-        LOG(INFO, "Root fsid: %s", common::DebugString(fsid_str).c_str());
     }
+
+    common::SplitString(store_path, ",", &store_path_list_);
     for (uint32_t i = 0; i < store_path_list_.size(); ++i) {
         std::string& disk_path = store_path_list_[i];
         disk_path = common::TrimString(disk_path, " ");
         if (disk_path.empty() || disk_path[disk_path.size() - 1] != '/') {
             disk_path += "/";
         }
-        if (0 == (stat_ret = statfs(disk_path.c_str(), &fs_info))
-          && (fsid_str.assign((const char*)&fs_info.f_fsid, sizeof(fs_info.f_fsid)),
-              fs_map.find(fsid_str) == fs_map.end())) {
-            int64_t disk_size = fs_info.f_blocks * fs_info.f_bsize;
-            int64_t user_quota = fs_info.f_bavail * fs_info.f_bsize;
-            int64_t super_quota = fs_info.f_bfree * fs_info.f_bsize;
-            fs_map[fsid_str] = disk_path;
-            LOG(INFO, "Use store path: %s block: %ld disk: %s available %s quota: %s",
-                disk_path.c_str(), fs_info.f_bsize,
-                common::HumanReadableString(disk_size).c_str(),
-                common::HumanReadableString(super_quota).c_str(),
-                common::HumanReadableString(user_quota).c_str());
-            disk_quota += user_quota;
-        } else {
+    }
+    std::sort(store_path_list_.begin(), store_path_list_.end());
+    auto it = std::unique(store_path_list_.begin(), store_path_list_.end());
+    store_path_list_.resize(std::distance(store_path_list_.begin(), it));
+
+    std::set<std::string> fsids;
+    int64_t disk_quota = 0;
+    for (uint32_t i = 0; i < store_path_list_.size(); ++i) {
+        std::string& disk_path = store_path_list_[i];
+        int stat_ret = statfs(disk_path.c_str(), &fs_info);
+        std::string fs_tmp((const char*)&fs_info.f_fsid, sizeof(fs_info.f_fsid));
+        if (stat_ret != 0 ||
+            (!FLAGS_chunkserver_multi_path_on_one_disk && fsids.find(fs_tmp) != fsids.end()) ||
+            (!FLAGS_chunkserver_use_root_partition && fs_tmp == home_fs)) {
+            // statfs failed
+            // do not allow multi data path on the same disk
+            // do not allow using root as data path
             if (stat_ret != 0) {
-                LOG(WARNING, "Stat store_path %s fail: %s, ignore it", disk_path.c_str(), strerror(errno));
+                LOG(WARNING, "Stat store_path %s fail: %s, ignore it",
+                        disk_path.c_str(), strerror(errno));
             } else {
-                LOG(WARNING, "%s's fsid is same to %s, ignore it",
-                    disk_path.c_str(), fs_map[fsid_str].c_str());
+                LOG(WARNING, "%s fsid has been used", disk_path.c_str());
             }
             store_path_list_[i] = store_path_list_[store_path_list_.size() - 1];
             store_path_list_.resize(store_path_list_.size() - 1);
             --i;
+        } else {
+            if (fsids.insert(fs_tmp).second) {
+                int64_t disk_size = fs_info.f_blocks * fs_info.f_bsize;
+                int64_t user_quota = fs_info.f_bavail * fs_info.f_bsize;
+                int64_t super_quota = fs_info.f_bfree * fs_info.f_bsize;
+                LOG(INFO, "Use store path: %s block: %ld disk: %s available %s quota: %s",
+                        disk_path.c_str(), fs_info.f_bsize,
+                        common::HumanReadableString(disk_size).c_str(),
+                        common::HumanReadableString(super_quota).c_str(),
+                        common::HumanReadableString(user_quota).c_str());
+                disk_quota += user_quota;
+            } else {
+                LOG(INFO, "Use store path: %s", disk_path.c_str());
+            }
         }
     }
-    std::sort(store_path_list_.begin(), store_path_list_.end());
-    std::vector<std::string>::iterator it
-       = std::unique(store_path_list_.begin(), store_path_list_.end());
-    store_path_list_.resize(std::distance(store_path_list_.begin(), it));
     LOG(INFO, "%lu store path used.", store_path_list_.size());
     assert(store_path_list_.size() > 0);
     disk_quota_ = disk_quota;

diff --git a/src/flags.cc b/src/flags.cc
@@ -69,6 +69,7 @@ DEFINE_int32(chunkserver_io_thread_num, 10, "Chunkserver io thread num");
 DEFINE_int32(chunkserver_recover_thread_num, 10, "Chunkserver work thread num");
 DEFINE_int32(chunkserver_file_cache_size, 1000, "Chunkserver file cache size");
 DEFINE_int32(chunkserver_use_root_partition, 1, "Should chunkserver use root partition, 0: forbidden");
+DEFINE_bool(chunkserver_multi_path_on_one_disk, false, "Allow multi data path on one disk");
 DEFINE_bool(chunkserver_auto_clean, true, "If namespace version mismatch, chunkserver clean itself");
 // SDK
 DEFINE_string(sdk_wirte_mode, "chains", "Sdk write strategy, choose from [chains, fanout]");