Skip to content

Commit

Permalink
curvefs/tool: list-topology output json file
Browse files Browse the repository at this point in the history 
1. add output json file which can be used for create-topology
2. add output json file which build topology like tree
  • Loading branch information
@Cyber-SiKu @YunhuiChen
Cyber-SiKu authored and YunhuiChen committed Jan 18, 2022
1 parent 34de287 commit 2a786da
Show file tree
Hide file tree
Showing 7 changed files with 447 additions and 31 deletions.
7 changes: 4 additions & 3 deletions curvefs/src/tools/create/curvefs_create_fs.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ int CreateFsTool::Init() {
mds::CreateFsRequest request;
request.set_fsname(FLAGS_fsName);
request.set_blocksize(FLAGS_blockSize);
if (FLAGS_fsType == "s3") {
if (FLAGS_fsType == kFsTypeS3) {
// s3
request.set_fstype(common::FSType::TYPE_S3);
auto s3 = new common::S3Info();
Expand All @@ -110,7 +110,7 @@ int CreateFsTool::Init() {
s3->set_blocksize(FLAGS_s3_blocksize);
s3->set_chunksize(FLAGS_s3_chunksize);
request.mutable_fsdetail()->set_allocated_s3info(s3);
} else if (FLAGS_fsType == "volume") {
} else if (FLAGS_fsType == kFsTypeVolume) {
// volume
request.set_fstype(common::FSType::TYPE_VOLUME);
auto volume = new common::Volume();
Expand All @@ -121,7 +121,8 @@ int CreateFsTool::Init() {
volume->set_password(FLAGS_volumePassword);
request.mutable_fsdetail()->set_allocated_volume(volume);
} else {
std::cerr << "-fsType should be s3 or volume." << std::endl;
std::cerr << "-fsType should be " << kFsTypeS3 << " or "
<< kFsTypeVolume << "." << std::endl;
ret = -1;
}

Expand Down
8 changes: 8 additions & 0 deletions curvefs/src/tools/curvefs_tool_define.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ DEFINE_string(s3_bucket_name, "bucketname", "s3 bucket name");
DEFINE_uint64(s3_blocksize, 1048576, "s3 block size");
DEFINE_uint64(s3_chunksize, 4194304, "s3 chunk size");

// list-topology
DEFINE_string(jsonPath, "/tmp/topology.json", "output json path");
DEFINE_string(jsonType, "build", "output json type(build or tree)");

// topology
DEFINE_string(mds_addr, "127.0.0.1:6700",
"mds ip and port, separated by \",\""); // NOLINT
Expand Down Expand Up @@ -215,6 +219,10 @@ std::function<bool(google::CommandLineFlagInfo*)> CheckPartitionIdDefault =
std::bind(&CheckFlagInfoDefault<fLS::clstring>, std::placeholders::_1,
"partitionId");

std::function<bool(google::CommandLineFlagInfo*)> CheckJsonPathDefault =
std::bind(&CheckFlagInfoDefault<fLS::clstring>, std::placeholders::_1,
"jsonPath");

/* translate to string */

auto StrVec2Str(const std::vector<std::string>& strVec) -> std::string {
Expand Down
27 changes: 27 additions & 0 deletions curvefs/src/tools/curvefs_tool_define.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,14 @@ const char kHostFollowerValue[] = "follower";
const char kEtcdLeaderValue[] = "StateLeader";
const char kEtcdFollowerValue[] = "StateFollower";

/* fs type */
const char kFsTypeS3[] = "s3";
const char kFsTypeVolume[] = "volume";

/* json type */
const char kJsonTypeBuild[] = "build";
const char kJsonTypeTree[] = "tree";

} // namespace tools
} // namespace curvefs

Expand All @@ -155,6 +163,23 @@ const char kZone[] = "zone";
const char kReplicasNum[] = "replicasnum";
const char kCopysetNum[] = "copysetnum";
const char kZoneNum[] = "zonenum";
const char kClusterId[] = "clusterid";
const char kPoolId[] = "poolid";
const char kPoolName[] = "poolname";
const char kCreateTime[] = "createtime";
const char kPolicy[] = "policy";
const char kPoollist[] = "poollist";
const char kZonelist[] = "zonelist";
const char kZoneId[] = "zoneid";
const char kZoneName[] = "zonename";
const char kServerlist[] = "serverlist";
const char kServerId[] = "serverid";
const char kHostName[] = "hostname";
const char kMetaserverList[] = "metaserverlist";
const char kMetaserverId[] = "metaserverid";
const char kHostIp[] = "hostip";
const char kPort[] = "port";
const char kOnlineState[] = "state";

} // namespace topology
} // namespace mds
Expand Down Expand Up @@ -245,6 +270,8 @@ extern std::function<bool(google::CommandLineFlagInfo*)> CheckCopysetIdDefault;
extern std::function<bool(google::CommandLineFlagInfo*)>
CheckPartitionIdDefault;

extern std::function<bool(google::CommandLineFlagInfo*)> CheckJsonPathDefault;

/* translate to string */
std::string StrVec2Str(const std::vector<std::string>&);

Expand Down
83 changes: 60 additions & 23 deletions curvefs/src/tools/list/curvefs_topology_list.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,18 +23,24 @@

#include <json/json.h>

#include <fstream>
#include <memory>

#include "src/common/string_util.h"

DECLARE_string(mdsAddr);
DECLARE_string(jsonPath);
DECLARE_string(jsonType);

namespace curvefs {
namespace tools {
namespace list {

void TopologyListTool::PrintHelp() {
CurvefsToolRpc::PrintHelp();
std::cout << " [-mdsAddr=" << FLAGS_mdsAddr << "]";
std::cout << std::endl;
std::cout << " [-mdsAddr=" << FLAGS_mdsAddr
<< "] [-jsonType=" << FLAGS_jsonType
<< " -jsonPath=" << FLAGS_jsonPath << "]" << std::endl;
}

void TopologyListTool::AddUpdateFlags() {
Expand Down Expand Up @@ -69,6 +75,9 @@ bool TopologyListTool::AfterSendRequestToHost(const std::string& host) {
ret = true;
// clusterId
clusterId_ = response_->clusterid();
clusterId2CLusterInfo_.insert(std::pair<std::string, ClusterInfo>(
clusterId_,
ClusterInfo(clusterId_, std::vector<mds::topology::PoolIdType>())));
// pool
if (!GetPoolInfoFromResponse()) {
ret = false;
Expand All @@ -89,39 +98,45 @@ bool TopologyListTool::AfterSendRequestToHost(const std::string& host) {
// show
if (show_) {
// cluster
std::cout << "[cluster]\nclusterId: " << clusterId_ << std::endl;
std::cout << "[cluster]\n"
<< mds::topology::kClusterId << ": " << clusterId_
<< std::endl;

// pool
std::cout << "[pool]" << std::endl;
for (auto const& i : poolId2PoolInfo) {
for (auto const& i : poolId2PoolInfo_) {
ShowPoolInfo(i.second);
}
// zone
std::cout << "[zone]" << std::endl;
for (auto const& i : zoneId2ZoneInfo) {
for (auto const& i : zoneId2ZoneInfo_) {
ShowZoneInfo(i.second);
}
// server
std::cout << "[server]" << std::endl;
for (auto const& i : serverId2ServerInfo) {
for (auto const& i : serverId2ServerInfo_) {
ShowServerInfo(i.second);
}
// metaserver
std::cout << "[metaserver]" << std::endl;
for (auto const& i : metaserverId2MetaserverInfo) {
for (auto const& i : metaserverId2MetaserverInfo_) {
ShowMetaserverInfo(i.second);
}
}

google::CommandLineFlagInfo info;
if (!CheckJsonPathDefault(&info) && ret) {
OutputFile();
}
}
return ret;
}

PoolPolicy::PoolPolicy(const std::string& jsonStr) {
Json::CharReaderBuilder reader;
std::stringstream ss(jsonStr);
std::string err;
Json::Value json;
bool parseCode = Json::parseFromStream(reader, ss, &json, &err);
bool parseCode = Json::parseFromStream(reader, ss, &json, nullptr);
if (parseCode && !json["replicaNum"].isNull() &&
!json["copysetNum"].isNull() && !json["zoneNum"].isNull()) {
replicaNum = json["replicaNum"].asUInt();
Expand All @@ -136,9 +151,9 @@ std::ostream& operator<<(std::ostream& os, const PoolPolicy& policy) {
if (policy.error) {
os << "policy has error!";
} else {
os << "copysetNum:" << policy.copysetNum
<< " replicaNum:" << policy.replicaNum
<< " zoneNum:" << policy.zoneNum;
os << mds::topology::kCopysetNum << ":" << policy.copysetNum << " "
<< mds::topology::kReplicasNum << ":" << policy.replicaNum << " "
<< mds::topology::kZoneNum << ":" << policy.zoneNum;
}
return os;
}
Expand All @@ -155,10 +170,11 @@ bool TopologyListTool::GetPoolInfoFromResponse() {
ret = false;
} else {
for (auto const& i : pools.poolinfos()) {
poolId2PoolInfo.insert(
poolId2PoolInfo_.insert(
std::pair<mds::topology::PoolIdType, PoolInfoType>(
i.poolid(),
PoolInfoType(i, std::vector<mds::topology::ZoneIdType>())));
clusterId2CLusterInfo_[clusterId_].second.emplace_back(i.poolid());
}
}
return ret;
Expand All @@ -176,13 +192,13 @@ bool TopologyListTool::GetZoneInfoFromResponse() {
ret = false;
} else {
for (auto const& i : zones.zoneinfos()) {
zoneId2ZoneInfo.insert(
zoneId2ZoneInfo_.insert(
std::pair<mds::topology::ZoneIdType, ZoneInfoType>(
i.zoneid(),
ZoneInfoType(i,
std::vector<mds::topology::ServerIdType>())));
if (poolId2PoolInfo.find(i.poolid()) != poolId2PoolInfo.end()) {
poolId2PoolInfo[i.poolid()].second.emplace_back(i.zoneid());
if (poolId2PoolInfo_.find(i.poolid()) != poolId2PoolInfo_.end()) {
poolId2PoolInfo_[i.poolid()].second.emplace_back(i.zoneid());
} else {
errorOutput_ << "zone:" << i.zoneid()
<< " has error: poolId:" << i.poolid()
Expand All @@ -206,14 +222,14 @@ bool TopologyListTool::GetServerInfoFromResponse() {
ret = false;
} else {
for (auto const& i : servers.serverinfos()) {
serverId2ServerInfo.insert(
serverId2ServerInfo_.insert(
std::pair<mds::topology::ServerIdType, ServerInfoType>(
i.serverid(),
ServerInfoType(
i, std::vector<mds::topology::MetaServerIdType>())));

if (zoneId2ZoneInfo.find(i.zoneid()) != zoneId2ZoneInfo.end()) {
zoneId2ZoneInfo[i.zoneid()].second.emplace_back(i.serverid());
if (zoneId2ZoneInfo_.find(i.zoneid()) != zoneId2ZoneInfo_.end()) {
zoneId2ZoneInfo_[i.zoneid()].second.emplace_back(i.serverid());
} else {
errorOutput_ << "server:" << i.serverid()
<< " has error: zoneId:" << i.zoneid()
Expand All @@ -237,12 +253,12 @@ bool TopologyListTool::GetMetaserverInfoFromResponse() {
ret = false;
} else {
for (auto const& i : metaservers.metaserverinfos()) {
metaserverId2MetaserverInfo.insert(
metaserverId2MetaserverInfo_.insert(
std::pair<mds::topology::MetaServerIdType, MetaserverInfoType>(
i.metaserverid(), MetaserverInfoType(i)));
if (serverId2ServerInfo.find(i.serverid()) !=
serverId2ServerInfo.end()) {
serverId2ServerInfo[i.serverid()].second.emplace_back(
if (serverId2ServerInfo_.find(i.serverid()) !=
serverId2ServerInfo_.end()) {
serverId2ServerInfo_[i.serverid()].second.emplace_back(
i.metaserverid());
} else {
errorOutput_ << "metaserver:" << i.metaserverid()
Expand Down Expand Up @@ -290,6 +306,27 @@ void TopologyListTool::ShowMetaserverInfo(
std::cout << MetaserverInfo2Str(metaserver) << std::endl;
}

bool TopologyListTool::OutputFile() {
Json::Value value;
topology::TopologyTreeJson treeJson(*this);
if (!treeJson.BuildJsonValue(&value, FLAGS_jsonType)) {
std::cerr << "build json file failed!" << std::endl;
return false;
}

std::ofstream jsonFile;
jsonFile.open(FLAGS_jsonPath.c_str(), std::ios::out);
if (!jsonFile) {
std::cerr << "open json file failed!" << std::endl;
return false;
}
Json::StreamWriterBuilder clusterMap;
std::unique_ptr<Json::StreamWriter> writer(clusterMap.newStreamWriter());
writer->write(value, &jsonFile);
jsonFile.close();
return true;
}

} // namespace list
} // namespace tools
} // namespace curvefs
27 changes: 22 additions & 5 deletions curvefs/src/tools/list/curvefs_topology_list.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@

#include <brpc/channel.h>
#include <gflags/gflags.h>
#include <json/json.h>

#include <map>
#include <string>
Expand All @@ -33,11 +34,19 @@
#include "curvefs/src/mds/common/mds_define.h"
#include "curvefs/src/tools/curvefs_tool.h"
#include "curvefs/src/tools/curvefs_tool_define.h"
#include "curvefs/src/tools/list/curvefs_topology_tree_json.h"

namespace curvefs {
namespace tools {

namespace topology {
class TopologyTreeJson;
}

namespace list {

using ClusterInfo =
std::pair<std::string, std::vector<mds::topology::PoolIdType>>;
using PoolInfoType =
std::pair<mds::topology::PoolInfo, std::vector<mds::topology::ZoneIdType>>;
using ZoneInfoType = std::pair<mds::topology::ZoneInfo,
Expand All @@ -63,7 +72,6 @@ struct PoolPolicy {
friend std::ostream& operator<<(std::ostream& os, const PoolPolicy& policy);
};

// TODO(chengyi01): output a json file which can be used to build topology
class TopologyListTool
: public CurvefsToolRpc<curvefs::mds::topology::ListTopologyRequest,
curvefs::mds::topology::ListTopologyResponse,
Expand All @@ -77,13 +85,17 @@ class TopologyListTool
void PrintHelp() override;
int Init() override;

friend class topology::TopologyTreeJson;

protected:
bool OutputFile();
void AddUpdateFlags() override;
bool AfterSendRequestToHost(const std::string& host) override;

/**
* @brief Get the PoolInfo From Response, fill into poolId2PoolInfo
* not include zoneId list (will be filled in GetZoneInfoFromResponse)
* will fill clusterId2CLusterInfo's poolId list
*
* @return true
* @return false
Expand Down Expand Up @@ -135,34 +147,39 @@ class TopologyListTool

protected:
std::string clusterId_;
/**
* poolId to clusterInfo and poolIds which belongs to pool
*/
std::map<std::string, ClusterInfo> clusterId2CLusterInfo_;

/**
* @brief poolId to poolInfo and zoneIds which belongs to pool
*
* @details
*/
std::map<mds::topology::PoolIdType, PoolInfoType> poolId2PoolInfo;
std::map<mds::topology::PoolIdType, PoolInfoType> poolId2PoolInfo_;

/**
* @brief zoneId to zoneInfo and serverIds which belongs to zone
*
* @details
*/
std::map<mds::topology::ZoneIdType, ZoneInfoType> zoneId2ZoneInfo;
std::map<mds::topology::ZoneIdType, ZoneInfoType> zoneId2ZoneInfo_;

/**
* @brief serverId to serverInfo and metaserverIds which belongs to server
*
* @details
*/
std::map<mds::topology::ServerIdType, ServerInfoType> serverId2ServerInfo;
std::map<mds::topology::ServerIdType, ServerInfoType> serverId2ServerInfo_;

/**
* @brief metaserverId to metaserverInfo
*
* @details
*/
std::map<mds::topology::MetaServerIdType, MetaserverInfoType>
metaserverId2MetaserverInfo;
metaserverId2MetaserverInfo_;
};

} // namespace list
Expand Down
Loading

0 comments on commit 2a786da

Please sign in to comment.