diff --git a/curvefs/conf/tools.conf b/curvefs/conf/tools.conf index 0a571f0603..b0d9235ab0 100644 --- a/curvefs/conf/tools.conf +++ b/curvefs/conf/tools.conf @@ -4,3 +4,7 @@ mdsAddr=127.0.0.1:6700 # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvar rpcTimeoutMs=500 # topo file path topoFilePath=curvefs/test/tools/topo_example.json # __ANSIBLE_TEMPLATE__ {{ project_root_dest }}/conf/topology.json __ANSIBLE_TEMPLATE__ +# metaserver +metaserverAddr=127.0.0.1:6701 # __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__ +# etcd +etcdAddr=127.0.0.1:12379 # __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_peer(hostvars, "etcd_listen_client_port") }} __ANSIBLE_TEMPLATE__ diff --git a/curvefs/proto/copyset.proto b/curvefs/proto/copyset.proto index 3e4d82324a..15229ed10f 100644 --- a/curvefs/proto/copyset.proto +++ b/curvefs/proto/copyset.proto @@ -80,7 +80,16 @@ message CopysetStatusResponse { optional CopysetStatus copysetStatus = 2; }; +message CopysetsStatusRequest { + repeated CopysetStatusRequest copysets = 1; +} + +message CopysetsStatusResponse { + repeated CopysetStatusResponse status = 1; +} + service CopysetService { rpc CreateCopysetNode(CreateCopysetRequest) returns (CreateCopysetResponse); rpc GetCopysetStatus(CopysetStatusRequest) returns (CopysetStatusResponse); + rpc GetCopysetsStatus(CopysetsStatusRequest) returns (CopysetsStatusResponse); } diff --git a/curvefs/proto/mds.proto b/curvefs/proto/mds.proto index 5d59575f09..6e4a1a82e5 100644 --- a/curvefs/proto/mds.proto +++ b/curvefs/proto/mds.proto @@ -146,22 +146,22 @@ message AllocateS3ChunkResponse { } -message StatClusterMetadataSpaceRequest{ +message StatMetadataUsageRequest{ } -message StatClusterMetadataSpaceResponse{ - required FSStatusCode status = 1; - required uint32 total = 2; - required uint32 used = 3; - required uint32 left = 4; +message StatMetadataUsageResponse{ + required FSStatusCode statusCode = 1; + // KB + required uint64 total = 2; + required uint64 used = 3; + required uint64 left = 4; } message ListClusterFsInfoRequest { } message ListClusterFsInfoResponse { - required FSStatusCode statusCode = 1; - repeated FsInfo fsInfo = 2; + repeated FsInfo fsInfo = 1; } service MdsService { @@ -173,6 +173,6 @@ service MdsService { // rpc UpdateFsInfo(UpdateFsInfoRequest) returns (UpdateFsInfoResponse); rpc DeleteFs(DeleteFsRequest) returns (DeleteFsResponse); rpc AllocateS3Chunk(AllocateS3ChunkRequest) returns (AllocateS3ChunkResponse); - rpc StatClusterMetadataSpace(StatClusterMetadataSpaceRequest) returns (StatClusterMetadataSpaceResponse); + rpc StatMetadataUsage(StatMetadataUsageRequest) returns (StatMetadataUsageResponse); rpc ListClusterFsInfo (ListClusterFsInfoRequest) returns (ListClusterFsInfoResponse); } diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index 5512bba129..3513a604c5 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -292,7 +292,15 @@ message MetaServerMetadata { required uint32 id = 2; required string token = 3; // required uint32 checksum = 4; -}; +} + +message GetAllCopysetsIdRequest { +} + +message GetAllCopysetsIdResponse { + required MetaStatusCode statusCode = 1; + repeated uint32 copysetsId = 2; +} service MetaServerService { // dentry interface @@ -313,4 +321,7 @@ service MetaServerService { // partition interface rpc CreatePartition(CreatePartitionRequest) returns (CreatePartitionResponse); rpc DeletePartition(DeletePartitionRequest) returns (DeletePartitionResponse); + + // get all copysets id + rpc GetAllCopysetId(GetAllCopysetsIdRequest) returns (GetAllCopysetsIdResponse); } diff --git a/curvefs/proto/topology.proto b/curvefs/proto/topology.proto index 71763b5153..ad07ed8b99 100644 --- a/curvefs/proto/topology.proto +++ b/curvefs/proto/topology.proto @@ -380,13 +380,41 @@ message CommitTxResponse { required TopoStatusCode statusCode = 1; } +message ListPartitionsRequest { + repeated uint32 fsId = 1; +} + +message PartitionInfoList { + repeated common.PartitionInfo partitionInfoList = 1; +} + +message ListPartitionsResponse { + required TopoStatusCode statusCode = 1; + map fsId2partitionList = 2; +} + message GetCopysetInfoRequest { - required uint32 copysetId = 1; + required uint32 poolId = 1; + required uint32 copysetId = 2; +} + +message CopysetInfo { + required uint32 poolId = 1; + required uint32 copysetId = 2; + repeated common.Peer peers = 3; } message GetCopysetInfoResponse { required TopoStatusCode statusCode = 1; - required curvefs.mds.heartbeat.CopySetInfo copysetInfo = 2; + optional CopysetInfo copysetInfo = 2; +} + +message GetCopysetsInfoRequest { + repeated GetCopysetInfoRequest copysets= 1; +} + +message GetCopysetsInfoResponse { + repeated GetCopysetInfoResponse copysetsInfo = 1; } service TopologyService { @@ -416,5 +444,7 @@ service TopologyService { rpc ListPartition(ListPartitionRequest) returns (ListPartitionResponse); rpc GetCopysetOfPartition(GetCopysetOfPartitionRequest) returns (GetCopysetOfPartitionResponse); rpc CommitTx(CommitTxRequest) returns (CommitTxResponse); + rpc ListPartitions(ListPartitionsRequest) returns (ListPartitionsResponse); rpc GetCopysetInfo (GetCopysetInfoRequest) returns (GetCopysetInfoResponse); + rpc GetCopysetsInfo (GetCopysetsInfoRequest) returns (GetCopysetsInfoResponse); } diff --git a/curvefs/src/mds/topology/deal_peerid.h b/curvefs/src/mds/topology/deal_peerid.h index 3f44fa0828..c1e7274944 100644 --- a/curvefs/src/mds/topology/deal_peerid.h +++ b/curvefs/src/mds/topology/deal_peerid.h @@ -65,6 +65,16 @@ inline bool SplitPeerId(const std::string &peerId, std::string *ip, return false; } +inline bool SplitPeerId(const std::string& peerId, std::string* addr) { + std::vector items; + curve::common::SplitString(peerId, ":", &items); + if (3 == items.size()) { + *addr = items[0] + ":" + items[1]; + return true; + } + return false; +} + } // namespace topology } // namespace mds } // namespace curvefs diff --git a/curvefs/src/tools/BUILD b/curvefs/src/tools/BUILD index f440441475..b62a88e30b 100644 --- a/curvefs/src/tools/BUILD +++ b/curvefs/src/tools/BUILD @@ -46,29 +46,54 @@ cc_binary( ], ) - # tools lib cc_library( name = "curvefs_tools_lib", srcs = glob([ + "curvefs_tool.cpp", "curvefs_tool_factory.cpp", - "version/curvefs_version_tool.cpp", "umountfs/curvefs_umountfs_tool.cpp", "curvefs_tool_define.cpp", - "topology/curvefs_build_topology_tool.cpp", + "curvefs_tool_metric.cpp", + "build/curvefs_build_topology_tool.cpp", + "version/curvefs_version_tool.cpp", + "space/curvefs_metadata_usage_tool.cpp", + "space/curvefs_space_base_tool.cpp", + "status/curvefs_status_base_tool.cpp", + "status/curvefs_mds_status.cpp", + "status/curvefs_metaserver_status.cpp", + "status/curvefs_etcd_status.cpp", + "status/curvefs_copyset_status.cpp", + "query/curvefs_copyset_query.cpp", + "list/curvefs_fs_partition_list.cpp", + "list/curvefs_fsinfo_list.cpp", + "status/curvefs_metaserver_status.cpp", + "list/curvefs_copysetid_list.cpp", + ]), hdrs = glob([ "curvefs_tool.h", "curvefs_tool_factory.h", "curvefs_tool_define.h", + "curvefs_tool_metric.h", "version/curvefs_version_tool.h", "curvefs_tool_abstract_creator.h", "umountfs/curvefs_umountfs_tool.h", - "topology/curvefs_build_topology_tool.h", + "build/curvefs_build_topology_tool.h", + "space/curvefs_metadata_usage_tool.h", + "space/curvefs_space_base_tool.h", + "status/curvefs_status_base_tool.h", + "status/curvefs_mds_status.h", + "status/curvefs_copyset_status.h", + "query/curvefs_copyset_query.h", + "list/curvefs_fs_partition_list.h", + "list/curvefs_fsinfo_list.h", + "status/curvefs_metaserver_status.h", + "status/curvefs_etcd_status.h", + "list/curvefs_copysetid_list.h", ]), copts = COPTS, linkopts = [ - "-lpthread" ], visibility = ["//visibility:public"], deps = [ @@ -77,8 +102,12 @@ cc_library( "//external:glog", "//external:json", "//curvefs/src/mds/common:fs_mds_common", - "//src/common:curve_common", + "//curvefs/src/mds/topology:curvefs_deal_peerid", + "//src/mds/topology:topology", "//curvefs/proto:mds_cc_proto", "//curvefs/proto:curvefs_topology_cc_proto", + "//curvefs/proto:space_cc_proto", + "//curvefs/proto:copyset_cc_proto", + "//curvefs/proto:metaserver_cc_proto", ], ) diff --git a/curvefs/src/tools/topology/curvefs_build_topology_tool.cpp b/curvefs/src/tools/build/curvefs_build_topology_tool.cpp similarity index 97% rename from curvefs/src/tools/topology/curvefs_build_topology_tool.cpp rename to curvefs/src/tools/build/curvefs_build_topology_tool.cpp index 3e8094e2ce..58ad352107 100644 --- a/curvefs/src/tools/topology/curvefs_build_topology_tool.cpp +++ b/curvefs/src/tools/build/curvefs_build_topology_tool.cpp @@ -20,11 +20,10 @@ * Author: wanghai01 */ -#include "curvefs/src/tools/topology/curvefs_build_topology_tool.h" +#include "curvefs/src/tools/build/curvefs_build_topology_tool.h" DECLARE_string(mds_addr); DECLARE_string(cluster_map); -DECLARE_uint32(rpcTimeOutMs); DECLARE_string(confPath); DECLARE_string(op); @@ -42,8 +41,8 @@ void UpdateFlagsFromConf(curve::common::Configuration* conf) { LOG(INFO) << "conf: " << FLAGS_mds_addr; } - if (GetCommandLineFlagInfo("rpcTimeOutMs", &info) && info.is_default) { - conf->GetUInt32Value("rpcTimeoutMs", &FLAGS_rpcTimeOutMs); + if (GetCommandLineFlagInfo("rpcTimeoutMs", &info) && info.is_default) { + conf->GetUInt32Value("rpcTimeoutMs", &FLAGS_rpcTimeoutMs); } if (GetCommandLineFlagInfo("cluster_map", &info) && info.is_default) { @@ -323,7 +322,7 @@ int CurvefsBuildTopologyTool::ListPool(std::list* poolInfos) { ListPoolRequest request; ListPoolResponse response; brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "ListPool send request: " << request.DebugString(); @@ -355,7 +354,7 @@ int CurvefsBuildTopologyTool::GetZonesInPool(PoolIdType poolid, request.set_poolid(poolid); brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "ListZoneInPool, send request: " << request.DebugString(); @@ -388,7 +387,7 @@ int CurvefsBuildTopologyTool::GetServersInZone( ListZoneServerResponse response; request.set_zoneid(zoneid); brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "ListZoneServer, send request: " << request.DebugString(); @@ -429,7 +428,7 @@ int CurvefsBuildTopologyTool::CreatePool() { request.set_redundanceandplacementpolicy(rapString); brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "CreatePool, send request: " << request.DebugString(); @@ -468,7 +467,7 @@ int CurvefsBuildTopologyTool::CreateZone() { request.set_poolname(it.poolName); brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "CreateZone, send request: " << request.DebugString(); @@ -511,7 +510,7 @@ int CurvefsBuildTopologyTool::CreateServer() { request.set_poolname(it.poolName); brpc::Controller cntl; - cntl.set_timeout_ms(FLAGS_rpcTimeOutMs); + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); cntl.set_log_id(1); LOG(INFO) << "CreateServer, send request: " << request.DebugString(); diff --git a/curvefs/src/tools/topology/curvefs_build_topology_tool.h b/curvefs/src/tools/build/curvefs_build_topology_tool.h similarity index 95% rename from curvefs/src/tools/topology/curvefs_build_topology_tool.h rename to curvefs/src/tools/build/curvefs_build_topology_tool.h index c32e1760ec..9cecfc3b98 100644 --- a/curvefs/src/tools/topology/curvefs_build_topology_tool.h +++ b/curvefs/src/tools/build/curvefs_build_topology_tool.h @@ -20,8 +20,8 @@ * Author: chengyi01 */ -#ifndef CURVEFS_SRC_TOOLS_TOPOLOGY_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ -#define CURVEFS_SRC_TOOLS_TOPOLOGY_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ +#ifndef CURVEFS_SRC_TOOLS_BUILD_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ +#define CURVEFS_SRC_TOOLS_BUILD_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ #include #include @@ -160,4 +160,4 @@ class CurvefsBuildTopologyTool : public curvefs::tools::CurvefsTool { } // namespace mds } // namespace curvefs -#endif // CURVEFS_SRC_TOOLS_TOPOLOGY_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ +#endif // CURVEFS_SRC_TOOLS_BUILD_CURVEFS_BUILD_TOPOLOGY_TOOL_H_ diff --git a/curvefs/src/tools/curvefs_tool.cpp b/curvefs/src/tools/curvefs_tool.cpp new file mode 100644 index 0000000000..846d0c65ad --- /dev/null +++ b/curvefs/src/tools/curvefs_tool.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-11-02 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/curvefs_tool.h" + +namespace curvefs { +namespace tools { + +void CurvefsTool::PrintHelp() { + std::cout << "Example :" << std::endl; + std::cout << programe_ << " " << command_ << " " << kConfPathHelp; +} + +int CurvefsTool::Run() { + if (Init() < 0) { + return -1; + } + int ret = RunCommand(); + return ret; +} + +int CurvefsToolMetric::Init(const std::shared_ptr& metricClient) { + metricClient_ = metricClient; + return 0; +} + +void CurvefsToolMetric::PrintHelp() { + CurvefsTool::PrintHelp(); + std::cout << " [-rpcTimeoutMs=" << FLAGS_rpcTimeoutMs << "]" + << " [-rpcRetryTimes=" << FLAGS_rpcRetryTimes << "]"; +} + +void CurvefsToolMetric::AddUpdateFlagsFunc( + const std::function& func) { + updateFlagsFunc_.push_back(func); +} + +int CurvefsToolMetric::RunCommand() { + int ret = 0; + for (auto const& i : addr2SubUri) { + std::string value; + MetricStatusCode statusCode = + metricClient_->GetMetric(i.first, i.second, &value); + if (statusCode != MetricStatusCode::kOK) { + std::cerr << "get metricName \"" << i.second << "\" from " + << i.first << " fail!" << std::endl; + ret = -1; + } + AfterGetMetric(i.first, i.second, value, statusCode); + } + + if (ProcessMetrics() != 0) { + ret = -1; + } + return ret; +} + +void CurvefsToolMetric::AddUpdateFlags() { + // rpcTimeout and rpcRetrytimes is default + AddUpdateFlagsFunc(SetRpcTimeoutMs); + AddUpdateFlagsFunc(SetRpcRetryTimes); +} + +void CurvefsToolMetric::UpdateFlags() { + curve::common::Configuration conf; + conf.SetConfigPath(FLAGS_confPath); + if (!conf.LoadConfig()) { + std::cerr << "load configure file " << FLAGS_confPath << " failed!" + << std::endl; + } + google::CommandLineFlagInfo info; + + for (auto& i : updateFlagsFunc_) { + i(&conf, &info); + } +} + +void CurvefsToolMetric::AddAddr2Suburi( + const std::pair& addrSubUri) { + addr2SubUri.push_back(addrSubUri); +} + +int CurvefsToolMetric::Init() { + // add need update flags + AddUpdateFlags(); + UpdateFlags(); + InitHostsAddr(); + return 0; +} + +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/curvefs_tool.h b/curvefs/src/tools/curvefs_tool.h index f04b2e2da8..9778cf5c17 100644 --- a/curvefs/src/tools/curvefs_tool.h +++ b/curvefs/src/tools/curvefs_tool.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 NetEase Inc. + * Copyright (c) 2021 NetEase Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,18 +25,26 @@ #include #include #include +#include +#include #include #include +#include #include +#include #include #include +#include #include #include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/curvefs_tool_metric.h" #include "src/common/configuration.h" DECLARE_string(confPath); +DECLARE_uint32(rpcTimeoutMs); +DECLARE_uint64(rpcRetryTimes); namespace curvefs { namespace tools { @@ -44,24 +52,16 @@ namespace tools { class CurvefsTool { public: CurvefsTool() {} - CurvefsTool(const std::string& command, const std::string& programe) - : command_(command), programe_(programe) {} + CurvefsTool(const std::string& command, + const std::string& programe = kProgrameName, bool show = true) + : command_(command), programe_(programe), show_(show) {} + virtual ~CurvefsTool() {} - virtual void PrintHelp() { - std::cout << "Example :" << std::endl; - std::cout << programe_ << " " << command_ << " " << kConfPathHelp; - } + virtual void PrintHelp(); - virtual int Run() { - if (Init() < 0) { - return -1; - } - int ret = RunCommand(); - return ret; - } + virtual int Run(); - private: /** * @brief configure the environment for the command * @@ -73,6 +73,7 @@ class CurvefsTool { protected: std::string command_; std::string programe_; + bool show_; // Control whether the command line is output }; /** @@ -86,29 +87,80 @@ class CurvefsTool { * @details * you can take umountfs as example */ -template +template class CurvefsToolRpc : public CurvefsTool { public: - CurvefsToolRpc(const std::string& command, const std::string& programe) - : CurvefsTool(command, programe) {} + CurvefsToolRpc(const std::string& command, + const std::string& programe = kProgrameName, + bool show = true) + : CurvefsTool(command, programe, show) {} int Init(const std::shared_ptr& channel, const std::shared_ptr& controller, - const std::shared_ptr& request, + const std::queue& requestQueue, const std::shared_ptr& response, - const std::shared_ptr& service_stub) { + const std::shared_ptr& service_stub, + const std::function& + service_stub_func) { channel_ = channel; controller_ = controller; - request_ = request; + requestQueue_ = requestQueue; response_ = response; service_stub_ = service_stub; + service_stub_func_ = service_stub_func; + InitHostsAddr(); return 0; } + virtual int Init() { + channel_ = std::make_shared(); + controller_ = std::make_shared(); + response_ = std::make_shared(); + service_stub_ = std::make_shared(channel_.get()); + + // add need update FlagInfos + AddUpdateFlags(); + UpdateFlags(); + InitHostsAddr(); + return 0; + } + + virtual void SetRequestQueue(const std::queue& requestQueue) { + requestQueue_ = requestQueue; + } + virtual void AddRequest(const RequestT& request) { + requestQueue_.push(request); + } + + virtual const std::shared_ptr& GetResponse() { + return response_; + } + + virtual int RunCommand() { + int ret = 0; + while (!requestQueue_.empty()) { + if (!SendRequestToServices()) { + std::cerr << "send request to host: [ "; + for (const auto& i : hostsAddr_) { + std::cerr << i << " "; + } + std::cerr << "] failed." << std::endl; + ret = -1; + } + + requestQueue_.pop(); + } + + return ret; + } + + virtual void InitHostsAddr() {} + protected: /** - * @brief send request to host in hostsAddressStr_ + * @brief send request to host in hostsAddr_ * * @return true * @return false @@ -116,16 +168,16 @@ class CurvefsToolRpc : public CurvefsTool { * as long as one succeeds, it returns true and ends sending */ virtual bool SendRequestToServices() { - for (const std::string& host : hostsAddressStr_) { + for (const std::string& host : hostsAddr_) { if (channel_->Init(host.c_str(), nullptr) != 0) { - std::cerr << "Fail init channel to host: " << host << std::endl; + std::cerr << "fail init channel to host: " << host << std::endl; continue; } // if service_stub_func_ does not assign a value // it will crash in there - service_stub_func_(controller_.get(), request_.get(), + service_stub_func_(controller_.get(), &requestQueue_.front(), response_.get()); - if (AfterSendRequestToService(host) == true) { + if (AfterSendRequestToHost(host) == true) { return true; } controller_->Reset(); @@ -134,28 +186,13 @@ class CurvefsToolRpc : public CurvefsTool { return false; } - virtual int Init() { - channel_ = std::make_shared(); - controller_ = std::make_shared(); - request_ = std::make_shared(); - response_ = std::make_shared(); - service_stub_ = std::make_shared(channel_.get()); - if (updateFlagsFunc_.size() > 0) { - // need update FlagInfos - AddUpdateFlagsFuncs(); - } - - UpdateFlagsFromConf(); - return 0; - } - void AddUpdateFlagsFunc( const std::function& func) { updateFlagsFunc_.push_back(func); } - virtual void UpdateFlagsFromConf() { + virtual void UpdateFlags() { curve::common::Configuration conf; conf.SetConfigPath(FLAGS_confPath); if (!conf.LoadConfig()) { @@ -174,20 +211,39 @@ class CurvefsToolRpc : public CurvefsTool { * * @details * use AddUpdateFlagsFunc to add UpdateFlagsFunc into updateFlagsFunc_; - * add this function will be called in UpdateFlagsFromConf; - * this function should be called before UpdateFlagsFromConf (like Init()). + * add this function will be called in UpdateFlags; + * this function should be called before UpdateFlags (like Init()). */ - virtual void AddUpdateFlagsFuncs() = 0; + virtual void AddUpdateFlags() = 0; /** * @brief deal with response info, include output err info * * @param host - * @return true: send request success - * @return false send request failed + * @return true: send one request success + * @return false send one request failed * @details */ - virtual bool AfterSendRequestToService(const std::string& host) = 0; + virtual bool AfterSendRequestToHost(const std::string& host) = 0; + + /** + * @brief + * + * @details + * If necessary, you can override RunCommand in a subclass: + * CurvefsToolRpc::RunCommand(); + * RemoveFailHostFromHostAddr(); + * Add the fail host in AfterSendRequestToHost: + * failHostsAddr_.push_back(); + */ + void RemoveFailHostFromHostAddr() { + for (auto const& i : failHostsAddr_) { + hostsAddr_.erase( + std::remove_if(hostsAddr_.begin(), hostsAddr_.end(), + [i](decltype(i) j) { return i == j; }), + hostsAddr_.end()); + } + } protected: /** @@ -196,10 +252,16 @@ class CurvefsToolRpc : public CurvefsTool { * * @details */ - std::vector hostsAddressStr_; + std::vector hostsAddr_; + /** + * @brief The hosts that failed to send the request + * + * @details + */ + std::vector failHostsAddr_; std::shared_ptr channel_; std::shared_ptr controller_; - std::shared_ptr request_; + std::queue requestQueue_; // should be defined in Init() std::shared_ptr response_; std::shared_ptr service_stub_; /** @@ -223,6 +285,78 @@ class CurvefsToolRpc : public CurvefsTool { updateFlagsFunc_; }; +class CurvefsToolMetric : public CurvefsTool { + public: + explicit CurvefsToolMetric(const std::string& command, + const std::string& programe = kProgrameName, + bool show = true) + : CurvefsTool(command, programe, show) { + metricClient_ = std::make_shared(); + } + + int Init(const std::shared_ptr& metricClient); + + virtual void PrintHelp(); + + virtual void InitHostsAddr() {} + + protected: + void AddUpdateFlagsFunc( + const std::function& func); + + virtual int RunCommand(); + + virtual int Init(); + + /** + * @brief add AddUpdateFlagsFunc in Subclass + * + * @details + * use AddUpdateFlagsFunc to add UpdateFlagsFunc into updateFlagsFunc_; + * add this function will be called in UpdateFlags; + * this function should be called before UpdateFlags (like Init()). + * + */ + virtual void AddUpdateFlags(); + + virtual void UpdateFlags(); + + virtual void AfterGetMetric(const std::string mdsAddr, + const std::string& subUri, + const std::string& Value, + const MetricStatusCode& statusCode) = 0; + + /** + * @brief + * + * @return int + * 0: no error + * -1: error + * @details + */ + virtual int ProcessMetrics() = 0; + + void AddAddr2Suburi(const std::pair& addrSubUri); + + protected: + std::shared_ptr metricClient_; + /** + * @brief get metricName from addr + * + * @details + * first: addr second: MetricName + */ + std::vector> addr2SubUri; + /** + * @brief save the functor which defined in curvefs_tool_define.h + * + * @details + */ + std::vector> + updateFlagsFunc_; +}; } // namespace tools } // namespace curvefs diff --git a/curvefs/src/tools/curvefs_tool_define.cpp b/curvefs/src/tools/curvefs_tool_define.cpp index cc3b3749f8..4cd13b270a 100644 --- a/curvefs/src/tools/curvefs_tool_define.cpp +++ b/curvefs/src/tools/curvefs_tool_define.cpp @@ -21,18 +21,26 @@ */ #include "curvefs/src/tools/curvefs_tool_define.h" -DEFINE_string(mdsAddr, "127.0.0.1:6700", "mds addr"); +DEFINE_string(mdsAddr, "127.0.0.1:16700,127.0.0.1:26700", "mds addr"); +DEFINE_string(metaserverAddr, "127.0.0.1:16701,127.0.0.1:26701", + "metaserver addr"); +DEFINE_string(etcdAddr, "127.0.0.1:2379", "etcd addr"); DEFINE_bool(example, false, "print the example of usage"); -DEFINE_string(confPath, "curvefs/conf/tools.conf", "config file path of tools"); +DEFINE_string(confPath, "/etc/curvefs/tools.conf", "config file path of tools"); DEFINE_string(fsname, "curvefs", "fs name"); +DEFINE_string(fsId, "1,2,3", "fs id"); DEFINE_string(mountpoint, "127.0.0.1:/mnt/curvefs-umount-test", "curvefs mount in local path"); +DEFINE_uint64(rpcRetryTimes, 5, "rpc retry times"); +DEFINE_uint32(rpcTimeoutMs, 5000u, "rpc time out"); +DEFINE_string(copysetsId, "1,2,3", "copysets id"); +DEFINE_string(poolsId, "1,2,3", "pools id"); +DEFINE_bool(detail, false, "show more infomation"); // topology DEFINE_string(mds_addr, "127.0.0.1:6700", "mds ip and port, separated by \",\""); // NOLINT DEFINE_string(cluster_map, "topo_example.json", "cluster topology map."); -DEFINE_uint32(rpcTimeOutMs, 5000u, "rpc time out"); namespace curvefs { @@ -50,5 +58,122 @@ void SetFlagInfo(curve::common::Configuration* conf, std::function SetMdsAddr = std::bind(&SetFlagInfo, std::placeholders::_1, std::placeholders::_2, "mdsAddr", &FLAGS_mdsAddr); + +std::function + SetRpcTimeoutMs = + std::bind(&SetFlagInfo, std::placeholders::_1, + std::placeholders::_2, "rpcTimeoutMs", &FLAGS_rpcTimeoutMs); + +std::function + SetRpcRetryTimes = + std::bind(&SetFlagInfo, std::placeholders::_1, + std::placeholders::_2, "rpcRetryTimes", &FLAGS_rpcRetryTimes); + +std::function + SetMetaserverAddr = std::bind(&SetFlagInfo, + std::placeholders::_1, std::placeholders::_2, + "metaserverAddr", &FLAGS_metaserverAddr); + +std::function + SetEtcdAddr = std::bind(&SetFlagInfo, std::placeholders::_1, + std::placeholders::_2, "etcdAddr", &FLAGS_etcdAddr); + +auto StrVec2Str(const std::vector& strVec) -> std::string { + std::stringstream ret; + for (auto const& i : strVec) { + ret << i << " "; + } + return ret.str(); +}; + +std::string HeartbeatCopysetInfo2Str( + const mds::heartbeat::CopySetInfo& copySetInfo) { + std::stringstream ret; + ret << "poolId: " << copySetInfo.poolid() + << " copysetId: " << copySetInfo.copysetid() << " peers: [ "; + for (auto const& i : copySetInfo.peers()) { + ret << CommomPeer2Str(i) << " "; + } + ret << "] epoch: " << copySetInfo.epoch() + << "leader peer: " << CommomPeer2Str(copySetInfo.leaderpeer()) + << " partitioninfo: [ "; + for (auto const& i : copySetInfo.partitioninfolist()) { + CommomPartitionInfo2Str(i); + } + ret << " ]"; + return ret.str(); +} + +std::string CommomPeer2Str(const common::Peer& peer) { + std::stringstream ret; + ret << "id: " << peer.id() << " address: " << peer.address(); + return ret.str(); +} + +std::string CommomPartitionInfo2Str(const common::PartitionInfo& partition) { + std::stringstream ret; + ret << "fsId: " << partition.fsid() << " poolId: " << partition.poolid() + << " copysetId: " << partition.copysetid() + << " partitionId: " << partition.partitionid() + << " start: " << partition.start() << " end: " << partition.end() + << " txId: " << partition.txid() << " nextId: " << partition.nextid() + << " status: "; + if (partition.status() == common::PartitionStatus::READWRITE) { + ret << "rw "; + } else if (partition.status() == common::PartitionStatus::READONLY) { + ret << "r "; + } else { + ret << "unknown "; + } + ret << "inodeNum: " << partition.inodenum() + << " dentryNum: " << partition.status(); + return ret.str(); +} + +std::string MetadataserverCopysetCopysetStatusResponse2Str( + const metaserver::copyset::CopysetStatusResponse& response) { + std::stringstream ret; + ret << "op_status: " << CopysetOpStatus2Str(response.status()); + if (response.has_copysetstatus()) { + ret << response.copysetstatus().DebugString(); + } + return ret.str(); +} + +std::string CopysetOpStatus2Str( + const metaserver::copyset::COPYSET_OP_STATUS& op_status) { + std::stringstream ret; + switch (op_status) { + case metaserver::copyset::COPYSET_OP_STATUS::COPYSET_OP_STATUS_SUCCESS: + ret << "success "; + break; + case metaserver::copyset::COPYSET_OP_STATUS::COPYSET_OP_STATUS_EXIST: + ret << "exist "; + break; + case metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_COPYSET_NOTEXIST: + ret << "no exist "; + break; + case metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_COPYSET_IS_HEALTHY: + ret << "health "; + break; + case metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_PARSE_PEER_ERROR: + ret << "parse peer error "; + break; + case metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_PEER_MISMATCH: + ret << "parse mismatch "; + break; + case metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_FAILURE_UNKNOWN: + default: + ret << "failure unknown "; + break; + } + return ret.str(); +} + } // namespace tools } // namespace curvefs diff --git a/curvefs/src/tools/curvefs_tool_define.h b/curvefs/src/tools/curvefs_tool_define.h index 1b2038b0f3..91b2431282 100644 --- a/curvefs/src/tools/curvefs_tool_define.h +++ b/curvefs/src/tools/curvefs_tool_define.h @@ -26,48 +26,95 @@ #include #include +#include #include +#include +#include "curvefs/proto/copyset.pb.h" +#include "curvefs/proto/heartbeat.pb.h" #include "src/common/configuration.h" namespace curvefs { namespace tools { +/* command */ // programe name const char kProgrameName[] = "curvefs-tool"; - // version const char kVersionCmd[] = "version"; - // build-topology const char kBuildTopologyCmd[] = "build-topology"; - // umount -const char kUmountCmd[] = "umountfs"; +const char kUmountCmd[] = "umount-fs"; +// metadata-usage +const char kMetedataUsageCmd[] = "usage-metadata"; +// mds-status +const char kMdsStatusCmd[] = "status-mds"; +// metaserver-status +const char kMetaserverStatusCmd[] = "status-metaserver"; +// etcd-status +const char kEtcdStatusCmd[] = "status-etcd"; +// copysets-status +// check the cluster of copysets status +const char kCopysetsStatusCmd[] = "status-copysets"; +// copyset-query +const char kCopysetQueryCmd[] = "query-copyset"; +// list-fs-partition +const char kFsPartitionListCmd[] = "list-fs-partition"; +// fsinfo-list +const char kFsInfoListCmd[] = "list-fs"; +// list-fs-copysetid +const char kFsCopysetIdListCmd[] = "list-fs-copysetid"; +// no-invoke Used for commands that are not directly invoked +const char kNoInvokeCmd[] = "no-invoke"; // configure const char kConfPathHelp[] = "[-confPath=/etc/curvefs/tools.conf]"; - // kHelp const char kHelpStr[] = - "Usage: curve_ops_tool [Command] [OPTIONS...]\n" + "Usage: curvefs_tool [Command] [OPTIONS...]\n" "COMMANDS:\n" // NOLINT "version: show the version of cluster\n" "build-topology: build cluster topology based on topo.json\n" - "umountfs: umount curvefs from local and cluster" + "umount-fs: umount curvefs from local and cluster\n" + "usage-metadata: show the metadata usage of cluster\n" + "status-mds: show the status of mds\n" + "status-metaserver: show the status of metaserver\n" + "status-etcd: show the status of etcd\n" + "query-copyset: query copyset by copysetId\n" + "list-fs-partition: list partition in fs \n" + "list-fs: list all fs in cluster\n" "You can specify the config path by -confPath to avoid typing too many " "options\n"; // NOLINT +/* Status Host Type */ +const char kHostTypeMds[] = "mds"; +const char kHostTypeMetaserver[] = "metaserver"; +const char kHostTypeEtcd[] = "etcd"; + +/* Metric */ +const char kVersionUri[] = "/version"; +const char kStatusUri[] = "/vars/status"; +const char kMdsStatusUri[] = "/vars/curvefs_mds_status"; +const char kEtcdVersionUri[] = "/version"; +const char kEtcdStatusUri[] = "/v2/stats/self"; +const char kEtcdClusterVersionKey[] = "etcdcluster"; +const char kMdsStatusKey[] = "curvefs_mds_status"; +const char kEtcdStateKey[] = "state"; +const char kHostLeaderValue[] = "leader"; +const char kHostFollowerValue[] = "follower"; +const char kEtcdLeaderValue[] = "StateLeader"; +const char kEtcdFollowerValue[] = "StateFollower"; + } // namespace tools } // namespace curvefs namespace curvefs { namespace mds { namespace topology { -// topology +/* topology */ const int kRetCodeCommonErr = -1; const int kRetCodeRedirectMds = -2; - const char kPools[] = "pools"; const char kPool[] = "pool"; const char kServers[] = "servers"; @@ -81,6 +128,7 @@ const char kZone[] = "zone"; const char kReplicasNum[] = "replicasnum"; const char kCopysetNum[] = "copysetnum"; const char kZoneNum[] = "zonenum"; + } // namespace topology } // namespace mds } // namespace curvefs @@ -93,9 +141,36 @@ void SetFlagInfo(curve::common::Configuration* conf, google::CommandLineFlagInfo* info, const std::string& key, FlagInfoT* flag); +/* update flags */ extern std::function SetMdsAddr; +extern std::function + SetRpcTimeoutMs; +extern std::function + SetRpcRetryTimes; +extern std::function + SetMetaserverAddr; +extern std::function + SetEtcdAddr; + +/* translate to string */ +std::string StrVec2Str(const std::vector&); + +std::string HeartbeatCopysetInfo2Str(const mds::heartbeat::CopySetInfo&); + +std::string CommomPeer2Str(const common::Peer&); + +std::string CommomPartitionInfo2Str(const common::PartitionInfo&); + +std::string MetadataserverCopysetCopysetStatusResponse2Str( + const metaserver::copyset::CopysetStatusResponse&); + +std::string CopysetOpStatus2Str(const metaserver::copyset::COPYSET_OP_STATUS&); } // namespace tools } // namespace curvefs diff --git a/curvefs/src/tools/curvefs_tool_factory.cpp b/curvefs/src/tools/curvefs_tool_factory.cpp index 686c035fbb..4fc1f92d99 100644 --- a/curvefs/src/tools/curvefs_tool_factory.cpp +++ b/curvefs/src/tools/curvefs_tool_factory.cpp @@ -30,7 +30,7 @@ CurvefsToolFactory::CurvefsToolFactory() { RegisterCurvefsTool(std::string(kVersionCmd), CurvefsToolCreator::Create); - // umountfs + // umount-fs RegisterCurvefsTool(std::string(kUmountCmd), CurvefsToolCreator::Create); @@ -38,6 +38,34 @@ CurvefsToolFactory::CurvefsToolFactory() { RegisterCurvefsTool( std::string(kBuildTopologyCmd), CurvefsToolCreator::Create); + + // usage-metadata + RegisterCurvefsTool(std::string(kMetedataUsageCmd), + CurvefsToolCreator::Create); + + // status-mds + RegisterCurvefsTool(std::string(kMdsStatusCmd), + CurvefsToolCreator::Create); + // status-metaserver + RegisterCurvefsTool( + std::string(kMetaserverStatusCmd), + CurvefsToolCreator::Create); + + // status-etcd + RegisterCurvefsTool(std::string(kEtcdStatusCmd), + CurvefsToolCreator::Create); + + // list-fs-partition + RegisterCurvefsTool(std::string(kFsPartitionListCmd), + CurvefsToolCreator::Create); + + // list-fs + RegisterCurvefsTool(std::string(kFsInfoListCmd), + CurvefsToolCreator::Create); + + // query-copyset + RegisterCurvefsTool(std::string(kCopysetQueryCmd), + CurvefsToolCreator::Create); } std::shared_ptr CurvefsToolFactory::GenerateCurvefsTool( diff --git a/curvefs/src/tools/curvefs_tool_factory.h b/curvefs/src/tools/curvefs_tool_factory.h index b055048be9..994bb5f12d 100644 --- a/curvefs/src/tools/curvefs_tool_factory.h +++ b/curvefs/src/tools/curvefs_tool_factory.h @@ -28,9 +28,16 @@ #include #include +#include "curvefs/src/tools/build/curvefs_build_topology_tool.h" #include "curvefs/src/tools/curvefs_tool.h" #include "curvefs/src/tools/curvefs_tool_define.h" -#include "curvefs/src/tools/topology/curvefs_build_topology_tool.h" +#include "curvefs/src/tools/list/curvefs_fs_partition_list.h" +#include "curvefs/src/tools/list/curvefs_fsinfo_list.h" +#include "curvefs/src/tools/query/curvefs_copyset_query.h" +#include "curvefs/src/tools/space/curvefs_metadata_usage_tool.h" +#include "curvefs/src/tools/status/curvefs_etcd_status.h" +#include "curvefs/src/tools/status/curvefs_mds_status.h" +#include "curvefs/src/tools/status/curvefs_metaserver_status.h" #include "curvefs/src/tools/umountfs/curvefs_umountfs_tool.h" #include "curvefs/src/tools/version/curvefs_version_tool.h" diff --git a/curvefs/src/tools/curvefs_tool_main.cpp b/curvefs/src/tools/curvefs_tool_main.cpp index 7aee0cbb33..9f42c095d0 100644 --- a/curvefs/src/tools/curvefs_tool_main.cpp +++ b/curvefs/src/tools/curvefs_tool_main.cpp @@ -41,6 +41,7 @@ DECLARE_int32(health_check_interval); int main(int argc, char** argv) { google::SetUsageMessage(curvefs::tools::kHelpStr); google::ParseCommandLineFlags(&argc, &argv, true); + google::InitGoogleLogging(argv[0]); if (argc < 2) { std::cout << curvefs::tools::kHelpStr << std::endl; diff --git a/curvefs/src/tools/curvefs_tool_metric.cpp b/curvefs/src/tools/curvefs_tool_metric.cpp new file mode 100644 index 0000000000..7745291d94 --- /dev/null +++ b/curvefs/src/tools/curvefs_tool_metric.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-25 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/curvefs_tool_metric.h" + +DECLARE_uint32(rpcTimeoutMs); +DECLARE_uint64(rpcRetryTimes); + +namespace curvefs { +namespace tools { + +MetricStatusCode MetricClient::GetMetric(const std::string& addr, + const std::string& subUri, + std::string* value) { + brpc::Channel httpChannel; + brpc::ChannelOptions options; + brpc::Controller cntl; + options.protocol = brpc::PROTOCOL_HTTP; + int res = httpChannel.Init(addr.c_str(), &options); + if (res != 0) { + std::cerr << "init httpChannel to " << addr << " fail!" << std::endl; + return MetricStatusCode::kOtherErr; + } + + cntl.http_request().uri() = addr + subUri; + + // TODO(chengyi01): move to while + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); + httpChannel.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr); + if (!cntl.Failed()) { + *value = cntl.response_attachment().to_string(); + + return MetricStatusCode::kOK; + } + + bool needRetry = + (cntl.Failed() && cntl.ErrorCode() != EHOSTDOWN && + cntl.ErrorCode() != ETIMEDOUT && cntl.ErrorCode() != brpc::ELOGOFF && + cntl.ErrorCode() != brpc::ERPCTIMEDOUT); + uint64_t retryTimes = 0; + while (needRetry && retryTimes < FLAGS_rpcRetryTimes) { + cntl.Reset(); + cntl.http_request().uri() = addr + subUri; + cntl.set_timeout_ms(FLAGS_rpcTimeoutMs); + httpChannel.CallMethod(nullptr, &cntl, nullptr, nullptr, nullptr); + if (cntl.Failed()) { + retryTimes++; + continue; + } + *value = cntl.response_attachment().to_string(); + } + // Hand over the printing process to the outside + bool notExist = cntl.ErrorCode() == brpc::EHTTP && + cntl.http_response().status_code() == kHttpCodeNotFound; + return notExist ? MetricStatusCode::kNotFound : MetricStatusCode::kOtherErr; +} + +int MetricClient::GetKeyValueFromJson(const std::string& strJson, + const std::string& key, + std::string* value) { + Json::CharReaderBuilder reader; + std::stringstream ss(strJson); + std::string err; + Json::Value json; + bool parseCode = Json::parseFromStream(reader, ss, &json, &err); + if (!parseCode) { + std::cerr << "parse " << ss.str() << " fail " << err << std::endl; + return -1; + } + + int ret = -1; + + if (json[key].isNull()) { + std::cerr << "there is no " << key << " in " << json.asString() + << std::endl; + } else if (json[key].isString()) { + *value = json[key].asString(); + ret = 0; + } else { + std::cerr << "the key " << key << " in " << json.asString() + << " is not a string value." << std::endl; + } + return ret; +} + +int MetricClient::GetKeyValueFromString(const std::string& str, + const std::string& key, + std::string* value) { + auto pos = str.find(":"); + if (pos == std::string::npos) { + std::cout << "parse response attachment fail!" << std::endl; + return -1; + } + *value = str.substr(pos + 1); + TrimMetricString(value); + return 0; +} + +void MetricClient::TrimMetricString(std::string* str) { + str->erase(0, str->find_first_not_of(" ")); + str->erase(str->find_last_not_of("\r\n") + 1); + str->erase(0, str->find_first_not_of("\"")); + str->erase(str->find_last_not_of("\"") + 1); +} + +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/curvefs_tool_metric.h b/curvefs/src/tools/curvefs_tool_metric.h new file mode 100644 index 0000000000..15db986444 --- /dev/null +++ b/curvefs/src/tools/curvefs_tool_metric.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-25 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_CURVEFS_TOOL_METRIC_H_ +#define CURVEFS_SRC_TOOLS_CURVEFS_TOOL_METRIC_H_ + +#include +#include + +#include +#include + +#include "curvefs/src/tools/curvefs_tool_define.h" + +namespace curvefs { +namespace tools { + +enum class MetricStatusCode { + // success + kOK = 0, + // metric not found + kNotFound = -1, + // other error + kOtherErr = -2, +}; + +class MetricClient { + public: + static MetricStatusCode GetMetric(const std::string& addr, + const std::string& subUri, + std::string* value); + static int GetKeyValueFromJson(const std::string& strJson, + const std::string& key, std::string* value); + static int GetKeyValueFromString(const std::string& str, + const std::string& key, + std::string* value); + static void TrimMetricString(std::string* str); + + protected: + static const int kHttpCodeNotFound = 404; +}; + +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_CURVEFS_TOOL_METRIC_H_ diff --git a/curvefs/src/tools/list/curvefs_copysetid_list.cpp b/curvefs/src/tools/list/curvefs_copysetid_list.cpp new file mode 100644 index 0000000000..1882b6ccf6 --- /dev/null +++ b/curvefs/src/tools/list/curvefs_copysetid_list.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/list/curvefs_copysetid_list.h" + +DECLARE_string(metaserverAddr); + +namespace curvefs { +namespace tools { +namespace list { + +void FsCopysetIdListTool::PrintHelp() { + CurvefsToolRpc::PrintHelp(); + std::cout << " [-metaserver=" << FLAGS_metaserverAddr; + std::cout << std::endl; +} + +int FsCopysetIdListTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + curve::common::SplitString(FLAGS_metaserverAddr, ",", &hostsAddr_); + curvefs::metaserver::GetAllCopysetsIdRequest request; + AddRequest(request); + + service_stub_func_ = + std::bind(&curvefs::metaserver::MetaServerService_Stub::GetAllCopysetId, + service_stub_.get(), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, nullptr); + return 0; +} + +void FsCopysetIdListTool::AddUpdateFlags() { + AddUpdateFlagsFunc(SetMetaserverAddr); +} + +bool FsCopysetIdListTool::AfterSendRequestToHost(const std::string& host) { + bool ret = true; + if (controller_->Failed()) { + std::cerr << "get all copysetId from [ " << FLAGS_metaserverAddr + << "] fail, errorcode= " << controller_->ErrorCode() + << ", error text " << controller_->ErrorText() << std::endl; + ret = false; + } else if (response_->statuscode() != + curvefs::metaserver::MetaStatusCode::OK) { + std::cerr << "get copysetId from [ " << FLAGS_metaserverAddr + << " fail, error code is " << response_->statuscode() + << std::endl; + ret = false; + } else if (show_) { + std::cout << "copysetsId: [ "; + for (auto const& i : response_->copysetsid()) { + std::cout << i << " "; + } + std::cout << "]." << std::endl; + } + return ret; +} + +} // namespace list +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/list/curvefs_copysetid_list.h b/curvefs/src/tools/list/curvefs_copysetid_list.h new file mode 100644 index 0000000000..0cc8797c63 --- /dev/null +++ b/curvefs/src/tools/list/curvefs_copysetid_list.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-11-04 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_LIST_CURVEFS_COPYSETID_LIST_H_ +#define CURVEFS_SRC_TOOLS_LIST_CURVEFS_COPYSETID_LIST_H_ + +#include + +#include + +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace list { + +class FsCopysetIdListTool + : public CurvefsToolRpc { + public: + explicit FsCopysetIdListTool(const std::string& cmd = kFsCopysetIdListCmd, + bool show = true) + : CurvefsToolRpc(cmd) { + show_ = show; + } + void PrintHelp() override; + int Init() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; +}; + +} // namespace list +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_LIST_CURVEFS_COPYSETID_LIST_H_ diff --git a/curvefs/src/tools/list/curvefs_fs_partition_list.cpp b/curvefs/src/tools/list/curvefs_fs_partition_list.cpp new file mode 100644 index 0000000000..69931b184c --- /dev/null +++ b/curvefs/src/tools/list/curvefs_fs_partition_list.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/list/curvefs_fs_partition_list.h" + +DECLARE_string(fsId); +DECLARE_string(mdsAddr); + +namespace curvefs { +namespace tools { +namespace list { + +void FsPartitionListTool::PrintHelp() { + CurvefsToolRpc::PrintHelp(); + std::cout << " -fsId=" << FLAGS_fsId << " [-mdsAddr=" << FLAGS_mdsAddr + << "]"; + std::cout << std::endl; +} + +void FsPartitionListTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); +} + +int FsPartitionListTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); + std::vector fsIds; + curve::common::SplitString(FLAGS_fsId, ",", &fsIds); + curvefs::mds::topology::ListPartitionsRequest request; + for (auto const& i : fsIds) { + request.add_fsid(std::stoul(i)); + } + AddRequest(request); + + service_stub_func_ = + std::bind(&curvefs::mds::topology::TopologyService_Stub::ListPartitions, + service_stub_.get(), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, nullptr); + return 0; +} + +bool FsPartitionListTool::AfterSendRequestToHost(const std::string& host) { + bool ret = true; + if (controller_->Failed()) { + std::cerr << "get fs [ " << FLAGS_fsId + << " ] partition from mds: " << host + << " failed, errorcode= " << controller_->ErrorCode() + << ", error text " << controller_->ErrorText() << std::endl; + ret = false; + } else if (response_->statuscode() != + curvefs::mds::topology::TopoStatusCode::TOPO_OK) { + std::cerr << "get fs [ " << FLAGS_fsId + << " ] partition from mds: " << host + << " fail, error code is " << response_->statuscode() + << std::endl; + ret = false; + } else if (show_) { + for (auto const& i : requestQueue_.front().fsid()) { + auto fsid2part = response_->fsid2partitionlist(); + auto iPosition = fsid2part.find(i); + if (iPosition != fsid2part.end()) { + std::cout << "fsId: " << i << " partitionlist: [ "; + for (auto const& j : fsid2part[i].partitioninfolist()) { + std::cout << j.DebugString() << " "; + } + std::cout << "]" << std::endl; + } else { + std::cerr << "fsId: " << i << "not found partition." + << std::endl; + } + } + } + return ret; +} + +} // namespace list +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/list/curvefs_fs_partition_list.h b/curvefs/src/tools/list/curvefs_fs_partition_list.h new file mode 100644 index 0000000000..1ec2b0bedc --- /dev/null +++ b/curvefs/src/tools/list/curvefs_fs_partition_list.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_LIST_CURVEFS_FS_PARTITION_LIST_H_ +#define CURVEFS_SRC_TOOLS_LIST_CURVEFS_FS_PARTITION_LIST_H_ + +#include + +#include +#include + +#include "curvefs/proto/topology.pb.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace list { + +class FsPartitionListTool + : public CurvefsToolRpc { + public: + explicit FsPartitionListTool(const std::string& cmd = kFsPartitionListCmd, + bool show = true) + : CurvefsToolRpc(cmd) { + show_ = show; + } + + void PrintHelp() override; + int Init() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; +}; + +} // namespace list +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_LIST_CURVEFS_FS_PARTITION_LIST_H_ diff --git a/curvefs/src/tools/list/curvefs_fsinfo_list.cpp b/curvefs/src/tools/list/curvefs_fsinfo_list.cpp new file mode 100644 index 0000000000..1f30b854bd --- /dev/null +++ b/curvefs/src/tools/list/curvefs_fsinfo_list.cpp @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/list/curvefs_fsinfo_list.h" + +DECLARE_string(mdsAddr); +DECLARE_uint32(rpcTimeoutMs); + +namespace curvefs { +namespace tools { +namespace list { + +void FsInfoListTool::PrintHelp() { + CurvefsToolRpc::PrintHelp(); + std::cout << " [-mdsAddr=" << FLAGS_mdsAddr << "]" + << " [-rpcTimeoutMs=" << FLAGS_rpcTimeoutMs << "]"; + std::cout << std::endl; +} + +void FsInfoListTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); + AddUpdateFlagsFunc(curvefs::tools::SetRpcTimeoutMs); +} + +int FsInfoListTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); + + service_stub_func_ = + std::bind(&curvefs::mds::MdsService_Stub::ListClusterFsInfo, + service_stub_.get(), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, nullptr); + + curvefs::mds::ListClusterFsInfoRequest request; + AddRequest(request); + + controller_->set_timeout_ms(FLAGS_rpcTimeoutMs); + + return 0; +} + +bool FsInfoListTool::AfterSendRequestToHost(const std::string& host) { + bool ret = false; + if (controller_->Failed()) { + std::cerr << "get fsinfo from mds: " << host + << " failed, errorcode= " << controller_->ErrorCode() + << ", error text " << controller_->ErrorText() << "\n"; + } else if (show_) { + for (auto const& i : response_->fsinfo()) { + std::cout << i.DebugString() << std::endl; + } + + ret = true; + } + return ret; +} + +} // namespace list +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/list/curvefs_fsinfo_list.h b/curvefs/src/tools/list/curvefs_fsinfo_list.h new file mode 100644 index 0000000000..35a7fbed56 --- /dev/null +++ b/curvefs/src/tools/list/curvefs_fsinfo_list.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_LIST_CURVEFS_FSINFO_LIST_H_ +#define CURVEFS_SRC_TOOLS_LIST_CURVEFS_FSINFO_LIST_H_ + +#include +#include + +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace list { + +class FsInfoListTool + : public CurvefsToolRpc { + public: + explicit FsInfoListTool(const std::string& cmd = kFsInfoListCmd, + bool show = true) + : CurvefsToolRpc(cmd) { + show_ = show; + } + void PrintHelp() override; + int Init() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; +}; +} // namespace list +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_LIST_CURVEFS_FSINFO_LIST_H_ diff --git a/curvefs/src/tools/query/curvefs_copyset_query.cpp b/curvefs/src/tools/query/curvefs_copyset_query.cpp new file mode 100644 index 0000000000..770c5e7b74 --- /dev/null +++ b/curvefs/src/tools/query/curvefs_copyset_query.cpp @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-30 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/query/curvefs_copyset_query.h" + +DECLARE_string(copysetsId); +DECLARE_string(poolsId); +DECLARE_string(mdsAddr); +DECLARE_bool(detail); + +// used for CopysetStatusTool +DECLARE_string(metaserverAddr); + +namespace curvefs { +namespace tools { +namespace query { + +void CopysetQueryTool::PrintHelp() { + CurvefsToolRpc::PrintHelp(); + std::cout << " -copysetsId=" << FLAGS_copysetsId + << " -poolsId=" << FLAGS_poolsId << " [-mdsAddr=" << FLAGS_mdsAddr + << "]" + << " [-detail=" << FLAGS_detail << "]"; + std::cout << std::endl; +} + +void CopysetQueryTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); +} + +int CopysetQueryTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); + std::vector copysetsId; + curve::common::SplitString(FLAGS_copysetsId, ",", ©setsId); + std::vector poolsId; + curve::common::SplitString(FLAGS_poolsId, ",", &poolsId); + if (copysetsId.size() != poolsId.size() || poolsId.empty()) { + std::cerr << "copysets not match pools." << std::endl; + return -1; + } + curvefs::mds::topology::GetCopysetsInfoRequest request; + for (unsigned i = 0; i < poolsId.size(); ++i) { + curvefs::mds::topology::GetCopysetInfoRequest copyset; + copyset.set_poolid(std::stoul(poolsId[i])); + copyset.set_copysetid(std::stoul(copysetsId[i])); + *request.add_copysets() = copyset; + } + AddRequest(request); + + service_stub_func_ = std::bind( + &curvefs::mds::topology::TopologyService_Stub::GetCopysetsInfo, + service_stub_.get(), std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3, nullptr); + return 0; +} + +bool CopysetQueryTool::AfterSendRequestToHost(const std::string& host) { + bool ret = true; + if (controller_->Failed()) { + std::cerr << "query copysets [ " << FLAGS_copysetsId + << " ] from mds: " << host + << " failed, errorcode= " << controller_->ErrorCode() + << ", error text " << controller_->ErrorText() << "\n"; + ret = false; + } else { + // copysetId and copysetstatus get from metaserver + if (FLAGS_detail) { + using StatusRequestType = + curvefs::metaserver::copyset::CopysetsStatusRequest; + // metaserver ip and send to metaserver request + std::map> ip2requests; + for (auto const& i : response_->copysetsinfo()) { + using tmpType = + curvefs::metaserver::copyset::CopysetStatusRequest; + + if (i.statuscode() != + curvefs::mds::topology::TopoStatusCode::TOPO_OK) { + // some error in copyset + std::cout << "query copyset [ " << i.DebugString() + << " ] from mds error." << std::endl; + continue; + } + avaliableCopysetId.insert(i.copysetinfo().copysetid()); + + tmpType tmp; + tmp.set_copysetid(i.copysetinfo().copysetid()); + tmp.set_poolid(i.copysetinfo().poolid()); + for (auto const& j : i.copysetinfo().peers()) { + tmp.set_allocated_peer(new curvefs::common::Peer(j)); + std::string addr; + if (!curvefs::mds::topology::SplitPeerId(j.address(), + &addr)) { + std::cerr << "copyset[" << tmp.copysetid() + << "] has error peerid: " << j.address() + << std::endl; + break; + } + auto& queueRequest = ip2requests[addr]; + if (queueRequest.empty()) { + queueRequest.push(curvefs::metaserver::copyset:: + CopysetsStatusRequest()); + } + *queueRequest.front().add_copysets() = tmp; + } + } + for (auto const& i : ip2requests) { + // set host + FLAGS_metaserverAddr = i.first; + status::CopysetStatusTool copysetStatustool("", false); + copysetStatustool.Init(); + auto copysets = i.second.front().copysets(); + copysetStatustool.SetRequestQueue(i.second); + ret = copysetStatustool.RunCommand(); + auto copysetsStatus = copysetStatustool.GetResponse()->status(); + for (int m = 0, n = 0; + m < copysets.size() && n < copysetsStatus.size(); + ++m, ++n) { + id2Status_[copysets[m].copysetid()].push_back( + copysetsStatus[n]); + } + } + if (show_) { + for (auto const& i : avaliableCopysetId) { + std::cout << "copyset: " << i << " status:[ "; + for (auto const& j : id2Status_[i]) { + std::cout << j.DebugString() << " "; + } + std::cout << "]." << std::endl; + } + } + } else if (show_) { + for (auto const& i : response_->copysetsinfo()) { + std::cout << i.DebugString() << std::endl; + } + } + } + return ret; +} + +std::map> CopysetQueryTool::GetId2Status() { + return id2Status_; +} + +} // namespace query +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/query/curvefs_copyset_query.h b/curvefs/src/tools/query/curvefs_copyset_query.h new file mode 100644 index 0000000000..3d802b265c --- /dev/null +++ b/curvefs/src/tools/query/curvefs_copyset_query.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-30 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_QUERY_CURVEFS_COPYSET_QUERY_H_ +#define CURVEFS_SRC_TOOLS_QUERY_CURVEFS_COPYSET_QUERY_H_ + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/proto/topology.pb.h" +#include "curvefs/src/mds/topology/deal_peerid.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/status/curvefs_copyset_status.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace query { + +using StatusType = curvefs::metaserver::copyset::CopysetStatusResponse; + +class CopysetQueryTool + : public CurvefsToolRpc { + public: + explicit CopysetQueryTool(const std::string& cmd = kCopysetQueryCmd, + bool show = true) + : CurvefsToolRpc(cmd) { + show_ = show; + } + void PrintHelp() override; + int Init() override; + std::map> GetId2Status(); + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; + std::map> id2Status_; + std::set avaliableCopysetId; // save no error copysetId +}; + +} // namespace query +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_QUERY_CURVEFS_COPYSET_QUERY_H_ diff --git a/curvefs/src/tools/space/curvefs_metadata_usage_tool.cpp b/curvefs/src/tools/space/curvefs_metadata_usage_tool.cpp new file mode 100644 index 0000000000..864d9bb150 --- /dev/null +++ b/curvefs/src/tools/space/curvefs_metadata_usage_tool.cpp @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-22 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/space/curvefs_metadata_usage_tool.h" + +DECLARE_string(mdsAddr); + +namespace curvefs { +namespace tools { +namespace space { + +void MatedataUsageTool::PrintHelp() { + CurvefsToolRpc::PrintHelp(); + std::cout << " [-mdsAddr=" << FLAGS_mdsAddr << "]"; + std::cout << std::endl; +} + +void MatedataUsageTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); +} + +bool MatedataUsageTool::AfterSendRequestToHost(const std::string& host) { + bool ret = true; + if (controller_->Failed() || + response_->statuscode() != curvefs::mds::FSStatusCode::OK) { + // connect error or mds internal error + std::cerr << "get metadata usage from mds: " << host + << " failed, errorcode= " << controller_->ErrorCode() + << ", error text: " << controller_->ErrorText() << std::endl; + ret = false; + } else { + std::cout << "total: " << ToReadableByte(response_->total()) + << std::endl + << "used: " << ToReadableByte(response_->used()) << std::endl + << "left: " << ToReadableByte(response_->left()) << std::endl; + } + + return ret; +} + +int MatedataUsageTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); + service_stub_func_ = + std::bind(&curvefs::mds::MdsService_Stub::StatMetadataUsage, + service_stub_.get(), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, nullptr); + + curvefs::mds::StatMetadataUsageRequest request; + AddRequest(request); + return 0; +} +} // namespace space +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/space/curvefs_metadata_usage_tool.h b/curvefs/src/tools/space/curvefs_metadata_usage_tool.h new file mode 100644 index 0000000000..55bb133477 --- /dev/null +++ b/curvefs/src/tools/space/curvefs_metadata_usage_tool.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-22 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_SPACE_CURVEFS_METADATA_USAGE_TOOL_H_ +#define CURVEFS_SRC_TOOLS_SPACE_CURVEFS_METADATA_USAGE_TOOL_H_ + +#include +#include +#include +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/proto/space.pb.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/space/curvefs_space_base_tool.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace space { + +/** + * @brief this class is used to query the metadata usage of cluster + * + * @details + */ +class MatedataUsageTool + : public CurvefsToolRpc { + public: + explicit MatedataUsageTool(const std::string& cmd = kMetedataUsageCmd) + : CurvefsToolRpc(cmd) {} + void PrintHelp() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; + + private: + int Init() override; +}; + +} // namespace space +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_SPACE_CURVEFS_METADATA_USAGE_TOOL_H_ diff --git a/curvefs/src/tools/space/curvefs_space_base_tool.cpp b/curvefs/src/tools/space/curvefs_space_base_tool.cpp new file mode 100644 index 0000000000..872fc8736d --- /dev/null +++ b/curvefs/src/tools/space/curvefs_space_base_tool.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-22 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/space/curvefs_space_base_tool.h" + +namespace curvefs { +namespace tools { +namespace space { + +const uint64_t TB = 1024 * 1024 * 1024; +const uint64_t GB = 1024 * 1024; +const uint64_t MB = 1024; + +std::string ToReadableByte(uint64_t byte) { + // Convert byte KB to a appropriate magnitude + // like 1024KB to 1 MB + std::stringstream ss; + ss.setf(std::ios::fixed); + ss.precision(2); // 2 decimal places + if (byte >= 1 * TB) { + // TB + ss << double(byte) / double(TB) << " TB"; + } else if (byte >= 1 * GB) { + // GB + ss << double(byte) / double(GB) << " GB"; + } else if (byte >= 1024) { + // MB + ss << double(byte) / double(MB) << " MB"; + } else { + // KB + ss << byte << " KB"; + } + return ss.str(); +} +} // namespace space +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/space/curvefs_space_base_tool.h b/curvefs/src/tools/space/curvefs_space_base_tool.h new file mode 100644 index 0000000000..1d3d3d8e08 --- /dev/null +++ b/curvefs/src/tools/space/curvefs_space_base_tool.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-22 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_SPACE_CURVEFS_SPACE_BASE_TOOL_H_ +#define CURVEFS_SRC_TOOLS_SPACE_CURVEFS_SPACE_BASE_TOOL_H_ + +#include +#include + +namespace curvefs { +namespace tools { +namespace space { + +std::string ToReadableByte(uint64_t byte); + +} // namespace space +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_SPACE_CURVEFS_SPACE_BASE_TOOL_H_ diff --git a/curvefs/src/tools/status/curvefs_copyset_status.cpp b/curvefs/src/tools/status/curvefs_copyset_status.cpp new file mode 100644 index 0000000000..81040ede72 --- /dev/null +++ b/curvefs/src/tools/status/curvefs_copyset_status.cpp @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/status/curvefs_copyset_status.h" + +DECLARE_string(metaserverAddr); + +namespace curvefs { +namespace tools { +namespace status { + +int CopysetStatusTool::Init() { + if (CurvefsToolRpc::Init() != 0) { + return -1; + } + + curve::common::SplitString(FLAGS_metaserverAddr, ",", &hostsAddr_); + + service_stub_func_ = std::bind( + &curvefs::metaserver::copyset::CopysetService_Stub::GetCopysetsStatus, + service_stub_.get(), std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3, nullptr); + curvefs::metaserver::copyset::CopysetsStatusRequest request; + AddRequest(request); + return 0; +} + +int CopysetStatusTool::RunCommand() { + return CurvefsToolRpc::RunCommand(); +} + +bool CopysetStatusTool::AfterSendRequestToHost(const std::string& host) { + bool ret = true; + if (controller_->Failed()) { + std::cerr << "get copyset status from metaserver: " << host + << " failed, errorcode= " << controller_->ErrorCode() + << ", error text " << controller_->ErrorText() << "\n"; + ret = false; + } else { + if (show_) { + for (auto const& i : response_->status()) { + std::cout << MetadataserverCopysetCopysetStatusResponse2Str(i) + << std::endl; + } + } + for (auto const& i : response_->status()) { + auto status = i.status(); + if (status == metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_COPYSET_NOTEXIST || + status == metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_PARSE_PEER_ERROR || + status == metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_PEER_MISMATCH || + status == metaserver::copyset::COPYSET_OP_STATUS:: + COPYSET_OP_STATUS_FAILURE_UNKNOWN) { + // As long as there is one unhealthy and report unhealthy + ret = false; + break; + } + } + } + return ret; +} + +void CopysetStatusTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMetaserverAddr); +} + +} // namespace status +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/status/curvefs_copyset_status.h b/curvefs/src/tools/status/curvefs_copyset_status.h new file mode 100644 index 0000000000..745667d240 --- /dev/null +++ b/curvefs/src/tools/status/curvefs_copyset_status.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-31 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_STATUS_CURVEFS_COPYSET_STATUS_H_ +#define CURVEFS_SRC_TOOLS_STATUS_CURVEFS_COPYSET_STATUS_H_ + +#include + +#include +#include + +#include "curvefs/proto/copyset.pb.h" +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/list/curvefs_fs_partition_list.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace status { + +class CopysetStatusTool + : public CurvefsToolRpc< + curvefs::metaserver::copyset::CopysetsStatusRequest, + curvefs::metaserver::copyset::CopysetsStatusResponse, + curvefs::metaserver::copyset::CopysetService_Stub> { + public: + explicit CopysetStatusTool(const std::string& cmd = kNoInvokeCmd, + bool show = false) + : CurvefsToolRpc(cmd) { + show_ = show; + } + + int RunCommand(); + int Init() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; +}; + +} // namespace status +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_STATUS_CURVEFS_COPYSET_STATUS_H_ diff --git a/curvefs/src/tools/status/curvefs_etcd_status.cpp b/curvefs/src/tools/status/curvefs_etcd_status.cpp new file mode 100644 index 0000000000..5021786aad --- /dev/null +++ b/curvefs/src/tools/status/curvefs_etcd_status.cpp @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-29 + * Author: chengyi01 + */ +#include "curvefs/src/tools/status/curvefs_etcd_status.h" + +DECLARE_string(etcdAddr); + +namespace curvefs { +namespace tools { +namespace status { + +void EtcdStatusTool::PrintHelp() { + StatusBaseTool::PrintHelp(); + std::cout << " [-etcdAddr=" << FLAGS_etcdAddr << "]"; + std::cout << std::endl; +} + +int EtcdStatusTool::Init() { + if (CurvefsToolMetric::Init() != 0) { + return -1; + } + InitHostsAddr(); + + if (!hostsAddr_.empty()) { + // get version from 1 host, just ok + AddAddr2Suburi({hostsAddr_[0], kEtcdVersionUri}); + } + + // get status(leader or not) from all host + for (auto const& i : hostsAddr_) { + AddAddr2Suburi({i, kEtcdStatusUri}); + } + + return 0; +} + +void EtcdStatusTool::AfterGetMetric(const std::string hostAddr, + const std::string& subUri, + const std::string& value, + const MetricStatusCode& statusCode) { + if (statusCode == MetricStatusCode::kOK) { + if (subUri == kEtcdStatusUri) { + std::string keyValue; + if (!metricClient_->GetKeyValueFromJson(value, kEtcdStateKey, + &keyValue)) { + if (keyValue == kEtcdFollowerValue) { + // standby host + standbyHost_.push_back(hostAddr); + } else if (keyValue == kEtcdLeaderValue) { + // leader host + leaderHosts_.push_back(hostAddr); + } else { + // state is unkown + std::cerr << "etcd' state in" << hostAddr + << "/v2/stats/self is unkown." << std::endl; + standbyHost_.push_back(hostAddr); + } + } else { + // etcd version is not compatible uri:/v2/stats/self + std::cerr << "etcd in" << hostAddr + << " is not compatible with /v2/stats/self." + << std::endl; + offlineHosts_.push_back(hostAddr); + } + } else if (subUri == kEtcdVersionUri) { + std::string keyValue; + + if (!metricClient_->GetKeyValueFromJson( + value, kEtcdClusterVersionKey, &keyValue)) { + version_ = keyValue; + } + } + } else { + // offline host + offlineHosts_.push_back(hostAddr); + } +} + +void EtcdStatusTool::InitHostsAddr() { + curve::common::SplitString(FLAGS_etcdAddr, ",", &hostsAddr_); +} + +void EtcdStatusTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetEtcdAddr); + StatusBaseTool::AddUpdateFlags(); +} +} // namespace status +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/status/curvefs_etcd_status.h b/curvefs/src/tools/status/curvefs_etcd_status.h new file mode 100644 index 0000000000..f382a54632 --- /dev/null +++ b/curvefs/src/tools/status/curvefs_etcd_status.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-29 + * Author: chengyi01 + */ + +#ifndef CURVEFS_SRC_TOOLS_STATUS_CURVEFS_ETCD_STATUS_H_ +#define CURVEFS_SRC_TOOLS_STATUS_CURVEFS_ETCD_STATUS_H_ + +#include + +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/status/curvefs_status_base_tool.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace status { + +class EtcdStatusTool : public StatusBaseTool { + public: + explicit EtcdStatusTool(const std::string& cmd = kEtcdStatusCmd, + const std::string hostType = kHostTypeEtcd) + : StatusBaseTool(cmd, hostType) {} + void PrintHelp() override; + void InitHostsAddr() override; + + protected: + void AfterGetMetric(const std::string hostAddr, const std::string& subUri, + const std::string& value, + const MetricStatusCode& statusCode) override; + int Init() override; + void AddUpdateFlags() override; +}; + +} // namespace status +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_STATUS_CURVEFS_ETCD_STATUS_H_ diff --git a/curvefs/src/tools/status/curvefs_mds_status.cpp b/curvefs/src/tools/status/curvefs_mds_status.cpp new file mode 100644 index 0000000000..a2cdad89cc --- /dev/null +++ b/curvefs/src/tools/status/curvefs_mds_status.cpp @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-28 + * Author: chengyi01 + */ +#include "curvefs/src/tools/status/curvefs_mds_status.h" + +DECLARE_string(mdsAddr); + +namespace curvefs { +namespace tools { +namespace status { + +void MdsStatusTool::PrintHelp() { + StatusBaseTool::PrintHelp(); + std::cout << " [-mdsAddr=" << FLAGS_mdsAddr << "]"; + std::cout << std::endl; +} + +int MdsStatusTool::Init() { + versionSubUri_ = kVersionUri; + StatusSubUri_ = kMdsStatusUri; + statusKey_ = kMdsStatusKey; + return StatusBaseTool::Init(); +} + +void MdsStatusTool::InitHostsAddr() { + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); +} + +void MdsStatusTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); + StatusBaseTool::AddUpdateFlags(); +} + +} // namespace status +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/status/curvefs_mds_status.h b/curvefs/src/tools/status/curvefs_mds_status.h new file mode 100644 index 0000000000..147d27fbce --- /dev/null +++ b/curvefs/src/tools/status/curvefs_mds_status.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-28 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_STATUS_CURVEFS_MDS_STATUS_H_ +#define CURVEFS_SRC_TOOLS_STATUS_CURVEFS_MDS_STATUS_H_ + +#include + +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/status/curvefs_status_base_tool.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace status { + +class MdsStatusTool : public StatusBaseTool { + public: + explicit MdsStatusTool(const std::string& cmd = kMdsStatusCmd, + const std::string hostType = kHostTypeMds) + : StatusBaseTool(cmd, hostType) {} + void PrintHelp() override; + int Init() override; + + protected: + void InitHostsAddr() override; + void AddUpdateFlags() override; +}; + +} // namespace status +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_STATUS_CURVEFS_MDS_STATUS_H_ diff --git a/curvefs/src/tools/status/curvefs_metaserver_status.cpp b/curvefs/src/tools/status/curvefs_metaserver_status.cpp new file mode 100644 index 0000000000..bc8e205a88 --- /dev/null +++ b/curvefs/src/tools/status/curvefs_metaserver_status.cpp @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-28 + * Author: chengyi01 + */ +#include "curvefs/src/tools/status/curvefs_metaserver_status.h" + +DECLARE_string(metaserverAddr); + +namespace curvefs { +namespace tools { +namespace status { + +void MetaserverStatusTool::PrintHelp() { + StatusBaseTool::PrintHelp(); + std::cout << " [-metaserverAddr=" << FLAGS_metaserverAddr << "]"; + std::cout << std::endl; +} + +void MetaserverStatusTool::InitHostsAddr() { + curve::common::SplitString(FLAGS_metaserverAddr, ",", &hostsAddr_); +} + +void MetaserverStatusTool::AddUpdateFlags() { + AddUpdateFlagsFunc(curvefs::tools::SetMetaserverAddr); + StatusBaseTool::AddUpdateFlags(); +} + +int MetaserverStatusTool::ProcessMetrics() { + int ret = 0; + + // version + if (show_) { + std::cout << hostType_ << " version: " << version_ << std::endl; + } + + // online host + if (onlineHosts_.empty()) { + if (show_) { + std::cerr << "no online " << hostType_ << "." << std::endl; + } + ret = -1; + } else if (show_) { + std::cout << "online " << hostType_ << ": [ "; + for (auto const& i : onlineHosts_) { + std::cerr << i << " "; + } + std::cout << "]." << std::endl; + } + + // offline host + if (!offlineHosts_.empty()) { + ret = -1; + if (show_) { + std::cout << "offline " << hostType_ << ": [ "; + for (auto const& i : offlineHosts_) { + std::cerr << i << " "; + } + std::cout << "]." << std::endl; + } + } + + return ret; +} + +} // namespace status +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/status/curvefs_metaserver_status.h b/curvefs/src/tools/status/curvefs_metaserver_status.h new file mode 100644 index 0000000000..8a5a1a5f6a --- /dev/null +++ b/curvefs/src/tools/status/curvefs_metaserver_status.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-29 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_STATUS_CURVEFS_METASERVER_STATUS_H_ +#define CURVEFS_SRC_TOOLS_STATUS_CURVEFS_METASERVER_STATUS_H_ + +#include +#include + +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/status/curvefs_status_base_tool.h" +#include "src/common/string_util.h" + +namespace curvefs { +namespace tools { +namespace status { + +class MetaserverStatusTool : public StatusBaseTool { + public: + explicit MetaserverStatusTool( + const std::string& cmd = kMetaserverStatusCmd, + const std::string& hostType = kHostTypeMetaserver) + : StatusBaseTool(cmd, hostType) {} + void PrintHelp() override; + void InitHostsAddr() override; + + protected: + void AddUpdateFlags() override; + int ProcessMetrics() override; +}; + +} // namespace status +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_STATUS_CURVEFS_METASERVER_STATUS_H_ diff --git a/curvefs/src/tools/status/curvefs_status_base_tool.cpp b/curvefs/src/tools/status/curvefs_status_base_tool.cpp new file mode 100644 index 0000000000..4b80a9f39c --- /dev/null +++ b/curvefs/src/tools/status/curvefs_status_base_tool.cpp @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-26 + * Author: chengyi01 + */ + +#include "curvefs/src/tools/status/curvefs_status_base_tool.h" + +namespace curvefs { +namespace tools { +namespace status { + +int StatusBaseTool::Init() { + if (CurvefsToolMetric::Init() != 0) { + return -1; + } + + // get version from one host + if (!hostsAddr_.empty()) { + // get version from 1 host, just ok + AddAddr2Suburi({hostsAddr_[0], versionSubUri_}); + } + + // get status(leader or not) from all host + for (auto const& i : hostsAddr_) { + AddAddr2Suburi({i, StatusSubUri_}); + } + + return 0; +} + +void StatusBaseTool::AfterGetMetric(const std::string hostAddr, + const std::string& subUri, + const std::string& value, + const MetricStatusCode& statusCode) { + if (statusCode == MetricStatusCode::kOK) { + onlineHosts_.push_back(hostAddr); + if (subUri == StatusSubUri_) { + std::string keyValue; + if (!metricClient_->GetKeyValueFromString(value, statusKey_, + &keyValue)) { + if (keyValue == hostStandbyValue_) { + // standby host + standbyHost_.push_back(hostAddr); + } else if (keyValue == hostLeaderValue_) { + // leader host + leaderHosts_.push_back(hostAddr); + } else { + // error host + errorHosts_.push_back(hostAddr); + } + } else { + std::cerr << "parse " << statusKey_ << " form " << hostAddr + << subUri << " error." << std::endl; + errorHosts_.push_back(hostAddr); + } + } else if (subUri == versionSubUri_) { + version_ = value; + } + + } else { + // offline host + offlineHosts_.push_back(hostAddr); + } +} + +int StatusBaseTool::ProcessMetrics() { + int ret = 0; + + // version + if (show_) { + std::cout << hostType_ << " version: " << version_ << std::endl; + } + + // leader host + if (leaderHosts_.empty()) { + ret = -1; + if (show_) { + std::cerr << "no leader " << hostType_ << "." << std::endl; + } + } else if (leaderHosts_.size() > 1) { + ret = -1; + if (show_) { + std::cerr << "more than 1 leader " << hostType_ << ":[ "; + for (auto const& i : leaderHosts_) { + std::cerr << i << " "; + } + std::cerr << "]." << std::endl; + } + } else if (show_) { + std::cout << "leader " << hostType_ << ": " << leaderHosts_[0] + << std::endl; + } + + // standby host + if (show_) { + std::cout << "standy " << hostType_ << ": [ "; + for (auto const& i : standbyHost_) { + std::cerr << i << " "; + } + std::cout << "]." << std::endl; + } + + // error host + if (!errorHosts_.empty()) { + ret = -1; + if (show_) { + std::cerr << "error " << hostType_ << ": ["; + for (auto const& i : errorHosts_) { + std::cerr << i << " "; + } + std::cout << "]." << std::endl; + } + } + + // offline host + if (!offlineHosts_.empty()) { + ret = -1; + if (show_) { + std::cout << "offline " << hostType_ << ": [ "; + for (auto const& i : offlineHosts_) { + std::cerr << i << " "; + } + std::cout << "]." << std::endl; + } + } + return ret; +} + +} // namespace status +} // namespace tools +} // namespace curvefs diff --git a/curvefs/src/tools/status/curvefs_status_base_tool.h b/curvefs/src/tools/status/curvefs_status_base_tool.h new file mode 100644 index 0000000000..f06305d8d4 --- /dev/null +++ b/curvefs/src/tools/status/curvefs_status_base_tool.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Created Date: 2021-10-26 + * Author: chengyi01 + */ +#ifndef CURVEFS_SRC_TOOLS_STATUS_CURVEFS_STATUS_BASE_TOOL_H_ +#define CURVEFS_SRC_TOOLS_STATUS_CURVEFS_STATUS_BASE_TOOL_H_ + +#include +#include + +#include "curvefs/src/tools/curvefs_tool.h" +#include "curvefs/src/tools/curvefs_tool_define.h" +#include "curvefs/src/tools/curvefs_tool_metric.h" + +namespace curvefs { +namespace tools { +namespace status { + +/** + * @brief + * + * @tparam hostType will defined in curvefs/src/tools/curvefs_tool_define.h + * @details + */ +class StatusBaseTool : public CurvefsToolMetric { + public: + explicit StatusBaseTool( + const std::string& cmd, const std::string& hostType, + const std::string& hostLeaderValue = kHostLeaderValue, + const std::string& hostFollowerValue = kHostFollowerValue) + : CurvefsToolMetric(cmd), + hostType_(hostType), + hostLeaderValue_(hostLeaderValue), + hostStandbyValue_(hostFollowerValue) {} + virtual ~StatusBaseTool() {} + + protected: + void AfterGetMetric(const std::string hostAddr, const std::string& subUri, + const std::string& value, + const MetricStatusCode& statusCode); + int Init() override; + virtual int ProcessMetrics(); + + protected: + std::vector hostsAddr_; + std::vector standbyHost_; + std::vector errorHosts_; // not leader,not standby + std::vector offlineHosts_; + std::vector leaderHosts_; + std::vector onlineHosts_; + std::string version_; + std::string hostType_; + std::string versionSubUri_; + std::string StatusSubUri_; + std::string versionKey_; + std::string statusKey_; + std::string hostLeaderValue_; + std::string hostStandbyValue_; +}; + +} // namespace status +} // namespace tools +} // namespace curvefs + +#endif // CURVEFS_SRC_TOOLS_STATUS_CURVEFS_STATUS_BASE_TOOL_H_ diff --git a/curvefs/src/tools/umountfs/curvefs_umountfs_tool.cpp b/curvefs/src/tools/umountfs/curvefs_umountfs_tool.cpp index 4c5ce1af5c..a3e0cea769 100644 --- a/curvefs/src/tools/umountfs/curvefs_umountfs_tool.cpp +++ b/curvefs/src/tools/umountfs/curvefs_umountfs_tool.cpp @@ -33,7 +33,8 @@ namespace umountfs { void UmountfsTool::PrintHelp() { CurvefsToolRpc::PrintHelp(); std::cout << " -fsname=" << FLAGS_fsname - << " -mountpoint=" << FLAGS_mountpoint; + << " -mountpoint=" << FLAGS_mountpoint + << " [-mdsAddr=" << FLAGS_mdsAddr << "]"; std::cout << std::endl; } @@ -55,13 +56,10 @@ int UmountfsTool::RunCommand() { std::thread sysUmount(std::system, command.c_str()); // umount from cluster - if (!SendRequestToServices()) { - std::cerr << "send request to all mds failed." << std::endl; - ret = -1; - } + ret = CurvefsToolRpc::RunCommand(); sysUmount.join(); } catch (std::exception& e) { - std::cerr << "umount " << localPath + std::cerr << "system umount " << localPath << " failed, error info: " << e.what() << std::endl; ret = -1; } @@ -69,25 +67,31 @@ int UmountfsTool::RunCommand() { } int UmountfsTool::Init() { - CurvefsToolRpc::Init(); + int ret = CurvefsToolRpc::Init(); - curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddressStr_); - - request_->set_fsname(FLAGS_fsname); - request_->set_mountpoint(FLAGS_mountpoint); + // adjust the unique element of the queue + curvefs::mds::UmountFsRequest request; + request.set_fsname(FLAGS_fsname); + request.set_mountpoint(FLAGS_mountpoint); + AddRequest(request); service_stub_func_ = std::bind(&curvefs::mds::MdsService_Stub::UmountFs, service_stub_.get(), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, nullptr); - return 0; + return ret; } -void UmountfsTool::AddUpdateFlagsFuncs() { +void UmountfsTool::InitHostsAddr() { + curve::common::SplitString(FLAGS_mdsAddr, ",", &hostsAddr_); +} + +void UmountfsTool::AddUpdateFlags() { AddUpdateFlagsFunc(curvefs::tools::SetMdsAddr); } -bool UmountfsTool::AfterSendRequestToService(const std::string& host) { +bool UmountfsTool::AfterSendRequestToHost(const std::string& host) { + bool ret = false; if (controller_->Failed()) { std::cerr << "umountfs " << FLAGS_mountpoint << " from mds: " << host << " failed, errorcode= " << controller_->ErrorCode() @@ -97,9 +101,9 @@ bool UmountfsTool::AfterSendRequestToService(const std::string& host) { << response_->statuscode() << "\n"; } else { std::cout << "umount fs from cluster success.\n"; - return true; + ret = true; } - return false; + return ret; } } // namespace umountfs diff --git a/curvefs/src/tools/umountfs/curvefs_umountfs_tool.h b/curvefs/src/tools/umountfs/curvefs_umountfs_tool.h index 7f06114e35..0413c6c1dd 100644 --- a/curvefs/src/tools/umountfs/curvefs_umountfs_tool.h +++ b/curvefs/src/tools/umountfs/curvefs_umountfs_tool.h @@ -22,6 +22,8 @@ #ifndef CURVEFS_SRC_TOOLS_UMOUNTFS_CURVEFS_UMOUNTFS_TOOL_H_ #define CURVEFS_SRC_TOOLS_UMOUNTFS_CURVEFS_UMOUNTFS_TOOL_H_ +#include +#include #include #include // std::system @@ -32,10 +34,10 @@ #include #include //NOLINT #include +#include #include "curvefs/proto/mds.pb.h" #include "curvefs/src/tools/curvefs_tool.h" -#include "curvefs/src/tools/curvefs_tool_abstract_creator.h" #include "curvefs/src/tools/curvefs_tool_define.h" #include "src/common/string_util.h" @@ -43,22 +45,22 @@ namespace curvefs { namespace tools { namespace umountfs { -class UmountfsTool - : public CurvefsToolRpc< - brpc::Channel, brpc::Controller, curvefs::mds::UmountFsRequest, - curvefs::mds::UmountFsResponse, curvefs::mds::MdsService_Stub> { +class UmountfsTool : public CurvefsToolRpc { public: - UmountfsTool() - : CurvefsToolRpc(std::string(kUmountCmd), std::string(kProgrameName)) {} + explicit UmountfsTool(const std::string& cmd = kUmountCmd) + : CurvefsToolRpc(cmd) {} void PrintHelp() override; - protected: - void AddUpdateFlagsFuncs() override; - bool AfterSendRequestToService(const std::string& host) override; - - private: int RunCommand() override; int Init() override; + + void InitHostsAddr() override; + + protected: + void AddUpdateFlags() override; + bool AfterSendRequestToHost(const std::string& host) override; }; } // namespace umountfs diff --git a/curvefs/src/tools/version/curvefs_version_tool.h b/curvefs/src/tools/version/curvefs_version_tool.h index 7027081f0e..b311e11009 100644 --- a/curvefs/src/tools/version/curvefs_version_tool.h +++ b/curvefs/src/tools/version/curvefs_version_tool.h @@ -39,13 +39,10 @@ namespace version { class VersionTool : public CurvefsTool { public: - VersionTool() - : CurvefsTool(std::string(kVersionCmd), std::string(kProgrameName)) {} - VersionTool(const std::string& command, const std::string& programe) - : CurvefsTool(command, programe) {} + explicit VersionTool(const std::string& command = kVersionCmd) + : CurvefsTool(command) {} void PrintHelp() override; - private: int RunCommand() override; int Init() override; }; diff --git a/curvefs/test/tools/BUILD b/curvefs/test/tools/BUILD index 6d5dfdbbcb..169fb0b936 100644 --- a/curvefs/test/tools/BUILD +++ b/curvefs/test/tools/BUILD @@ -96,6 +96,35 @@ cc_test( "curvefs_build_topology_tool_test.cpp", "mock_topology_service.h", ]), + copts = COPTS, + deps = [ + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + "//curvefs/src/tools:curvefs_tools_lib", + ], +) + +# space-base-tool +cc_test( + name = "curvefs_space_base_tool_test", + srcs = glob([ + "curvefs_space_base_tool_test.cpp", + ]), + copts = COPTS, + deps = [ + "@com_google_googletest//:gtest_main", + "@com_google_googletest//:gtest", + "//curvefs/src/tools:curvefs_tools_lib", + ], +) + + +# space-status +cc_test( + name = "curvefs_space_status_tool_test", + srcs = glob([ + "curvefs_space_status_tool_test.cpp", + ]), copts = ["-g","-DHAVE_ZLIB=1"], deps = [ "@com_google_googletest//:gtest_main", diff --git a/curvefs/test/tools/curvefs_build_topology_tool_test.cpp b/curvefs/test/tools/curvefs_build_topology_tool_test.cpp index 58fea5447b..7e14c96f48 100644 --- a/curvefs/test/tools/curvefs_build_topology_tool_test.cpp +++ b/curvefs/test/tools/curvefs_build_topology_tool_test.cpp @@ -20,7 +20,7 @@ * @Author: chengyi01 */ -#include "curvefs/src/tools/topology/curvefs_build_topology_tool.h" +#include "curvefs/src/tools/build/curvefs_build_topology_tool.h" #include #include diff --git a/curvefs/test/tools/curvefs_space_base_tool_test.cpp b/curvefs/test/tools/curvefs_space_base_tool_test.cpp new file mode 100644 index 0000000000..0b083da972 --- /dev/null +++ b/curvefs/test/tools/curvefs_space_base_tool_test.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * @Project: curve + * @Date: 2021-10-22 + * @Author: chengyi01 + */ + +#include "curvefs/src/tools/space/curvefs_space_base_tool.h" + +#include +#include + +namespace curvefs { +namespace tools { +namespace space { + +class BaseToolTest : public testing::Test {}; + +TEST(BaseToolTest, KB_test) { + ASSERT_EQ(ToReadableByte(521), "521 KB"); +} + +TEST(BaseToolTest, MB_test) { + ASSERT_EQ(ToReadableByte(2 * 1024 + 1), "2.00 MB"); +} + +TEST(BaseToolTest, GB_test) { + ASSERT_EQ(ToReadableByte(3 * 1024 * 1024 + 1023 * 1024), "4.00 GB"); +} + +TEST(BaseToolTest, TB_test) { + uint64_t byte = 100ull * 1024 * 1024 * 1024; + ASSERT_EQ(ToReadableByte(byte), "100.00 TB"); +} + +} // namespace space +} // namespace tools +} // namespace curvefs diff --git a/curvefs/test/tools/curvefs_umountfs_tool_test.cpp b/curvefs/test/tools/curvefs_umountfs_tool_test.cpp index aa9760d720..0746e1056b 100644 --- a/curvefs/test/tools/curvefs_umountfs_tool_test.cpp +++ b/curvefs/test/tools/curvefs_umountfs_tool_test.cpp @@ -28,6 +28,7 @@ #include #include +#include #include "curvefs/proto/mds.pb.h" #include "curvefs/src/mds/common/mds_define.h" @@ -94,7 +95,7 @@ TEST_F(UmountfsToolTest, test_umount_failed) { ASSERT_EQ(ret, -1); } -// connect to nds failed +// connect to mds failed TEST_F(UmountfsToolTest, test_umount_connect_failed) { FLAGS_mdsAddr = "127.0.0.1:6700"; int ret = ut_.Run(); @@ -124,8 +125,11 @@ TEST_F(UmountfsToolTest, test_umount_tool_init) { std::shared_ptr channel = std::make_shared(); std::shared_ptr controller = std::make_shared(); - std::shared_ptr request = - std::make_shared(); + curvefs::mds::UmountFsRequest request; + request.set_fsname("123"); + request.set_mountpoint(FLAGS_mountpoint); + std::queue requestQueue; + requestQueue.push(request); std::shared_ptr response = std::make_shared(); @@ -136,9 +140,14 @@ TEST_F(UmountfsToolTest, test_umount_tool_init) { std::shared_ptr service_stub = std::make_shared(channel.get()); - ut_.CurvefsToolRpc::Init(channel, controller, request, response, - service_stub); - int ret = ut_.Run(); + + ut_.CurvefsToolRpc::Init( + channel, controller, requestQueue, response, service_stub, + std::bind(&curvefs::mds::MdsService_Stub::UmountFs, service_stub.get(), + std::placeholders::_1, std::placeholders::_2, + std::placeholders::_3, nullptr)); + + int ret = ut_.RunCommand(); ASSERT_EQ(ret, 0); }