diff --git a/include/pika_acl.h b/include/pika_acl.h index cd2d942547..8d830581f8 100644 --- a/include/pika_acl.h +++ b/include/pika_acl.h @@ -19,8 +19,8 @@ class PikaAclCmd : public Cmd { subCmdName_ = {"cat", "deluser", "dryrun", "genpass", "getuser", "list", "load", "log", "save", "setuser", "users", "whoami", "help"}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PikaAclCmd(*this); } diff --git a/include/pika_admin.h b/include/pika_admin.h index 601040f4a5..a61c77cebe 100644 --- a/include/pika_admin.h +++ b/include/pika_admin.h @@ -27,8 +27,8 @@ class SlaveofCmd : public Cmd { public: SlaveofCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlaveofCmd(*this); } @@ -48,8 +48,8 @@ class DbSlaveofCmd : public Cmd { public: DbSlaveofCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DbSlaveofCmd(*this); } @@ -72,8 +72,8 @@ class DbSlaveofCmd : public Cmd { class AuthCmd : public Cmd { public: AuthCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new AuthCmd(*this); } @@ -85,8 +85,8 @@ class BgsaveCmd : public Cmd { public: BgsaveCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new BgsaveCmd(*this); } @@ -100,8 +100,8 @@ class CompactCmd : public Cmd { public: CompactCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new CompactCmd(*this); } @@ -119,8 +119,8 @@ class CompactCmd : public Cmd { class CompactRangeCmd : public Cmd { public: CompactRangeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new CompactRangeCmd(*this); } @@ -142,8 +142,8 @@ class PurgelogstoCmd : public Cmd { public: PurgelogstoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PurgelogstoCmd(*this); } @@ -156,8 +156,8 @@ class PurgelogstoCmd : public Cmd { class PingCmd : public Cmd { public: PingCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PingCmd(*this); } @@ -168,8 +168,8 @@ class PingCmd : public Cmd { class SelectCmd : public Cmd { public: SelectCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SelectCmd(*this); } @@ -177,25 +177,26 @@ class SelectCmd : public Cmd { void DoInitial() override; void Clear() override { db_name_.clear(); } std::string db_name_; - std::shared_ptr select_db_; }; class FlushallCmd : public Cmd { public: FlushallCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)) {} - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new FlushallCmd(*this); } void Execute() override; void FlushAllWithoutLock(); + void DoBinlog(std::shared_ptr sync_db_); + private: void DoInitial() override; std::string ToRedisProtocol() override; - void DoWithoutLock(std::shared_ptr slot); + void DoWithoutLock(std::shared_ptr db); }; class FlushdbCmd : public Cmd { @@ -204,13 +205,13 @@ class FlushdbCmd : public Cmd { : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)) {} // The flush command belongs to the write categories, so the key cannot be empty std::vector current_key() const override { return {""}; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new FlushdbCmd(*this); } - void FlushAllSlotsWithoutLock(std::shared_ptr db); + void FlushAllDBsWithoutLock(); void Execute() override; std::string GetFlushDname() { return db_name_; } @@ -218,7 +219,7 @@ class FlushdbCmd : public Cmd { std::string db_name_; void DoInitial() override; void Clear() override { db_name_.clear(); } - void DoWithoutLock(std::shared_ptr slot); + void DoWithoutLock(); }; class ClientCmd : public Cmd { @@ -226,10 +227,10 @@ class ClientCmd : public Cmd { ClientCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) { subCmdName_ = {"getname", "setname", "list", "addr", "kill"}; } - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; const static std::string CLIENT_LIST_S; const static std::string CLIENT_KILL_S; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ClientCmd(*this); } @@ -260,8 +261,8 @@ class InfoCmd : public Cmd { }; InfoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new InfoCmd(*this); } void Execute() override; @@ -301,14 +302,13 @@ class InfoCmd : public Cmd { void InfoStats(std::string& info); void InfoExecCount(std::string& info); void InfoCPU(std::string& info); - void InfoShardingReplication(std::string& info); void InfoReplication(std::string& info); void InfoKeyspace(std::string& info); void InfoData(std::string& info); void InfoRocksDB(std::string& info); void InfoDebug(std::string& info); void InfoCommandStats(std::string& info); - void InfoCache(std::string& info, std::shared_ptr slot); + void InfoCache(std::string& info, std::shared_ptr db); std::string CacheStatusToString(int status); }; @@ -317,8 +317,8 @@ class ShutdownCmd : public Cmd { public: ShutdownCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ShutdownCmd(*this); } @@ -332,8 +332,8 @@ class ConfigCmd : public Cmd { : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) { subCmdName_ = {"get", "set", "rewrite", "resetstat"}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ConfigCmd(*this); } void Execute() override; @@ -342,7 +342,7 @@ class ConfigCmd : public Cmd { std::vector config_args_v_; void DoInitial() override; void ConfigGet(std::string& ret); - void ConfigSet(std::string& ret, std::shared_ptr slot); + void ConfigSet(std::string& ret, std::shared_ptr db); void ConfigRewrite(std::string& ret); void ConfigResetstat(std::string& ret); void ConfigRewriteReplicationID(std::string& ret); @@ -352,8 +352,8 @@ class MonitorCmd : public Cmd { public: MonitorCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new MonitorCmd(*this); } @@ -365,8 +365,8 @@ class DbsizeCmd : public Cmd { public: DbsizeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DbsizeCmd(*this); } @@ -377,8 +377,8 @@ class DbsizeCmd : public Cmd { class TimeCmd : public Cmd { public: TimeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new TimeCmd(*this); } @@ -389,8 +389,8 @@ class TimeCmd : public Cmd { class LastsaveCmd : public Cmd { public: LastsaveCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new LastsaveCmd(*this); } @@ -402,8 +402,8 @@ class DelbackupCmd : public Cmd { public: DelbackupCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DelbackupCmd(*this); } @@ -415,8 +415,8 @@ class EchoCmd : public Cmd { public: EchoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} void Merge() override{}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; Cmd* Clone() override { return new EchoCmd(*this); } private: @@ -427,8 +427,8 @@ class EchoCmd : public Cmd { class ScandbCmd : public Cmd { public: ScandbCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ScandbCmd(*this); } @@ -443,8 +443,8 @@ class SlowlogCmd : public Cmd { enum SlowlogCondition { kGET, kLEN, kRESET }; SlowlogCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlowlogCmd(*this); } @@ -462,8 +462,8 @@ class PaddingCmd : public Cmd { public: PaddingCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PaddingCmd(*this); } @@ -476,8 +476,8 @@ class PKPatternMatchDelCmd : public Cmd { public: PKPatternMatchDelCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKPatternMatchDelCmd(*this); } @@ -492,8 +492,8 @@ class DummyCmd : public Cmd { DummyCmd() : Cmd("", 0, 0) {} DummyCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DummyCmd(*this); } @@ -504,8 +504,8 @@ class DummyCmd : public Cmd { class QuitCmd : public Cmd { public: QuitCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new QuitCmd(*this); } @@ -516,8 +516,8 @@ class QuitCmd : public Cmd { class HelloCmd : public Cmd { public: HelloCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HelloCmd(*this); } @@ -529,8 +529,8 @@ class DiskRecoveryCmd : public Cmd { public: DiskRecoveryCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DiskRecoveryCmd(*this); } @@ -543,11 +543,10 @@ class ClearReplicationIDCmd : public Cmd { public: ClearReplicationIDCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::ADMIN)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ClearReplicationIDCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override {}; private: void DoInitial() override; @@ -556,8 +555,8 @@ class ClearReplicationIDCmd : public Cmd { class DisableWalCmd : public Cmd { public: DisableWalCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DisableWalCmd(*this); } @@ -569,8 +568,8 @@ class CacheCmd : public Cmd { public: enum CacheCondition {kCLEAR_DB, kCLEAR_HITRATIO, kDEL_KEYS, kRANDOM_KEY}; CacheCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new CacheCmd(*this); } @@ -587,8 +586,8 @@ class CacheCmd : public Cmd { class ClearCacheCmd : public Cmd { public: ClearCacheCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ClearCacheCmd(*this); } @@ -600,8 +599,8 @@ class ClearCacheCmd : public Cmd { class CommandCmd : public Cmd { public: CommandCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new CommandCmd(*this); } diff --git a/include/pika_binlog.h b/include/pika_binlog.h index c2b4f9f650..6a1e8aa1ca 100644 --- a/include/pika_binlog.h +++ b/include/pika_binlog.h @@ -37,7 +37,7 @@ class Version final : public pstd::noncopyable { void debug() { std::shared_lock l(rwlock_); - printf("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); + printf("Current pro_num %u pro_offset %llu\n", pro_num_, pro_offset_); } private: @@ -63,12 +63,8 @@ class Binlog : public pstd::noncopyable { // Need to hold Lock(); pstd::Status Truncate(uint32_t pro_num, uint64_t pro_offset, uint64_t index); - uint64_t file_size() { return file_size_; } - std::string filename() { return filename_; } - bool IsBinlogIoError() { return binlog_io_error_; } - // need to hold mutex_ void SetTerm(uint32_t term) { std::lock_guard l(version_->rwlock_); @@ -85,11 +81,9 @@ class Binlog : public pstd::noncopyable { private: pstd::Status Put(const char* item, int len); + pstd::Status EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, int* temp_pro_offset); static pstd::Status AppendPadding(pstd::WritableFile* file, uint64_t* len); - // pstd::WritableFile *queue() { return queue_; } - void InitLogFile(); - pstd::Status EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, int* temp_pro_offset); /* * Produce @@ -109,8 +103,6 @@ class Binlog : public pstd::noncopyable { int block_offset_ = 0; - char* pool_ = nullptr; - bool exit_all_consume_ = false; const std::string binlog_path_; uint64_t file_size_ = 0; @@ -118,8 +110,6 @@ class Binlog : public pstd::noncopyable { std::string filename_; std::atomic binlog_io_error_; - // Not use - // int32_t retry_; }; #endif diff --git a/include/pika_binlog_reader.h b/include/pika_binlog_reader.h index a3192def4d..1d604b02f7 100644 --- a/include/pika_binlog_reader.h +++ b/include/pika_binlog_reader.h @@ -16,9 +16,6 @@ #include "include/pika_binlog.h" -// using pstd::Slice; -// using pstd::Status; - class PikaBinlogReader { public: PikaBinlogReader(uint32_t cur_filenum, uint64_t cur_offset); diff --git a/include/pika_bit.h b/include/pika_bit.h index fc3c29a226..94e7767b16 100644 --- a/include/pika_bit.h +++ b/include/pika_bit.h @@ -11,7 +11,6 @@ #include "include/acl.h" #include "include/pika_command.h" #include "include/pika_kv.h" -#include "include/pika_slot.h" /* * bitoperation @@ -25,11 +24,11 @@ class BitGetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new BitGetCmd(*this); } @@ -53,10 +52,10 @@ class BitSetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new BitSetCmd(*this); } @@ -82,11 +81,11 @@ class BitCountCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new BitCountCmd(*this); } @@ -114,11 +113,11 @@ class BitPosCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new BitPosCmd(*this); } @@ -145,7 +144,7 @@ class BitOpCmd : public Cmd { public: BitOpCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::BITMAP)) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); }; BitOpCmd(const BitOpCmd& other) : Cmd(other), @@ -153,17 +152,17 @@ class BitOpCmd : public Cmd { src_keys_(other.src_keys_), op_(other.op_), value_to_dest_(other.value_to_dest_) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } std::vector current_key() const override { return {dest_key_}; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new BitOpCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::string dest_key_; diff --git a/include/pika_cache.h b/include/pika_cache.h index e96b8a4522..fa5e090af6 100644 --- a/include/pika_cache.h +++ b/include/pika_cache.h @@ -10,14 +10,19 @@ #include #include -#include "include/pika_server.h" #include "include/pika_define.h" #include "include/pika_zset.h" +#include "include/pika_command.h" #include "pstd/include/pstd_mutex.h" #include "pstd/include/pstd_status.h" #include "cache/include/cache.h" #include "storage/storage.h" +class PikaCacheLoadThread; +class ZIncrbyCmd; +class ZRangebyscoreCmd; +class ZRevrangebyscoreCmd; +class ZCountCmd; enum RangeStatus { RangeError = 1, RangeHit, RangeMiss }; struct CacheInfo { @@ -41,10 +46,8 @@ struct CacheInfo { } }; -class PikaCacheLoadThread; class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this { public: - PikaCache(int zset_cache_start_pos, int zset_cache_field_num_per_key); ~PikaCache(); @@ -57,159 +60,155 @@ class PikaCache : public pstd::noncopyable, public std::enable_shared_from_this< int CacheStatus(void); void ClearHitRatio(void); // Normal Commands - void Info(CacheInfo &info); - long long DbSize(void); - bool Exists(std::string &key); - void FlushSlot(void); - void ActiveExpireCycle(); + void Info(CacheInfo& info); + bool Exists(std::string& key); + void FlushCache(void); void ProcessCronTask(void); - rocksdb::Status Del(const std::vector &keys); - rocksdb::Status Expire(std::string &key, int64_t ttl); - rocksdb::Status Expireat(std::string &key, int64_t ttl); - rocksdb::Status TTL(std::string &key, int64_t *ttl); - rocksdb::Status Persist(std::string &key); - rocksdb::Status Type(std::string &key, std::string *value); - rocksdb::Status RandomKey(std::string *key); + rocksdb::Status Del(const std::vector& keys); + rocksdb::Status Expire(std::string& key, int64_t ttl); + rocksdb::Status Expireat(std::string& key, int64_t ttl); + rocksdb::Status TTL(std::string& key, int64_t* ttl); + rocksdb::Status Persist(std::string& key); + rocksdb::Status Type(std::string& key, std::string* value); + rocksdb::Status RandomKey(std::string* key); rocksdb::Status GetType(const std::string& key, bool single, std::vector& types); // String Commands - rocksdb::Status Set(std::string &key, std::string &value, int64_t ttl); - rocksdb::Status SetWithoutTTL(std::string &key, std::string &value); - rocksdb::Status Setnx(std::string &key, std::string &value, int64_t ttl); - rocksdb::Status SetnxWithoutTTL(std::string &key, std::string &value); - rocksdb::Status Setxx(std::string &key, std::string &value, int64_t ttl); - rocksdb::Status SetxxWithoutTTL(std::string &key, std::string &value); + rocksdb::Status Set(std::string& key, std::string& value, int64_t ttl); + rocksdb::Status Setnx(std::string& key, std::string& value, int64_t ttl); + rocksdb::Status SetnxWithoutTTL(std::string& key, std::string& value); + rocksdb::Status Setxx(std::string& key, std::string& value, int64_t ttl); + rocksdb::Status SetxxWithoutTTL(std::string& key, std::string& value); rocksdb::Status MSet(const std::vector& kvs); - rocksdb::Status Get(std::string &key, std::string *value); + rocksdb::Status Get(std::string& key, std::string* value); rocksdb::Status MGet(const std::vector& keys, std::vector* vss); - rocksdb::Status Incrxx(std::string &key); - rocksdb::Status Decrxx(std::string &key); - rocksdb::Status IncrByxx(std::string &key, uint64_t incr); - rocksdb::Status DecrByxx(std::string &key, uint64_t incr); - rocksdb::Status Incrbyfloatxx(std::string &key, long double incr); - rocksdb::Status Appendxx(std::string &key, std::string &value); - rocksdb::Status GetRange(std::string &key, int64_t start, int64_t end, std::string *value); - rocksdb::Status SetRangexx(std::string &key, int64_t start, std::string &value); - rocksdb::Status Strlen(std::string &key, int32_t *len); + rocksdb::Status Incrxx(std::string& key); + rocksdb::Status Decrxx(std::string& key); + rocksdb::Status IncrByxx(std::string& key, uint64_t incr); + rocksdb::Status DecrByxx(std::string& key, uint64_t incr); + rocksdb::Status Incrbyfloatxx(std::string& key, long double incr); + rocksdb::Status Appendxx(std::string& key, std::string& value); + rocksdb::Status GetRange(std::string& key, int64_t start, int64_t end, std::string* value); + rocksdb::Status SetRangexx(std::string& key, int64_t start, std::string& value); + rocksdb::Status Strlen(std::string& key, int32_t* len); // Hash Commands - rocksdb::Status HDel(std::string &key, std::vector &fields); - rocksdb::Status HSet(std::string &key, std::string &field, std::string &value); - rocksdb::Status HSetIfKeyExist(std::string &key, std::string &field, std::string &value); - rocksdb::Status HSetIfKeyExistAndFieldNotExist(std::string &key, std::string &field, std::string &value); - rocksdb::Status HMSet(std::string &key, std::vector &fvs); - rocksdb::Status HMSetnx(std::string &key, std::vector &fvs, int64_t ttl); - rocksdb::Status HMSetnxWithoutTTL(std::string &key, std::vector &fvs); - rocksdb::Status HMSetxx(std::string &key, std::vector &fvs); - rocksdb::Status HGet(std::string &key, std::string &field, std::string *value); - rocksdb::Status HMGet(std::string &key, std::vector &fields, std::vector *vss); - rocksdb::Status HGetall(std::string &key, std::vector *fvs); - rocksdb::Status HKeys(std::string &key, std::vector *fields); - rocksdb::Status HVals(std::string &key, std::vector *values); - rocksdb::Status HExists(std::string &key, std::string &field); - rocksdb::Status HIncrbyxx(std::string &key, std::string &field, int64_t value); - rocksdb::Status HIncrbyfloatxx(std::string &key, std::string &field, long double value); - rocksdb::Status HLen(std::string &key, uint64_t *len); - rocksdb::Status HStrlen(std::string &key, std::string &field, uint64_t *len); + rocksdb::Status HDel(std::string& key, std::vector& fields); + rocksdb::Status HSet(std::string& key, std::string& field, std::string& value); + rocksdb::Status HSetIfKeyExist(std::string& key, std::string& field, std::string& value); + rocksdb::Status HSetIfKeyExistAndFieldNotExist(std::string& key, std::string& field, std::string& value); + rocksdb::Status HMSet(std::string& key, std::vector& fvs); + rocksdb::Status HMSetnx(std::string& key, std::vector& fvs, int64_t ttl); + rocksdb::Status HMSetnxWithoutTTL(std::string& key, std::vector& fvs); + rocksdb::Status HMSetxx(std::string& key, std::vector& fvs); + rocksdb::Status HGet(std::string& key, std::string& field, std::string* value); + rocksdb::Status HMGet(std::string& key, std::vector& fields, std::vector* vss); + rocksdb::Status HGetall(std::string& key, std::vector* fvs); + rocksdb::Status HKeys(std::string& key, std::vector* fields); + rocksdb::Status HVals(std::string& key, std::vector* values); + rocksdb::Status HExists(std::string& key, std::string& field); + rocksdb::Status HIncrbyxx(std::string& key, std::string& field, int64_t value); + rocksdb::Status HIncrbyfloatxx(std::string& key, std::string& field, long double value); + rocksdb::Status HLen(std::string& key, uint64_t* len); + rocksdb::Status HStrlen(std::string& key, std::string& field, uint64_t* len); // List Commands - rocksdb::Status LIndex(std::string &key, int64_t index, std::string *element); - rocksdb::Status LInsert(std::string &key, storage::BeforeOrAfter &before_or_after, std::string &pivot, std::string &value); - rocksdb::Status LLen(std::string &key, uint64_t *len); - rocksdb::Status LPop(std::string &key, std::string *element); - rocksdb::Status LPush(std::string &key, std::vector &values); - rocksdb::Status LPushx(std::string &key, std::vector &values); - rocksdb::Status LRange(std::string &key, int64_t start, int64_t stop, std::vector *values); - rocksdb::Status LRem(std::string &key, int64_t count, std::string &value); - rocksdb::Status LSet(std::string &key, int64_t index, std::string &value); - rocksdb::Status LTrim(std::string &key, int64_t start, int64_t stop); - rocksdb::Status RPop(std::string &key, std::string *element); - rocksdb::Status RPush(std::string &key, std::vector &values); - rocksdb::Status RPushx(std::string &key, std::vector &values); - rocksdb::Status RPushnx(std::string &key, std::vector &values, int64_t ttl); - rocksdb::Status RPushnxWithoutTTL(std::string &key, std::vector &values); + rocksdb::Status LIndex(std::string& key, int64_t index, std::string* element); + rocksdb::Status LInsert(std::string& key, storage::BeforeOrAfter& before_or_after, std::string& pivot, std::string& value); + rocksdb::Status LLen(std::string& key, uint64_t* len); + rocksdb::Status LPop(std::string& key, std::string* element); + rocksdb::Status LPush(std::string& key, std::vector &values); + rocksdb::Status LPushx(std::string& key, std::vector &values); + rocksdb::Status LRange(std::string& key, int64_t start, int64_t stop, std::vector* values); + rocksdb::Status LRem(std::string& key, int64_t count, std::string& value); + rocksdb::Status LSet(std::string& key, int64_t index, std::string& value); + rocksdb::Status LTrim(std::string& key, int64_t start, int64_t stop); + rocksdb::Status RPop(std::string& key, std::string* element); + rocksdb::Status RPush(std::string& key, std::vector &values); + rocksdb::Status RPushx(std::string& key, std::vector &values); + rocksdb::Status RPushnx(std::string& key, std::vector &values, int64_t ttl); + rocksdb::Status RPushnxWithoutTTL(std::string& key, std::vector &values); // Set Commands - rocksdb::Status SAdd(std::string &key, std::vector &members); - rocksdb::Status SAddIfKeyExist(std::string &key, std::vector &members); - rocksdb::Status SAddnx(std::string &key, std::vector &members, int64_t ttl); - rocksdb::Status SAddnxWithoutTTL(std::string &key, std::vector &members); - rocksdb::Status SCard(std::string &key, uint64_t *len); - rocksdb::Status SIsmember(std::string &key, std::string &member); - rocksdb::Status SMembers(std::string &key, std::vector *members); - rocksdb::Status SRem(std::string &key, std::vector &members); - rocksdb::Status SRandmember(std::string &key, int64_t count, std::vector *members); + rocksdb::Status SAdd(std::string& key, std::vector& members); + rocksdb::Status SAddIfKeyExist(std::string& key, std::vector& members); + rocksdb::Status SAddnx(std::string& key, std::vector& members, int64_t ttl); + rocksdb::Status SAddnxWithoutTTL(std::string& key, std::vector& members); + rocksdb::Status SCard(std::string& key, uint64_t* len); + rocksdb::Status SIsmember(std::string& key, std::string& member); + rocksdb::Status SMembers(std::string& key, std::vector* members); + rocksdb::Status SRem(std::string& key, std::vector& members); + rocksdb::Status SRandmember(std::string& key, int64_t count, std::vector* members); // ZSet Commands - rocksdb::Status ZAdd(std::string &key, std::vector &score_members); - rocksdb::Status ZAddIfKeyExist(std::string &key, std::vector &score_members); - rocksdb::Status ZAddnx(std::string &key, std::vector &score_members, int64_t ttl); - rocksdb::Status ZAddnxWithoutTTL(std::string &key, std::vector &score_members); - rocksdb::Status ZCard(std::string &key, uint32_t *len, const std::shared_ptr &slot); - rocksdb::Status ZCount(std::string &key, std::string &min, std::string &max, uint64_t *len, ZCountCmd *cmd); - rocksdb::Status ZIncrby(std::string &key, std::string &member, double increment); - rocksdb::Status ZIncrbyIfKeyExist(std::string &key, std::string &member, double increment, ZIncrbyCmd *cmd, - const std::shared_ptr &slot); - rocksdb::Status ZRange(std::string &key, int64_t start, int64_t stop, std::vector *score_members, - const std::shared_ptr &slot); - rocksdb::Status ZRangebyscore(std::string &key, std::string &min, std::string &max, - std::vector *score_members, ZRangebyscoreCmd *cmd); - rocksdb::Status ZRank(std::string &key, std::string &member, int64_t *rank, const std::shared_ptr &slot); - rocksdb::Status ZRem(std::string &key, std::vector &members, std::shared_ptr slot = nullptr); - rocksdb::Status ZRemrangebyrank(std::string &key, std::string &min, std::string &max, int32_t ele_deleted = 0, - const std::shared_ptr &slot = nullptr); - rocksdb::Status ZRemrangebyscore(std::string &key, std::string &min, std::string &max, const std::shared_ptr &slot); - rocksdb::Status ZRevrange(std::string &key, int64_t start, int64_t stop, std::vector *score_members, - const std::shared_ptr &slot); - rocksdb::Status ZRevrangebyscore(std::string &key, std::string &min, std::string &max, - std::vector *score_members, ZRevrangebyscoreCmd *cmd, - const std::shared_ptr &slot); - rocksdb::Status ZRevrangebylex(std::string &key, std::string &min, std::string &max, std::vector *members, - const std::shared_ptr &slot); - rocksdb::Status ZRevrank(std::string &key, std::string &member, int64_t *rank, const std::shared_ptr &slot); - rocksdb::Status ZScore(std::string &key, std::string &member, double *score, const std::shared_ptr &slot); - rocksdb::Status ZRangebylex(std::string &key, std::string &min, std::string &max, std::vector *members, const std::shared_ptr &slot); - rocksdb::Status ZLexcount(std::string &key, std::string &min, std::string &max, uint64_t *len, - const std::shared_ptr &slot); - rocksdb::Status ZRemrangebylex(std::string &key, std::string &min, std::string &max, const std::shared_ptr &slot); + rocksdb::Status ZAdd(std::string& key, std::vector& score_members); + rocksdb::Status ZAddIfKeyExist(std::string& key, std::vector& score_members); + rocksdb::Status ZAddnx(std::string& key, std::vector& score_members, int64_t ttl); + rocksdb::Status ZAddnxWithoutTTL(std::string& key, std::vector& score_members); + rocksdb::Status ZCard(std::string& key, uint32_t* len, const std::shared_ptr& db); + rocksdb::Status ZCount(std::string& key, std::string& min, std::string& max, uint64_t* len, ZCountCmd* cmd); + rocksdb::Status ZIncrby(std::string& key, std::string& member, double increment); + rocksdb::Status ZIncrbyIfKeyExist(std::string& key, std::string& member, double increment, ZIncrbyCmd* cmd, const std::shared_ptr& db); + rocksdb::Status ZRange(std::string& key, int64_t start, int64_t stop, std::vector* score_members, + const std::shared_ptr& db); + rocksdb::Status ZRangebyscore(std::string& key, std::string& min, std::string& max, + std::vector* score_members, ZRangebyscoreCmd* cmd); + rocksdb::Status ZRank(std::string& key, std::string& member, int64_t* rank, const std::shared_ptr& db); + rocksdb::Status ZRem(std::string& key, std::vector& members, std::shared_ptr db); + rocksdb::Status ZRemrangebyrank(std::string& key, std::string& min, std::string& max, int32_t ele_deleted = 0, + const std::shared_ptr& db = nullptr); + rocksdb::Status ZRemrangebyscore(std::string& key, std::string& min, std::string& max, const std::shared_ptr& db); + rocksdb::Status ZRevrange(std::string& key, int64_t start, int64_t stop, std::vector* score_members, + const std::shared_ptr& db); + rocksdb::Status ZRevrangebyscore(std::string& key, std::string& min, std::string& max, + std::vector* score_members, ZRevrangebyscoreCmd* cmd, + const std::shared_ptr& db); + rocksdb::Status ZRevrangebylex(std::string& key, std::string& min, std::string& max, std::vector* members, + const std::shared_ptr& db); + rocksdb::Status ZRevrank(std::string& key, std::string& member, int64_t *rank, const std::shared_ptr& db); + rocksdb::Status ZScore(std::string& key, std::string& member, double* score, const std::shared_ptr& db); + rocksdb::Status ZRangebylex(std::string& key, std::string& min, std::string& max, std::vector* members, const std::shared_ptr& db); + rocksdb::Status ZLexcount(std::string& key, std::string& min, std::string& max, uint64_t* len, + const std::shared_ptr& db); + rocksdb::Status ZRemrangebylex(std::string& key, std::string& min, std::string& max, const std::shared_ptr& db); // Bit Commands - rocksdb::Status SetBit(std::string &key, size_t offset, int64_t value); - rocksdb::Status SetBitIfKeyExist(std::string &key, size_t offset, int64_t value); - rocksdb::Status GetBit(std::string &key, size_t offset, int64_t *value); - rocksdb::Status BitCount(std::string &key, int64_t start, int64_t end, int64_t *value, bool have_offset); - rocksdb::Status BitPos(std::string &key, int64_t bit, int64_t *value); - rocksdb::Status BitPos(std::string &key, int64_t bit, int64_t start, int64_t *value); - rocksdb::Status BitPos(std::string &key, int64_t bit, int64_t start, int64_t end, int64_t *value); + rocksdb::Status SetBit(std::string& key, size_t offset, int64_t value); + rocksdb::Status SetBitIfKeyExist(std::string& key, size_t offset, int64_t value); + rocksdb::Status GetBit(std::string& key, size_t offset, int64_t* value); + rocksdb::Status BitCount(std::string& key, int64_t start, int64_t end, int64_t* value, bool have_offset); + rocksdb::Status BitPos(std::string& key, int64_t bit, int64_t* value); + rocksdb::Status BitPos(std::string& key, int64_t bit, int64_t start, int64_t* value); + rocksdb::Status BitPos(std::string& key, int64_t bit, int64_t start, int64_t end, int64_t* value); // Cache - rocksdb::Status WriteKVToCache(std::string &key, std::string &value, int64_t ttl); - rocksdb::Status WriteHashToCache(std::string &key, std::vector &fvs, int64_t ttl); - rocksdb::Status WriteListToCache(std::string &key, std::vector &values, int64_t ttl); - rocksdb::Status WriteSetToCache(std::string &key, std::vector &members, int64_t ttl); - rocksdb::Status WriteZSetToCache(std::string &key, std::vector &score_members, int64_t ttl); - void PushKeyToAsyncLoadQueue(const char key_type, std::string &key, const std::shared_ptr &slot); - rocksdb::Status CacheZCard(std::string &key, uint64_t *len); + rocksdb::Status WriteKVToCache(std::string& key, std::string& value, int64_t ttl); + rocksdb::Status WriteHashToCache(std::string& key, std::vector& fvs, int64_t ttl); + rocksdb::Status WriteListToCache(std::string& key, std::vector &values, int64_t ttl); + rocksdb::Status WriteSetToCache(std::string& key, std::vector& members, int64_t ttl); + rocksdb::Status WriteZSetToCache(std::string& key, std::vector& score_members, int64_t ttl); + void PushKeyToAsyncLoadQueue(const char key_type, std::string& key, const std::shared_ptr& db); + rocksdb::Status CacheZCard(std::string& key, uint64_t* len); private: - rocksdb::Status InitWithoutLock(uint32_t cache_num, cache::CacheConfig *cache_cfg); + rocksdb::Status InitWithoutLock(uint32_t cache_num, cache::CacheConfig* cache_cfg); void DestroyWithoutLock(void); - int CacheIndex(const std::string &key); - RangeStatus CheckCacheRange(int32_t cache_len, int32_t db_len, int64_t start, int64_t stop, int64_t &out_start, - int64_t &out_stop); - RangeStatus CheckCacheRevRange(int32_t cache_len, int32_t db_len, int64_t start, int64_t stop, int64_t &out_start, - int64_t &out_stop); + int CacheIndex(const std::string& key); + RangeStatus CheckCacheRange(int32_t cache_len, int32_t db_len, int64_t start, int64_t stop, int64_t& out_start, + int64_t& out_stop); + RangeStatus CheckCacheRevRange(int32_t cache_len, int32_t db_len, int64_t start, int64_t stop, int64_t& out_start, + int64_t& out_stop); RangeStatus CheckCacheRangeByScore(uint64_t cache_len, double cache_min, double cache_max, double min, double max, bool left_close, bool right_close); - bool CacheSizeEqsDB(std::string &key, const std::shared_ptr &slot); - void GetMinMaxScore(std::vector &score_members, double &min, double &max); - bool GetCacheMinMaxSM(cache::RedisCache *cache_obj, std::string &key, storage::ScoreMember &min_m, + bool CacheSizeEqsDB(std::string& key, const std::shared_ptr& db); + void GetMinMaxScore(std::vector& score_members, double &min, double &max); + bool GetCacheMinMaxSM(cache::RedisCache* cache_obj, std::string& key, storage::ScoreMember &min_m, storage::ScoreMember &max_m); - bool ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string &key, int mem_len = -1, int db_len = -1, - const std::shared_ptr &slot = nullptr); - rocksdb::Status CleanCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string &key); + bool ReloadCacheKeyIfNeeded(cache::RedisCache* cache_obj, std::string& key, int mem_len = -1, int db_len = -1, + const std::shared_ptr& db = nullptr); + rocksdb::Status CleanCacheKeyIfNeeded(cache::RedisCache* cache_obj, std::string& key); private: std::atomic cache_status_; diff --git a/include/pika_cache_load_thread.h b/include/pika_cache_load_thread.h index e470d0b000..00e73a9c81 100644 --- a/include/pika_cache_load_thread.h +++ b/include/pika_cache_load_thread.h @@ -13,7 +13,6 @@ #include #include "include/pika_cache.h" -#include "include/pika_slot.h" #include "include/pika_define.h" #include "net/include/net_thread.h" #include "storage/storage.h" @@ -25,20 +24,20 @@ class PikaCacheLoadThread : public net::Thread { uint64_t AsyncLoadKeysNum(void) { return async_load_keys_num_; } uint32_t WaittingLoadKeysNum(void) { return waitting_load_keys_num_; } - void Push(const char key_type, std::string& key, const std::shared_ptr &slot); + void Push(const char key_type, std::string& key, const std::shared_ptr& db); private: - bool LoadKV(std::string& key, const std::shared_ptr& slot); - bool LoadHash(std::string& key, const std::shared_ptr& slot); - bool LoadList(std::string& key, const std::shared_ptr& slot); - bool LoadSet(std::string& key, const std::shared_ptr& slot); - bool LoadZset(std::string& key, const std::shared_ptr& slot); - bool LoadKey(const char key_type, std::string& key, const std::shared_ptr& slot); + bool LoadKV(std::string& key, const std::shared_ptr& db); + bool LoadHash(std::string& key, const std::shared_ptr& db); + bool LoadList(std::string& key, const std::shared_ptr& db); + bool LoadSet(std::string& key, const std::shared_ptr& db); + bool LoadZset(std::string& key, const std::shared_ptr& db); + bool LoadKey(const char key_type, std::string& key, const std::shared_ptr& db); virtual void* ThreadMain() override; private: std::atomic_bool should_exit_; - std::deque>> loadkeys_queue_; + std::deque>> loadkeys_queue_; pstd::CondVar loadkeys_cond_; pstd::Mutex loadkeys_mutex_; diff --git a/include/pika_client_conn.h b/include/pika_client_conn.h index 89e5c121b4..6b5dbab419 100644 --- a/include/pika_client_conn.h +++ b/include/pika_client_conn.h @@ -11,7 +11,7 @@ #include "acl.h" #include "include/pika_command.h" - +#include "include/pika_define.h" // TODO: stat time costing in write out data to connfd struct TimeStat { @@ -53,7 +53,6 @@ class PikaClientConn : public net::RedisConn { std::shared_ptr resp_ptr; LogOffset offset; std::string db_name; - uint32_t slot_id; }; struct TxnStateBitMask { @@ -73,13 +72,12 @@ class PikaClientConn : public net::RedisConn { void BatchExecRedisCmd(const std::vector& argvs); int DealMessage(const net::RedisCmdArgsType& argv, std::string* response) override { return 0; } static void DoBackgroundTask(void* arg); - static void DoExecTask(void* arg); bool IsPubSub() { return is_pubsub_; } void SetIsPubSub(bool is_pubsub) { is_pubsub_ = is_pubsub; } void SetCurrentDb(const std::string& db_name) { current_db_ = db_name; } - const std::string& GetCurrentTable() override { return current_db_; } void SetWriteCompleteCallback(WriteCompleteCallback cb) { write_completed_cb_ = std::move(cb); } + const std::string& GetCurrentTable() override { return current_db_; } void DoAuth(const std::shared_ptr& user); @@ -92,24 +90,22 @@ class PikaClientConn : public net::RedisConn { std::string UserName() const; // Txn - void PushCmdToQue(std::shared_ptr cmd); std::queue> GetTxnCmdQue(); + void PushCmdToQue(std::shared_ptr cmd); void ClearTxnCmdQue(); - bool IsInTxn(); - bool IsTxnFailed(); - bool IsTxnInitFailed(); - bool IsTxnWatchFailed(); - bool IsTxnExecing(void); void SetTxnWatchFailState(bool is_failed); void SetTxnInitFailState(bool is_failed); void SetTxnStartState(bool is_start); - - void AddKeysToWatch(const std::vector &db_keys); + void AddKeysToWatch(const std::vector& db_keys); void RemoveWatchedKeys(); - void SetTxnFailedFromKeys(const std::vector &db_keys); + void SetTxnFailedFromKeys(const std::vector& db_keys); void SetAllTxnFailed(); void SetTxnFailedFromDBs(std::string db_name); void ExitTxn(); + bool IsInTxn(); + bool IsTxnInitFailed(); + bool IsTxnWatchFailed(); + bool IsTxnExecing(void); net::ServerThread* server_thread() { return server_thread_; } void ClientInfoToString(std::string* info, const std::string& cmdName); diff --git a/include/pika_cmd_table_manager.h b/include/pika_cmd_table_manager.h index 184c0ee0a8..1b0c162807 100644 --- a/include/pika_cmd_table_manager.h +++ b/include/pika_cmd_table_manager.h @@ -31,7 +31,6 @@ class PikaCmdTableManager { virtual ~PikaCmdTableManager() = default; void InitCmdTable(void); std::shared_ptr GetCmd(const std::string& opt); - uint32_t DistributeKey(const std::string& key, uint32_t slot_num); bool CmdExist(const std::string& cmd) const; CmdTable* GetCmdTable(); uint32_t GetCmdId(); diff --git a/include/pika_command.h b/include/pika_command.h index c7250c870e..d6bd22be93 100644 --- a/include/pika_command.h +++ b/include/pika_command.h @@ -12,16 +12,17 @@ #include #include +#include "rocksdb/status.h" + #include "net/include/net_conn.h" #include "net/include/redis_conn.h" #include "pstd/include/pstd_string.h" -#include "include/pika_slot.h" #include "net/src/dispatch_thread.h" -class SyncMasterSlot; -class SyncSlaveSlot; - +class SyncMasterDB; +class SyncSlaveDB; +class DB; // Constant for command name // Admin const std::string kCmdNameSlaveof = "slaveof"; @@ -77,6 +78,7 @@ const std::string kCmdNameSlotsMgrtSlotAsync = "slotsmgrtslot-async"; const std::string kCmdNameSlotsMgrtExecWrapper = "slotsmgrt-exec-wrapper"; const std::string kCmdNameSlotsMgrtAsyncStatus = "slotsmgrt-async-status"; const std::string kCmdNameSlotsMgrtAsyncCancel = "slotsmgrt-async-cancel"; + // Kv const std::string kCmdNameSet = "set"; const std::string kCmdNameGet = "get"; @@ -245,9 +247,20 @@ const std::string kCmdNameXTrim = "xtrim"; const std::string kCmdNameXInfo = "xinfo"; const std::string kClusterPrefix = "pkcluster"; + using PikaCmdArgsType = net::RedisCmdArgsType; static const int RAW_ARGS_LEN = 1024 * 1024; +enum CmdFlagsMask { + kCmdFlagsMaskRW = 1, + kCmdFlagsMaskLocal = (1 << 1), + kCmdFlagsMaskSuspend = (1 << 2), + kCmdFlagsMaskReadCache = (1 << 3), + kCmdFlagsMaskAdminRequire = (1 << 4), + kCmdFlagsMaskUpdateCache = (1 << 5), + kCmdFlagsMaskDoThrouhDB = (1 << 6), +}; + enum CmdFlags { kCmdFlagsRead = 1, // default rw kCmdFlagsWrite = (1 << 1), @@ -264,17 +277,14 @@ enum CmdFlags { kCmdFlagsLocal = (1 << 12), kCmdFlagsSuspend = (1 << 13), kCmdFlagsAdminRequire = (1 << 14), - kCmdFlagsSingleSlot = (1 << 15), - kCmdFlagsMultiSlot = (1 << 16), - kCmdFlagsNoAuth = (1 << 17), // command no auth can also be executed - kCmdFlagsReadCache = (1 << 18), - kCmdFlagsUpdateCache = (1 << 19), - kCmdFlagsDoThroughDB = (1 << 20), - kCmdFlagsOperateKey = (1 << 21), // redis keySpace - kCmdFlagsPreDo = (1 << 22), - kCmdFlagsStream = (1 << 23), - kCmdFlagsFast = (1 << 24), - kCmdFlagsSlow = (1 << 25), + kCmdFlagsNoAuth = (1 << 15), // command no auth can also be executed + kCmdFlagsReadCache = (1 << 16), + kCmdFlagsUpdateCache = (1 << 17), + kCmdFlagsDoThroughDB = (1 << 18), + kCmdFlagsOperateKey = (1 << 19), // redis keySpace + kCmdFlagsStream = (1 << 20), + kCmdFlagsFast = (1 << 21), + kCmdFlagsSlow = (1 << 22), }; void inline RedisAppendContent(std::string& str, const std::string& value); @@ -446,7 +456,6 @@ class CmdRes { message_ = content; } } - CmdRet GetCmdRet() const { return ret_; } private: std::string message_; @@ -459,10 +468,10 @@ class CmdRes { */ struct UnblockTaskArgs { std::string key; - std::shared_ptr slot; - net::DispatchThread* dispatchThread{nullptr}; - UnblockTaskArgs(std::string key_, std::shared_ptr slot_, net::DispatchThread* dispatchThread_) - : key(std::move(key_)), slot(std::move(slot_)), dispatchThread(dispatchThread_) {} + std::shared_ptr db; + net::DispatchThread* dispatchThread{ nullptr }; + UnblockTaskArgs(std::string key_, std::shared_ptr db_, net::DispatchThread* dispatchThread_) + : key(std::move(key_)), db(db_), dispatchThread(dispatchThread_) {} }; class Cmd : public std::enable_shared_from_this { @@ -470,20 +479,17 @@ class Cmd : public std::enable_shared_from_this { enum CmdStage { kNone, kBinlogStage, kExecuteStage }; struct HintKeys { HintKeys() = default; - void Push(const std::string& key, int hint) { - keys.push_back(key); - hints.push_back(hint); - } + bool empty() const { return keys.empty() && hints.empty(); } std::vector keys; std::vector hints; }; struct ProcessArg { ProcessArg() = default; - ProcessArg(std::shared_ptr _slot, std::shared_ptr _sync_slot, HintKeys _hint_keys) - : slot(std::move(_slot)), sync_slot(std::move(_sync_slot)), hint_keys(std::move(_hint_keys)) {} - std::shared_ptr slot; - std::shared_ptr sync_slot; + ProcessArg(std::shared_ptr _db, std::shared_ptr _sync_db, HintKeys _hint_keys) + : db(std::move(_db)), sync_db(std::move(_sync_db)), hint_keys(std::move(_hint_keys)) {} + std::shared_ptr db; + std::shared_ptr sync_db; HintKeys hint_keys; }; struct CommandStatistics { @@ -501,22 +507,18 @@ class Cmd : public std::enable_shared_from_this { virtual std::vector current_key() const; virtual void Execute(); - virtual void ProcessSingleSlotCmd(); - virtual void ProcessMultiSlotCmd(); - virtual void Do(std::shared_ptr slot = nullptr) = 0; - virtual void DoThroughDB(std::shared_ptr slot = nullptr) {} - virtual void DoUpdateCache(std::shared_ptr slot = nullptr) {} - virtual void ReadCache(std::shared_ptr slot = nullptr) {} - rocksdb::Status CmdStatus() { return s_; }; + virtual void Do() {}; + virtual void DoThroughDB() {} + virtual void DoUpdateCache() {} + virtual void ReadCache() {} virtual Cmd* Clone() = 0; // used for execute multikey command into different slots - virtual void Split(std::shared_ptr slot, const HintKeys& hint_keys) = 0; + virtual void Split(const HintKeys& hint_keys) = 0; virtual void Merge() = 0; int8_t SubCmdIndex(const std::string& cmdName); // if the command no subCommand,return -1; void Initial(const PikaCmdArgsType& argv, const std::string& db_name); - uint32_t flag() const; bool hasFlag(uint32_t flag) const; bool is_read() const; @@ -525,16 +527,14 @@ class Cmd : public std::enable_shared_from_this { bool IsLocal() const; bool IsSuspend() const; bool IsAdminRequire() const; - bool is_single_slot() const; - bool is_multi_slot() const; bool HasSubCommand() const; // The command is there a sub command std::vector SubCommand() const; // Get command is there a sub command bool IsNeedUpdateCache() const; - bool is_only_from_cache() const; bool IsNeedReadCache() const; bool IsNeedCacheDo() const; bool HashtagIsConsistent(const std::string& lhs, const std::string& rhs) const; uint64_t GetDoDuration() const { return do_duration_; }; + std::shared_ptr GetDB() const { return db_; }; uint32_t AclCategory() const; void AddAclCategory(uint32_t aclCategory); void SetDbName(const std::string& db_name) { db_name_ = db_name; } @@ -543,7 +543,6 @@ class Cmd : public std::enable_shared_from_this { std::string name() const; CmdRes& res(); std::string db_name() const; - BinlogOffset binlog_offset() const; PikaCmdArgsType& argv(); virtual std::string ToRedisProtocol(); @@ -555,7 +554,7 @@ class Cmd : public std::enable_shared_from_this { void SetStage(CmdStage stage); - virtual void DoBinlog(const std::shared_ptr& slot); + virtual void DoBinlog(); uint32_t GetCmdId() const { return cmdId_; }; bool CheckArg(uint64_t num) const; @@ -563,11 +562,9 @@ class Cmd : public std::enable_shared_from_this { protected: // enable copy, used default copy // Cmd(const Cmd&); - void ProcessCommand(const std::shared_ptr& slot, const std::shared_ptr& sync_slot, - const HintKeys& hint_key = HintKeys()); - void InternalProcessCommand(const std::shared_ptr& slot, const std::shared_ptr& sync_slot, - const HintKeys& hint_key); - void DoCommand(const std::shared_ptr& slot, const HintKeys& hint_key); + void ProcessCommand(const HintKeys& hint_key = HintKeys()); + void InternalProcessCommand(const HintKeys& hint_key); + void DoCommand(const HintKeys& hint_key); void LogCommand() const; std::string name_; @@ -581,7 +578,8 @@ class Cmd : public std::enable_shared_from_this { PikaCmdArgsType argv_; std::string db_name_; rocksdb::Status s_; - + std::shared_ptr db_; + std::shared_ptr sync_db_; std::weak_ptr conn_; std::weak_ptr resp_; CmdStage stage_ = kNone; @@ -615,6 +613,4 @@ void RedisAppendLen(std::string& str, int64_t ori, const std::string& prefix) { str.append(kNewLine); } -void TryAliasChange(std::vector* argv); - #endif diff --git a/include/pika_conf.h b/include/pika_conf.h index a0e23189f1..b6a9788e5c 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -485,7 +485,7 @@ class PikaConf : public pstd::BaseConf { } void SetSlotMigrate(const std::string& value) { std::lock_guard l(rwlock_); - slotmigrate_ = (value == "yes") ? true : false; + slotmigrate_ = (value == "yes"); } void SetExpireLogsNums(const int value) { std::lock_guard l(rwlock_); @@ -629,6 +629,7 @@ class PikaConf : public pstd::BaseConf { acl_Log_max_len_ = value; } + int64_t cache_maxmemory() { return cache_maxmemory_; } void SetSlowCmd(const std::string& value) { std::lock_guard l(rwlock_); std::string lower_value = value; @@ -641,40 +642,28 @@ class PikaConf : public pstd::BaseConf { void SetCacheDisableFlag() { tmp_cache_disable_flag_ = true; } int zset_cache_start_pos() { return zset_cache_start_pos_; } int zset_cache_field_num_per_key() { return zset_cache_field_num_per_key_; } - int64_t cache_maxmemory() { return cache_maxmemory_; } int cache_maxmemory_policy() { return cache_maxmemory_policy_; } int cache_maxmemory_samples() { return cache_maxmemory_samples_; } int cache_lfu_decay_time() { return cache_lfu_decay_time_; } - pstd::Status DBSlotsSanityCheck(const std::string& db_name, const std::set& slot_ids, - bool is_add); - pstd::Status AddDBSlots(const std::string& db_name, const std::set& slot_ids); - pstd::Status RemoveDBSlots(const std::string& db_name, const std::set& slot_ids); - pstd::Status AddDB(const std::string& db_name, uint32_t slot_num); - pstd::Status AddDBSanityCheck(const std::string& db_name); - pstd::Status DelDB(const std::string& db_name); - pstd::Status DelDBSanityCheck(const std::string& db_name); - int Load(); int ConfigRewrite(); int ConfigRewriteReplicationID(); private: - pstd::Status InternalGetTargetDB(const std::string& db_name, uint32_t* target); - int port_ = 0; - std::string slaveof_; int slave_priority_ = 0; int thread_num_ = 0; int thread_pool_size_ = 0; int slow_cmd_thread_pool_size_ = 0; std::unordered_set slow_cmd_set_; int sync_thread_num_ = 0; + int expire_dump_days_ = 3; + int db_sync_speed_ = 0; + std::string slaveof_; std::string log_path_; std::string log_level_; std::string db_path_; std::string db_sync_path_; - int expire_dump_days_ = 3; - int db_sync_speed_ = 0; std::string compact_cron_; std::string compact_interval_; bool disable_auto_compactions_ = false; @@ -765,10 +754,11 @@ class PikaConf : public pstd::BaseConf { int binlog_file_size_ = 0; // cache + std::vector cache_type_; + std::atomic_bool tmp_cache_disable_flag_; + std::atomic_int64_t cache_maxmemory_; std::atomic_int cache_num_; std::atomic_int cache_model_; - std::atomic_bool tmp_cache_disable_flag_; - std::vector cache_type_; std::atomic_int cache_string_; std::atomic_int cache_set_; std::atomic_int cache_zset_; @@ -777,25 +767,22 @@ class PikaConf : public pstd::BaseConf { std::atomic_int cache_bit_; std::atomic_int zset_cache_start_pos_; std::atomic_int zset_cache_field_num_per_key_; - std::atomic_int64_t cache_maxmemory_; std::atomic_int cache_maxmemory_policy_; std::atomic_int cache_maxmemory_samples_; std::atomic_int cache_lfu_decay_time_; - // rocksdb blob bool enable_blob_files_ = false; - int64_t min_blob_size_ = 4096; // 4K - int64_t blob_file_size_ = 256 * 1024 * 1024; // 256M - std::string blob_compression_type_ = "none"; bool enable_blob_garbage_collection_ = false; double blob_garbage_collection_age_cutoff_ = 0.25; double blob_garbage_collection_force_threshold_ = 1.0; + int64_t min_blob_size_ = 4096; // 4K int64_t blob_cache_ = 0; int64_t blob_num_shard_bits_ = 0; + int64_t blob_file_size_ = 256 * 1024 * 1024; // 256M + std::string blob_compression_type_ = "none"; std::unique_ptr local_meta_; - std::shared_mutex rwlock_; // Rsync Rate limiting configuration diff --git a/include/pika_consensus.h b/include/pika_consensus.h index c146fcba28..d40e4efec4 100644 --- a/include/pika_consensus.h +++ b/include/pika_consensus.h @@ -21,7 +21,6 @@ class Context : public pstd::noncopyable { pstd::Status Init(); // RWLock should be held when access members. pstd::Status StableSave(); - void PrepareUpdateAppliedIndex(const LogOffset& offset); void UpdateAppliedIndex(const LogOffset& offset); void Reset(const LogOffset& offset); @@ -48,16 +47,13 @@ class SyncProgress { ~SyncProgress() = default; std::shared_ptr GetSlaveNode(const std::string& ip, int port); std::unordered_map> GetAllSlaveNodes(); - std::unordered_map GetAllMatchIndex(); - pstd::Status AddSlaveNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, - int session_id); + pstd::Status AddSlaveNode(const std::string& ip, int port, const std::string& db_name, int session_id); pstd::Status RemoveSlaveNode(const std::string& ip, int port); pstd::Status Update(const std::string& ip, int port, const LogOffset& start, const LogOffset& end, LogOffset* committed_index); int SlaveSize(); private: - std::shared_mutex rwlock_; std::unordered_map> slaves_; std::unordered_map match_index_; @@ -82,8 +78,6 @@ class MemLog { logs_.push_back(item); last_offset_ = item.offset; } - pstd::Status PurgeLogs(const LogOffset& offset, std::vector* logs); - pstd::Status GetRangeLogs(int start, int end, std::vector* logs); pstd::Status TruncateTo(const LogOffset& offset); void Reset(const LogOffset& offset); @@ -108,7 +102,7 @@ class MemLog { class ConsensusCoordinator { public: - ConsensusCoordinator(const std::string& db_name, uint32_t slot_id); + ConsensusCoordinator(const std::string& db_name); ~ConsensusCoordinator(); // since it is invoked in constructor all locks not hold void Init(); @@ -121,7 +115,6 @@ class ConsensusCoordinator { pstd::Status RemoveSlaveNode(const std::string& ip, int port); void UpdateTerm(uint32_t term); uint32_t term(); - pstd::Status CheckEnoughFollower(); // invoked by follower pstd::Status ProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute); @@ -139,11 +132,6 @@ class ConsensusCoordinator { return committed_index_; } - LogOffset applied_index() { - std::shared_lock lock(context_->rwlock_); - return context_->applied_index_; - } - std::shared_ptr context() { return context_; } // redis parser cb @@ -177,9 +165,6 @@ class ConsensusCoordinator { } private: - pstd::Status ScheduleApplyLog(const LogOffset& committed_index); - pstd::Status ScheduleApplyFollowerLog(const LogOffset& committed_index); - bool MatchConsensusLevel(); pstd::Status TruncateTo(const LogOffset& offset); pstd::Status InternalAppendLog(const std::shared_ptr& cmd_ptr); @@ -209,7 +194,6 @@ class ConsensusCoordinator { uint32_t term_ = 0; std::string db_name_; - uint32_t slot_id_ = 0; SyncProgress sync_pros_; std::shared_ptr stable_logger_; diff --git a/include/pika_data_distribution.h b/include/pika_data_distribution.h index bff392f5b3..7f8d494fe0 100644 --- a/include/pika_data_distribution.h +++ b/include/pika_data_distribution.h @@ -17,28 +17,12 @@ class PikaDataDistribution { virtual ~PikaDataDistribution() = default; // Initialization virtual void Init() = 0; - // key map to slot id - virtual uint32_t Distribute(const std::string& str, uint32_t slot_num) = 0; }; class HashModulo : public PikaDataDistribution { public: ~HashModulo() override = default; void Init() override; - uint32_t Distribute(const std::string& str, uint32_t slot_num) override; }; -class Crc32 : public PikaDataDistribution { - public: - void Init() override; - uint32_t Distribute(const std::string& str, uint32_t slot_num) override; - - private: - void Crc32TableInit(uint32_t poly); - uint32_t Crc32Update(uint32_t crc, const char* buf, int len); - uint32_t crc32tab[256]; -}; - -std::string GetHashkey(const std::string& key); - #endif diff --git a/include/pika_db.h b/include/pika_db.h index 339718d2f7..c7828a5722 100644 --- a/include/pika_db.h +++ b/include/pika_db.h @@ -9,13 +9,80 @@ #include #include "storage/storage.h" - #include "include/pika_command.h" -#include "include/pika_slot.h" +#include "lock_mgr.h" +#include "pika_cache.h" +#include "pika_define.h" +#include "storage/backupable.h" + +class PikaCache; +class CacheInfo; +/* + *Keyscan used + */ +struct KeyScanInfo { + time_t start_time = 0; + std::string s_start_time; + int32_t duration = -3; + std::vector key_infos; // the order is strings, hashes, lists, zsets, sets + bool key_scaning_ = false; + KeyScanInfo() : + s_start_time("0"), + key_infos({{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}) + {} +}; + +struct BgSaveInfo { + bool bgsaving = false; + time_t start_time = 0; + std::string s_start_time; + std::string path; + LogOffset offset; + BgSaveInfo() = default; + void Clear() { + bgsaving = false; + path.clear(); + offset = LogOffset(); + } +}; + +struct DisplayCacheInfo { + int status = 0; + uint32_t cache_num = 0; + uint64_t keys_num = 0; + uint64_t used_memory = 0; + uint64_t hits = 0; + uint64_t misses = 0; + uint64_t hits_per_sec = 0; + uint64_t read_cmd_per_sec = 0; + double hitratio_per_sec = 0.0; + double hitratio_all = 0.0; + uint64_t load_keys_per_sec = 0; + uint64_t last_time_us = 0; + uint64_t last_load_keys_num = 0; + uint32_t waitting_load_keys_num = 0; + DisplayCacheInfo& operator=(const DisplayCacheInfo &obj) { + status = obj.status; + cache_num = obj.cache_num; + keys_num = obj.keys_num; + used_memory = obj.used_memory; + hits = obj.hits; + misses = obj.misses; + hits_per_sec = obj.hits_per_sec; + read_cmd_per_sec = obj.read_cmd_per_sec; + hitratio_per_sec = obj.hitratio_per_sec; + hitratio_all = obj.hitratio_all; + load_keys_per_sec = obj.load_keys_per_sec; + last_time_us = obj.last_time_us; + last_load_keys_num = obj.last_load_keys_num; + waitting_load_keys_num = obj.waitting_load_keys_num; + return *this; + } +}; class DB : public std::enable_shared_from_this, public pstd::noncopyable { public: - DB(std::string db_name, uint32_t slot_num, const std::string& db_path, const std::string& log_path); + DB(std::string db_name, const std::string& db_path, const std::string& log_path); virtual ~DB(); friend class Cmd; @@ -24,35 +91,29 @@ class DB : public std::enable_shared_from_this, public pstd::noncopyable { friend class PikaServer; std::string GetDBName(); + std::shared_ptr storage() const; + void GetBgSaveMetaData(std::vector* fileNames, std::string* snapshot_uuid); void BgSaveDB(); - void CompactDB(const storage::DataType& type); - bool FlushSlotDB(); - bool FlushSlotSubDB(const std::string& db_name); void SetBinlogIoError(); void SetBinlogIoErrorrelieve(); bool IsBinlogIoError(); - uint32_t SlotNum(); - void GetAllSlots(std::set& slot_ids); - std::shared_mutex& GetSlotLock() { - return slots_rw_; + std::shared_ptr cache() const; + std::shared_mutex& GetDBLock() { + return dbs_rw_; } - void SlotLock() { - slots_rw_.lock(); + void DBLock() { + dbs_rw_.lock(); } - void SlotLockShared() { - slots_rw_.lock_shared(); + void DBLockShared() { + dbs_rw_.lock_shared(); } - void SlotUnlock() { - slots_rw_.unlock(); + void DBUnlock() { + dbs_rw_.unlock(); } - void SlotUnlockShared() { - slots_rw_.unlock_shared(); + void DBUnlockShared() { + dbs_rw_.unlock_shared(); } - // Dynamic change slot - pstd::Status AddSlots(const std::set& slot_ids); - pstd::Status RemoveSlots(const std::set& slot_ids); - // KeyScan use; void KeyScan(); bool IsKeyScaning(); @@ -60,35 +121,54 @@ class DB : public std::enable_shared_from_this, public pstd::noncopyable { void StopKeyScan(); void ScanDatabase(const storage::DataType& type); KeyScanInfo GetKeyScanInfo(); - pstd::Status GetSlotsKeyScanInfo(std::map* infos); // Compact use; void Compact(const storage::DataType& type); void CompactRange(const storage::DataType& type, const std::string& start, const std::string& end); - void LeaveAllSlot(); - std::set GetSlotIDs(); - std::shared_ptr GetSlotById(uint32_t slot_id); - std::shared_ptr GetSlotByKey(const std::string& key); - bool DBIsEmpty(); - pstd::Status MovetoToTrash(const std::string& path); - pstd::Status Leave(); - std::map> GetSlots() { - return slots_; - } + std::shared_ptr LockMgr(); + void DbRWLockWriter(); + void DbRWLockReader(); + void DbRWUnLock(); + /* + * Cache used + */ + DisplayCacheInfo GetCacheInfo(); + void UpdateCacheInfo(CacheInfo& cache_info); + void ResetDisplayCacheInfo(int status); + uint64_t cache_usage_; + void Init(); + bool TryUpdateMasterOffset(); + /* + * FlushDB & FlushSubDB use + */ + bool FlushDB(); + bool FlushSubDB(const std::string& db_name); + bool FlushDBWithoutLock(); + bool FlushSubDBWithoutLock(const std::string& db_name); + bool ChangeDb(const std::string& new_path); + pstd::Status GetBgSaveUUID(std::string* snapshot_uuid); + void PrepareRsync(); + bool IsBgSaving(); + BgSaveInfo bgsave_info(); + pstd::Status GetKeyNum(std::vector* key_info); + private: + bool opened_ = false; + std::string dbsync_path_; std::string db_name_; - uint32_t slot_num_ = 0; std::string db_path_; + std::string snapshot_uuid_; std::string log_path_; - + std::string bgsave_sub_path_; + pstd::Mutex key_info_protector_; + std::shared_mutex db_rwlock_; std::atomic binlog_io_error_; - // lock order - // slots_rw_ > key_scan_protector_ - - std::shared_mutex slots_rw_; - std::map> slots_; - + std::shared_mutex dbs_rw_; + // class may be shared, using shared_ptr would be a better choice + std::shared_ptr lock_mgr_; + std::shared_ptr storage_; + std::shared_ptr cache_; /* * KeyScan use */ @@ -96,11 +176,28 @@ class DB : public std::enable_shared_from_this, public pstd::noncopyable { void InitKeyScan(); pstd::Mutex key_scan_protector_; KeyScanInfo key_scan_info_; + /* + * Cache used + */ + DisplayCacheInfo cache_info_; + std::shared_mutex cache_info_rwlock_; + /* + * BgSave use + */ + static void DoBgSave(void* arg); + bool RunBgsaveEngine(); + + bool InitBgsaveEnv(); + bool InitBgsaveEngine(); + void ClearBgsave(); + void FinishBgsave(); + BgSaveInfo bgsave_info_; + pstd::Mutex bgsave_protector_; + std::shared_ptr bgsave_engine_; }; struct BgTaskArg { std::shared_ptr db; - std::shared_ptr slot; }; #endif diff --git a/include/pika_define.h b/include/pika_define.h index b81e2797f8..1faec2edec 100644 --- a/include/pika_define.h +++ b/include/pika_define.h @@ -34,7 +34,6 @@ class PikaServer; /* Port shift */ const int kPortShiftRSync = 1000; const int kPortShiftReplServer = 2000; -//TODO: Temporarily used for rsync server port shift. will be deleted. const int kPortShiftRsync2 = 10001; const std::string kPikaPidFile = "pika.pid"; const std::string kPikaSecretFile = "rsync.secret"; @@ -44,27 +43,15 @@ const std::string kDefaultRsyncAuth = "default"; const int kMaxRsyncParallelNum = 4; struct DBStruct { - DBStruct(std::string tn, const uint32_t pn, std::set pi) - : db_name(std::move(tn)), slot_num(pn), slot_ids(std::move(pi)) {} + DBStruct(std::string tn) + : db_name(std::move(tn)) {} bool operator==(const DBStruct& db_struct) const { - return db_name == db_struct.db_name && slot_num == db_struct.slot_num && - slot_ids == db_struct.slot_ids; + return db_name == db_struct.db_name; } std::string db_name; - uint32_t slot_num = 0; - std::set slot_ids; }; -struct WorkerCronTask { - int task; - std::string ip_port; -}; -using MonitorCronTask = WorkerCronTask; -// task define -#define TASK_KILL 0 -#define TASK_KILLALL 1 - // slave item struct SlaveItem { std::string ip_port; @@ -92,11 +79,6 @@ enum ReplState { const std::string ReplStateMsg[] = {"kNoConnect", "kTryConnect", "kTryDBSync", "kWaitDBSync", "kWaitReply", "kConnected", "kError", "kDBNoConnect"}; -enum SlotState { - INFREE = 0, - INBUSY = 1, -}; - struct LogicOffset { uint32_t term{0}; uint64_t index{0}; @@ -166,10 +148,8 @@ struct DBSyncArg { std::string ip; int port; std::string db_name; - uint32_t slot_id; - DBSyncArg(PikaServer* const _p, std::string _ip, int _port, std::string _db_name, - uint32_t _slot_id) - : p(_p), ip(std::move(_ip)), port(_port), db_name(std::move(_db_name)), slot_id(_slot_id) {} + DBSyncArg(PikaServer* const _p, std::string _ip, int _port, std::string _db_name) + : p(_p), ip(std::move(_ip)), port(_port), db_name(std::move(_db_name)) {} }; // rm define @@ -201,28 +181,30 @@ struct BinlogChip { } }; -struct SlotInfo { - SlotInfo(std::string db_name, uint32_t slot_id) - : db_name_(std::move(db_name)), slot_id_(slot_id) {} +struct DBInfo { + DBInfo(std::string db_name) + : db_name_(std::move(db_name)) {} - SlotInfo() = default; + DBInfo() = default; - bool operator==(const SlotInfo& other) const { - return db_name_ == other.db_name_ && slot_id_ == other.slot_id_; + bool operator==(const DBInfo& other) const { + return db_name_ == other.db_name_; } - bool operator<(const SlotInfo& other) const { - return db_name_ < other.db_name_ || (db_name_ == other.db_name_ && slot_id_ < other.slot_id_); + bool operator<(const DBInfo& other) const { + return db_name_ < other.db_name_ || (db_name_ == other.db_name_); } - std::string ToString() const { return "(" + db_name_ + ":" + std::to_string(slot_id_) + ")"; } + std::string ToString() const { return "(" + db_name_ + ")"; } std::string db_name_; - uint32_t slot_id_{0}; }; -struct hash_slot_info { - size_t operator()(const SlotInfo& n) const { - return std::hash()(n.db_name_) ^ std::hash()(n.slot_id_); +/* + * Used to define the sorting rule of the db in the map + */ +struct hash_db_info { + size_t operator()(const DBInfo& n) const { + return std::hash()(n.db_name_); } }; @@ -242,37 +224,36 @@ class Node { class RmNode : public Node { public: - RmNode(const std::string& ip, int port, SlotInfo slot_info) - : Node(ip, port), slot_info_(std::move(slot_info)) {} + RmNode(const std::string& ip, int port, DBInfo db_info) + : Node(ip, port), db_info_(std::move(db_info)) {} - RmNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id) + RmNode(const std::string& ip, int port, const std::string& db_name) : Node(ip, port), - slot_info_(db_name, slot_id) + db_info_(db_name) {} - RmNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, int32_t session_id) + RmNode(const std::string& ip, int port, const std::string& db_name, int32_t session_id) : Node(ip, port), - slot_info_(db_name, slot_id), + db_info_(db_name), session_id_(session_id) {} - RmNode(const std::string& db_name, uint32_t slot_id) - : slot_info_(db_name, slot_id) {} + RmNode(const std::string& db_name) + : db_info_(db_name) {} RmNode() = default; ~RmNode() override = default; bool operator==(const RmNode& other) const { - return slot_info_.db_name_ == other.DBName() && slot_info_.slot_id_ == other.SlotId() && + return db_info_.db_name_ == other.DBName() && Ip() == other.Ip() && Port() == other.Port(); } - const std::string& DBName() const { return slot_info_.db_name_; } - uint32_t SlotId() const { return slot_info_.slot_id_; } - const SlotInfo& NodeSlotInfo() const { return slot_info_; } + const std::string& DBName() const { return db_info_.db_name_; } + const DBInfo& NodeDBInfo() const { return db_info_; } void SetSessionId(int32_t session_id) { session_id_ = session_id; } int32_t SessionId() const { return session_id_; } std::string ToString() const { - return "slot=" + DBName() + "_" + std::to_string(SlotId()) + ",ip_port=" + Ip() + ":" + + return "db=" + DBName() + "_,ip_port=" + Ip() + ":" + std::to_string(Port()) + ",session id=" + std::to_string(SessionId()); } void SetLastSendTime(uint64_t last_send_time) { last_send_time_ = last_send_time; } @@ -281,19 +262,12 @@ class RmNode : public Node { uint64_t LastRecvTime() const { return last_recv_time_; } private: - SlotInfo slot_info_; + DBInfo db_info_; int32_t session_id_ = 0; uint64_t last_send_time_ = 0; uint64_t last_recv_time_ = 0; }; -struct hash_rm_node { - size_t operator()(const RmNode& n) const { - return std::hash()(n.DBName()) ^ std::hash()(n.SlotId()) ^ - std::hash()(n.Ip()) ^ std::hash()(n.Port()); - } -}; - struct WriteTask { struct RmNode rm_node_; struct BinlogChip binlog_chip_; @@ -342,12 +316,6 @@ constexpr int PIKA_CACHE_READ = 1; #define PIKA_CACHE_SIZE_MIN 536870912 // 512M #define PIKA_CACHE_SIZE_DEFAULT 10737418240 // 10G -/* - * The size of Binlogfile - */ -// static uint64_t kBinlogSize = 128; -// static const uint64_t kBinlogSize = 1024 * 1024 * 100; - enum RecordType { kZeroType = 0, kFullType = 1, @@ -385,13 +353,11 @@ const std::string kContext = "context"; /* * define common character - * */ #define COMMA ',' /* * define reply between master and slave - * */ const std::string kInnerReplOk = "ok"; const std::string kInnerReplWait = "wait"; @@ -425,6 +391,7 @@ const int PIKA_CACHE_STATUS_DESTROY = 4; const int PIKA_CACHE_STATUS_CLEAR = 5; const int CACHE_START_FROM_BEGIN = 0; const int CACHE_START_FROM_END = -1; + /* * key type */ @@ -434,7 +401,6 @@ const char PIKA_KEY_TYPE_LIST = 'l'; const char PIKA_KEY_TYPE_SET = 's'; const char PIKA_KEY_TYPE_ZSET = 'z'; - /* * cache task type */ diff --git a/include/pika_geo.h b/include/pika_geo.h index 9a077e9ff8..70b287da03 100644 --- a/include/pika_geo.h +++ b/include/pika_geo.h @@ -6,9 +6,10 @@ #ifndef PIKA_GEO_H_ #define PIKA_GEO_H_ +#include "include/pika_db.h" #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" +#include "storage/storage.h" /* * zset @@ -58,8 +59,8 @@ class GeoAddCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new GeoAddCmd(*this); } @@ -78,8 +79,8 @@ class GeoPosCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new GeoPosCmd(*this); } @@ -98,8 +99,8 @@ class GeoDistCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new GeoDistCmd(*this); } @@ -117,8 +118,8 @@ class GeoHashCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override{}; Cmd* Clone() override { return new GeoHashCmd(*this); } @@ -132,8 +133,8 @@ class GeoRadiusCmd : public Cmd { public: GeoRadiusCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::GEO)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new GeoRadiusCmd(*this); } @@ -158,8 +159,8 @@ class GeoRadiusByMemberCmd : public Cmd { public: GeoRadiusByMemberCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::GEO)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new GeoRadiusByMemberCmd(*this); } diff --git a/include/pika_geohash.h b/include/pika_geohash.h index 2971351f68..1ba348515e 100644 --- a/include/pika_geohash.h +++ b/include/pika_geohash.h @@ -43,7 +43,7 @@ extern "C" { #define RANGEISZERO(r) (!(r).max && !(r).min) #define RANGEPISZERO(r) ((r) == nullptr || RANGEISZERO(*(r))) -#define GEO_STEP_MAX 26 /* 26*2 = 52 bits. */ +#define GEO_STEP_MAX 26 /* 26 * 2 = 52 bits. */ /* Limits from EPSG:900913 / EPSG:3785 / OSGEO:41001 */ constexpr double GEO_LAT_MIN{-85.05112878}; @@ -51,18 +51,6 @@ constexpr double GEO_LAT_MAX{85.05112878}; constexpr int64_t GEO_LONG_MIN{-180}; constexpr int64_t GEO_LONG_MAX{180}; - -enum GeoDirection { - GEOHASH_NORTH = 0, - GEOHASH_EAST, - GEOHASH_WEST, - GEOHASH_SOUTH, - GEOHASH_SOUTH_WEST, - GEOHASH_SOUTH_EAST, - GEOHASH_NORT_WEST, - GEOHASH_NORT_EAST -}; - struct GeoHashBits { uint64_t bits; uint8_t step; @@ -102,11 +90,9 @@ int geohashEncodeWGS84(double longitude, double latitude, uint8_t step, GeoHashB int geohashDecode(GeoHashRange long_range, GeoHashRange lat_range, GeoHashBits hash, GeoHashArea* area); int geohashDecodeType(GeoHashBits hash, GeoHashArea* area); -int geohashDecodeWGS84(GeoHashBits hash, GeoHashArea* area); int geohashDecodeAreaToLongLat(const GeoHashArea* area, double* xy); int geohashDecodeToLongLatType(GeoHashBits hash, double* xy); int geohashDecodeToLongLatWGS84(GeoHashBits hash, double* xy); -int geohashDecodeToLongLatMercator(GeoHashBits hash, double* xy); void geohashNeighbors(const GeoHashBits* hash, GeoHashNeighbors* neighbors); #if defined(__cplusplus) diff --git a/include/pika_geohash_helper.h b/include/pika_geohash_helper.h index 8478ea60ac..63ad4782a2 100644 --- a/include/pika_geohash_helper.h +++ b/include/pika_geohash_helper.h @@ -35,11 +35,8 @@ #include "include/pika_geohash.h" #define GZERO(s) s.bits = (s).step = 0; -#define GISZERO(s) (!(s).bits && !(s).step) -#define GISNOTZERO(s) ((s).bits || (s).step) using GeoHashFix52Bits = uint64_t; -using GeoHashVarBits = uint64_t; struct GeoHashRadius { GeoHashBits hash; @@ -47,12 +44,10 @@ struct GeoHashRadius { GeoHashNeighbors neighbors; }; -int GeoHashBitsComparator(const GeoHashBits* a, const GeoHashBits* b); uint8_t geohashEstimateStepsByRadius(double range_meters, double lat); int geohashBoundingBox(double longitude, double latitude, double radius_meters, double* bounds); GeoHashRadius geohashGetAreasByRadius(double longitude, double latitude, double radius_meters); GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude, double radius_meters); -GeoHashRadius geohashGetAreasByRadiusMercator(double longitude, double latitude, double radius_meters); GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits& hash); double geohashGetDistance(double lon1d, double lat1d, double lon2d, double lat2d); int geohashGetDistanceIfInRadius(double x1, double y1, double x2, double y2, double radius, double* distance); diff --git a/include/pika_hash.h b/include/pika_hash.h index 40d5d1c5fb..2f31885520 100644 --- a/include/pika_hash.h +++ b/include/pika_hash.h @@ -7,10 +7,10 @@ #define PIKA_HASH_H_ #include "storage/storage.h" - #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" +#include "include/pika_db.h" +#include "storage/storage.h" /* * hash @@ -24,10 +24,10 @@ class HDelCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HDelCmd(*this); } @@ -48,11 +48,11 @@ class HGetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HGetCmd(*this); } @@ -71,11 +71,11 @@ class HGetallCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HGetallCmd(*this); } @@ -94,10 +94,10 @@ class HSetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HSetCmd(*this); } @@ -116,11 +116,11 @@ class HExistsCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HExistsCmd(*this); } @@ -139,10 +139,10 @@ class HIncrbyCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HIncrbyCmd(*this); } @@ -162,10 +162,10 @@ class HIncrbyfloatCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HIncrbyfloatCmd(*this); } @@ -184,11 +184,11 @@ class HKeysCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HKeysCmd(*this); } @@ -207,11 +207,11 @@ class HLenCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HLenCmd(*this); } @@ -230,11 +230,11 @@ class HMgetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HMgetCmd(*this); } @@ -254,10 +254,10 @@ class HMsetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HMsetCmd(*this); } @@ -277,10 +277,10 @@ class HSetnxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HSetnxCmd(*this); } @@ -299,11 +299,11 @@ class HStrlenCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HStrlenCmd(*this); } @@ -322,11 +322,11 @@ class HValsCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HValsCmd(*this); } @@ -345,8 +345,8 @@ class HScanCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HScanCmd(*this); } @@ -371,8 +371,8 @@ class HScanxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new HScanxCmd(*this); } @@ -397,8 +397,8 @@ class PKHScanRangeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKHScanRangeCmd(*this); } @@ -424,8 +424,8 @@ class PKHRScanRangeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKHRScanRangeCmd(*this); } diff --git a/include/pika_hyperloglog.h b/include/pika_hyperloglog.h index 2a096abf05..77c374642f 100644 --- a/include/pika_hyperloglog.h +++ b/include/pika_hyperloglog.h @@ -7,7 +7,6 @@ #define PIKA_HYPERLOGLOG_H_ #include "include/pika_command.h" -#include "include/pika_slot.h" #include "include/pika_kv.h" /* * hyperloglog @@ -20,8 +19,8 @@ class PfAddCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PfAddCmd(*this); } @@ -35,8 +34,8 @@ class PfAddCmd : public Cmd { class PfCountCmd : public Cmd { public: PfCountCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PfCountCmd(*this); } @@ -49,20 +48,20 @@ class PfCountCmd : public Cmd { class PfMergeCmd : public Cmd { public: PfMergeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } PfMergeCmd(const PfMergeCmd& other) : Cmd(other), keys_(other.keys_), value_to_dest_(other.value_to_dest_) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } std::vector current_key() const override { return keys_; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PfMergeCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector keys_; diff --git a/include/pika_kv.h b/include/pika_kv.h index 977c469bb8..e64b6feff2 100644 --- a/include/pika_kv.h +++ b/include/pika_kv.h @@ -7,10 +7,9 @@ #define PIKA_KV_H_ #include "storage/storage.h" - +#include "include/pika_db.h" #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" /* * kv @@ -25,12 +24,13 @@ class SetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SetCmd(*this); } + private: std::string key_; std::string value_; @@ -58,11 +58,11 @@ class GetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void ReadCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new GetCmd(*this); } @@ -78,14 +78,14 @@ class DelCmd : public Cmd { public: DelCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)){}; - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; std::vector current_key() const override { return keys_; } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override; + void Split(const HintKeys& hint_keys) override; void Merge() override; Cmd* Clone() override { return new DelCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector keys_; @@ -103,10 +103,10 @@ class IncrCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new IncrCmd(*this); } @@ -126,10 +126,10 @@ class IncrbyCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new IncrbyCmd(*this); } @@ -149,10 +149,10 @@ class IncrbyfloatCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new IncrbyfloatCmd(*this); } @@ -172,10 +172,10 @@ class DecrCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new DecrCmd(*this); } @@ -195,10 +195,10 @@ class DecrbyCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new DecrbyCmd(*this); } @@ -218,10 +218,10 @@ class GetsetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new GetsetCmd(*this); } @@ -241,10 +241,10 @@ class AppendCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new AppendCmd(*this); } @@ -259,12 +259,12 @@ class MgetCmd : public Cmd { public: MgetCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STRING)){}; - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; std::vector current_key() const override { return keys_; } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override; + void Split(const HintKeys& hint_keys) override; void Merge() override; Cmd* Clone() override { return new MgetCmd(*this); } @@ -283,8 +283,8 @@ class KeysCmd : public Cmd { public: KeysCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STRING)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new KeysCmd(*this); } @@ -305,8 +305,8 @@ class SetnxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SetnxCmd(*this); } @@ -328,10 +328,10 @@ class SetexCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SetexCmd(*this); } @@ -353,10 +353,10 @@ class PsetexCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PsetexCmd(*this); } @@ -378,8 +378,8 @@ class DelvxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new DelvxCmd(*this); } @@ -395,15 +395,15 @@ class MsetCmd : public Cmd { public: MsetCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STRING)) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } MsetCmd(const MsetCmd& other) : Cmd(other), kvs_(other.kvs_) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; std::vector current_key() const override { std::vector res; for (auto& kv : kvs_) { @@ -411,10 +411,10 @@ class MsetCmd : public Cmd { } return res; } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override; + void Split(const HintKeys& hint_keys) override; void Merge() override; Cmd* Clone() override { return new MsetCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector kvs_; @@ -428,11 +428,11 @@ class MsetnxCmd : public Cmd { public: MsetnxCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STRING)) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } MsetnxCmd(const MsetnxCmd& other) : Cmd(other), kvs_(other.kvs_), success_(other.success_) { - set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv); + set_cmd_ = std::make_shared(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv); } std::vector current_key() const override { std::vector res; @@ -441,11 +441,11 @@ class MsetnxCmd : public Cmd { } return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new MsetnxCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector kvs_; @@ -464,11 +464,11 @@ class GetrangeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new GetrangeCmd(*this); } @@ -491,10 +491,10 @@ class SetrangeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SetrangeCmd(*this); } @@ -515,11 +515,11 @@ class StrlenCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new StrlenCmd(*this); } @@ -535,11 +535,11 @@ class ExistsCmd : public Cmd { public: ExistsCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)) {} - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; std::vector current_key() const override { return keys_; } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override; + void Split(const HintKeys& hint_keys) override; void Merge() override; Cmd* Clone() override { return new ExistsCmd(*this); } @@ -558,10 +558,10 @@ class ExpireCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ExpireCmd(*this); } @@ -582,10 +582,10 @@ class PexpireCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PexpireCmd(*this); } @@ -606,10 +606,10 @@ class ExpireatCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ExpireatCmd(*this); } @@ -629,10 +629,10 @@ class PexpireatCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PexpireatCmd(*this); } @@ -653,10 +653,10 @@ class TtlCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new TtlCmd(*this); } @@ -675,10 +675,10 @@ class PttlCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PttlCmd(*this); } @@ -697,10 +697,10 @@ class PersistCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PersistCmd(*this); } @@ -719,10 +719,10 @@ class TypeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new TypeCmd(*this); } @@ -741,8 +741,8 @@ class PTypeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PTypeCmd(*this); } @@ -756,8 +756,8 @@ class ScanCmd : public Cmd { public: ScanCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)), pattern_("*") {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ScanCmd(*this); } @@ -779,8 +779,8 @@ class ScanxCmd : public Cmd { public: ScanxCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::KEYSPACE)), pattern_("*") {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ScanxCmd(*this); } @@ -806,8 +806,8 @@ class PKSetexAtCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKSetexAtCmd(*this); } @@ -829,8 +829,8 @@ class PKScanRangeCmd : public Cmd { res.push_back(key_start_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKScanRangeCmd(*this); } @@ -859,8 +859,8 @@ class PKRScanRangeCmd : public Cmd { res.push_back(key_start_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new PKRScanRangeCmd(*this); } diff --git a/include/pika_list.h b/include/pika_list.h index d7ac1752de..5cb9d465c4 100644 --- a/include/pika_list.h +++ b/include/pika_list.h @@ -8,7 +8,6 @@ #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" #include "storage/storage.h" /* @@ -23,11 +22,11 @@ class LIndexCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LIndexCmd(*this); } @@ -48,10 +47,10 @@ class LInsertCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LInsertCmd(*this); } @@ -73,11 +72,11 @@ class LLenCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LLenCmd(*this); } @@ -96,15 +95,15 @@ class BlockingBaseCmd : public Cmd { struct WriteBinlogOfPopArgs { BlockKeyType block_type; std::string key; - std::shared_ptr slot; + std::shared_ptr db; std::shared_ptr conn; WriteBinlogOfPopArgs() = default; - WriteBinlogOfPopArgs(BlockKeyType block_type_, const std::string& key_, std::shared_ptr slot_, + WriteBinlogOfPopArgs(BlockKeyType block_type_, const std::string& key_, std::shared_ptr db_, std::shared_ptr conn_) - : block_type(block_type_), key(key_), slot(slot_), conn(conn_) {} + : block_type(block_type_), key(key_), db(db_), conn(conn_) {} }; void BlockThisClientToWaitLRPush(BlockKeyType block_pop_type, std::vector& keys, int64_t expire_time); - void TryToServeBLrPopWithThisKey(const std::string& key, std::shared_ptr slot); + void TryToServeBLrPopWithThisKey(const std::string& key, std::shared_ptr db); static void ServeAndUnblockConns(void* args); static void WriteBinlogOfPop(std::vector& pop_args); void removeDuplicates(std::vector& keys_); @@ -116,12 +115,12 @@ class BLPopCmd final : public BlockingBaseCmd { BLPopCmd(const std::string& name, int arity, uint32_t flag) : BlockingBaseCmd(name, arity, flag, static_cast(AclCategory::BLOCKING)){}; std::vector current_key() const override { return {keys_}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new BLPopCmd(*this); } void DoInitial() override; - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector keys_; @@ -140,10 +139,10 @@ class LPopCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LPopCmd(*this); } @@ -162,10 +161,10 @@ class LPushCmd : public BlockingBaseCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LPushCmd(*this); } @@ -186,10 +185,10 @@ class LPushxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LPushxCmd(*this); } @@ -210,11 +209,11 @@ class LRangeCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LRangeCmd(*this); } @@ -235,10 +234,10 @@ class LRemCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LRemCmd(*this); } @@ -259,10 +258,10 @@ class LSetCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LSetCmd(*this); } @@ -283,10 +282,10 @@ class LTrimCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new LTrimCmd(*this); } @@ -303,12 +302,12 @@ class BRPopCmd final : public BlockingBaseCmd { BRPopCmd(const std::string& name, int arity, uint32_t flag) : BlockingBaseCmd(name, arity, flag, static_cast(AclCategory::BLOCKING)){}; std::vector current_key() const override { return {keys_}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new BRPopCmd(*this); } void DoInitial() override; - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::vector keys_; @@ -326,10 +325,10 @@ class RPopCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new RPopCmd(*this); } @@ -344,8 +343,8 @@ class RPopLPushCmd : public BlockingBaseCmd { public: RPopLPushCmd(const std::string& name, int arity, uint32_t flag) : BlockingBaseCmd(name, arity, flag, static_cast(AclCategory::BLOCKING)) { - rpop_cmd_ = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); - lpush_cmd_ = std::make_shared(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); + rpop_cmd_ = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsList); + lpush_cmd_ = std::make_shared(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsList); }; RPopLPushCmd(const RPopLPushCmd& other) : BlockingBaseCmd(other), @@ -353,8 +352,8 @@ class RPopLPushCmd : public BlockingBaseCmd { receiver_(other.receiver_), value_poped_from_source_(other.value_poped_from_source_), is_write_binlog_(other.is_write_binlog_) { - rpop_cmd_ = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); - lpush_cmd_ = std::make_shared(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); + rpop_cmd_ = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsList); + lpush_cmd_ = std::make_shared(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsList); } std::vector current_key() const override { std::vector res; @@ -362,12 +361,12 @@ class RPopLPushCmd : public BlockingBaseCmd { res.push_back(source_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new RPopLPushCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::string source_; @@ -389,11 +388,11 @@ class RPushCmd : public BlockingBaseCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new RPushCmd(*this); } @@ -414,10 +413,10 @@ class RPushxCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoThroughDB() override; + void DoUpdateCache() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new RPushxCmd(*this); } diff --git a/include/pika_migrate_thread.h b/include/pika_migrate_thread.h index 82a5ac4c2b..d7b75dc2b5 100644 --- a/include/pika_migrate_thread.h +++ b/include/pika_migrate_thread.h @@ -3,31 +3,31 @@ #include "include/pika_client_conn.h" #include "include/pika_command.h" -#include "include/pika_slot.h" #include "net/include/net_cli.h" #include "net/include/net_thread.h" #include "pika_client_conn.h" +#include "pika_db.h" #include "storage/storage.h" #include "strings.h" -void WriteDelKeyToBinlog(const std::string &key, const std::shared_ptr& slot); -static int DoMigrate(net::NetCli *cli, std::string send_str); +void WriteDelKeyToBinlog(const std::string& key, const std::shared_ptr& db); class PikaMigrateThread; +class DB; class PikaParseSendThread : public net::Thread { public: - PikaParseSendThread(PikaMigrateThread *migrate_thread, const std::shared_ptr& slot_); + PikaParseSendThread(PikaMigrateThread* migrate_thread, const std::shared_ptr& db_); ~PikaParseSendThread() override; - - bool Init(const std::string &ip, int64_t port, int64_t timeout_ms, int64_t mgrtkeys_num); + bool Init(const std::string& ip, int64_t port, int64_t timeout_ms, int64_t mgrtkeys_num); void ExitThread(void); private: - int MigrateOneKey(net::NetCli *cli, const std::string& key, const char key_type, bool async); - void DelKeysAndWriteBinlog(std::deque> &send_keys, const std::shared_ptr& slot); + int MigrateOneKey(net::NetCli* cli, const std::string& key, const char key_type, bool async); + void DelKeysAndWriteBinlog(std::deque>& send_keys, const std::shared_ptr& db); bool CheckMigrateRecv(int64_t need_receive_num); void *ThreadMain() override; + private: std::string dest_ip_; int64_t dest_port_ = 0; @@ -37,18 +37,18 @@ class PikaParseSendThread : public net::Thread { PikaMigrateThread *migrate_thread_ = nullptr; net::NetCli *cli_ = nullptr; pstd::Mutex working_mutex_; - std::shared_ptr slot_; + std::shared_ptr db_; }; class PikaMigrateThread : public net::Thread { public: PikaMigrateThread(); ~PikaMigrateThread() override; - bool ReqMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_num, int64_t keys_num, - const std::shared_ptr& slot); - int ReqMigrateOne(const std::string &key, const std::shared_ptr& slot); - void GetMigrateStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, - int64_t *remained); + bool ReqMigrateBatch(const std::string& ip, int64_t port, int64_t time_out, int64_t keys_num, int64_t slot_id, + const std::shared_ptr& db); + int ReqMigrateOne(const std::string& key, const std::shared_ptr& db); + void GetMigrateStatus(std::string* ip, int64_t* port, int64_t* slot, bool* migrating, int64_t* moved, + int64_t* remained); void CancelMigrate(void); void IncWorkingThreadNum(void); void DecWorkingThreadNum(void); @@ -63,8 +63,8 @@ class PikaMigrateThread : public net::Thread { void ResetThread(void); void DestroyThread(bool is_self_exit); void NotifyRequestMigrate(void); - bool IsMigrating(std::pair &kpair); - void ReadSlotKeys(const std::string &slotKey, int64_t need_read_num, int64_t &real_read_num, int32_t *finish); + bool IsMigrating(std::pair& kpair); + void ReadSlotKeys(const std::string& slotKey, int64_t need_read_num, int64_t& real_read_num, int32_t* finish); bool CreateParseSendThreads(int32_t dispatch_num); void DestroyParseSendThreads(void); void *ThreadMain() override; @@ -73,31 +73,29 @@ class PikaMigrateThread : public net::Thread { std::string dest_ip_; int64_t dest_port_ = 0; int64_t timeout_ms_ = 60; - int64_t slot_id_ = 0; int64_t keys_num_ = 0; time_t start_time_ = 0; time_t end_time_ = 0; std::string s_start_time_; - std::shared_ptr slot_; + std::shared_ptr db_; std::atomic is_migrating_; std::atomic should_exit_; std::atomic is_task_success_; std::atomic send_num_; std::atomic response_num_; std::atomic moved_num_; - std::shared_ptr slot; bool request_migrate_ = false; pstd::CondVar request_migrate_cond_; std::mutex request_migrate_mutex_; int32_t workers_num_ = 0; - std::vector workers_; + std::vector workers_; std::atomic working_thread_num_; pstd::CondVar workers_cond_; std::mutex workers_mutex_; - + int64_t slot_id_ = 0; std::deque> mgrtone_queue_; std::mutex mgrtone_queue_mutex_; diff --git a/include/pika_monitor_thread.h b/include/pika_monitor_thread.h deleted file mode 100644 index 42bab74783..0000000000 --- a/include/pika_monitor_thread.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_MONITOR_THREAD_H_ -#define PIKA_MONITOR_THREAD_H_ - -#include -#include -#include -#include - -#include "net/include/net_thread.h" -#include "pstd/include/pstd_mutex.h" - -#include "include/pika_client_conn.h" -#include "include/pika_define.h" - -class PikaMonitorThread : public net::Thread { - public: - PikaMonitorThread(); - ~PikaMonitorThread() override; - - void AddMonitorClient(const std::shared_ptr& client_ptr); - void AddMonitorMessage(const std::string& monitor_message); - int32_t ThreadClientList(std::vector* client = nullptr); - bool ThreadClientKill(const std::string& ip_port = "all"); - bool HasMonitorClients(); - - private: - void AddCronTask(const MonitorCronTask& task); - bool FindClient(const std::string& ip_port); - net::WriteStatus SendMessage(int32_t fd, std::string& message); - void RemoveMonitorClient(const std::string& ip_port); - - std::atomic has_monitor_clients_; - pstd::Mutex monitor_mutex_protector_; - pstd::CondVar monitor_cond_; - - std::list monitor_clients_; - std::deque monitor_messages_; - std::queue cron_tasks_; - - void* ThreadMain() override; - void RemoveMonitorClient(int32_t client_fd); -}; -#endif diff --git a/include/pika_monotonic_time.h b/include/pika_monotonic_time.h index c2e0a8f8b6..909fadfaec 100644 --- a/include/pika_monotonic_time.h +++ b/include/pika_monotonic_time.h @@ -12,11 +12,9 @@ * holding a monotonic time. This will help distinguish & document that the * variable is associated with the monotonic clock and should not be confused * with other types of time.*/ -typedef uint64_t monotime; +using monotime = uint64_t; // Get monotonic time in microseconds monotime getMonotonicUs(); -#endif // PIKA_MONOTONIC_TIME_H - - +#endif // PIKA_MONOTONIC_TIME_H \ No newline at end of file diff --git a/include/pika_pubsub.h b/include/pika_pubsub.h index d440e02bf4..f9f7d85a30 100644 --- a/include/pika_pubsub.h +++ b/include/pika_pubsub.h @@ -16,8 +16,8 @@ class PublishCmd : public Cmd { public: PublishCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PublishCmd(*this); } std::vector current_key() const override { return {channel_}; } @@ -32,8 +32,8 @@ class SubscribeCmd : public Cmd { public: SubscribeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SubscribeCmd(*this); } std::vector current_key() const override { return channels_; } @@ -47,8 +47,8 @@ class UnSubscribeCmd : public Cmd { public: UnSubscribeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new UnSubscribeCmd(*this); } std::vector current_key() const override { return channels_; } @@ -62,8 +62,8 @@ class PUnSubscribeCmd : public Cmd { public: PUnSubscribeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PUnSubscribeCmd(*this); } std::vector current_key() const override { return {channels_}; } @@ -77,8 +77,8 @@ class PSubscribeCmd : public Cmd { public: PSubscribeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PSubscribeCmd(*this); } std::vector current_key() const override { return {channels_}; } @@ -92,8 +92,8 @@ class PubSubCmd : public Cmd { public: PubSubCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::PUBSUB)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new PubSubCmd(*this); } diff --git a/include/pika_repl_bgworker.h b/include/pika_repl_bgworker.h index 156f8d833f..2401d72009 100644 --- a/include/pika_repl_bgworker.h +++ b/include/pika_repl_bgworker.h @@ -16,6 +16,7 @@ #include "pika_inner_message.pb.h" #include "include/pika_binlog_transverter.h" +#include "include/pika_define.h" #include "include/pika_command.h" class PikaReplBgWorker { @@ -32,7 +33,6 @@ class PikaReplBgWorker { net::RedisParser redis_parser_; std::string ip_port_; std::string db_name_; - uint32_t slot_id_ = 0; private: net::BGThread bg_thread_; diff --git a/include/pika_repl_client.h b/include/pika_repl_client.h index 04a358d8d7..2389f35978 100644 --- a/include/pika_repl_client.h +++ b/include/pika_repl_client.h @@ -19,7 +19,6 @@ #include "include/pika_define.h" #include "include/pika_repl_bgworker.h" #include "include/pika_repl_client_thread.h" -#include "include/pika_slot.h" #include "net/include/thread_pool.h" #include "pika_inner_message.pb.h" @@ -37,7 +36,7 @@ struct ReplClientWriteBinlogTaskArg { std::shared_ptr conn; void* res_private_data; PikaReplBgWorker* worker; - ReplClientWriteBinlogTaskArg(const std::shared_ptr& _res, + ReplClientWriteBinlogTaskArg(const std::shared_ptr& _res, const std::shared_ptr& _conn, void* _res_private_data, PikaReplBgWorker* _worker) : res(_res), conn(_conn), res_private_data(_res_private_data), worker(_worker) {} @@ -47,13 +46,10 @@ struct ReplClientWriteDBTaskArg { const std::shared_ptr cmd_ptr; LogOffset offset; std::string db_name; - uint32_t slot_id; - ReplClientWriteDBTaskArg(std::shared_ptr _cmd_ptr, const LogOffset& _offset, std::string _db_name, - uint32_t _slot_id) + ReplClientWriteDBTaskArg(std::shared_ptr _cmd_ptr, const LogOffset& _offset, std::string _db_name) : cmd_ptr(std::move(_cmd_ptr)), offset(_offset), - db_name(std::move(_db_name)), - slot_id(_slot_id) {} + db_name(std::move(_db_name)) {} ~ReplClientWriteDBTaskArg() = default; }; @@ -69,21 +65,19 @@ class PikaReplClient { pstd::Status Close(const std::string& ip, int port); void Schedule(net::TaskFunc func, void* arg); - void ScheduleWriteBinlogTask(const std::string& db_slot, const std::shared_ptr& res, + void ScheduleWriteBinlogTask(const std::string& db_name, const std::shared_ptr& res, const std::shared_ptr& conn, void* res_private_data); - void ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, const std::string& db_name, - uint32_t slot_id); + void ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, const std::string& db_name); pstd::Status SendMetaSync(); - pstd::Status SendSlotDBSync(const std::string& ip, uint32_t port, const std::string& db_name, uint32_t slot_id, + pstd::Status SendDBSync(const std::string& ip, uint32_t port, const std::string& db_name, const BinlogOffset& boffset, const std::string& local_ip); - pstd::Status SendSlotTrySync(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const BinlogOffset& boffset, const std::string& local_ip); - pstd::Status SendSlotBinlogSync(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const LogOffset& ack_start, const LogOffset& ack_end, + pstd::Status SendTrySync(const std::string& ip, uint32_t port, const std::string& db_name, + const BinlogOffset& boffset, const std::string& local_ip); + pstd::Status SendBinlogSync(const std::string& ip, uint32_t port, const std::string& db_name, + const LogOffset& ack_start, const LogOffset& ack_end, const std::string& local_ip, bool is_first_send); - pstd::Status SendRemoveSlaveNode(const std::string& ip, uint32_t port, const std::string& db_name, uint32_t slot_id, - const std::string& local_ip); + pstd::Status SendRemoveSlaveNode(const std::string& ip, uint32_t port, const std::string& db_name, const std::string& local_ip); private: size_t GetHashIndex(const std::string& key, bool upper_half); diff --git a/include/pika_repl_client_conn.h b/include/pika_repl_client_conn.h index fcafe2d05e..a071dfdec4 100644 --- a/include/pika_repl_client_conn.h +++ b/include/pika_repl_client_conn.h @@ -14,8 +14,8 @@ #include "include/pika_conf.h" #include "pika_inner_message.pb.h" -class SyncMasterSlot; -class SyncSlaveSlot; +class SyncMasterDB; +class SyncSlaveDB; class PikaReplClientConn : public net::PbConn { public: @@ -29,22 +29,15 @@ class PikaReplClientConn : public net::PbConn { static void HandleRemoveSlaveNodeResponse(void* arg); static pstd::Status TrySyncConsensusCheck(const InnerMessage::ConsensusMeta& consensus_meta, - const std::shared_ptr& slot, - const std::shared_ptr& slave_slot); + const std::shared_ptr& db, + const std::shared_ptr& slave_db); static bool IsDBStructConsistent(const std::vector& current_dbs, const std::vector& expect_tables); int DealMessage() override; private: - // dispatch binlog by its table_name + slot + // dispatch binlog by its db_name void DispatchBinlogRes(const std::shared_ptr& response); - - struct ReplRespArg { - std::shared_ptr resp; - std::shared_ptr conn; - ReplRespArg(std::shared_ptr _resp, std::shared_ptr _conn) - : resp(std::move(_resp)), conn(std::move(_conn)) {} - }; }; #endif diff --git a/include/pika_repl_client_thread.h b/include/pika_repl_client_thread.h index 92c29ee0a2..fe8213b090 100644 --- a/include/pika_repl_client_thread.h +++ b/include/pika_repl_client_thread.h @@ -18,7 +18,6 @@ class PikaReplClientThread : public net::ClientThread { public: PikaReplClientThread(int cron_interval, int keepalive_timeout); ~PikaReplClientThread() override = default; - int Start(); private: class ReplClientConnFactory : public net::ConnFactory { @@ -36,11 +35,6 @@ class PikaReplClientThread : public net::ClientThread { void FdTimeoutHandle(int fd, const std::string& ip_port) const override; void FdClosedHandle(int fd, const std::string& ip_port) const override; bool AccessHandle(std::string& ip) const override { - // ban 127.0.0.1 if you want to test this routine - // if (ip.find("127.0.0.2") != std::string::npos) { - // std::cout << "AccessHandle " << ip << std::endl; - // return false; - // } return true; } int CreateWorkerSpecificData(void** data) const override { return 0; } diff --git a/include/pika_repl_server.h b/include/pika_repl_server.h index 68daf1ee4d..4a12f99cb9 100644 --- a/include/pika_repl_server.h +++ b/include/pika_repl_server.h @@ -32,10 +32,10 @@ class PikaReplServer { int Stop(); pstd::Status SendSlaveBinlogChips(const std::string& ip, int port, const std::vector& tasks); - void BuildBinlogOffset(const LogOffset& offset, InnerMessage::BinlogOffset* boffset); - void BuildBinlogSyncResp(const std::vector& tasks, InnerMessage::InnerResponse* resp); pstd::Status Write(const std::string& ip, int port, const std::string& msg); + void BuildBinlogOffset(const LogOffset& offset, InnerMessage::BinlogOffset* boffset); + void BuildBinlogSyncResp(const std::vector& tasks, InnerMessage::InnerResponse* resp); void Schedule(net::TaskFunc func, void* arg); void UpdateClientConnMap(const std::string& ip_port, int fd); void RemoveClientConn(int fd); @@ -44,7 +44,6 @@ class PikaReplServer { private: std::unique_ptr server_tp_ = nullptr; std::unique_ptr pika_repl_server_thread_ = nullptr; - std::shared_mutex client_conn_rwlock_; std::map client_conn_map_; }; diff --git a/include/pika_repl_server_conn.h b/include/pika_repl_server_conn.h index 271509fced..223773e134 100644 --- a/include/pika_repl_server_conn.h +++ b/include/pika_repl_server_conn.h @@ -14,7 +14,7 @@ #include "include/pika_define.h" #include "pika_inner_message.pb.h" -class SyncMasterSlot; +class SyncMasterDB; class PikaReplServerConn : public net::PbConn { public: @@ -25,14 +25,14 @@ class PikaReplServerConn : public net::PbConn { static void HandleMetaSyncRequest(void* arg); static void HandleTrySyncRequest(void* arg); - static bool TrySyncOffsetCheck(const std::shared_ptr& slot, + static bool TrySyncOffsetCheck(const std::shared_ptr& db, const InnerMessage::InnerRequest::TrySync& try_sync_request, InnerMessage::InnerResponse::TrySync* try_sync_response); - static bool TrySyncConsensusOffsetCheck(const std::shared_ptr& slot, + static bool TrySyncConsensusOffsetCheck(const std::shared_ptr& db, const InnerMessage::ConsensusMeta& meta, InnerMessage::InnerResponse* response, InnerMessage::InnerResponse::TrySync* try_sync_response); - static bool TrySyncUpdateSlaveNode(const std::shared_ptr& slot, + static bool TrySyncUpdateSlaveNode(const std::shared_ptr& db, const InnerMessage::InnerRequest::TrySync& try_sync_request, const std::shared_ptr& conn, InnerMessage::InnerResponse::TrySync* try_sync_response); diff --git a/include/pika_repl_server_thread.h b/include/pika_repl_server_thread.h index 013dd869a0..c4e356839b 100644 --- a/include/pika_repl_server_thread.h +++ b/include/pika_repl_server_thread.h @@ -14,12 +14,8 @@ class PikaReplServerThread : public net::HolyThread { public: PikaReplServerThread(const std::set& ips, int port, int cron_interval); ~PikaReplServerThread() override = default; - int ListenPort(); - // for ProcessBinlogData use - uint64_t GetnPlusSerial() { return serial_++; } - private: class ReplServerConnFactory : public net::ConnFactory { public: diff --git a/include/pika_rm.h b/include/pika_rm.h index 335248b719..4e624b5538 100644 --- a/include/pika_rm.h +++ b/include/pika_rm.h @@ -31,73 +31,55 @@ #define kRecvKeepAliveTimeout (20 * 1000000) -class SyncSlot { +class SyncDB { public: - SyncSlot(const std::string& db_name, uint32_t slot_id); - virtual ~SyncSlot() = default; - - SlotInfo& SyncSlotInfo() { return slot_info_; } - - std::string SlotName(); + SyncDB(const std::string& db_name); + virtual ~SyncDB() = default; + DBInfo& SyncDBInfo() { return db_info_; } + std::string DBName(); protected: - SlotInfo slot_info_; + DBInfo db_info_; }; -class SyncMasterSlot : public SyncSlot { +class SyncMasterDB : public SyncDB { public: - SyncMasterSlot(const std::string& db_name, uint32_t slot_id); + SyncMasterDB(const std::string& db_name); pstd::Status AddSlaveNode(const std::string& ip, int port, int session_id); pstd::Status RemoveSlaveNode(const std::string& ip, int port); - pstd::Status ActivateSlaveBinlogSync(const std::string& ip, int port, const LogOffset& offset); pstd::Status ActivateSlaveDbSync(const std::string& ip, int port); - pstd::Status SyncBinlogToWq(const std::string& ip, int port); - pstd::Status GetSlaveSyncBinlogInfo(const std::string& ip, int port, BinlogOffset* sent_offset, BinlogOffset* acked_offset); pstd::Status GetSlaveState(const std::string& ip, int port, SlaveState* slave_state); - - pstd::Status SetLastSendTime(const std::string& ip, int port, uint64_t time); - pstd::Status GetLastSendTime(const std::string& ip, int port, uint64_t* time); - pstd::Status SetLastRecvTime(const std::string& ip, int port, uint64_t time); - pstd::Status GetLastRecvTime(const std::string& ip, int port, uint64_t* time); - pstd::Status GetSafetyPurgeBinlog(std::string* safety_purge); - bool BinlogCloudPurge(uint32_t index); - pstd::Status WakeUpSlaveBinlogSync(); pstd::Status CheckSyncTimeout(uint64_t now); - + pstd::Status GetSlaveNodeSession(const std::string& ip, int port, int32_t* session); + void GetValidSlaveNames(std::vector* slavenames); int GetNumberOfSlaveNode(); + bool BinlogCloudPurge(uint32_t index); bool CheckSlaveNodeExist(const std::string& ip, int port); - pstd::Status GetSlaveNodeSession(const std::string& ip, int port, int32_t* session); - void GetValidSlaveNames(std::vector* slavenames); // display use pstd::Status GetInfo(std::string* info); // debug use std::string ToStringStatus(); - int32_t GenSessionId(); - bool CheckSessionId(const std::string& ip, int port, const std::string& db_name, uint64_t slot_id, - int session_id); + bool CheckSessionId(const std::string& ip, int port, const std::string& db_name, int session_id); // consensus use pstd::Status ConsensusUpdateSlave(const std::string& ip, int port, const LogOffset& start, const LogOffset& end); pstd::Status ConsensusProposeLog(const std::shared_ptr& cmd_ptr); - pstd::Status ConsensusSanityCheck(); pstd::Status ConsensusProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute); LogOffset ConsensusCommittedIndex(); LogOffset ConsensusLastIndex(); uint32_t ConsensusTerm(); void ConsensusUpdateTerm(uint32_t term); pstd::Status ConsensusUpdateAppliedIndex(const LogOffset& offset); - LogOffset ConsensusAppliedIndex(); pstd::Status ConsensusLeaderNegotiate(const LogOffset& f_last_offset, bool* reject, std::vector* hints); pstd::Status ConsensusFollowerNegotiate(const std::vector& hints, LogOffset* reply_offset); - pstd::Status ConsensusReset(const LogOffset& applied_offset); void CommitPreviousLogs(const uint32_t& term); std::shared_ptr StableLogger() { return coordinator_.StableLogger(); } @@ -110,7 +92,6 @@ class SyncMasterSlot : public SyncSlot { } private: - bool CheckReadBinlogFromCache(); // invoker need to hold slave_mu_ pstd::Status ReadBinlogFileToWq(const std::shared_ptr& slave_ptr); @@ -119,51 +100,36 @@ class SyncMasterSlot : public SyncSlot { pstd::Mutex session_mu_; int32_t session_id_ = 0; - ConsensusCoordinator coordinator_; }; -class SyncSlaveSlot : public SyncSlot { +class SyncSlaveDB : public SyncDB { public: - SyncSlaveSlot(const std::string& db_name, uint32_t slot_id); - + SyncSlaveDB(const std::string& db_name); void Activate(const RmNode& master, const ReplState& repl_state); void Deactivate(); - void SetLastRecvTime(uint64_t time); - uint64_t LastRecvTime(); - void SetReplState(const ReplState& repl_state); ReplState State(); - pstd::Status CheckSyncTimeout(uint64_t now); // For display pstd::Status GetInfo(std::string* info); // For debug std::string ToStringStatus(); - + std::string LocalIp(); + int32_t MasterSessionId(); const std::string& MasterIp(); - int MasterPort(); - void SetMasterSessionId(int32_t session_id); - - int32_t MasterSessionId(); - void SetLocalIp(const std::string& local_ip); - - std::string LocalIp(); - void StopRsync(); - void ActivateRsync(); - bool IsRsyncRunning() {return rsync_cli_->IsRunning();} private: std::unique_ptr rsync_cli_; - pstd::Mutex slot_mu_; + pstd::Mutex db_mu_; RmNode m_info_; ReplState repl_state_{kNoConnect}; std::string local_ip_; @@ -173,114 +139,89 @@ class PikaReplicaManager { public: PikaReplicaManager(); ~PikaReplicaManager() = default; - friend Cmd; - void Start(); void Stop(); - bool CheckMasterSyncFinished(); - - pstd::Status AddSyncSlotSanityCheck(const std::set& p_infos); - pstd::Status AddSyncSlot(const std::set& p_infos); - pstd::Status RemoveSyncSlotSanityCheck(const std::set& p_infos); - pstd::Status RemoveSyncSlot(const std::set& p_infos); - pstd::Status ActivateSyncSlaveSlot(const RmNode& node, const ReplState& repl_state); - pstd::Status DeactivateSyncSlaveSlot(const SlotInfo& p_info); - pstd::Status SyncDBSanityCheck(const std::string& db_name); - pstd::Status DelSyncDB(const std::string& db_name); + pstd::Status ActivateSyncSlaveDB(const RmNode& node, const ReplState& repl_state); // For Pika Repl Client Thread pstd::Status SendMetaSyncRequest(); - pstd::Status SendRemoveSlaveNodeRequest(const std::string& table, uint32_t slot_id); - pstd::Status SendSlotTrySyncRequest(const std::string& db_name, size_t slot_id); - pstd::Status SendSlotDBSyncRequest(const std::string& db_name, size_t slot_id); - pstd::Status SendSlotBinlogSyncAckRequest(const std::string& table, uint32_t slot_id, const LogOffset& ack_start, - const LogOffset& ack_end, bool is_first_send = false); + pstd::Status SendRemoveSlaveNodeRequest(const std::string& table); + pstd::Status SendTrySyncRequest(const std::string& db_name); + pstd::Status SendDBSyncRequest(const std::string& db_name); + pstd::Status SendBinlogSyncAckRequest(const std::string& table, const LogOffset& ack_start, + const LogOffset& ack_end, bool is_first_send = false); pstd::Status CloseReplClientConn(const std::string& ip, int32_t port); // For Pika Repl Server Thread pstd::Status SendSlaveBinlogChipsRequest(const std::string& ip, int port, const std::vector& tasks); - // For SyncMasterSlot - std::shared_ptr GetSyncMasterSlotByName(const SlotInfo& p_info); + // For SyncMasterDB + std::shared_ptr GetSyncMasterDBByName(const DBInfo& p_info); - // For SyncSlaveSlot - std::shared_ptr GetSyncSlaveSlotByName(const SlotInfo& p_info); + // For SyncSlaveDB + std::shared_ptr GetSyncSlaveDBByName(const DBInfo& p_info); - pstd::Status RunSyncSlaveSlotStateMachine(); + pstd::Status RunSyncSlaveDBStateMachine(); pstd::Status CheckSyncTimeout(uint64_t now); - // To check slot info + // To check db info // For pkcluster info command - pstd::Status GetSlotInfo(const std::string& table, uint32_t slot_id, std::string* info); - - void FindCompleteReplica(std::vector* replica); + static bool CheckSlaveDBState(const std::string& ip, int port); void FindCommonMaster(std::string* master); - pstd::Status CheckSlotRole(const std::string& table, uint32_t slot_id, int* role); - void RmStatus(std::string* debug_info); - - static bool CheckSlaveSlotState(const std::string& ip, int port); - + pstd::Status CheckDBRole(const std::string& table, int* role); pstd::Status LostConnection(const std::string& ip, int port); // Update binlog win and try to send next binlog pstd::Status UpdateSyncBinlogStatus(const RmNode& slave, const LogOffset& offset_start, const LogOffset& offset_end); - pstd::Status WakeUpBinlogSync(); // write_queue related - void ProduceWriteQueue(const std::string& ip, int port, uint32_t slot_id, const std::vector& tasks); - int ConsumeWriteQueue(); + void ProduceWriteQueue(const std::string& ip, int port, std::string db_name, const std::vector& tasks); void DropItemInWriteQueue(const std::string& ip, int port); + int ConsumeWriteQueue(); // Schedule Task void ScheduleReplServerBGTask(net::TaskFunc func, void* arg); void ScheduleReplClientBGTask(net::TaskFunc func, void* arg); - void ScheduleWriteBinlogTask(const std::string& db_slot, + void ScheduleWriteBinlogTask(const std::string& db_name, const std::shared_ptr& res, const std::shared_ptr& conn, void* res_private_data); - void ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, const std::string& db_name, - uint32_t slot_id); - + void ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, const std::string& db_name); void ReplServerRemoveClientConn(int fd); void ReplServerUpdateClientConnMap(const std::string& ip_port, int fd); - std::shared_mutex& GetSlotLock() { return slots_rw_; } - void SlotLock() { - slots_rw_.lock(); - } - void SlotLockShared() { - slots_rw_.lock_shared(); - } - void SlotUnlock() { - slots_rw_.unlock(); + std::shared_mutex& GetDBLock() { return dbs_rw_; } + + void DBLock() { + dbs_rw_.lock(); } - void SlotUnlockShared() { - slots_rw_.unlock_shared(); + void DBUnlock() { + dbs_rw_.unlock(); } - std::unordered_map, hash_slot_info>& GetSyncMasterSlots() { - return sync_master_slots_; + std::unordered_map, hash_db_info>& GetSyncMasterDBs() { + return sync_master_dbs_; } - std::unordered_map, hash_slot_info>& GetSyncSlaveSlots() { - return sync_slave_slots_; + std::unordered_map, hash_db_info>& GetSyncSlaveDBs() { + return sync_slave_dbs_; } private: - void InitSlot(); + void InitDB(); pstd::Status SelectLocalIp(const std::string& remote_ip, int remote_port, std::string* local_ip); - std::shared_mutex slots_rw_; - std::unordered_map, hash_slot_info> sync_master_slots_; - std::unordered_map, hash_slot_info> sync_slave_slots_; + std::shared_mutex dbs_rw_; + std::unordered_map, hash_db_info> sync_master_dbs_; + std::unordered_map, hash_db_info> sync_slave_dbs_; pstd::Mutex write_queue_mu_; - // every host owns a queue, the key is "ip+port" - std::unordered_map>> write_queues_; + // every host owns a queue, the key is "ip + port" + std::unordered_map>> write_queues_; std::unique_ptr pika_repl_client_; std::unique_ptr pika_repl_server_; }; diff --git a/include/pika_server.h b/include/pika_server.h index e3478d992b..33a0dfa589 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -29,8 +29,10 @@ #include "acl.h" #include "include/pika_auxiliary_thread.h" #include "include/pika_binlog.h" +#include "include/pika_cache.h" #include "include/pika_client_processor.h" #include "include/pika_cmd_table_manager.h" +#include "include/pika_command.h" #include "include/pika_conf.h" #include "include/pika_db.h" #include "include/pika_define.h" @@ -43,84 +45,8 @@ #include "include/pika_slot_command.h" #include "include/pika_statistic.h" #include "include/pika_transaction.h" -#include "include/pika_cmd_table_manager.h" -#include "include/pika_cache.h" -#include "include/pika_slot.h" - - #include "include/rsync_server.h" -/* -static std::set MultiKvCommands {kCmdNameDel, - kCmdNameMget, kCmdNameKeys, kCmdNameMset, - kCmdNameMsetnx, kCmdNameExists, kCmdNameScan, - kCmdNameScanx, kCmdNamePKScanRange, kCmdNamePKRScanRange, - kCmdNameRPopLPush, kCmdNameZUnionstore, kCmdNameZInterstore, - kCmdNameSUnion, kCmdNameSUnionstore, kCmdNameSInter, - kCmdNameSInterstore, kCmdNameSDiff, kCmdNameSDiffstore, - kCmdNameSMove, kCmdNameBitOp, kCmdNamePfAdd, - kCmdNamePfCount, kCmdNamePfMerge, kCmdNameGeoAdd, - kCmdNameGeoPos, kCmdNameGeoDist, kCmdNameGeoHash, - kCmdNameGeoRadius, kCmdNameGeoRadiusByMember}; -*/ - -static std::set ConsensusNotSupportCommands{kCmdNameMsetnx, - kCmdNameScan, - kCmdNameKeys, - kCmdNameRPopLPush, - kCmdNameZUnionstore, - kCmdNameZInterstore, - kCmdNameSUnion, - kCmdNameSUnionstore, - kCmdNameSInter, - kCmdNameSInterstore, - kCmdNameSDiff, - kCmdNameSDiffstore, - kCmdNameSMove, - kCmdNameBitOp, - kCmdNamePfAdd, - kCmdNamePfCount, - kCmdNamePfMerge, - kCmdNameGeoAdd, - kCmdNameGeoPos, - kCmdNameGeoDist, - kCmdNameGeoHash, - kCmdNameGeoRadius, - kCmdNameGeoRadiusByMember, - kCmdNamePKPatternMatchDel, - kCmdNameSlaveof, - kCmdNameDbSlaveof, - kCmdNameMset, - kCmdNameMget, - kCmdNameScanx}; - -static std::set ShardingModeNotSupportCommands{kCmdNameMsetnx, - kCmdNameScan, - kCmdNameKeys, - kCmdNameScanx, - kCmdNameZUnionstore, - kCmdNameZInterstore, - kCmdNameSUnion, - kCmdNameSUnionstore, - kCmdNameSInter, - kCmdNameSInterstore, - kCmdNameSDiff, - kCmdNameSDiffstore, - kCmdNameSMove, - kCmdNameBitOp, - kCmdNamePfAdd, - kCmdNamePfCount, - kCmdNamePfMerge, - kCmdNameGeoAdd, - kCmdNameGeoPos, - kCmdNameGeoDist, - kCmdNameGeoHash, - kCmdNameGeoRadius, - kCmdNameGeoRadiusByMember, - kCmdNamePKPatternMatchDel, - kCmdNameSlaveof, - kCmdNameDbSlaveof}; - extern std::unique_ptr g_pika_conf; enum TaskType { @@ -161,7 +87,6 @@ class PikaServer : public pstd::noncopyable { * Server init info */ bool ServerInit(); - void Start(); void Exit(); @@ -182,19 +107,15 @@ class PikaServer : public pstd::noncopyable { storage::StorageOptions storage_options(); /* - * Table use + * DB use */ void InitDBStruct(); - pstd::Status AddDBStruct(const std::string& db_name, uint32_t num); - pstd::Status DelDBStruct(const std::string& db_name); - std::shared_ptr GetDB(const std::string& db_name); - std::set GetDBSlotIds(const std::string& db_name); bool IsBgSaving(); bool IsKeyScaning(); bool IsCompacting(); bool IsDBExist(const std::string& db_name); - bool IsDBSlotExist(const std::string& db_name, uint32_t slot_id); bool IsDBBinlogIoError(const std::string& db_name); + std::shared_ptr GetDB(const std::string& db_name); std::set GetAllDBName(); pstd::Status DoSameThingSpecificDB(const std::set& dbs, const TaskArg& arg); std::shared_mutex& GetDBLock() { @@ -214,17 +135,14 @@ class PikaServer : public pstd::noncopyable { } /* - * Slot use + * DB use */ - void PrepareSlotTrySync(); - void SlotSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys); - void SlotSetSmallCompactionThreshold(uint32_t small_compaction_threshold); - void SlotSetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold); - bool GetDBSlotBinlogOffset(const std::string& db_name, uint32_t slot_id, BinlogOffset* boffset); - std::shared_ptr GetSlotByDBName(const std::string& db_name); - std::shared_ptr GetDBSlotById(const std::string& db_name, uint32_t slot_id); - std::shared_ptr GetDBSlotByKey(const std::string& db_name, const std::string& key); - pstd::Status DoSameThingEverySlot(const TaskType& type); + void PrepareDBTrySync(); + void DBSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys); + void DBSetSmallCompactionThreshold(uint32_t small_compaction_threshold); + void DBSetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold); + bool GetDBBinlogOffset(const std::string& db_name, BinlogOffset* boffset); + pstd::Status DoSameThingEveryDB(const TaskType& type); /* * Master use @@ -233,7 +151,6 @@ class PikaServer : public pstd::noncopyable { void DeleteSlave(int fd); // conn fd int32_t CountSyncSlaves(); int32_t GetSlaveListString(std::string& slave_list_str); - int32_t GetShardingSlaveListString(std::string& slave_list_str); bool TryAddSlave(const std::string& ip, int64_t port, int fd, const std::vector& table_structs); pstd::Mutex slave_mutex_; // protect slaves_; std::vector slaves_; @@ -257,9 +174,7 @@ class PikaServer : public pstd::noncopyable { void FinishMetaSync(); bool MetaSyncDone(); void ResetMetaSyncStatus(); - bool AllSlotConnectSuccess(); - bool LoopSlotStateMachine(); - void SetLoopSlotStateMachine(bool need_loop); + void SetLoopDBStateMachine(bool need_loop); int GetMetaSyncTimestamp(); void UpdateMetaSyncTimestamp(); void UpdateMetaSyncTimestampWithoutLock(); @@ -296,13 +211,12 @@ class PikaServer : public pstd::noncopyable { /* * DBSync used */ - pstd::Status GetDumpUUID(const std::string& db_name, const uint32_t slot_id, std::string* snapshot_uuid); - pstd::Status GetDumpMeta(const std::string& db_name, const uint32_t slot_id, std::vector* files, - std::string* snapshot_uuid); - void DBSync(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id); - void DoBgSaveSlot(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, int32_t top); - void DbSyncSendFile(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id); - std::string DbSyncTaskIndex(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id); + pstd::Status GetDumpUUID(const std::string& db_name, std::string* snapshot_uuid); + pstd::Status GetDumpMeta(const std::string& db_name, std::vector* files, std::string* snapshot_uuid); + void DBSync(const std::string& ip, int port, const std::string& db_name); + void TryDBSync(const std::string& ip, int port, const std::string& db_name, int32_t top); + void DbSyncSendFile(const std::string& ip, int port, const std::string& db_name); + std::string DbSyncTaskIndex(const std::string& ip, int port, const std::string& db_name); /* * Keyscan used @@ -329,23 +243,22 @@ class PikaServer : public pstd::noncopyable { */ void SlowlogTrim(); void SlowlogReset(); - uint32_t SlowlogLen(); void SlowlogObtain(int64_t number, std::vector* slowlogs); - void SlowlogPushEntry(const PikaCmdArgsType& argv, int64_t time, int64_t duration); + void SlowlogPushEntry(const std::vector& argv, int64_t time, int64_t duration); + uint32_t SlowlogLen(); uint64_t SlowlogCount(); /* * Statistic used */ - void ResetStat(); uint64_t ServerQueryNum(); uint64_t ServerCurrentQps(); uint64_t accumulative_connections(); + void ResetStat(); void incr_accumulative_connections(); void ResetLastSecQuerynum(); void UpdateQueryNumAndExecCountDB(const std::string& db_name, const std::string& command, bool is_write); std::unordered_map ServerExecCountDB(); - QpsStatistic ServerDBStat(const std::string& db_name); std::unordered_map ServerAllDBStat(); /* @@ -379,7 +292,6 @@ class PikaServer : public pstd::noncopyable { std::vector>* result); void PubSubChannels(const std::string& pattern, std::vector* result); void PubSubNumSub(const std::vector& channels, std::vector>* result); - int ClientPubSubChannelSize(const std::shared_ptr& conn); int ClientPubSubChannelPatternSize(const std::shared_ptr& conn); @@ -389,15 +301,12 @@ class PikaServer : public pstd::noncopyable { void ServerStatus(std::string* info); /* - * * Async migrate used + * Async migrate used */ - int SlotsMigrateOne(const std::string& key, const std::shared_ptr& slot); - bool SlotsMigrateBatch(const std::string& ip, int64_t port, int64_t time_out, int64_t slots, int64_t keys_num, - const std::shared_ptr& slot); - void GetSlotsMgrtSenderStatus(std::string* ip, int64_t* port, int64_t* slot, bool* migrating, int64_t* moved, - int64_t* remained); + int SlotsMigrateOne(const std::string& key, const std::shared_ptr &db); + bool SlotsMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slots, int64_t keys_num, const std::shared_ptr& db); + void GetSlotsMgrtSenderStatus(std::string *ip, int64_t* port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained); bool SlotsMigrateAsyncCancel(); - std::shared_mutex bgsave_protector_; BgSaveInfo bgsave_info_; @@ -412,7 +321,7 @@ class PikaServer : public pstd::noncopyable { int64_t cursor = 0; std::string pattern = "*"; int64_t count = 100; - std::shared_ptr slot; + std::shared_ptr db; BGSlotsReload() = default; void Clear() { reloading = false; @@ -449,7 +358,7 @@ class PikaServer : public pstd::noncopyable { std::lock_guard ml(bgsave_protector_); bgslots_reload_.end_time = time(nullptr); } - void Bgslotsreload(const std::shared_ptr& slot); + void Bgslotsreload(const std::shared_ptr& db); // Revoke the authorization of the specified account, when handle Cmd deleteUser void AllClientUnAuth(const std::set& users); @@ -468,7 +377,7 @@ class PikaServer : public pstd::noncopyable { int64_t cursor = 0; std::string pattern = "*"; int64_t count = 100; - std::shared_ptr slot; + std::shared_ptr db; storage::DataType type_; std::vector cleanup_slots; BGSlotsCleanup() = default; @@ -502,10 +411,6 @@ class PikaServer : public pstd::noncopyable { std::lock_guard ml(bgsave_protector_); bgslots_cleanup_.cursor = cursor; } - int64_t GetSlotscleaningupCursor() { - std::lock_guard ml(bgsave_protector_); - return bgslots_cleanup_.cursor; - } void SetCleanupSlots(std::vector cleanup_slots) { std::lock_guard ml(bgsave_protector_); bgslots_cleanup_.cleanup_slots.swap(cleanup_slots); @@ -514,11 +419,8 @@ class PikaServer : public pstd::noncopyable { std::lock_guard ml(bgsave_protector_); return bgslots_cleanup_.cleanup_slots; } - void SetSlotscleaningupEndtime() { - std::lock_guard ml(bgsave_protector_); - bgslots_cleanup_.end_time = time(nullptr); - } - void Bgslotscleanup(std::vector cleanup_slots, const std::shared_ptr& slot); + + void Bgslotscleanup(std::vector cleanup_slots, const std::shared_ptr& db); void StopBgslotscleanup() { std::lock_guard ml(bgsave_protector_); bgslots_cleanup_.cleaningup = false; @@ -559,26 +461,26 @@ class PikaServer : public pstd::noncopyable { struct BGCacheTaskArg { BGCacheTaskArg() : conf(nullptr), reenable_cache(false) {} int task_type; - std::shared_ptr slot; + std::shared_ptr db; uint32_t cache_num; cache::CacheConfig cache_cfg; std::unique_ptr conf; bool reenable_cache; }; + /* * Cache used */ - void ResetCacheAsync(uint32_t cache_num, std::shared_ptr slot, cache::CacheConfig *cache_cfg = nullptr); - void ClearCacheDbAsync(std::shared_ptr slot); - void ClearCacheDbAsyncV2(std::shared_ptr slot); - void ResetCacheConfig(std::shared_ptr slot); - void ClearHitRatio(std::shared_ptr slot); - void OnCacheStartPosChanged(int zset_cache_start_pos, std::shared_ptr slot); static void DoCacheBGTask(void* arg); + void ResetCacheAsync(uint32_t cache_num, std::shared_ptr db, cache::CacheConfig *cache_cfg = nullptr); + void ClearCacheDbAsync(std::shared_ptr db); + void ClearCacheDbAsyncV2(std::shared_ptr db); + void ResetCacheConfig(std::shared_ptr db); + void ClearHitRatio(std::shared_ptr db); + void OnCacheStartPosChanged(int zset_cache_start_pos, std::shared_ptr db); void UpdateCacheInfo(void); - void ResetDisplayCacheInfo(int status, std::shared_ptr slot); + void ResetDisplayCacheInfo(int status, std::shared_ptr db); void CacheConfigInit(cache::CacheConfig &cache_cfg); - void ClearHitRatio(void); void ProcessCronTask(); double HitRatio(); @@ -592,11 +494,9 @@ class PikaServer : public pstd::noncopyable { * TimingTask use */ void DoTimingTask(); - void AutoResumeDB(); void AutoCompactRange(); void AutoPurge(); void AutoDeleteExpiredDump(); - void AutoKeepAliveRSync(); void AutoUpdateNetworkMetric(); void PrintThreadPoolQueueStatus(); int64_t GetLastSaveTime(const std::string& dump_dir); @@ -613,9 +513,8 @@ class PikaServer : public pstd::noncopyable { std::timed_mutex exit_mutex_; /* - * Table used + * DB used */ - std::atomic slot_state_; std::shared_mutex dbs_rw_; std::map> dbs_; @@ -647,7 +546,6 @@ class PikaServer : public pstd::noncopyable { int role_ = PIKA_ROLE_SINGLE; int last_meta_sync_timestamp_ = 0; bool first_meta_sync_ = false; - bool loop_slot_state_machine_ = false; bool force_full_sync_ = false; bool leader_protected_mode_ = false; // reject request after master slave sync done std::shared_mutex state_protector_; // protect below, use for master-slave mode diff --git a/include/pika_set.h b/include/pika_set.h index 9a84d3e8c9..fb41d568e4 100644 --- a/include/pika_set.h +++ b/include/pika_set.h @@ -8,7 +8,6 @@ #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" #include "pika_kv.h" /* @@ -23,10 +22,10 @@ class SAddCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SAddCmd(*this); } @@ -46,10 +45,10 @@ class SPopCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SPopCmd(*this); } @@ -70,11 +69,11 @@ class SCardCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SCardCmd(*this); } @@ -93,11 +92,11 @@ class SMembersCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SMembersCmd(*this); } @@ -116,8 +115,8 @@ class SScanCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SScanCmd(*this); } @@ -141,10 +140,10 @@ class SRemCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SRemCmd(*this); } @@ -160,8 +159,8 @@ class SUnionCmd : public Cmd { public: SUnionCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SUnionCmd(*this); } @@ -174,17 +173,17 @@ class SetOperationCmd : public Cmd { public: SetOperationCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SET)) { - sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); - del_cmd_ = std::make_shared(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv); + sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSet); + del_cmd_ = std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache); } SetOperationCmd(const SetOperationCmd& other) : Cmd(other), dest_key_(other.dest_key_), value_to_dest_(other.value_to_dest_) { - sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); - del_cmd_ = std::make_shared(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv); + sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSet); + del_cmd_ = std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache); } std::vector current_key() const override { return {dest_key_}; } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; protected: std::string dest_key_; @@ -199,10 +198,10 @@ class SUnionstoreCmd : public SetOperationCmd { public: SUnionstoreCmd(const std::string& name, int arity, uint32_t flag) : SetOperationCmd(name, arity, flag) {} // current_key() is override in base class : SetOperationCmd - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SUnionstoreCmd(*this); } @@ -215,8 +214,8 @@ class SInterCmd : public Cmd { public: SInterCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SInterCmd(*this); } @@ -228,10 +227,10 @@ class SInterCmd : public Cmd { class SInterstoreCmd : public SetOperationCmd { public: SInterstoreCmd(const std::string& name, int arity, uint32_t flag) : SetOperationCmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SInterstoreCmd(*this); } @@ -249,11 +248,11 @@ class SIsmemberCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SIsmemberCmd(*this); } @@ -268,8 +267,8 @@ class SDiffCmd : public Cmd { public: SDiffCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SET)) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SDiffCmd(*this); } @@ -281,10 +280,10 @@ class SDiffCmd : public Cmd { class SDiffstoreCmd : public SetOperationCmd { public: SDiffstoreCmd(const std::string& name, int arity, uint32_t flag) : SetOperationCmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SDiffstoreCmd(*this); } @@ -297,8 +296,8 @@ class SMoveCmd : public Cmd { public: SMoveCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SET)) { - srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); - sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); + srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSet); + sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSet); } SMoveCmd(const SMoveCmd& other) : Cmd(other), @@ -306,17 +305,17 @@ class SMoveCmd : public Cmd { dest_key_(other.dest_key_), member_(other.member_), move_success_(other.move_success_) { - srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); - sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet); + srem_cmd_ = std::make_shared(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSet); + sadd_cmd_ = std::make_shared(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSet); } std::vector current_key() const override { return {src_key_, dest_key_}; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SMoveCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: std::string src_key_, dest_key_, member_; @@ -336,11 +335,11 @@ class SRandmemberCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new SRandmemberCmd(*this); } diff --git a/include/pika_slave_node.h b/include/pika_slave_node.h index 401d7fe8e6..e37325b521 100644 --- a/include/pika_slave_node.h +++ b/include/pika_slave_node.h @@ -59,7 +59,7 @@ class SyncWindow { // role master use class SlaveNode : public RmNode { public: - SlaveNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, int session_id); + SlaveNode(const std::string& ip, int port, const std::string& db_name, int session_id); ~SlaveNode() override; void Lock() { slave_mu.lock(); } void Unlock() { slave_mu.unlock(); } diff --git a/include/pika_slaveping_thread.h b/include/pika_slaveping_thread.h index 35b726fb12..a79200782e 100644 --- a/include/pika_slaveping_thread.h +++ b/include/pika_slaveping_thread.h @@ -33,10 +33,8 @@ class PikaSlavepingThread : public net::Thread { private: int64_t sid_ = 0; bool is_first_send_ = true; - int sockfd_ = -1; net::NetCli* cli_ = nullptr; - virtual void* ThreadMain(); }; diff --git a/include/pika_slot.h b/include/pika_slot.h deleted file mode 100644 index 1b04dd8547..0000000000 --- a/include/pika_slot.h +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_SLOT_H_ -#define PIKA_SLOT_H_ - -#include - -#include "pstd/include/scope_record_lock.h" - -#include "storage/backupable.h" -#include "storage/storage.h" -#include "include/pika_binlog.h" - -class Cmd; -class PikaCache; -class CacheInfo; - -/* - *Keyscan used - */ -struct KeyScanInfo { - time_t start_time = 0; - std::string s_start_time; - int32_t duration = -3; - std::vector key_infos; // the order is strings, hashes, lists, zsets, sets - bool key_scaning_ = false; - KeyScanInfo() : - s_start_time("0"), - key_infos({{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}) - {} -}; - -struct BgSaveInfo { - bool bgsaving = false; - time_t start_time = 0; - std::string s_start_time; - std::string path; - LogOffset offset; - BgSaveInfo() = default; - void Clear() { - bgsaving = false; - path.clear(); - offset = LogOffset(); - } -}; - -struct DisplayCacheInfo { - int status = 0; - uint32_t cache_num = 0; - uint64_t keys_num = 0; - uint64_t used_memory = 0; - uint64_t hits = 0; - uint64_t misses = 0; - uint64_t hits_per_sec = 0; - uint64_t read_cmd_per_sec = 0; - double hitratio_per_sec = 0.0; - double hitratio_all = 0.0; - uint64_t load_keys_per_sec = 0; - uint64_t last_time_us = 0; - uint64_t last_load_keys_num = 0; - uint32_t waitting_load_keys_num = 0; - DisplayCacheInfo& operator=(const DisplayCacheInfo &obj) { - status = obj.status; - cache_num = obj.cache_num; - keys_num = obj.keys_num; - used_memory = obj.used_memory; - hits = obj.hits; - misses = obj.misses; - hits_per_sec = obj.hits_per_sec; - read_cmd_per_sec = obj.read_cmd_per_sec; - hitratio_per_sec = obj.hitratio_per_sec; - hitratio_all = obj.hitratio_all; - load_keys_per_sec = obj.load_keys_per_sec; - last_time_us = obj.last_time_us; - last_load_keys_num = obj.last_load_keys_num; - waitting_load_keys_num = obj.waitting_load_keys_num; - return *this; - } -}; - -class Slot : public std::enable_shared_from_this,public pstd::noncopyable { - public: - Slot(const std::string& db_name, uint32_t slot_id, const std::string& table_db_path); - virtual ~Slot(); - - std::string GetDBName() const; - uint32_t GetSlotID() const; - std::string GetSlotName() const; - std::shared_ptr db() const; - std::shared_ptr cache() const; - - void Init(); - - void Compact(const storage::DataType& type); - void CompactRange(const storage::DataType& type, const std::string& start, const std::string& end); - - void DbRWLockWriter(); - void DbRWLockReader(); - void DbRWUnLock(); - - std::shared_ptr LockMgr(); - - void PrepareRsync(); - bool TryUpdateMasterOffset(); - bool ChangeDb(const std::string& new_path); - - void Leave(); - void Close(); - void MoveToTrash(); - - // BgSave use; - bool IsBgSaving(); - void BgSaveSlot(); - BgSaveInfo bgsave_info(); - void GetBgSaveMetaData(std::vector* fileNames, std::string* snapshot_uuid); - pstd::Status GetBgSaveUUID(std::string* snapshot_uuid); - - // FlushDB & FlushSubDB use - bool FlushDB(); - bool FlushSubDB(const std::string& db_name); - bool FlushDBWithoutLock(); - bool FlushSubDBWithoutLock(const std::string& db_name); - - // key scan info use - pstd::Status GetKeyNum(std::vector* key_info); - KeyScanInfo GetKeyScanInfo(); - - /* - * Cache used - */ - DisplayCacheInfo GetCacheInfo(); - void UpdateCacheInfo(CacheInfo& cache_info); - void ResetDisplayCacheInfo(int status); - uint64_t cache_usage_; - - private: - std::string db_name_; - uint32_t slot_id_ = 0; - std::string snapshot_uuid_; - - std::string db_path_; - std::string bgsave_sub_path_; - std::string dbsync_path_; - std::string slot_name_; - - bool opened_ = false; - - std::shared_mutex db_rwlock_; - // class may be shared, using shared_ptr would be a better choice - std::shared_ptr lock_mgr_; - std::shared_ptr db_; - std::shared_ptr cache_; - - bool full_sync_ = false; - - pstd::Mutex key_info_protector_; - KeyScanInfo key_scan_info_; - - /* - * BgSave use - */ - static void DoBgSave(void* arg); - bool RunBgsaveEngine(); - bool InitBgsaveEnv(); - bool InitBgsaveEngine(); - void ClearBgsave(); - void FinishBgsave(); - BgSaveInfo bgsave_info_; - pstd::Mutex bgsave_protector_; - std::shared_ptr bgsave_engine_; - - // key scan info use - void InitKeyScan(); - /* - * Cache used - */ - DisplayCacheInfo cache_info_; - std::shared_mutex cache_info_rwlock_; -}; - -#endif - diff --git a/include/pika_slot_command.h b/include/pika_slot_command.h index 996c2ef267..644483b932 100644 --- a/include/pika_slot_command.h +++ b/include/pika_slot_command.h @@ -3,7 +3,6 @@ #include "include/pika_client_conn.h" #include "include/pika_command.h" -#include "include/pika_slot.h" #include "net/include/net_cli.h" #include "net/include/net_thread.h" #include "storage/storage.h" @@ -18,26 +17,26 @@ void CRC32TableInit(uint32_t poly); extern void InitCRC32Table(); -extern uint32_t CRC32Update(uint32_t crc, const char *buf, int len); -extern uint32_t CRC32CheckSum(const char *buf, int len); +extern uint32_t CRC32Update(uint32_t crc, const char* buf, int len); +extern uint32_t CRC32CheckSum(const char* buf, int len); int GetSlotID(const std::string &str); -int GetKeyType(const std::string& key, std::string &key_type, const std::shared_ptr& slot); -void AddSlotKey(const std::string& type, const std::string& key, const std::shared_ptr& slot); -void RemSlotKey(const std::string& key, const std::shared_ptr& slot); -int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr& slot); +int GetKeyType(const std::string& key, std::string& key_type, const std::shared_ptr& db); +int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr& db); +int GetSlotsID(const std::string& str, uint32_t* pcrc, int* phastag); +void AddSlotKey(const std::string& type, const std::string& key, const std::shared_ptr& db); +void RemSlotKey(const std::string& key, const std::shared_ptr& db); +void RemSlotKeyByType(const std::string& type, const std::string& key, const std::shared_ptr& db); std::string GetSlotKey(int slot); std::string GetSlotsTagKey(uint32_t crc); -int GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag); -void RemSlotKeyByType(const std::string &type, const std::string &key, const std::shared_ptr& slot); class PikaMigrate { public: PikaMigrate(); virtual ~PikaMigrate(); - int MigrateKey(const std::string &host, const int port, int timeout, const std::string &key, const char type, - std::string &detail, const std::shared_ptr& slot); + int MigrateKey(const std::string& host, const int port, int timeout, const std::string& key, const char type, + std::string& detail, const std::shared_ptr& db); void CleanMigrateClient(); void Lock() { @@ -49,53 +48,51 @@ class PikaMigrate { void Unlock() { mutex_.unlock(); } - net::NetCli *GetMigrateClient(const std::string &host, const int port, int timeout); + net::NetCli* GetMigrateClient(const std::string& host, const int port, int timeout); private: - std::map migrate_clients_; + std::map migrate_clients_; pstd::Mutex mutex_; - - void KillMigrateClient(net::NetCli *migrate_cli); + void KillMigrateClient(net::NetCli* migrate_cli); void KillAllMigrateClient(); - - int MigrateSend(net::NetCli *migrate_cli, const std::string &key, const char type, std::string &detail, - const std::shared_ptr& slot); - bool MigrateRecv(net::NetCli *migrate_cli, int need_receive, std::string &detail); - - int ParseKey(const std::string &key, const char type, std::string &wbuf_str, const std::shared_ptr& slot); - int64_t TTLByType(const char key_type, const std::string &key, const std::shared_ptr& slot); - int ParseKKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); - int ParseZKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); - int ParseSKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); - int ParseHKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); - int ParseLKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot); - bool SetTTL(const std::string &key, std::string &wbuf_str, int64_t ttl); + int64_t TTLByType(const char key_type, const std::string& key, const std::shared_ptr& db); + int MigrateSend(net::NetCli* migrate_cli, const std::string& key, const char type, std::string& detail, + const std::shared_ptr& db); + bool MigrateRecv(net::NetCli* migrate_cli, int need_receive, std::string& detail); + int ParseKey(const std::string& key, const char type, std::string& wbuf_str, const std::shared_ptr& db); + int ParseKKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db); + int ParseZKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db); + int ParseSKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db); + int ParseHKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db); + int ParseLKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db); + bool SetTTL(const std::string& key, std::string& wbuf_str, int64_t ttl); }; class SlotsMgrtTagSlotCmd : public Cmd { public: SlotsMgrtTagSlotCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtTagSlotCmd(*this); } + private: std::string dest_ip_; int64_t dest_port_ = 0; int64_t timeout_ms_ = 60; int64_t slot_id_ = 0; std::basic_string, std::allocator> key_; - void DoInitial() override; }; class SlotsMgrtTagSlotAsyncCmd : public Cmd { public: SlotsMgrtTagSlotAsyncCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag){} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtTagSlotAsyncCmd(*this); } + private: std::string dest_ip_; int64_t dest_port_ = 0; @@ -104,17 +101,17 @@ class SlotsMgrtTagSlotAsyncCmd : public Cmd { int64_t max_bytes_ = 0; int64_t slot_id_ = 0; int64_t keys_num_ = 0; - void DoInitial() override; }; class SlotsMgrtTagOneCmd : public Cmd { public: SlotsMgrtTagOneCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtTagOneCmd(*this); } + private: std::string dest_ip_; int64_t dest_port_ = 0; @@ -123,14 +120,14 @@ class SlotsMgrtTagOneCmd : public Cmd { int64_t slot_id_ = 0; char key_type_ = '\0'; void DoInitial() override; - int KeyTypeCheck(const std::shared_ptr& slot); + int KeyTypeCheck(const std::shared_ptr& db); }; class SlotsMgrtAsyncStatusCmd : public Cmd { public: SlotsMgrtAsyncStatusCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtAsyncStatusCmd(*this); } @@ -141,10 +138,11 @@ class SlotsMgrtAsyncStatusCmd : public Cmd { class SlotsInfoCmd : public Cmd { public: SlotsInfoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsInfoCmd(*this); } + private: void DoInitial() override; @@ -155,10 +153,11 @@ class SlotsInfoCmd : public Cmd { class SlotsMgrtAsyncCancelCmd : public Cmd { public: SlotsMgrtAsyncCancelCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtAsyncCancelCmd(*this); } + private: void DoInitial() override; }; @@ -166,10 +165,11 @@ class SlotsMgrtAsyncCancelCmd : public Cmd { class SlotsDelCmd : public Cmd { public: SlotsDelCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsDelCmd(*this); } + private: std::vector slots_; void DoInitial() override; @@ -178,10 +178,11 @@ class SlotsDelCmd : public Cmd { class SlotsHashKeyCmd : public Cmd { public: SlotsHashKeyCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsHashKeyCmd(*this); } + private: std::vector keys_; void DoInitial() override; @@ -190,10 +191,11 @@ class SlotsHashKeyCmd : public Cmd { class SlotsScanCmd : public Cmd { public: SlotsScanCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsScanCmd(*this); } + private: std::string key_; std::string pattern_ = "*"; @@ -214,10 +216,11 @@ class SlotsScanCmd : public Cmd { class SlotsMgrtExecWrapperCmd : public Cmd { public: SlotsMgrtExecWrapperCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsMgrtExecWrapperCmd(*this); } + private: std::string key_; std::vector args; @@ -228,10 +231,11 @@ class SlotsMgrtExecWrapperCmd : public Cmd { class SlotsReloadCmd : public Cmd { public: SlotsReloadCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsReloadCmd(*this); } + private: void DoInitial() override; }; @@ -239,10 +243,11 @@ class SlotsReloadCmd : public Cmd { class SlotsReloadOffCmd : public Cmd { public: SlotsReloadOffCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptrslot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsReloadOffCmd(*this); } + private: void DoInitial() override; }; @@ -250,11 +255,12 @@ class SlotsReloadOffCmd : public Cmd { class SlotsCleanupCmd : public Cmd { public: SlotsCleanupCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsCleanupCmd(*this); } std::vector cleanup_slots_; + private: void DoInitial() override; }; @@ -262,10 +268,11 @@ class SlotsCleanupCmd : public Cmd { class SlotsCleanupOffCmd : public Cmd { public: SlotsCleanupOffCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag) {} - void Do(std::shared_ptr slot) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new SlotsCleanupOffCmd(*this); } + private: void DoInitial() override; }; diff --git a/include/pika_stable_log.h b/include/pika_stable_log.h index a4aae0e7a3..300e0d0fc5 100644 --- a/include/pika_stable_log.h +++ b/include/pika_stable_log.h @@ -13,7 +13,7 @@ class StableLog : public std::enable_shared_from_this { public: - StableLog(std::string table_name, uint32_t slot_id, std::string log_path); + StableLog(std::string table_name, std::string log_path); ~StableLog(); std::shared_ptr Logger() { return stable_logger_; } void Leave(); @@ -46,7 +46,6 @@ class StableLog : public std::enable_shared_from_this { std::atomic purging_; std::string db_name_; - uint32_t slot_id_ = 0; std::string log_path_; std::shared_ptr stable_logger_; diff --git a/include/pika_statistic.h b/include/pika_statistic.h index c6d03d2e3e..dcfe97d652 100644 --- a/include/pika_statistic.h +++ b/include/pika_statistic.h @@ -16,9 +16,7 @@ class QpsStatistic { QpsStatistic(); QpsStatistic(const QpsStatistic& other); ~QpsStatistic() = default; - void IncreaseQueryNum(bool is_write); - void ResetLastSecQuerynum(); std::atomic querynum; diff --git a/include/pika_stream.h b/include/pika_stream.h index da539c0d5f..f9c6366acb 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -7,7 +7,6 @@ #define PIKA_STREAM_H_ #include "include/pika_command.h" -#include "include/pika_slot.h" #include "include/pika_stream_base.h" #include "include/pika_stream_meta_value.h" #include "include/pika_stream_types.h" @@ -26,15 +25,15 @@ inline void ParseReadOrReadGroupArgsOrReply(CmdRes& res, const PikaCmdArgsType& // @field_values is the result of ScanStream. // field is the serialized message id, // value is the serialized message. -inline void AppendMessagesToRes(CmdRes& res, std::vector& field_values, const Slot* slot); +inline void AppendMessagesToRes(CmdRes& res, std::vector& field_values, const DB* db); class XAddCmd : public Cmd { public: XAddCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; std::vector current_key() const override { return {key_}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XAddCmd(*this); } @@ -52,8 +51,8 @@ class XDelCmd : public Cmd { XDelCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; std::vector current_key() const override { return {key_}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XDelCmd(*this); } @@ -63,17 +62,17 @@ class XDelCmd : public Cmd { void DoInitial() override; void Clear() override { ids_.clear(); } - inline void SetFirstOrLastIDOrReply(StreamMetaValue& stream_meta, const Slot* slot, bool is_set_first); - inline void SetFirstIDOrReply(StreamMetaValue& stream_meta, const Slot* slot); - inline void SetLastIDOrReply(StreamMetaValue& stream_meta, const Slot* slot); + inline void SetFirstOrLastIDOrReply(StreamMetaValue& stream_meta, const DB* db, bool is_set_first); + inline void SetFirstIDOrReply(StreamMetaValue& stream_meta, const DB* db); + inline void SetLastIDOrReply(StreamMetaValue& stream_meta, const DB* db); }; class XReadCmd : public Cmd { public: XReadCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XReadCmd(*this); } @@ -91,8 +90,8 @@ class XRangeCmd : public Cmd { public: XRangeCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XRangeCmd(*this); } @@ -110,8 +109,8 @@ class XRangeCmd : public Cmd { class XRevrangeCmd : public XRangeCmd { public: XRevrangeCmd(const std::string& name, int arity, uint32_t flag) : XRangeCmd(name, arity, flag){}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XRevrangeCmd(*this); } }; @@ -120,8 +119,8 @@ class XLenCmd : public Cmd { public: XLenCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XLenCmd(*this); } @@ -135,8 +134,8 @@ class XTrimCmd : public Cmd { public: XTrimCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag){}; std::vector current_key() const override { return {key_}; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XTrimCmd(*this); } @@ -151,8 +150,8 @@ class XInfoCmd : public Cmd { public: XInfoCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::STREAM)){}; - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new XInfoCmd(*this); } @@ -165,9 +164,9 @@ class XInfoCmd : public Cmd { bool is_full_{false}; void DoInitial() override; - void StreamInfo(std::shared_ptr& slot); - void GroupsInfo(std::shared_ptr& slot); - void ConsumersInfo(std::shared_ptr& slot); + void StreamInfo(std::shared_ptr& db); + void GroupsInfo(std::shared_ptr& db); + void ConsumersInfo(std::shared_ptr& db); }; #endif // PIKA_STREAM_H_ diff --git a/include/pika_stream_base.h b/include/pika_stream_base.h index 1091e19ebc..eb9dc6b34c 100644 --- a/include/pika_stream_base.h +++ b/include/pika_stream_base.h @@ -81,7 +81,7 @@ class TreeIDGenerator { return instance; } - storage::Status GetNextTreeID(const Slot *slot, treeID &tid); + storage::Status GetNextTreeID(const DB *db, treeID &tid); private: static const treeID START_TREE_ID = 0; @@ -112,63 +112,63 @@ class StreamStorage { }; static storage::Status ScanStream(const ScanStreamOptions &option, std::vector &field_values, - std::string &next_field, const Slot *slot); + std::string &next_field, const DB *db); // get and parse the stream meta if found // @return ok only when the stream meta exists - static storage::Status GetStreamMeta(StreamMetaValue &tream_meta, const std::string &key, const Slot *slot); + static storage::Status GetStreamMeta(StreamMetaValue &tream_meta, const std::string &key, const DB *db); // will create stream meta hash if it dosent't exist. // return !s.ok() only when insert failed - static storage::Status SetStreamMeta(const std::string &key, std::string &meta_value, const Slot *slot); + static storage::Status SetStreamMeta(const std::string &key, std::string &meta_value, const DB *db); static storage::Status InsertStreamMessage(const std::string &key, const streamID &id, const std::string &message, - const Slot *slot); + const DB *db); static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &ids, int32_t &ret, - const Slot *slot); + const DB *db); static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &serialized_ids, - int32_t &ret, const Slot *slot); + int32_t &ret, const DB *db); static storage::Status GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const Slot *slot); + const DB *db); - static storage::Status DeleteStreamData(const std::string &key, const Slot *slot); + static storage::Status DeleteStreamData(const std::string &key, const DB *db); static storage::Status TrimStream(int32_t &res, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const Slot *slot); + StreamAddTrimArgs &args, const DB *db); // get the abstracted tree node, e.g. get a message in pel, get a consumer meta or get a cgroup meta. // the behavior of abstracted tree is similar to radix-tree in redis; // in cgroup tree, field is groupname // in consumer tree, field is consumername // in pel tree, field is messageID - static storage::Status GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, const Slot *slot); + static storage::Status GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, const DB *db); static storage::Status InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const Slot *slot); - static storage::Status DeleteTreeNode(const treeID tid, const std::string &field, const Slot *slot); + const DB *db); + static storage::Status DeleteTreeNode(const treeID tid, const std::string &field, const DB *db); static storage::Status GetAllTreeNode(const treeID tid, std::vector &field_values, - const Slot *slot); + const DB *db); // delete the stream meta // @return true if the stream meta exists and deleted - static storage::Status DeleteStreamMeta(const std::string &key, const Slot *slot); + static storage::Status DeleteStreamMeta(const std::string &key, const DB *db); // note: the tree must exist // @return true if the tree exists and is deleted - static storage::Status DeleteTree(const treeID tid, const Slot *slot); + static storage::Status DeleteTree(const treeID tid, const DB *db); // get consumer meta value. // if the consumer meta value does not exist, create a new one and return it. - static storage::Status GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot, + static storage::Status GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const DB *db, StreamConsumerMetaValue &consumer_meta); - static storage::Status CreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot); + static storage::Status CreateConsumer(treeID consumer_tid, std::string &consumername, const DB *db); // delete the pels, consumers, cgroups and stream meta of a stream // note: this function do not delete the stream data value - static storage::Status DestoryStreams(std::vector &keys, const Slot *slot); + static storage::Status DestoryStreams(std::vector &keys, const DB *db); private: StreamStorage(); @@ -183,10 +183,10 @@ class StreamStorage { }; static storage::Status TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args); + const DB *db, const StreamAddTrimArgs &args); static storage::Status TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args); + const DB *db, const StreamAddTrimArgs &args); }; // Helper function of stream command. diff --git a/include/pika_transaction.h b/include/pika_transaction.h index dd25fb12a8..9957475b21 100644 --- a/include/pika_transaction.h +++ b/include/pika_transaction.h @@ -16,9 +16,9 @@ class MultiCmd : public Cmd { public: MultiCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::TRANSACTION)) {} - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; Cmd* Clone() override { return new MultiCmd(*this); } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {} + void Split(const HintKeys& hint_keys) override {} void Merge() override {} private: @@ -29,23 +29,21 @@ class ExecCmd : public Cmd { public: ExecCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::TRANSACTION)) {} - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; Cmd* Clone() override { return new ExecCmd(*this); } void Execute() override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {} + void Split(const HintKeys& hint_keys) override {} void Merge() override {} std::vector current_key() const override { return {}; } private: struct CmdInfo { public: - CmdInfo(std::shared_ptr cmd, std::shared_ptr db, std::shared_ptr slot, - std::shared_ptr sync_slot) - : cmd_(cmd), db_(db), slot_(slot), sync_slot_(sync_slot) {} + CmdInfo(std::shared_ptr cmd, std::shared_ptr db, + std::shared_ptr sync_db) : cmd_(cmd), db_(db), sync_db_(sync_db) {} std::shared_ptr cmd_; std::shared_ptr db_; - std::shared_ptr slot_; - std::shared_ptr sync_slot_; + std::shared_ptr sync_db_; }; void DoInitial() override; void Lock(); @@ -54,9 +52,9 @@ class ExecCmd : public Cmd { void SetCmdsVec(); void ServeToBLrPopWithKeys(); std::unordered_set> lock_db_{}; - std::unordered_map, std::vector> lock_slot_keys_{}; - std::unordered_set> r_lock_slots_{}; - bool is_lock_rm_slots_{false}; // g_pika_rm->slots_rw_; + std::unordered_map, std::vector> lock_db_keys_{}; + std::unordered_set> r_lock_dbs_ {}; + bool is_lock_rm_dbs_{false}; // g_pika_rm->dbs_rw_; std::vector cmds_; std::vector list_cmd_; std::vector keys_; @@ -66,9 +64,9 @@ class DiscardCmd : public Cmd { public: DiscardCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::TRANSACTION)) {} - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; Cmd* Clone() override { return new DiscardCmd(*this); } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {} + void Split(const HintKeys& hint_keys) override {} void Merge() override {} private: @@ -80,9 +78,9 @@ class WatchCmd : public Cmd { WatchCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::TRANSACTION)) {} - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; void Execute() override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {} + void Split(const HintKeys& hint_keys) override {} Cmd* Clone() override { return new WatchCmd(*this); } void Merge() override {} std::vector current_key() const override { return keys_; } @@ -98,9 +96,9 @@ class UnwatchCmd : public Cmd { UnwatchCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::TRANSACTION)) {} - void Do(std::shared_ptr slot = nullptr) override; + void Do() override; Cmd* Clone() override { return new UnwatchCmd(*this); } - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {} + void Split(const HintKeys& hint_keys) override {} void Merge() override {} private: diff --git a/include/pika_zset.h b/include/pika_zset.h index 00a1b5bc28..a74ee026fc 100644 --- a/include/pika_zset.h +++ b/include/pika_zset.h @@ -7,10 +7,8 @@ #define PIKA_ZSET_H_ #include "storage/storage.h" - #include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_slot.h" #include "pika_kv.h" /* @@ -25,10 +23,10 @@ class ZAddCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZAddCmd(*this); } @@ -48,11 +46,11 @@ class ZCardCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZCardCmd(*this); } @@ -70,8 +68,8 @@ class ZScanCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ZScanCmd(*this); } @@ -94,10 +92,10 @@ class ZIncrbyCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZIncrbyCmd(*this); } double Score() { return score_; } @@ -131,11 +129,11 @@ class ZRangeCmd : public ZsetRangeParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRangeCmd(*this); } @@ -152,11 +150,11 @@ class ZRevrangeCmd : public ZsetRangeParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRevrangeCmd(*this); } @@ -176,6 +174,7 @@ class ZsetRangebyscoreParentCmd : public Cmd { bool RightClose() { return right_close_; } int64_t Offset() { return offset_; } int64_t Count() { return count_; } + protected: std::string key_; std::string min_, max_; @@ -199,11 +198,11 @@ class ZRangebyscoreCmd : public ZsetRangebyscoreParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRangebyscoreCmd(*this); } @@ -221,11 +220,11 @@ class ZRevrangebyscoreCmd : public ZsetRangebyscoreParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRevrangebyscoreCmd(*this); } @@ -243,11 +242,11 @@ class ZCountCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZCountCmd(*this); } double MinScore() { return min_score_; } @@ -277,10 +276,10 @@ class ZRemCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRemCmd(*this); } @@ -296,8 +295,7 @@ class ZsetUIstoreParentCmd : public Cmd { public: ZsetUIstoreParentCmd(const std::string& name, int arity, uint32_t flag) : Cmd(name, arity, flag, static_cast(AclCategory::SORTEDSET)) { - zadd_cmd_ = std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset); - del_cmd_ = std::make_shared(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv); + zadd_cmd_ = std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsZset); } ZsetUIstoreParentCmd(const ZsetUIstoreParentCmd& other) : Cmd(other), @@ -306,8 +304,7 @@ class ZsetUIstoreParentCmd : public Cmd { aggregate_(other.aggregate_), keys_(other.keys_), weights_(other.weights_) { - zadd_cmd_ = std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset); - del_cmd_ = std::make_shared(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv); + zadd_cmd_ = std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsZset); } std::vector current_key() const override { return {dest_key_}; } @@ -322,16 +319,15 @@ class ZsetUIstoreParentCmd : public Cmd { void Clear() override { aggregate_ = storage::SUM; } // used for write binlog std::shared_ptr zadd_cmd_; - std::shared_ptr del_cmd_; }; class ZUnionstoreCmd : public ZsetUIstoreParentCmd { public: ZUnionstoreCmd(const std::string& name, int arity, uint32_t flag) : ZsetUIstoreParentCmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZUnionstoreCmd(*this); } @@ -340,19 +336,19 @@ class ZUnionstoreCmd : public ZsetUIstoreParentCmd { // used for write binlog std::map value_to_dest_; rocksdb::Status s_; - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; }; class ZInterstoreCmd : public ZsetUIstoreParentCmd { public: ZInterstoreCmd(const std::string& name, int arity, uint32_t flag) : ZsetUIstoreParentCmd(name, arity, flag) {} - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZInterstoreCmd(*this); } - void DoBinlog(const std::shared_ptr& slot) override; + void DoBinlog() override; private: void DoInitial() override; @@ -379,11 +375,11 @@ class ZRankCmd : public ZsetRankParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRankCmd(*this); } @@ -400,11 +396,11 @@ class ZRevrankCmd : public ZsetRankParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRevrankCmd(*this); } @@ -421,11 +417,11 @@ class ZScoreCmd : public ZsetRankParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZScoreCmd(*this); } @@ -461,11 +457,11 @@ class ZRangebylexCmd : public ZsetRangebylexParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRangebylexCmd(*this); } @@ -482,11 +478,11 @@ class ZRevrangebylexCmd : public ZsetRangebylexParentCmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRevrangebylexCmd(*this); } @@ -504,11 +500,11 @@ class ZLexcountCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void ReadCache(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void ReadCache() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZLexcountCmd(*this); } @@ -530,10 +526,10 @@ class ZRemrangebyrankCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRemrangebyrankCmd(*this); } @@ -554,10 +550,10 @@ class ZRemrangebyscoreCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRemrangebyscoreCmd(*this); } @@ -579,10 +575,10 @@ class ZRemrangebylexCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void DoUpdateCache(std::shared_ptr slot = nullptr) override; - void DoThroughDB(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override{}; + void Do() override; + void DoUpdateCache() override; + void DoThroughDB() override; + void Split(const HintKeys& hint_keys) override{}; void Merge() override{}; Cmd* Clone() override { return new ZRemrangebylexCmd(*this); } @@ -604,8 +600,8 @@ class ZPopmaxCmd : public Cmd { res.emplace_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ZPopmaxCmd(*this); } @@ -624,8 +620,8 @@ class ZPopminCmd : public Cmd { res.push_back(key_); return res; } - void Do(std::shared_ptr slot = nullptr) override; - void Split(std::shared_ptr slot, const HintKeys& hint_keys) override {}; + void Do() override; + void Split(const HintKeys& hint_keys) override {}; void Merge() override {}; Cmd* Clone() override { return new ZPopminCmd(*this); } diff --git a/include/rsync_client.h b/include/rsync_client.h index 388bd8ff0e..02a33b398e 100644 --- a/include/rsync_client.h +++ b/include/rsync_client.h @@ -43,6 +43,7 @@ class WaitObject; class WaitObjectManager; using pstd::Status; + using ResponseSPtr = std::shared_ptr; class RsyncClient : public net::Thread { public: @@ -51,7 +52,7 @@ class RsyncClient : public net::Thread { RUNNING, STOP, }; - RsyncClient(const std::string& dir, const std::string& db_name, const uint32_t slot_id); + RsyncClient(const std::string& dir, const std::string& db_name); void* ThreadMain() override; void Copy(const std::set& file_set, int index); bool Init(); @@ -81,13 +82,11 @@ class RsyncClient : public net::Thread { private: typedef std::unique_ptr NetThreadUPtr; - std::map meta_table_; std::set file_set_; std::string snapshot_uuid_; std::string dir_; std::string db_name_; - uint32_t slot_id_ = 0; NetThreadUPtr client_thread_; std::vector work_threads_; @@ -105,7 +104,7 @@ class RsyncClient : public net::Thread { }; class RsyncWriter { -public: + public: RsyncWriter(const std::string& filepath) { filepath_ = filepath; fd_ = open(filepath.c_str(), O_RDWR | O_APPEND | O_CREAT, 0644); @@ -139,13 +138,13 @@ class RsyncWriter { return Status::OK(); } -private: + private: std::string filepath_; int fd_ = -1; }; class WaitObject { -public: + public: WaitObject() : filename_(""), type_(RsyncService::kRsyncMeta), offset_(0), resp_(nullptr) {} ~WaitObject() {} @@ -183,7 +182,7 @@ class WaitObject { std::string Filename() {return filename_;} RsyncService::Type Type() {return type_;} size_t Offset() {return offset_;} -private: + private: std::string filename_; RsyncService::Type type_; size_t offset_ = kInvalidOffset; @@ -193,7 +192,7 @@ class WaitObject { }; class WaitObjectManager { -public: + public: WaitObjectManager() { wo_vec_.resize(kMaxRsyncParallelNum); for (int i = 0; i < kMaxRsyncParallelNum; i++) { @@ -230,7 +229,7 @@ class WaitObjectManager { wo_vec_[index]->WakeUp(resp); } -private: + private: std::vector wo_vec_; std::mutex mu_; }; diff --git a/include/rsync_client_thread.h b/include/rsync_client_thread.h index 91d5cf383f..19bebcb56d 100644 --- a/include/rsync_client_thread.h +++ b/include/rsync_client_thread.h @@ -17,19 +17,19 @@ using namespace net; namespace rsync { class RsyncClientConn : public PbConn { -public: + public: RsyncClientConn(int fd, const std::string& ip_port, net::Thread* thread, void* cb_handler, NetMultiplexer* mpx); ~RsyncClientConn() override; int DealMessage() override; -private: + private: void* cb_handler_ = nullptr; }; class RsyncClientConnFactory : public ConnFactory { -public: + public: RsyncClientConnFactory(void* scheduler) : cb_handler_(scheduler) {} std::shared_ptr NewNetConn(int connfd, const std::string& ip_port, net::Thread* thread, void* cb_handler, @@ -37,15 +37,15 @@ class RsyncClientConnFactory : public ConnFactory { return std::static_pointer_cast( std::make_shared(connfd, ip_port, thread, cb_handler_, net)); } -private: + private: void* cb_handler_ = nullptr; }; class RsyncClientThread : public ClientThread { -public: + public: RsyncClientThread(int cron_interval, int keepalive_timeout, void* scheduler); ~RsyncClientThread() override; -private: + private: RsyncClientConnFactory conn_factory_; ClientHandle handle_; }; diff --git a/include/rsync_server.h b/include/rsync_server.h index 4899890e2d..560585f3c8 100644 --- a/include/rsync_server.h +++ b/include/rsync_server.h @@ -34,19 +34,19 @@ class RsyncReader; class RsyncServerThread; class RsyncServer { -public: + public: RsyncServer(const std::set& ips, const int port); ~RsyncServer(); void Schedule(net::TaskFunc func, void* arg); int Start(); int Stop(); -private: + private: std::unique_ptr work_thread_; std::unique_ptr rsync_server_thread_; }; class RsyncServerConn : public net::PbConn { -public: + public: RsyncServerConn(int connfd, const std::string& ip_port, net::Thread* thread, void* worker_specific_data, net::NetMultiplexer* mpx); @@ -54,20 +54,20 @@ class RsyncServerConn : public net::PbConn { int DealMessage() override; static void HandleMetaRsyncRequest(void* arg); static void HandleFileRsyncRequest(void* arg); -private: + private: std::vector > readers_; std::mutex mu_; void* data_ = nullptr; }; class RsyncServerThread : public net::HolyThread { -public: + public: RsyncServerThread(const std::set& ips, int port, int cron_internal, RsyncServer* arg); ~RsyncServerThread(); -private: + private: class RsyncServerConnFactory : public net::ConnFactory { - public: + public: explicit RsyncServerConnFactory(RsyncServer* sched) : scheduler_(sched) {} std::shared_ptr NewNetConn(int connfd, const std::string& ip_port, @@ -76,23 +76,23 @@ class RsyncServerThread : public net::HolyThread { return std::static_pointer_cast( std::make_shared(connfd, ip_port, thread, scheduler_, net)); } - private: + private: RsyncServer* scheduler_ = nullptr; }; class RsyncServerHandle : public net::ServerHandle { - public: + public: void FdClosedHandle(int fd, const std::string& ip_port) const override; void FdTimeoutHandle(int fd, const std::string& ip_port) const override; bool AccessHandle(int fd, std::string& ip) const override; void CronHandle() const override; }; -private: + private: RsyncServerConnFactory conn_factory_; RsyncServerHandle handle_; }; class RsyncReader { -public: + public: RsyncReader() { block_data_ = new char[kBlockSize]; } @@ -117,6 +117,7 @@ class RsyncReader { *is_eof = (offset + copy_count == total_size_); return pstd::Status::OK(); } + private: pstd::Status readAhead(const std::string filepath, const size_t offset) { if (filepath == filepath_ && offset >= start_offset_ && offset < end_offset_) { @@ -167,7 +168,7 @@ class RsyncReader { fd_ = -1; } -private: + private: std::mutex mu_; const size_t kBlockSize = 16 << 20; diff --git a/include/throttle.h b/include/throttle.h index cb0dc7d638..2bdbe6ed71 100644 --- a/include/throttle.h +++ b/include/throttle.h @@ -15,25 +15,25 @@ extern std::unique_ptr g_pika_conf; namespace rsync { class Throttle { public: - Throttle() {} - Throttle(size_t throttle_throughput_bytes, size_t check_cycle); - ~Throttle(); - size_t ThrottledByThroughput(size_t bytes); - void ReturnUnusedThroughput(size_t acquired, size_t consumed, size_t elaspe_time_us); - static Throttle& GetInstance() { - static Throttle instance(g_pika_conf->throttle_bytes_per_second(), 10); - return instance; - } + Throttle() {} + Throttle(size_t throttle_throughput_bytes, size_t check_cycle); + ~Throttle(); + size_t ThrottledByThroughput(size_t bytes); + void ReturnUnusedThroughput(size_t acquired, size_t consumed, size_t elaspe_time_us); + static Throttle& GetInstance() { + static Throttle instance(g_pika_conf->throttle_bytes_per_second(), 10); + return instance; + } private: - std::atomic throttle_throughput_bytes_ = 100 * 1024 * 1024; - // the num of tasks doing install_snapshot - std::atomic last_throughput_check_time_us_; - std::atomic cur_throughput_bytes_; - // user defined check cycles of throughput per second - size_t check_cycle_ = 10; - pstd::Mutex keys_mutex_; - size_t caculate_check_time_us_(int64_t current_time_us, int64_t check_cycle) { + std::atomic throttle_throughput_bytes_ = 100 * 1024 * 1024; + // the num of tasks doing install_snapshot + std::atomic last_throughput_check_time_us_; + std::atomic cur_throughput_bytes_; + // user defined check cycles of throughput per second + size_t check_cycle_ = 10; + pstd::Mutex keys_mutex_; + size_t caculate_check_time_us_(int64_t current_time_us, int64_t check_cycle) { size_t base_aligning_time_us = 1000 * 1000 / check_cycle; return current_time_us / base_aligning_time_us * base_aligning_time_us; } diff --git a/src/acl.cc b/src/acl.cc index c784d99ebb..3c78adf0e8 100644 --- a/src/acl.cc +++ b/src/acl.cc @@ -10,6 +10,7 @@ #include "include/acl.h" #include "include/pika_cmd_table_manager.h" +#include "include/pika_command.h" #include "include/pika_server.h" #include "pstd_defer.h" #include "pstd_hash.h" diff --git a/src/build_version.cc.in b/src/build_version.cc.in index b4897fd02f..1d341ef321 100644 --- a/src/build_version.cc.in +++ b/src/build_version.cc.in @@ -3,8 +3,6 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#include "include/build_version.h" const char* pika_build_git_sha = "pika_git_sha:@PIKA_GIT_SHA@"; -const char* pika_build_git_date = "pika_build_git_date:@PIKA_GIT_DATE@"; const char* pika_build_compile_date = "@PIKA_BUILD_DATE@"; diff --git a/src/cache/include/cache.h b/src/cache/include/cache.h index 7c7d96294f..869cb5aa1b 100644 --- a/src/cache/include/cache.h +++ b/src/cache/include/cache.h @@ -42,119 +42,119 @@ class RedisCache { int32_t ActiveExpireCycle(void); // Normal Commands - bool Exists(std::string &key); + bool Exists(std::string& key); int64_t DbSize(void); - void FlushDb(void); - - Status Del(const std::string &key); - Status Expire(std::string &key, int64_t ttl); - Status Expireat(std::string &key, int64_t ttl); - Status TTL(std::string &key, int64_t *ttl); - Status Persist(std::string &key); - Status Type(std::string &key, std::string *value); + void FlushCache(void); + + Status Del(const std::string& key); + Status Expire(std::string& key, int64_t ttl); + Status Expireat(std::string& key, int64_t ttl); + Status TTL(std::string& key, int64_t *ttl); + Status Persist(std::string& key); + Status Type(std::string& key, std::string *value); Status RandomKey(std::string *key); // String Commands - Status Set(std::string &key, std::string &value, int64_t ttl); - Status SetWithoutTTL(std::string &key, std::string &value); - Status Setnx(std::string &key, std::string &value, int64_t ttl); - Status SetnxWithoutTTL(std::string &key, std::string &value); - Status Setxx(std::string &key, std::string &value, int64_t ttl); - Status SetxxWithoutTTL(std::string &key, std::string &value); - Status Get(const std::string &key, std::string *value); - Status Incr(std::string &key); - Status Decr(std::string &key); - Status IncrBy(std::string &key, int64_t incr); - Status DecrBy(std::string &key, int64_t incr); - Status Incrbyfloat(std::string &key, double incr); - Status Append(std::string &key, std::string &value); - Status GetRange(std::string &key, int64_t start, int64_t end, std::string *value); - Status SetRange(std::string &key, int64_t start, std::string &value); - Status Strlen(std::string &key, int32_t *len); + Status Set(std::string& key, std::string &value, int64_t ttl); + Status SetWithoutTTL(std::string& key, std::string &value); + Status Setnx(std::string& key, std::string &value, int64_t ttl); + Status SetnxWithoutTTL(std::string& key, std::string &value); + Status Setxx(std::string& key, std::string &value, int64_t ttl); + Status SetxxWithoutTTL(std::string& key, std::string &value); + Status Get(const std::string& key, std::string *value); + Status Incr(std::string& key); + Status Decr(std::string& key); + Status IncrBy(std::string& key, int64_t incr); + Status DecrBy(std::string& key, int64_t incr); + Status Incrbyfloat(std::string& key, double incr); + Status Append(std::string& key, std::string &value); + Status GetRange(std::string& key, int64_t start, int64_t end, std::string *value); + Status SetRange(std::string& key, int64_t start, std::string &value); + Status Strlen(std::string& key, int32_t *len); // Hash Commands Status HDel(std::string& key, std::vector &fields); - Status HSet(std::string &key, std::string &field, std::string &value); - Status HSetnx(std::string &key, std::string &field, std::string &value); - Status HMSet(std::string &key, std::vector &fvs); - Status HGet(std::string &key, std::string &field, std::string *value); - Status HMGet(std::string &key, + Status HSet(std::string& key, std::string &field, std::string &value); + Status HSetnx(std::string& key, std::string &field, std::string &value); + Status HMSet(std::string& key, std::vector &fvs); + Status HGet(std::string& key, std::string &field, std::string *value); + Status HMGet(std::string& key, std::vector &fields, std::vector* vss); - Status HGetall(std::string &key, std::vector *fvs); - Status HKeys(std::string &key, std::vector *fields); - Status HVals(std::string &key, std::vector *values); - Status HExists(std::string &key, std::string &field); - Status HIncrby(std::string &key, std::string &field, int64_t value); - Status HIncrbyfloat(std::string &key, std::string &field, double value); - Status HLen(std::string &key, uint64_t *len); - Status HStrlen(std::string &key, std::string &field, uint64_t *len); + Status HGetall(std::string& key, std::vector *fvs); + Status HKeys(std::string& key, std::vector *fields); + Status HVals(std::string& key, std::vector *values); + Status HExists(std::string& key, std::string &field); + Status HIncrby(std::string& key, std::string &field, int64_t value); + Status HIncrbyfloat(std::string& key, std::string &field, double value); + Status HLen(std::string& key, uint64_t *len); + Status HStrlen(std::string& key, std::string &field, uint64_t *len); // List Commands - Status LIndex(std::string &key, int64_t index, std::string *element); - Status LInsert(std::string &key, storage::BeforeOrAfter &before_or_after, + Status LIndex(std::string& key, int64_t index, std::string *element); + Status LInsert(std::string& key, storage::BeforeOrAfter &before_or_after, std::string &pivot, std::string &value); - Status LLen(std::string &key, uint64_t *len); - Status LPop(std::string &key, std::string *element); - Status LPush(std::string &key, std::vector &values); - Status LPushx(std::string &key, std::vector &values); - Status LRange(std::string &key, int64_t start, int64_t stop, std::vector *values); - Status LRem(std::string &key, int64_t count, std::string &value); - Status LSet(std::string &key, int64_t index, std::string &value); - Status LTrim(std::string &key, int64_t start, int64_t stop); - Status RPop(std::string &key, std::string *element); - Status RPush(std::string &key, std::vector &values); - Status RPushx(std::string &key, std::vector &values); + Status LLen(std::string& key, uint64_t *len); + Status LPop(std::string& key, std::string *element); + Status LPush(std::string& key, std::vector &values); + Status LPushx(std::string& key, std::vector &values); + Status LRange(std::string& key, int64_t start, int64_t stop, std::vector *values); + Status LRem(std::string& key, int64_t count, std::string &value); + Status LSet(std::string& key, int64_t index, std::string &value); + Status LTrim(std::string& key, int64_t start, int64_t stop); + Status RPop(std::string& key, std::string *element); + Status RPush(std::string& key, std::vector &values); + Status RPushx(std::string& key, std::vector &values); // Set Commands - Status SAdd(std::string &key, std::vector &members); - Status SCard(std::string &key, uint64_t *len); - Status SIsmember(std::string &key, std::string &member); - Status SMembers(std::string &key, std::vector *members); - Status SRem(std::string &key, std::vector &members); - Status SRandmember(std::string &key, int64_t count, std::vector *members); + Status SAdd(std::string& key, std::vector &members); + Status SCard(std::string& key, uint64_t *len); + Status SIsmember(std::string& key, std::string& member); + Status SMembers(std::string& key, std::vector *members); + Status SRem(std::string& key, std::vector &members); + Status SRandmember(std::string& key, int64_t count, std::vector *members); // Zset Commands - Status ZAdd(std::string &key, std::vector &score_members); - Status ZCard(std::string &key, uint64_t *len); - Status ZCount(std::string &key, std::string &min, std::string &max, uint64_t *len); - Status ZIncrby(std::string &key, std::string &member, double increment); - Status ZRange(std::string &key, + Status ZAdd(std::string& key, std::vector &score_members); + Status ZCard(std::string& key, uint64_t *len); + Status ZCount(std::string& key, std::string &min, std::string &max, uint64_t *len); + Status ZIncrby(std::string& key, std::string& member, double increment); + Status ZRange(std::string& key, int64_t start, int64_t stop, std::vector *score_members); - Status ZRangebyscore(std::string &key, + Status ZRangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, int64_t offset = 0, int64_t count = -1); - Status ZRank(std::string &key, std::string &member, int64_t *rank); - Status ZRem(std::string &key, std::vector &members); - Status ZRemrangebyrank(std::string &key, std::string &min, std::string &max); - Status ZRemrangebyscore(std::string &key, std::string &min, std::string &max); - Status ZRevrange(std::string &key, + Status ZRank(std::string& key, std::string& member, int64_t *rank); + Status ZRem(std::string& key, std::vector &members); + Status ZRemrangebyrank(std::string& key, std::string &min, std::string &max); + Status ZRemrangebyscore(std::string& key, std::string &min, std::string &max); + Status ZRevrange(std::string& key, int64_t start, int64_t stop, std::vector *score_members); - Status ZRevrangebyscore(std::string &key, + Status ZRevrangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, int64_t offset = 0, int64_t count = -1); - Status ZRevrangebylex(std::string &key, + Status ZRevrangebylex(std::string& key, std::string &min, std::string &max, std::vector *members); - Status ZRevrank(std::string &key, std::string &member, int64_t *rank); - Status ZScore(std::string &key, std::string &member, double *score); - Status ZRangebylex(std::string &key, + Status ZRevrank(std::string& key, std::string& member, int64_t *rank); + Status ZScore(std::string& key, std::string& member, double *score); + Status ZRangebylex(std::string& key, std::string &min, std::string &max, std::vector *members); - Status ZLexcount(std::string &key, std::string &min, std::string &max, uint64_t *len); - Status ZRemrangebylex(std::string &key, std::string &min, std::string &max); + Status ZLexcount(std::string& key, std::string &min, std::string &max, uint64_t *len); + Status ZRemrangebylex(std::string& key, std::string &min, std::string &max); // Bit Commands - Status SetBit(std::string &key, size_t offset, int64_t value); - Status GetBit(std::string &key, size_t offset, int64_t *value); - Status BitCount(std::string &key, int64_t start, int64_t end, int64_t *value, bool have_offset); - Status BitPos(std::string &key, int64_t bit, int64_t *value); - Status BitPos(std::string &key, int64_t bit, int64_t start, int64_t *value); - Status BitPos(std::string &key, int64_t bit, int64_t start, int64_t end, int64_t *value); + Status SetBit(std::string& key, size_t offset, int64_t value); + Status GetBit(std::string& key, size_t offset, int64_t *value); + Status BitCount(std::string& key, int64_t start, int64_t end, int64_t *value, bool have_offset); + Status BitPos(std::string& key, int64_t bit, int64_t *value); + Status BitPos(std::string& key, int64_t bit, int64_t start, int64_t *value); + Status BitPos(std::string& key, int64_t bit, int64_t start, int64_t end, int64_t *value); protected: void DecrObjectsRefCount(robj *argv1, robj *argv2 = nullptr, robj *argv3 = nullptr); diff --git a/src/cache/src/bit.cc b/src/cache/src/bit.cc index 75072592ee..d8955875a2 100644 --- a/src/cache/src/bit.cc +++ b/src/cache/src/bit.cc @@ -9,7 +9,7 @@ namespace cache { -Status RedisCache::SetBit(std::string &key, size_t offset, int64_t value) { +Status RedisCache::SetBit(std::string& key, size_t offset, int64_t value) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -28,7 +28,7 @@ Status RedisCache::SetBit(std::string &key, size_t offset, int64_t value) { return Status::OK(); } -Status RedisCache::GetBit(std::string &key, size_t offset, int64_t *value) { +Status RedisCache::GetBit(std::string& key, size_t offset, int64_t *value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -45,7 +45,7 @@ Status RedisCache::GetBit(std::string &key, size_t offset, int64_t *value) { return Status::OK(); } -Status RedisCache::BitCount(std::string &key, int64_t start, int64_t end, int64_t *value, bool have_offset) { +Status RedisCache::BitCount(std::string& key, int64_t start, int64_t end, int64_t *value, bool have_offset) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -62,7 +62,7 @@ Status RedisCache::BitCount(std::string &key, int64_t start, int64_t end, int64_ return Status::OK(); } -Status RedisCache::BitPos(std::string &key, int64_t bit, int64_t *value) { +Status RedisCache::BitPos(std::string& key, int64_t bit, int64_t *value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -78,7 +78,7 @@ Status RedisCache::BitPos(std::string &key, int64_t bit, int64_t *value) { return Status::OK(); } -Status RedisCache::BitPos(std::string &key, int64_t bit, int64_t start, int64_t *value) { +Status RedisCache::BitPos(std::string& key, int64_t bit, int64_t start, int64_t *value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -94,7 +94,7 @@ Status RedisCache::BitPos(std::string &key, int64_t bit, int64_t start, int64_t return Status::OK(); } -Status RedisCache::BitPos(std::string &key, int64_t bit, int64_t start, int64_t end, int64_t *value) { +Status RedisCache::BitPos(std::string& key, int64_t bit, int64_t start, int64_t end, int64_t *value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/cache/src/cache.cc b/src/cache/src/cache.cc index 80c10aa104..ef0b3103f3 100644 --- a/src/cache/src/cache.cc +++ b/src/cache/src/cache.cc @@ -82,7 +82,7 @@ int32_t RedisCache::ActiveExpireCycle(void) { return RcActiveExpireCycle(cache_) /*----------------------------------------------------------------------------- * Normal Commands *----------------------------------------------------------------------------*/ -bool RedisCache::Exists(std::string &key) { +bool RedisCache::Exists(std::string& key) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { decrRefCount(kobj); @@ -98,9 +98,9 @@ int64_t RedisCache::DbSize(void) { return dbsize; } -void RedisCache::FlushDb(void) { RcFlushCache(cache_); } +void RedisCache::FlushCache(void) { RcFlushCache(cache_); } -Status RedisCache::Del(const std::string &key) { +Status RedisCache::Del(const std::string& key) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { decrRefCount(kobj); @@ -117,7 +117,7 @@ Status RedisCache::Del(const std::string &key) { return Status::OK(); } -Status RedisCache::Expire(std::string &key, int64_t ttl) { +Status RedisCache::Expire(std::string& key, int64_t ttl) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *tobj = createStringObjectFromLongLong(ttl); DEFER { @@ -135,7 +135,7 @@ Status RedisCache::Expire(std::string &key, int64_t ttl) { return Status::OK(); } -Status RedisCache::Expireat(std::string &key, int64_t ttl) { +Status RedisCache::Expireat(std::string& key, int64_t ttl) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *tobj = createStringObjectFromLongLong(ttl); DEFER { @@ -152,7 +152,7 @@ Status RedisCache::Expireat(std::string &key, int64_t ttl) { return Status::OK(); } -Status RedisCache::TTL(std::string &key, int64_t *ttl) { +Status RedisCache::TTL(std::string& key, int64_t *ttl) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -168,7 +168,7 @@ Status RedisCache::TTL(std::string &key, int64_t *ttl) { return Status::OK(); } -Status RedisCache::Persist(std::string &key) { +Status RedisCache::Persist(std::string& key) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -184,7 +184,7 @@ Status RedisCache::Persist(std::string &key) { return Status::OK(); } -Status RedisCache::Type(std::string &key, std::string *value) { +Status RedisCache::Type(std::string& key, std::string *value) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { diff --git a/src/cache/src/hash.cc b/src/cache/src/hash.cc index 1576de418b..3b7b019a06 100644 --- a/src/cache/src/hash.cc +++ b/src/cache/src/hash.cc @@ -9,7 +9,7 @@ namespace cache { -Status RedisCache::HDel(std::string &key, std::vector &fields) { +Status RedisCache::HDel(std::string& key, std::vector &fields) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **fields_obj = (robj **)zcallocate(sizeof(robj *) * fields.size()); for (unsigned int i = 0; i < fields.size(); ++i) { @@ -31,7 +31,7 @@ Status RedisCache::HDel(std::string &key, std::vector &fields) { return Status::OK(); } -Status RedisCache::HSet(std::string &key, std::string &field, std::string &value) { +Status RedisCache::HSet(std::string& key, std::string &field, std::string &value) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -51,7 +51,7 @@ Status RedisCache::HSet(std::string &key, std::string &field, std::string &value return Status::OK(); } -Status RedisCache::HSetnx(std::string &key, std::string &field, std::string &value) { +Status RedisCache::HSetnx(std::string& key, std::string &field, std::string &value) { if (C_OK != RcFreeMemoryIfNeeded(cache_)) { return Status::Corruption("[error] Free memory faild !"); } @@ -69,7 +69,7 @@ Status RedisCache::HSetnx(std::string &key, std::string &field, std::string &val return Status::OK(); } -Status RedisCache::HMSet(std::string &key, std::vector &fvs) { +Status RedisCache::HMSet(std::string& key, std::vector &fvs) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -93,7 +93,7 @@ Status RedisCache::HMSet(std::string &key, std::vector &fvs return Status::OK(); } -Status RedisCache::HGet(std::string &key, std::string &field, std::string *value) { +Status RedisCache::HGet(std::string& key, std::string &field, std::string *value) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); @@ -117,7 +117,7 @@ Status RedisCache::HGet(std::string &key, std::string &field, std::string *value return Status::OK(); } -Status RedisCache::HMGet(std::string &key, std::vector &fields, std::vector *vss) { +Status RedisCache::HMGet(std::string& key, std::vector &fields, std::vector *vss) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); hitem *items = (hitem *)zcallocate(sizeof(hitem) * fields.size()); for (unsigned int i = 0; i < fields.size(); ++i) { @@ -148,7 +148,7 @@ Status RedisCache::HMGet(std::string &key, std::vector &fields, std return Status::OK(); } -Status RedisCache::HGetall(std::string &key, std::vector *fvs) { +Status RedisCache::HGetall(std::string& key, std::vector *fvs) { hitem *items = nullptr; unsigned long items_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -174,7 +174,7 @@ Status RedisCache::HGetall(std::string &key, std::vector *f return Status::OK(); } -Status RedisCache::HKeys(std::string &key, std::vector *fields) { +Status RedisCache::HKeys(std::string& key, std::vector *fields) { hitem *items = nullptr; unsigned long items_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -197,7 +197,7 @@ Status RedisCache::HKeys(std::string &key, std::vector *fields) { return Status::OK(); } -Status RedisCache::HVals(std::string &key, std::vector *values) { +Status RedisCache::HVals(std::string& key, std::vector *values) { hitem *items = nullptr; unsigned long items_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -220,7 +220,7 @@ Status RedisCache::HVals(std::string &key, std::vector *values) { return Status::OK(); } -Status RedisCache::HExists(std::string &key, std::string &field) { +Status RedisCache::HExists(std::string& key, std::string &field) { int is_exist = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); @@ -238,7 +238,7 @@ Status RedisCache::HExists(std::string &key, std::string &field) { return is_exist ? Status::OK() : Status::NotFound("field not exist"); } -Status RedisCache::HIncrby(std::string &key, std::string &field, int64_t value) { +Status RedisCache::HIncrby(std::string& key, std::string &field, int64_t value) { int64_t result = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); @@ -256,7 +256,7 @@ Status RedisCache::HIncrby(std::string &key, std::string &field, int64_t value) return Status::OK(); } -Status RedisCache::HIncrbyfloat(std::string &key, std::string &field, double value) { +Status RedisCache::HIncrbyfloat(std::string& key, std::string &field, double value) { long double result = .0f; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); @@ -274,7 +274,7 @@ Status RedisCache::HIncrbyfloat(std::string &key, std::string &field, double val return Status::OK(); } -Status RedisCache::HLen(std::string &key, uint64_t *len) { +Status RedisCache::HLen(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -290,7 +290,7 @@ Status RedisCache::HLen(std::string &key, uint64_t *len) { return Status::OK(); } -Status RedisCache::HStrlen(std::string &key, std::string &field, uint64_t *len) { +Status RedisCache::HStrlen(std::string& key, std::string &field, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *fobj = createObject(OBJ_STRING, sdsnewlen(field.data(), field.size())); DEFER { diff --git a/src/cache/src/list.cc b/src/cache/src/list.cc index 08e774961f..03c7d9149a 100644 --- a/src/cache/src/list.cc +++ b/src/cache/src/list.cc @@ -8,7 +8,7 @@ namespace cache { -Status RedisCache::LIndex(std::string &key, int64_t index, std::string *element) { +Status RedisCache::LIndex(std::string& key, int64_t index, std::string *element) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -31,7 +31,7 @@ Status RedisCache::LIndex(std::string &key, int64_t index, std::string *element) return Status::OK(); } -Status RedisCache::LInsert(std::string &key, storage::BeforeOrAfter &before_or_after, std::string &pivot, +Status RedisCache::LInsert(std::string& key, storage::BeforeOrAfter &before_or_after, std::string &pivot, std::string &value) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { @@ -56,7 +56,7 @@ Status RedisCache::LInsert(std::string &key, storage::BeforeOrAfter &before_or_a return Status::OK(); } -Status RedisCache::LLen(std::string &key, uint64_t *len) { +Status RedisCache::LLen(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -72,7 +72,7 @@ Status RedisCache::LLen(std::string &key, uint64_t *len) { return Status::OK(); } -Status RedisCache::LPop(std::string &key, std::string *element) { +Status RedisCache::LPop(std::string& key, std::string *element) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -93,7 +93,7 @@ Status RedisCache::LPop(std::string &key, std::string *element) { return Status::OK(); } -Status RedisCache::LPush(std::string &key, std::vector &values) { +Status RedisCache::LPush(std::string& key, std::vector &values) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -116,7 +116,7 @@ Status RedisCache::LPush(std::string &key, std::vector &values) { return Status::OK(); } -Status RedisCache::LPushx(std::string &key, std::vector &values) { +Status RedisCache::LPushx(std::string& key, std::vector &values) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -142,7 +142,7 @@ Status RedisCache::LPushx(std::string &key, std::vector &values) { return Status::OK(); } -Status RedisCache::LRange(std::string &key, int64_t start, int64_t stop, std::vector *values) { +Status RedisCache::LRange(std::string& key, int64_t start, int64_t stop, std::vector *values) { sds *vals = nullptr; uint64_t vals_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -165,7 +165,7 @@ Status RedisCache::LRange(std::string &key, int64_t start, int64_t stop, std::ve return Status::OK(); } -Status RedisCache::LRem(std::string &key, int64_t count, std::string &value) { +Status RedisCache::LRem(std::string& key, int64_t count, std::string &value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); DEFER { @@ -182,7 +182,7 @@ Status RedisCache::LRem(std::string &key, int64_t count, std::string &value) { return Status::OK(); } -Status RedisCache::LSet(std::string &key, int64_t index, std::string &value) { +Status RedisCache::LSet(std::string& key, int64_t index, std::string &value) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); DEFER { @@ -201,7 +201,7 @@ Status RedisCache::LSet(std::string &key, int64_t index, std::string &value) { return Status::OK(); } -Status RedisCache::LTrim(std::string &key, int64_t start, int64_t stop) { +Status RedisCache::LTrim(std::string& key, int64_t start, int64_t stop) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -218,7 +218,7 @@ Status RedisCache::LTrim(std::string &key, int64_t start, int64_t stop) { return Status::OK(); } -Status RedisCache::RPop(std::string &key, std::string *element) { +Status RedisCache::RPop(std::string& key, std::string *element) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -239,7 +239,7 @@ Status RedisCache::RPop(std::string &key, std::string *element) { return Status::OK(); } -Status RedisCache::RPush(std::string &key, std::vector &values) { +Status RedisCache::RPush(std::string& key, std::vector &values) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -262,7 +262,7 @@ Status RedisCache::RPush(std::string &key, std::vector &values) { return Status::OK(); } -Status RedisCache::RPushx(std::string &key, std::vector &values) { +Status RedisCache::RPushx(std::string& key, std::vector &values) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); diff --git a/src/cache/src/set.cc b/src/cache/src/set.cc index daac7dc332..aa1610e00b 100644 --- a/src/cache/src/set.cc +++ b/src/cache/src/set.cc @@ -8,7 +8,7 @@ namespace cache { -Status RedisCache::SAdd(std::string &key, std::vector &members) { +Status RedisCache::SAdd(std::string& key, std::vector &members) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -31,7 +31,7 @@ Status RedisCache::SAdd(std::string &key, std::vector &members) { return Status::OK(); } -Status RedisCache::SCard(std::string &key, uint64_t *len) { +Status RedisCache::SCard(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -47,7 +47,7 @@ Status RedisCache::SCard(std::string &key, uint64_t *len) { return Status::OK(); } -Status RedisCache::SIsmember(std::string &key, std::string &member) { +Status RedisCache::SIsmember(std::string& key, std::string& member) { int is_member = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *mobj = createObject(OBJ_STRING, sdsnewlen(member.data(), member.size())); @@ -65,7 +65,7 @@ Status RedisCache::SIsmember(std::string &key, std::string &member) { return is_member ? Status::OK() : Status::NotFound("member not exist"); } -Status RedisCache::SMembers(std::string &key, std::vector *members) { +Status RedisCache::SMembers(std::string& key, std::vector *members) { sds *vals = nullptr; unsigned long vals_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -88,7 +88,7 @@ Status RedisCache::SMembers(std::string &key, std::vector *members) return Status::OK(); } -Status RedisCache::SRem(std::string &key, std::vector &members) { +Status RedisCache::SRem(std::string& key, std::vector &members) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **vals = (robj **)zcallocate(sizeof(robj *) * members.size()); for (unsigned int i = 0; i < members.size(); ++i) { @@ -110,7 +110,7 @@ Status RedisCache::SRem(std::string &key, std::vector &members) { return Status::OK(); } -Status RedisCache::SRandmember(std::string &key, int64_t count, std::vector *members) { +Status RedisCache::SRandmember(std::string& key, int64_t count, std::vector *members) { sds *vals = nullptr; unsigned long vals_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); diff --git a/src/cache/src/string.cc b/src/cache/src/string.cc index df7d847e4c..4195fe7b6e 100644 --- a/src/cache/src/string.cc +++ b/src/cache/src/string.cc @@ -10,7 +10,7 @@ namespace cache { -Status RedisCache::Set(std::string &key, std::string &value, int64_t ttl) { +Status RedisCache::Set(std::string& key, std::string &value, int64_t ttl) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -30,7 +30,7 @@ Status RedisCache::Set(std::string &key, std::string &value, int64_t ttl) { return Status::OK(); } -Status RedisCache::SetWithoutTTL(std::string &key, std::string &value) { +Status RedisCache::SetWithoutTTL(std::string& key, std::string &value) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -49,7 +49,7 @@ Status RedisCache::SetWithoutTTL(std::string &key, std::string &value) { return Status::OK(); } -Status RedisCache::Setnx(std::string &key, std::string &value, int64_t ttl) { +Status RedisCache::Setnx(std::string& key, std::string &value, int64_t ttl) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -69,7 +69,7 @@ Status RedisCache::Setnx(std::string &key, std::string &value, int64_t ttl) { return Status::OK(); } -Status RedisCache::SetnxWithoutTTL(std::string &key, std::string &value) { +Status RedisCache::SetnxWithoutTTL(std::string& key, std::string &value) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -88,7 +88,7 @@ Status RedisCache::SetnxWithoutTTL(std::string &key, std::string &value) { return Status::OK(); } -Status RedisCache::Setxx(std::string &key, std::string &value, int64_t ttl) { +Status RedisCache::Setxx(std::string& key, std::string &value, int64_t ttl) { int ret = RcFreeMemoryIfNeeded(cache_); if (C_OK != ret) { return Status::Corruption("[error] Free memory faild !"); @@ -108,7 +108,7 @@ Status RedisCache::Setxx(std::string &key, std::string &value, int64_t ttl) { return Status::OK(); } -Status RedisCache::SetxxWithoutTTL(std::string &key, std::string &value) { +Status RedisCache::SetxxWithoutTTL(std::string& key, std::string &value) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -127,7 +127,7 @@ Status RedisCache::SetxxWithoutTTL(std::string &key, std::string &value) { return Status::OK(); } -Status RedisCache::Get(const std::string &key, std::string *value) { +Status RedisCache::Get(const std::string& key, std::string *value) { robj *val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -148,7 +148,7 @@ Status RedisCache::Get(const std::string &key, std::string *value) { return Status::OK(); } -Status RedisCache::Incr(std::string &key) { +Status RedisCache::Incr(std::string& key) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -162,7 +162,7 @@ Status RedisCache::Incr(std::string &key) { return Status::OK(); } -Status RedisCache::Decr(std::string &key) { +Status RedisCache::Decr(std::string& key) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -176,7 +176,7 @@ Status RedisCache::Decr(std::string &key) { return Status::OK(); } -Status RedisCache::IncrBy(std::string &key, int64_t incr) { +Status RedisCache::IncrBy(std::string& key, int64_t incr) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -190,7 +190,7 @@ Status RedisCache::IncrBy(std::string &key, int64_t incr) { return Status::OK(); } -Status RedisCache::DecrBy(std::string &key, int64_t incr) { +Status RedisCache::DecrBy(std::string& key, int64_t incr) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -204,7 +204,7 @@ Status RedisCache::DecrBy(std::string &key, int64_t incr) { return Status::OK(); } -Status RedisCache::Incrbyfloat(std::string &key, double incr) { +Status RedisCache::Incrbyfloat(std::string& key, double incr) { long double ret = .0f; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -218,7 +218,7 @@ Status RedisCache::Incrbyfloat(std::string &key, double incr) { return Status::OK(); } -Status RedisCache::Append(std::string &key, std::string &value) { +Status RedisCache::Append(std::string& key, std::string &value) { uint64_t ret = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *vobj = createObject(OBJ_STRING, sdsnewlen(value.data(), value.size())); @@ -233,7 +233,7 @@ Status RedisCache::Append(std::string &key, std::string &value) { return Status::OK(); } -Status RedisCache::GetRange(std::string &key, int64_t start, int64_t end, std::string *value) { +Status RedisCache::GetRange(std::string& key, int64_t start, int64_t end, std::string *value) { sds val; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { @@ -255,7 +255,7 @@ Status RedisCache::GetRange(std::string &key, int64_t start, int64_t end, std::s return Status::OK(); } -Status RedisCache::SetRange(std::string &key, int64_t start, std::string &value) { +Status RedisCache::SetRange(std::string& key, int64_t start, std::string &value) { if (C_OK != RcFreeMemoryIfNeeded(cache_)) { return Status::Corruption("[error] Free memory faild !"); } @@ -274,7 +274,7 @@ Status RedisCache::SetRange(std::string &key, int64_t start, std::string &value) return Status::OK(); } -Status RedisCache::Strlen(std::string &key, int32_t *len) { +Status RedisCache::Strlen(std::string& key, int32_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); diff --git a/src/cache/src/zset.cc b/src/cache/src/zset.cc index 6961b7b133..3333cc6854 100644 --- a/src/cache/src/zset.cc +++ b/src/cache/src/zset.cc @@ -8,7 +8,7 @@ namespace cache { -Status RedisCache::ZAdd(std::string &key, std::vector &score_members) { +Status RedisCache::ZAdd(std::string& key, std::vector &score_members) { int res = RcFreeMemoryIfNeeded(cache_); if (C_OK != res) { return Status::Corruption("[error] Free memory faild !"); @@ -34,7 +34,7 @@ Status RedisCache::ZAdd(std::string &key, std::vector &sco return Status::OK(); } -Status RedisCache::ZCard(std::string &key, uint64_t *len) { +Status RedisCache::ZCard(std::string& key, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); DEFER { DecrObjectsRefCount(kobj); @@ -50,7 +50,7 @@ Status RedisCache::ZCard(std::string &key, uint64_t *len) { return Status::OK(); } -Status RedisCache::ZCount(std::string &key, std::string &min, std::string &max, uint64_t *len) { +Status RedisCache::ZCount(std::string& key, std::string &min, std::string &max, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *minobj = createObject(OBJ_STRING, sdsnewlen(min.data(), min.size())); robj *maxobj = createObject(OBJ_STRING, sdsnewlen(max.data(), max.size())); @@ -68,7 +68,7 @@ Status RedisCache::ZCount(std::string &key, std::string &min, std::string &max, return Status::OK(); } -Status RedisCache::ZIncrby(std::string &key, std::string &member, double increment) { +Status RedisCache::ZIncrby(std::string& key, std::string& member, double increment) { if (C_OK != RcFreeMemoryIfNeeded(cache_)) { return Status::Corruption("[error] Free memory faild !"); } @@ -89,7 +89,7 @@ Status RedisCache::ZIncrby(std::string &key, std::string &member, double increme return Status::OK(); } -Status RedisCache::ZRange(std::string &key, int64_t start, int64_t stop, std::vector *score_members) { +Status RedisCache::ZRange(std::string& key, int64_t start, int64_t stop, std::vector *score_members) { zitem *items = nullptr; uint64_t items_size = 0; robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); @@ -115,7 +115,7 @@ Status RedisCache::ZRange(std::string &key, int64_t start, int64_t stop, std::ve return Status::OK(); } -Status RedisCache::ZRangebyscore(std::string &key, std::string &min, std::string &max, +Status RedisCache::ZRangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, int64_t offset, int64_t count) { zitem *items = nullptr; uint64_t items_size = 0; @@ -145,7 +145,7 @@ Status RedisCache::ZRangebyscore(std::string &key, std::string &min, std::string return Status::OK(); } -Status RedisCache::ZRank(std::string &key, std::string &member, int64_t *rank) { +Status RedisCache::ZRank(std::string& key, std::string& member, int64_t *rank) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *mobj = createObject(OBJ_STRING, sdsnewlen(member.data(), member.size())); DEFER { @@ -164,7 +164,7 @@ Status RedisCache::ZRank(std::string &key, std::string &member, int64_t *rank) { return Status::OK(); } -Status RedisCache::ZRem(std::string &key, std::vector &members) { +Status RedisCache::ZRem(std::string& key, std::vector &members) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj **members_obj = (robj **)zcallocate(sizeof(robj *) * members.size()); for (unsigned int i = 0; i < members.size(); ++i) { @@ -186,7 +186,7 @@ Status RedisCache::ZRem(std::string &key, std::vector &members) { return Status::OK(); } -Status RedisCache::ZRemrangebyrank(std::string &key, std::string &min, std::string &max) { +Status RedisCache::ZRemrangebyrank(std::string& key, std::string &min, std::string &max) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *minobj = createObject(OBJ_STRING, sdsnewlen(min.data(), min.size())); robj *maxobj = createObject(OBJ_STRING, sdsnewlen(max.data(), max.size())); @@ -204,7 +204,7 @@ Status RedisCache::ZRemrangebyrank(std::string &key, std::string &min, std::stri return Status::OK(); } -Status RedisCache::ZRemrangebyscore(std::string &key, std::string &min, std::string &max) { +Status RedisCache::ZRemrangebyscore(std::string& key, std::string &min, std::string &max) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *minobj = createObject(OBJ_STRING, sdsnewlen(min.data(), min.size())); robj *maxobj = createObject(OBJ_STRING, sdsnewlen(max.data(), max.size())); @@ -222,7 +222,7 @@ Status RedisCache::ZRemrangebyscore(std::string &key, std::string &min, std::str return Status::OK(); } -Status RedisCache::ZRevrange(std::string &key, int64_t start, int64_t stop, +Status RedisCache::ZRevrange(std::string& key, int64_t start, int64_t stop, std::vector *score_members) { zitem *items = nullptr; uint64_t items_size = 0; @@ -249,7 +249,7 @@ Status RedisCache::ZRevrange(std::string &key, int64_t start, int64_t stop, return Status::OK(); } -Status RedisCache::ZRevrangebyscore(std::string &key, std::string &min, std::string &max, +Status RedisCache::ZRevrangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, int64_t offset, int64_t count) { zitem *items = nullptr; uint64_t items_size = 0; @@ -279,7 +279,7 @@ Status RedisCache::ZRevrangebyscore(std::string &key, std::string &min, std::str return Status::OK(); } -Status RedisCache::ZRevrangebylex(std::string &key, std::string &min, std::string &max, +Status RedisCache::ZRevrangebylex(std::string& key, std::string &min, std::string &max, std::vector *members) { sds *vals = nullptr; uint64_t vals_size = 0; @@ -305,7 +305,7 @@ Status RedisCache::ZRevrangebylex(std::string &key, std::string &min, std::strin return Status::OK(); } -Status RedisCache::ZRevrank(std::string &key, std::string &member, int64_t *rank) { +Status RedisCache::ZRevrank(std::string& key, std::string& member, int64_t *rank) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *mobj = createObject(OBJ_STRING, sdsnewlen(member.data(), member.size())); DEFER { @@ -324,7 +324,7 @@ Status RedisCache::ZRevrank(std::string &key, std::string &member, int64_t *rank return Status::OK(); } -Status RedisCache::ZScore(std::string &key, std::string &member, double *score) { +Status RedisCache::ZScore(std::string& key, std::string& member, double *score) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *mobj = createObject(OBJ_STRING, sdsnewlen(member.data(), member.size())); DEFER { @@ -343,7 +343,7 @@ Status RedisCache::ZScore(std::string &key, std::string &member, double *score) return Status::OK(); } -Status RedisCache::ZRangebylex(std::string &key, std::string &min, std::string &max, +Status RedisCache::ZRangebylex(std::string& key, std::string &min, std::string &max, std::vector *members) { sds *vals = nullptr; uint64_t vals_size = 0; @@ -369,7 +369,7 @@ Status RedisCache::ZRangebylex(std::string &key, std::string &min, std::string & return Status::OK(); } -Status RedisCache::ZLexcount(std::string &key, std::string &min, std::string &max, uint64_t *len) { +Status RedisCache::ZLexcount(std::string& key, std::string &min, std::string &max, uint64_t *len) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *minobj = createObject(OBJ_STRING, sdsnewlen(min.data(), min.size())); robj *maxobj = createObject(OBJ_STRING, sdsnewlen(max.data(), max.size())); @@ -387,7 +387,7 @@ Status RedisCache::ZLexcount(std::string &key, std::string &min, std::string &ma return Status::OK(); } -Status RedisCache::ZRemrangebylex(std::string &key, std::string &min, std::string &max) { +Status RedisCache::ZRemrangebylex(std::string& key, std::string &min, std::string &max) { robj *kobj = createObject(OBJ_STRING, sdsnewlen(key.data(), key.size())); robj *minobj = createObject(OBJ_STRING, sdsnewlen(min.data(), min.size())); robj *maxobj = createObject(OBJ_STRING, sdsnewlen(max.data(), max.size())); diff --git a/src/pika.cc b/src/pika.cc index 520e2c8f6f..2d62d3d6b9 100644 --- a/src/pika.cc +++ b/src/pika.cc @@ -104,7 +104,6 @@ static void create_pid_file() { size_t pos = path.find_last_of('/'); if (pos != std::string::npos) { - // mkpath(path.substr(0, pos).c_str(), 0755); pstd::CreateDir(path.substr(0, pos)); } else { path = kPikaPidFile; @@ -240,7 +239,7 @@ int main(int argc, char* argv[]) { } // stop PikaReplicaManager first,avoid internal threads - // may references to dead PikaServer + // may reference to dead PikaServer g_pika_rm->Stop(); return 0; diff --git a/src/pika_acl.cc b/src/pika_acl.cc index 0a3a1f3e14..296cbe4206 100644 --- a/src/pika_acl.cc +++ b/src/pika_acl.cc @@ -8,12 +8,13 @@ #include "include/pika_acl.h" #include "include/pika_client_conn.h" #include "include/pika_cmd_table_manager.h" +#include "include/pika_command.h" const static int AclGenPassMaxBit = 4096; extern std::unique_ptr g_pika_cmd_table_manager; -void PikaAclCmd::Do(std::shared_ptr slot) { +void PikaAclCmd::Do() { if (subCmd_ == "cat") { Cat(); } else if (subCmd_ == "deluser") { diff --git a/src/pika_admin.cc b/src/pika_admin.cc index d6b4755f15..22f1bba421 100644 --- a/src/pika_admin.cc +++ b/src/pika_admin.cc @@ -111,7 +111,6 @@ void SlaveofCmd::DoInitial() { is_none_ = true; return; } - // self is master of A , want to slaveof B if ((g_pika_server->role() & PIKA_ROLE_MASTER) != 0) { res_.SetRes(CmdRes::kErrOther, "already master of others, invalid usage"); @@ -139,7 +138,7 @@ void SlaveofCmd::DoInitial() { } } -void SlaveofCmd::Do(std::shared_ptr slot) { +void SlaveofCmd::Do() { // Check if we are already connected to the specified master if ((master_ip_ == "127.0.0.1" || g_pika_server->master_ip() == master_ip_) && g_pika_server->master_port() == master_port_) { @@ -163,7 +162,7 @@ void SlaveofCmd::Do(std::shared_ptr slot) { if (sm_ret) { res_.SetRes(CmdRes::kOk); - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db_); g_pika_conf->SetSlaveof(master_ip_ + ":" + std::to_string(master_port_)); g_pika_server->SetFirstMetaSync(true); } else { @@ -221,26 +220,25 @@ void DbSlaveofCmd::DoInitial() { } } -void DbSlaveofCmd::Do(std::shared_ptr slot) { - std::shared_ptr slave_slot = g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name_, 0)); - if (!slave_slot) { +void DbSlaveofCmd::Do() { + std::shared_ptr slave_db = g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name_)); + if (!slave_db) { res_.SetRes(CmdRes::kErrOther, "Db not found"); return; } Status s; if (is_none_) { - // In classic mode a db has only one slot - s = g_pika_rm->SendRemoveSlaveNodeRequest(db_name_, 0); + s = g_pika_rm->SendRemoveSlaveNodeRequest(db_name_); } else { - if (slave_slot->State() == ReplState::kNoConnect || slave_slot->State() == ReplState::kError || - slave_slot->State() == ReplState::kDBNoConnect) { + if (slave_db->State() == ReplState::kNoConnect || slave_db->State() == ReplState::kError || + slave_db->State() == ReplState::kDBNoConnect) { if (have_offset_) { - std::shared_ptr db_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name_, 0)); - db_slot->Logger()->SetProducerStatus(filenum_, offset_); + std::shared_ptr db = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); + db->Logger()->SetProducerStatus(filenum_, offset_); } ReplState state = force_sync_ ? ReplState::kTryDBSync : ReplState::kTryConnect; - s = g_pika_rm->ActivateSyncSlaveSlot( + s = g_pika_rm->ActivateSyncSlaveDB( RmNode(g_pika_server->master_ip(), g_pika_server->master_port(), db_name_, 0), state); } } @@ -259,7 +257,7 @@ void AuthCmd::DoInitial() { } } -void AuthCmd::Do(std::shared_ptr slot) { +void AuthCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNamePing); @@ -318,7 +316,7 @@ void BgsaveCmd::DoInitial() { } } -void BgsaveCmd::Do(std::shared_ptr slot) { +void BgsaveCmd::Do() { g_pika_server->DoSameThingSpecificDB(bgsave_dbs_, {TaskType::kBgSave}); LogCommand(); res_.AppendContent("+Background saving started"); @@ -356,7 +354,7 @@ void CompactCmd::DoInitial() { } } -void CompactCmd::Do(std::shared_ptr slot) { +void CompactCmd::Do() { if (strcasecmp(struct_type_.data(), "all") == 0) { g_pika_server->DoSameThingSpecificDB(compact_dbs_, {TaskType::kCompactAll}); } else if (strcasecmp(struct_type_.data(), "string") == 0) { @@ -403,7 +401,7 @@ void CompactRangeCmd::DoInitial() { end_key_ = argv_[4]; } -void CompactRangeCmd::Do(std::shared_ptr slot) { +void CompactRangeCmd::Do() { if (strcasecmp(struct_type_.data(), "string") == 0) { g_pika_server->DoSameThingSpecificDB(compact_dbs_, {TaskType::kCompactRangeStrings, {start_key_, end_key_}}); } else if (strcasecmp(struct_type_.data(), "hash") == 0) { @@ -447,12 +445,12 @@ void PurgelogstoCmd::DoInitial() { } } -void PurgelogstoCmd::Do(std::shared_ptr slot) { - std::shared_ptr sync_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_, 0)); - if (!sync_slot) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); +void PurgelogstoCmd::Do() { + std::shared_ptr sync_db = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_)); + if (!sync_db) { + res_.SetRes(CmdRes::kErrOther, "DB not found"); } else { - sync_slot->StableLogger()->PurgeStableLogs(num_, true); + sync_db->StableLogger()->PurgeStableLogs(num_, true); res_.SetRes(CmdRes::kOk); } } @@ -464,7 +462,7 @@ void PingCmd::DoInitial() { } } -void PingCmd::Do(std::shared_ptr slot) { +void PingCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNamePing); @@ -484,17 +482,8 @@ void SelectCmd::DoInitial() { return; } db_name_ = "db" + argv_[1]; - select_db_ = g_pika_server->GetDB(db_name_); - return; -} - -void SelectCmd::Do(std::shared_ptr slot) { - std::shared_ptr conn = std::dynamic_pointer_cast(GetConn()); - if (!conn) { - res_.SetRes(CmdRes::kErrOther, kCmdNameSelect); - LOG(WARNING) << name_ << " weak ptr is empty"; - return; - } + db_ = g_pika_server->GetDB(db_name_); + sync_db_ = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); int index = atoi(argv_[1].data()); if (std::to_string(index) != argv_[1]) { res_.SetRes(CmdRes::kInvalidIndex, kCmdNameSelect); @@ -504,10 +493,19 @@ void SelectCmd::Do(std::shared_ptr slot) { res_.SetRes(CmdRes::kInvalidIndex, kCmdNameSelect + " DB index is out of range"); return; } - if (select_db_ == nullptr) { + if (db_ == nullptr || sync_db_ == nullptr) { res_.SetRes(CmdRes::kInvalidDB, kCmdNameSelect); return; } +} + +void SelectCmd::Do() { + std::shared_ptr conn = std::dynamic_pointer_cast(GetConn()); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNameSelect); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } conn->SetCurrentDb(db_name_); res_.SetRes(CmdRes::kOk); } @@ -518,26 +516,26 @@ void FlushallCmd::DoInitial() { return; } } -void FlushallCmd::Do(std::shared_ptr slot) { - if (!slot) { - LOG(INFO) << "Flushall, but Slot not found"; +void FlushallCmd::Do() { + if (!db_) { + LOG(INFO) << "Flushall, but DB not found"; } else { - slot->FlushDB(); + db_->FlushDB(); } } -void FlushallCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void FlushallCmd::DoThroughDB() { + Do(); } -void FlushallCmd::DoUpdateCache(std::shared_ptr slot) { +void FlushallCmd::DoUpdateCache() { // clear cache if (PIKA_CACHE_NONE != g_pika_conf->cache_model()) { - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db_); } } -// flushall convert flushdb writes to every slot binlog +// flushall convert flushdb writes to every db binlog std::string FlushallCmd::ToRedisProtocol() { std::string content; content.reserve(RAW_ARGS_LEN); @@ -558,15 +556,15 @@ void FlushallCmd::Execute() { return; } } - g_pika_rm->SlotLock(); + g_pika_rm->DBLock(); for (const auto& db_item : g_pika_server->GetDB()) { - db_item.second->SlotLock(); + db_item.second->DBLock(); } FlushAllWithoutLock(); for (const auto& db_item : g_pika_server->GetDB()) { - db_item.second->SlotUnlock(); + db_item.second->DBUnlock(); } - g_pika_rm->SlotUnlock(); + g_pika_rm->DBUnlock(); if (res_.ok()) { res_.SetRes(CmdRes::kOk); } @@ -574,28 +572,52 @@ void FlushallCmd::Execute() { void FlushallCmd::FlushAllWithoutLock() { for (const auto& db_item : g_pika_server->GetDB()) { - for (const auto& slot_item : db_item.second->GetSlots()) { - std::shared_ptr slot = slot_item.second; - SlotInfo p_info(slot->GetDBName(), slot->GetSlotID()); - if (g_pika_rm->GetSyncMasterSlots().find(p_info) == g_pika_rm->GetSyncMasterSlots().end()) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - DoWithoutLock(slot); - DoBinlog(g_pika_rm->GetSyncMasterSlots()[p_info]); + std::shared_ptr db = db_item.second; + DBInfo p_info(db->GetDBName()); + if (g_pika_rm->GetSyncMasterDBs().find(p_info) == g_pika_rm->GetSyncMasterDBs().end()) { + res_.SetRes(CmdRes::kErrOther, "DB not found"); + return; } + DoWithoutLock(db); + DoBinlog(g_pika_rm->GetSyncMasterDBs()[p_info]); } if (res_.ok()) { res_.SetRes(CmdRes::kOk); } } -void FlushallCmd::DoWithoutLock(std::shared_ptr slot) { - if (!slot) { - LOG(INFO) << "Flushall, but Slot not found"; +void FlushallCmd::DoBinlog(std::shared_ptr sync_db) { + if (res().ok() && is_write() && g_pika_conf->write_binlog()) { + std::shared_ptr conn_ptr = GetConn(); + std::shared_ptr resp_ptr = GetResp(); + // Consider that dummy cmd appended by system, both conn and resp are null. + if ((!conn_ptr || !resp_ptr) && (name_ != kCmdDummy)) { + if (!conn_ptr) { + LOG(WARNING) << sync_db->SyncDBInfo().ToString() << " conn empty."; + } + if (!resp_ptr) { + LOG(WARNING) << sync_db->SyncDBInfo().ToString() << " resp empty."; + } + res().SetRes(CmdRes::kErrOther); + return; + } + + Status s = sync_db->ConsensusProposeLog(shared_from_this()); + if (!s.ok()) { + LOG(WARNING) << sync_db->SyncDBInfo().ToString() << " Writing binlog failed, maybe no space left on device " + << s.ToString(); + res().SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + } +} + +void FlushallCmd::DoWithoutLock(std::shared_ptr db) { + if (!db) { + LOG(INFO) << "Flushall, but DB not found"; } else { - slot->FlushDBWithoutLock(); - DoUpdateCache(slot); + db->FlushDBWithoutLock(); + DoUpdateCache(); } } @@ -624,67 +646,63 @@ void FlushdbCmd::DoInitial() { } } -void FlushdbCmd::Do(std::shared_ptr slot) { - if (!slot) { - LOG(INFO) << "Flushdb, but Slot not found"; +void FlushdbCmd::Do() { + if (!db_) { + LOG(INFO) << "Flushdb, but DB not found"; } else { if (db_name_ == "all") { - slot->FlushDB(); + db_->FlushDB(); } else { - slot->FlushSubDB(db_name_); + db_->FlushSubDB(db_name_); } } } -void FlushdbCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void FlushdbCmd::DoThroughDB() { + Do(); } -void FlushdbCmd::DoUpdateCache(std::shared_ptr slot) { +void FlushdbCmd::DoUpdateCache() { // clear cache if (g_pika_conf->cache_model() != PIKA_CACHE_NONE) { - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db_); } } -void FlushdbCmd::FlushAllSlotsWithoutLock(std::shared_ptr db) { - for (const auto& slot_item : db->GetSlots()) { - std::shared_ptr slot = slot_item.second; - SlotInfo p_info(slot->GetDBName(), slot->GetSlotID()); - if (g_pika_rm->GetSyncMasterSlots().find(p_info) == g_pika_rm->GetSyncMasterSlots().end()) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - DoWithoutLock(slot); - DoBinlog(g_pika_rm->GetSyncMasterSlots()[p_info]); +void FlushdbCmd::FlushAllDBsWithoutLock() { + DBInfo p_info(db_->GetDBName()); + if (g_pika_rm->GetSyncMasterDBs().find(p_info) == g_pika_rm->GetSyncMasterDBs().end()) { + res_.SetRes(CmdRes::kErrOther, "DB not found"); + return; } + DoWithoutLock(); + DoBinlog(); } -void FlushdbCmd::DoWithoutLock(std::shared_ptr slot) { - if (!slot) { - LOG(INFO) << "Flushdb, but Slot not found"; +void FlushdbCmd::DoWithoutLock() { + if (!db_) { + LOG(INFO) << "Flushdb, but DB not found"; } else { if (db_name_ == "all") { - slot->FlushDBWithoutLock(); + db_->FlushDBWithoutLock(); } else { - slot->FlushSubDBWithoutLock(db_name_); + db_->FlushSubDBWithoutLock(db_name_); } - DoUpdateCache(slot); + DoUpdateCache(); } } void FlushdbCmd::Execute() { - std::shared_ptr db = g_pika_server->GetDB(Cmd::db_name_); - if (!db) { + if (!db_) { res_.SetRes(CmdRes::kInvalidDB); } else { - if (db->IsKeyScaning()) { + if (db_->IsKeyScaning()) { res_.SetRes(CmdRes::kErrOther, "The keyscan operation is executing, Try again later"); } else { - std::lock_guard l_prw(db->GetSlotLock()); - std::lock_guard s_prw(g_pika_rm->GetSlotLock()); - FlushAllSlotsWithoutLock(db); + std::lock_guard l_prw(db_->GetDBLock()); + std::lock_guard s_prw(g_pika_rm->GetDBLock()); + FlushAllDBsWithoutLock(); res_.SetRes(CmdRes::kOk); } } @@ -730,7 +748,7 @@ void ClientCmd::DoInitial() { operation_ = argv_[1]; } -void ClientCmd::Do(std::shared_ptr slot) { +void ClientCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNameClient); @@ -803,7 +821,7 @@ void ShutdownCmd::DoInitial() { } } // no return -void ShutdownCmd::Do(std::shared_ptr slot) { +void ShutdownCmd::Do() { DLOG(WARNING) << "handle \'shutdown\'"; g_pika_server->Exit(); res_.SetRes(CmdRes::kNone); @@ -905,7 +923,7 @@ void InfoCmd::DoInitial() { } } -void InfoCmd::Do(std::shared_ptr slot) { +void InfoCmd::Do() { std::string info; switch (info_section_) { case kInfo: @@ -936,7 +954,7 @@ void InfoCmd::Do(std::shared_ptr slot) { info.append("\r\n"); InfoCommandStats(info); info.append("\r\n"); - InfoCache(info, slot); + InfoCache(info, db_); info.append("\r\n"); InfoCPU(info); info.append("\r\n"); @@ -980,7 +998,7 @@ void InfoCmd::Do(std::shared_ptr slot) { InfoCommandStats(info); break; case kInfoCache: - InfoCache(info, slot); + InfoCache(info, db_); break; default: // kInfoErr is nothing @@ -1122,37 +1140,34 @@ void InfoCmd::InfoReplication(std::string& info) { std::stringstream tmp_stream; std::stringstream out_of_sync; - bool all_slot_sync = true; + bool all_db_sync = true; std::shared_lock db_rwl(g_pika_server->dbs_rw_); for (const auto& db_item : g_pika_server->GetDB()) { - std::shared_lock slot_rwl(db_item.second->slots_rw_); - for (const auto& slot_item : db_item.second->slots_) { - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); - if (!slave_slot) { - out_of_sync << "(" << slot_item.second->GetSlotName() << ": InternalError)"; - continue; - } - if (slave_slot->State() != ReplState::kConnected) { - all_slot_sync = false; - out_of_sync << "(" << slot_item.second->GetSlotName() << ":"; - if (slave_slot->State() == ReplState::kNoConnect) { - out_of_sync << "NoConnect)"; - } else if (slave_slot->State() == ReplState::kWaitDBSync) { - out_of_sync << "WaitDBSync)"; - } else if (slave_slot->State() == ReplState::kError) { - out_of_sync << "Error)"; - } else if (slave_slot->State() == ReplState::kWaitReply) { - out_of_sync << "kWaitReply)"; - } else if (slave_slot->State() == ReplState::kTryConnect) { - out_of_sync << "kTryConnect)"; - } else if (slave_slot->State() == ReplState::kTryDBSync) { - out_of_sync << "kTryDBSync)"; - } else if (slave_slot->State() == ReplState::kDBNoConnect) { - out_of_sync << "kDBNoConnect)"; - } else { - out_of_sync << "Other)"; - } + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_item.second->GetDBName())); + if (!slave_db) { + out_of_sync << "(" << db_item.first << ": InternalError)"; + continue; + } + if (slave_db->State() != ReplState::kConnected) { + all_db_sync = false; + out_of_sync << "(" << db_item.first << ":"; + if (slave_db->State() == ReplState::kNoConnect) { + out_of_sync << "NoConnect)"; + } else if (slave_db->State() == ReplState::kWaitDBSync) { + out_of_sync << "WaitDBSync)"; + } else if (slave_db->State() == ReplState::kError) { + out_of_sync << "Error)"; + } else if (slave_db->State() == ReplState::kWaitReply) { + out_of_sync << "kWaitReply)"; + } else if (slave_db->State() == ReplState::kTryConnect) { + out_of_sync << "kTryConnect)"; + } else if (slave_db->State() == ReplState::kTryDBSync) { + out_of_sync << "kTryDBSync)"; + } else if (slave_db->State() == ReplState::kDBNoConnect) { + out_of_sync << "kDBNoConnect)"; + } else { + out_of_sync << "Other)"; } } } @@ -1180,11 +1195,11 @@ void InfoCmd::InfoReplication(std::string& info) { tmp_stream << "master_host:" << g_pika_server->master_ip() << "\r\n"; tmp_stream << "master_port:" << g_pika_server->master_port() << "\r\n"; tmp_stream << "master_link_status:" - << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) && all_slot_sync) ? "up" : "down") + << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) && all_db_sync) ? "up" : "down") << "\r\n"; tmp_stream << "slave_priority:" << g_pika_conf->slave_priority() << "\r\n"; tmp_stream << "slave_read_only:" << g_pika_conf->slave_read_only() << "\r\n"; - if (!all_slot_sync) { + if (!all_db_sync) { tmp_stream << "db_repl_state:" << out_of_sync.str() << "\r\n"; } break; @@ -1192,10 +1207,10 @@ void InfoCmd::InfoReplication(std::string& info) { tmp_stream << "master_host:" << g_pika_server->master_ip() << "\r\n"; tmp_stream << "master_port:" << g_pika_server->master_port() << "\r\n"; tmp_stream << "master_link_status:" - << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) && all_slot_sync) ? "up" : "down") + << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) && all_db_sync) ? "up" : "down") << "\r\n"; tmp_stream << "slave_read_only:" << g_pika_conf->slave_read_only() << "\r\n"; - if (!all_slot_sync) { + if (!all_db_sync) { tmp_stream << "db_repl_state:" << out_of_sync.str() << "\r\n"; } case PIKA_ROLE_SINGLE: @@ -1208,22 +1223,19 @@ void InfoCmd::InfoReplication(std::string& info) { uint32_t filenum = 0; uint64_t offset = 0; std::string safety_purge; - std::shared_ptr master_slot = nullptr; + std::shared_ptr master_db = nullptr; for (const auto& t_item : g_pika_server->dbs_) { - std::shared_lock slot_rwl(t_item.second->slots_rw_); - for (const auto& p_item : t_item.second->slots_) { - std::string db_name = p_item.second->GetDBName(); - uint32_t slot_id = p_item.second->GetSlotID(); - master_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!master_slot) { - LOG(WARNING) << "Sync Master Slot: " << db_name << ":" << slot_id << ", NotFound"; - continue; - } - master_slot->Logger()->GetProducerStatus(&filenum, &offset); - tmp_stream << db_name << " binlog_offset=" << filenum << " " << offset; - s = master_slot->GetSafetyPurgeBinlog(&safety_purge); - tmp_stream << ",safety_purge=" << (s.ok() ? safety_purge : "error") << "\r\n"; + std::shared_lock db_rwl(t_item.second->dbs_rw_); + std::string db_name = t_item.first; + master_db = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!master_db) { + LOG(WARNING) << "Sync Master DB: " << db_name << " NotFound"; + continue; } + master_db->Logger()->GetProducerStatus(&filenum, &offset); + tmp_stream << db_name << " binlog_offset=" << filenum << " " << offset; + s = master_db->GetSafetyPurgeBinlog(&safety_purge); + tmp_stream << ",safety_purge=" << (s.ok() ? safety_purge : "error") << "\r\n"; } info.append(tmp_stream.str()); @@ -1331,23 +1343,20 @@ void InfoCmd::InfoData(std::string& info) { if (!db_item.second) { continue; } - std::shared_lock slot_rwl(db_item.second->slots_rw_); - for (const auto& slot_item : db_item.second->slots_) { - background_errors.clear(); - memtable_usage = table_reader_usage = 0; - slot_item.second->DbRWLockReader(); - slot_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES, &memtable_usage); - slot_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM, &table_reader_usage); - slot_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors); - slot_item.second->DbRWUnLock(); - total_memtable_usage += memtable_usage; - total_table_reader_usage += table_reader_usage; - for (const auto& item : background_errors) { - if (item.second != 0) { - db_fatal_msg_stream << (total_background_errors != 0 ? "," : ""); - db_fatal_msg_stream << slot_item.second->GetSlotName() << "/" << item.first; - total_background_errors += item.second; - } + background_errors.clear(); + memtable_usage = table_reader_usage = 0; + db_item.second->DbRWLockReader(); + db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_CUR_SIZE_ALL_MEM_TABLES, &memtable_usage); + db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_ESTIMATE_TABLE_READER_MEM, &table_reader_usage); + db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors); + db_item.second->DbRWUnLock(); + total_memtable_usage += memtable_usage; + total_table_reader_usage += table_reader_usage; + for (const auto& item : background_errors) { + if (item.second != 0) { + db_fatal_msg_stream << (total_background_errors != 0 ? "," : ""); + db_fatal_msg_stream << db_item.first << "/" << item.first; + total_background_errors += item.second; } } } @@ -1368,21 +1377,17 @@ void InfoCmd::InfoRocksDB(std::string& info) { tmp_stream << "# RocksDB" << "\r\n"; - std::shared_lock table_rwl(g_pika_server->dbs_rw_); - for (const auto& table_item : g_pika_server->dbs_) { - if (!table_item.second) { + std::shared_lock db_rwl(g_pika_server->dbs_rw_); + for (const auto& db_item : g_pika_server->dbs_) { + if (!db_item.second) { continue; } - std::shared_lock slot_rwl(table_item.second->slots_rw_); - for (const auto& slot_item : table_item.second->slots_) { - std::string rocksdb_info; - slot_item.second->DbRWLockReader(); - slot_item.second->db()->GetRocksDBInfo(rocksdb_info); - slot_item.second->DbRWUnLock(); - tmp_stream << rocksdb_info; - } + std::string rocksdb_info; + db_item.second->DbRWLockReader(); + db_item.second->storage()->GetRocksDBInfo(rocksdb_info); + db_item.second->DbRWUnLock(); + tmp_stream << rocksdb_info; } - info.append(tmp_stream.str()); } @@ -1425,13 +1430,13 @@ void InfoCmd::InfoCommandStats(std::string& info) { info.append(tmp_stream.str()); } -void InfoCmd::InfoCache(std::string& info, std::shared_ptr slot) { +void InfoCmd::InfoCache(std::string& info, std::shared_ptr db) { std::stringstream tmp_stream; tmp_stream << "# Cache" << "\r\n"; if (PIKA_CACHE_NONE == g_pika_conf->cache_model()) { tmp_stream << "cache_status:Disable" << "\r\n"; } else { - auto cache_info = slot->GetCacheInfo(); + auto cache_info = db->GetCacheInfo(); tmp_stream << "cache_status:" << CacheStatusToString(cache_info.status) << "\r\n"; tmp_stream << "cache_db_num:" << cache_info.cache_num << "\r\n"; tmp_stream << "cache_keys:" << cache_info.keys_num << "\r\n"; @@ -1469,8 +1474,8 @@ std::string InfoCmd::CacheStatusToString(int status) { } void InfoCmd::Execute() { - std::shared_ptr slot = g_pika_server->GetSlotByDBName(db_name_); - Do(slot); + std::shared_ptr db = g_pika_server->GetDB(db_name_); + Do(); } void ConfigCmd::DoInitial() { @@ -1509,12 +1514,12 @@ void ConfigCmd::DoInitial() { config_args_v_.assign(argv_.begin() + 1, argv_.end()); } -void ConfigCmd::Do(std::shared_ptr slot) { +void ConfigCmd::Do() { std::string config_ret; if (strcasecmp(config_args_v_[0].data(), "get") == 0) { ConfigGet(config_ret); } else if (strcasecmp(config_args_v_[0].data(), "set") == 0) { - ConfigSet(config_ret, slot); + ConfigSet(config_ret, db_); } else if (strcasecmp(config_args_v_[0].data(), "rewrite") == 0) { ConfigRewrite(config_ret); } else if (strcasecmp(config_args_v_[0].data(), "resetstat") == 0) { @@ -2074,7 +2079,7 @@ void ConfigCmd::ConfigGet(std::string& ret) { } // Remember to sync change PikaConf::ConfigRewrite(); -void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { +void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr db) { std::string set_item = config_args_v_[1]; if (set_item == "*") { ret = "*29\r\n"; @@ -2216,7 +2221,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { return; } g_pika_conf->SetMaxCacheStatisticKeys(static_cast(ival)); - g_pika_server->SlotSetMaxCacheStatisticKeys(static_cast(ival)); + g_pika_server->DBSetMaxCacheStatisticKeys(static_cast(ival)); ret = "+OK\r\n"; } else if (set_item == "small-compaction-threshold") { if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival < 0) { @@ -2224,7 +2229,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { return; } g_pika_conf->SetSmallCompactionThreshold(static_cast(ival)); - g_pika_server->SlotSetSmallCompactionThreshold(static_cast(ival)); + g_pika_server->DBSetSmallCompactionThreshold(static_cast(ival)); ret = "+OK\r\n"; } else if (set_item == "small-compaction-duration-threshold") { if ((pstd::string2int(value.data(), value.size(), &ival) == 0) || ival < 0) { @@ -2232,7 +2237,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { return; } g_pika_conf->SetSmallCompactionDurationThreshold(static_cast(ival)); - g_pika_server->SlotSetSmallCompactionDurationThreshold(static_cast(ival)); + g_pika_server->DBSetSmallCompactionDurationThreshold(static_cast(ival)); ret = "+OK\r\n"; } else if (set_item == "disable_auto_compactions") { if (value != "true" && value != "false") { @@ -2453,7 +2458,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { int cache_num = (ival <= 0 || ival > 48) ? 16 : ival; if (cache_num != g_pika_conf->GetCacheNum()) { g_pika_conf->SetCacheNum(cache_num); - g_pika_server->ResetCacheAsync(cache_num, slot); + g_pika_server->ResetCacheAsync(cache_num, db); } ret = "+OK\r\n"; } else if (set_item == "cache-model") { @@ -2466,7 +2471,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { } else { g_pika_conf->SetCacheModel(ival); if (PIKA_CACHE_NONE == ival) { - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db); } ret = "+OK\r\n"; } @@ -2497,7 +2502,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { auto origin_start_pos = g_pika_conf->zset_cache_start_pos(); if (origin_start_pos != ival) { g_pika_conf->SetCacheStartPos(ival); - g_pika_server->OnCacheStartPosChanged(ival, slot); + g_pika_server->OnCacheStartPosChanged(ival, db); } ret = "+OK\r\n"; } else if (set_item == "zset-cache-field-num-per-key") { @@ -2506,7 +2511,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { return; } g_pika_conf->SetCacheItemsPerKey(ival); - g_pika_server->ResetCacheConfig(slot); + g_pika_server->ResetCacheConfig(db); ret = "+OK\r\n"; } else if (set_item == "cache-maxmemory") { if (!pstd::string2int(value.data(), value.size(), &ival) || ival < 0) { @@ -2515,7 +2520,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { } int64_t cache_maxmemory = (PIKA_CACHE_SIZE_MIN > ival) ? PIKA_CACHE_SIZE_DEFAULT : ival; g_pika_conf->SetCacheMaxmemory(cache_maxmemory); - g_pika_server->ResetCacheConfig(slot); + g_pika_server->ResetCacheConfig(db); ret = "+OK\r\n"; } else if (set_item == "cache-maxmemory-policy") { if (!pstd::string2int(value.data(), value.size(), &ival) || ival < 0) { @@ -2524,7 +2529,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { } int cache_maxmemory_policy_ = (ival < 0|| ival > 5) ? 3 : ival; // default allkeys-lru g_pika_conf->SetCacheMaxmemoryPolicy(cache_maxmemory_policy_); - g_pika_server->ResetCacheConfig(slot); + g_pika_server->ResetCacheConfig(db); ret = "+OK\r\n"; } else if (set_item == "cache-maxmemory-samples") { if (!pstd::string2int(value.data(), value.size(), &ival) || ival < 0) { @@ -2533,7 +2538,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { } int cache_maxmemory_samples = (ival > 1) ? 5 : ival; g_pika_conf->SetCacheMaxmemorySamples(cache_maxmemory_samples); - g_pika_server->ResetCacheConfig(slot); + g_pika_server->ResetCacheConfig(db); ret = "+OK\r\n"; } else if (set_item == "cache-lfu-decay-time") { if (!pstd::string2int(value.data(), value.size(), &ival) || ival < 0) { @@ -2542,7 +2547,7 @@ void ConfigCmd::ConfigSet(std::string& ret, std::shared_ptr slot) { } int cache_lfu_decay_time = (ival < 0) ? 1 : ival; g_pika_conf->SetCacheLFUDecayTime(cache_lfu_decay_time); - g_pika_server->ResetCacheConfig(slot); + g_pika_server->ResetCacheConfig(db); ret = "+OK\r\n"; } else if (set_item == "acl-pubsub-default") { std::string v(value); @@ -2587,9 +2592,7 @@ void ConfigCmd::ConfigResetstat(std::string& ret) { } void ConfigCmd::Execute() { - std::shared_ptr slot; - slot = g_pika_server->GetSlotByDBName(db_name_); - Do(slot); + Do(); } void MonitorCmd::DoInitial() { @@ -2599,7 +2602,7 @@ void MonitorCmd::DoInitial() { } } -void MonitorCmd::Do(std::shared_ptr slot) { +void MonitorCmd::Do() { std::shared_ptr conn_repl = GetConn(); if (!conn_repl) { res_.SetRes(CmdRes::kErrOther, kCmdNameMonitor); @@ -2618,12 +2621,12 @@ void DbsizeCmd::DoInitial() { } } -void DbsizeCmd::Do(std::shared_ptr slot) { - std::shared_ptr db = g_pika_server->GetDB(db_name_); - if (!db) { +void DbsizeCmd::Do() { + std::shared_ptr dbs = g_pika_server->GetDB(db_name_); + if (!dbs) { res_.SetRes(CmdRes::kInvalidDB); } else { - KeyScanInfo key_scan_info = db->GetKeyScanInfo(); + KeyScanInfo key_scan_info = dbs->GetKeyScanInfo(); std::vector key_infos = key_scan_info.key_infos; if (key_infos.size() != 5) { res_.SetRes(CmdRes::kErrOther, "keyspace error"); @@ -2641,7 +2644,7 @@ void TimeCmd::DoInitial() { } } -void TimeCmd::Do(std::shared_ptr slot) { +void TimeCmd::Do() { struct timeval tv; if (gettimeofday(&tv, nullptr) == 0) { res_.AppendArrayLen(2); @@ -2665,7 +2668,7 @@ void LastsaveCmd::DoInitial() { } } -void LastsaveCmd::Do(std::shared_ptr slot) { +void LastsaveCmd::Do() { res_.AppendInteger(g_pika_server->GetLastSave()); } @@ -2676,7 +2679,7 @@ void DelbackupCmd::DoInitial() { } } -void DelbackupCmd::Do(std::shared_ptr slot) { +void DelbackupCmd::Do() { std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); std::string db_sync_path = g_pika_conf->bgsave_path(); std::vector dump_dir; @@ -2725,7 +2728,7 @@ void EchoCmd::DoInitial() { body_ = argv_[1]; } -void EchoCmd::Do(std::shared_ptr slot) { res_.AppendString(body_); } +void EchoCmd::Do() { res_.AppendString(body_); } void ScandbCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -2751,12 +2754,12 @@ void ScandbCmd::DoInitial() { } } -void ScandbCmd::Do(std::shared_ptr slot) { - std::shared_ptr db = g_pika_server->GetDB(db_name_); - if (!db) { +void ScandbCmd::Do() { + std::shared_ptr dbs = g_pika_server->GetDB(db_name_); + if (!dbs) { res_.SetRes(CmdRes::kInvalidDB); } else { - db->ScanDatabase(type_); + dbs->ScanDatabase(type_); res_.SetRes(CmdRes::kOk); } } @@ -2782,7 +2785,7 @@ void SlowlogCmd::DoInitial() { } } -void SlowlogCmd::Do(std::shared_ptr slot) { +void SlowlogCmd::Do() { if (condition_ == SlowlogCmd::kRESET) { g_pika_server->SlowlogReset(); res_.SetRes(CmdRes::kOk); @@ -2812,7 +2815,7 @@ void PaddingCmd::DoInitial() { } } -void PaddingCmd::Do(std::shared_ptr slot) { res_.SetRes(CmdRes::kOk); } +void PaddingCmd::Do() { res_.SetRes(CmdRes::kOk); } std::string PaddingCmd::ToRedisProtocol() { return PikaBinlogTransverter::ConstructPaddingBinlog( @@ -2842,9 +2845,9 @@ void PKPatternMatchDelCmd::DoInitial() { } } -void PKPatternMatchDelCmd::Do(std::shared_ptr slot) { +void PKPatternMatchDelCmd::Do() { int ret = 0; - rocksdb::Status s = slot->db()->PKPatternMatchDel(type_, pattern_, &ret); + rocksdb::Status s = db_->storage()->PKPatternMatchDel(type_, pattern_, &ret); if (s.ok()) { res_.AppendInteger(ret); } else { @@ -2854,7 +2857,7 @@ void PKPatternMatchDelCmd::Do(std::shared_ptr slot) { void DummyCmd::DoInitial() {} -void DummyCmd::Do(std::shared_ptr slot) {} +void DummyCmd::Do() {} void QuitCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -2862,7 +2865,7 @@ void QuitCmd::DoInitial() { } } -void QuitCmd::Do(std::shared_ptr slot) { +void QuitCmd::Do() { res_.SetRes(CmdRes::kOk); LOG(INFO) << "QutCmd will close connection " << GetConn()->String(); GetConn()->SetClose(true); @@ -2878,7 +2881,7 @@ void HelloCmd::DoInitial() { } } -void HelloCmd::Do(std::shared_ptr slot) { +void HelloCmd::Do() { size_t next_arg = 1; long ver = 0; if (argv_.size() >= 2) { @@ -2984,7 +2987,7 @@ void DiskRecoveryCmd::DoInitial() { } } -void DiskRecoveryCmd::Do(std::shared_ptr slot) { +void DiskRecoveryCmd::Do() { struct statvfs disk_info; int ret = statvfs(g_pika_conf->db_path().c_str(), &disk_info); if (ret == -1) { @@ -3000,7 +3003,6 @@ void DiskRecoveryCmd::Do(std::shared_ptr slot) { res_.SetRes(CmdRes::kErrOther, "The available disk capacity is insufficient"); return; } - std::shared_mutex slots_rw; std::shared_mutex dbs_rw; std::shared_lock db_rwl(dbs_rw); // loop every db @@ -3009,19 +3011,15 @@ void DiskRecoveryCmd::Do(std::shared_ptr slot) { continue; } db_item.second->SetBinlogIoErrorrelieve(); - std::shared_lock slot_rwl(slots_rw); - // loop every slot - for (const auto& slot_item : db_item.second->GetSlots()) { - background_errors_.clear(); - slot_item.second->DbRWLockReader(); - slot_item.second->db()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors_); - slot_item.second->DbRWUnLock(); - for (const auto& item : background_errors_) { - if (item.second != 0) { - rocksdb::Status s = slot_item.second->db()->GetDBByType(item.first)->Resume(); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "The restore operation failed."); - } + background_errors_.clear(); + db_item.second->DbRWLockReader(); + db_item.second->storage()->GetUsage(storage::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &background_errors_); + db_item.second->DbRWUnLock(); + for (const auto &item: background_errors_) { + if (item.second != 0) { + rocksdb::Status s = db_item.second->storage()->GetDBByType(item.first)->Resume(); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, "The restore operation failed."); } } } @@ -3036,7 +3034,7 @@ void ClearReplicationIDCmd::DoInitial() { } } -void ClearReplicationIDCmd::Do(std::shared_ptr slot) { +void ClearReplicationIDCmd::Do() { g_pika_conf->SetReplicationID(""); g_pika_conf->ConfigRewriteReplicationID(); res_.SetRes(CmdRes::kOk, "ReplicationID is cleared"); @@ -3049,7 +3047,7 @@ void DisableWalCmd::DoInitial() { } } -void DisableWalCmd::Do(std::shared_ptr slot) { +void DisableWalCmd::Do() { std::string option = argv_[1].data(); bool is_wal_disable = false; if (option.compare("true") == 0) { @@ -3060,7 +3058,7 @@ void DisableWalCmd::Do(std::shared_ptr slot) { res_.SetRes(CmdRes::kErrOther, "Invalid parameter"); return; } - slot->db()->DisableWal(is_wal_disable); + db_->storage()->DisableWal(is_wal_disable); res_.SetRes(CmdRes::kOk, "Wal options is changed"); } @@ -3088,23 +3086,23 @@ void CacheCmd::DoInitial() { return; } -void CacheCmd::Do(std::shared_ptr slot) { +void CacheCmd::Do() { std::string key; switch (condition_) { case kCLEAR_DB: - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db_); res_.SetRes(CmdRes::kOk); break; case kCLEAR_HITRATIO: - g_pika_server->ClearHitRatio(slot); + g_pika_server->ClearHitRatio(db_); res_.SetRes(CmdRes::kOk); break; case kDEL_KEYS: - slot->cache()->Del(keys_); + db_->cache()->Del(keys_); res_.SetRes(CmdRes::kOk); break; case kRANDOM_KEY: - s_ = slot->cache()->RandomKey(&key); + s_ = db_->cache()->RandomKey(&key); if (!s_.ok()) { res_.AppendStringLen(-1); } else { @@ -3126,10 +3124,10 @@ void ClearCacheCmd::DoInitial() { } } -void ClearCacheCmd::Do(std::shared_ptr slot) { +void ClearCacheCmd::Do() { // clean cache if (PIKA_CACHE_NONE != g_pika_conf->cache_model()) { - g_pika_server->ClearCacheDbAsync(slot); + g_pika_server->ClearCacheDbAsync(db_); } res_.SetRes(CmdRes::kOk, "Cache is cleared"); } @@ -3293,7 +3291,7 @@ void CommandCmd::DoInitial() { extern std::unique_ptr g_pika_cmd_table_manager; -void CommandCmd::Do(std::shared_ptr slots) { +void CommandCmd::Do(std::shared_ptr dbs) { std::unordered_map cmds; std::unordered_map specializations; if (cmds_begin_ == cmds_end_) { diff --git a/src/pika_auxiliary_thread.cc b/src/pika_auxiliary_thread.cc index 5368a901be..e94104b442 100644 --- a/src/pika_auxiliary_thread.cc +++ b/src/pika_auxiliary_thread.cc @@ -24,7 +24,7 @@ void* PikaAuxiliaryThread::ThreadMain() { if (g_pika_server->ShouldMetaSync()) { g_pika_rm->SendMetaSyncRequest(); } else if (g_pika_server->MetaSyncDone()) { - g_pika_rm->RunSyncSlaveSlotStateMachine(); + g_pika_rm->RunSyncSlaveDBStateMachine(); } pstd::Status s = g_pika_rm->CheckSyncTimeout(pstd::NowMicros()); diff --git a/src/pika_binlog_transverter.cc b/src/pika_binlog_transverter.cc index 41207cfe93..a6f3d2b271 100644 --- a/src/pika_binlog_transverter.cc +++ b/src/pika_binlog_transverter.cc @@ -12,6 +12,7 @@ #include "pstd/include/pstd_coding.h" #include "include/pika_command.h" +#include "include/pika_define.h" #include "storage/storage.h" diff --git a/src/pika_bit.cc b/src/pika_bit.cc index cec5903f9d..aa11a11dad 100644 --- a/src/pika_bit.cc +++ b/src/pika_bit.cc @@ -6,6 +6,8 @@ #include "include/pika_bit.h" #include "pstd/include/pstd_string.h" +#include "include/pika_db.h" + #include "include/pika_define.h" #include "include/pika_slot_command.h" @@ -40,26 +42,26 @@ void BitSetCmd::DoInitial() { } } -void BitSetCmd::Do(std::shared_ptr slot) { +void BitSetCmd::Do() { std::string value; int32_t bit_val = 0; - s_ = slot->db()->SetBit(key_, bit_offset_, static_cast(on_), &bit_val); + s_ = db_->storage()->SetBit(key_, bit_offset_, static_cast(on_), &bit_val); if (s_.ok()) { res_.AppendInteger(static_cast(bit_val)); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void BitSetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void BitSetCmd::DoThroughDB() { + Do(); } -void BitSetCmd::DoUpdateCache(std::shared_ptr slot) { +void BitSetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyB = PCacheKeyPrefixB + key_; - slot->cache()->SetBitIfKeyExist(CachePrefixKeyB, bit_offset_, on_); + db_->cache()->SetBitIfKeyExist(CachePrefixKeyB, bit_offset_, on_); } } @@ -80,9 +82,9 @@ void BitGetCmd::DoInitial() { } } -void BitGetCmd::Do(std::shared_ptr slot) { +void BitGetCmd::Do() { int32_t bit_val = 0; - s_ = slot->db()->GetBit(key_, bit_offset_, &bit_val); + s_ = db_->storage()->GetBit(key_, bit_offset_, &bit_val); if (s_.ok()) { res_.AppendInteger(static_cast(bit_val)); } else { @@ -90,10 +92,10 @@ void BitGetCmd::Do(std::shared_ptr slot) { } } -void BitGetCmd::ReadCache(std::shared_ptr slot) { +void BitGetCmd::ReadCache() { int64_t bit_val = 0; std::string CachePrefixKeyB = PCacheKeyPrefixB + key_; - auto s = slot->cache()->GetBit(CachePrefixKeyB, bit_offset_, &bit_val); + auto s = db_->cache()->GetBit(CachePrefixKeyB, bit_offset_, &bit_val); if (s.ok()) { res_.AppendInteger(bit_val); } else if (s.IsNotFound()) { @@ -103,14 +105,14 @@ void BitGetCmd::ReadCache(std::shared_ptr slot) { } } -void BitGetCmd::DoThroughDB(std::shared_ptr slot) { +void BitGetCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void BitGetCmd::DoUpdateCache(std::shared_ptr slot){ +void BitGetCmd::DoUpdateCache(){ if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, db_); } } @@ -137,12 +139,12 @@ void BitCountCmd::DoInitial() { } } -void BitCountCmd::Do(std::shared_ptr slot) { +void BitCountCmd::Do() { int32_t count = 0; if (count_all_) { - s_ = slot->db()->BitCount(key_, start_offset_, end_offset_, &count, false); + s_ = db_->storage()->BitCount(key_, start_offset_, end_offset_, &count, false); } else { - s_ = slot->db()->BitCount(key_, start_offset_, end_offset_, &count, true); + s_ = db_->storage()->BitCount(key_, start_offset_, end_offset_, &count, true); } if (s_.ok() || s_.IsNotFound()) { @@ -152,16 +154,16 @@ void BitCountCmd::Do(std::shared_ptr slot) { } } -void BitCountCmd::ReadCache(std::shared_ptr slot) { +void BitCountCmd::ReadCache() { int64_t count = 0; int64_t start = static_cast(start_offset_); int64_t end = static_cast(end_offset_); rocksdb::Status s; std::string CachePrefixKeyB = PCacheKeyPrefixB + key_; if (count_all_) { - s = slot->cache()->BitCount(CachePrefixKeyB, start, end, &count, 0); + s = db_->cache()->BitCount(CachePrefixKeyB, start, end, &count, 0); } else { - s = slot->cache()->BitCount(CachePrefixKeyB, start, end, &count, 1); + s = db_->cache()->BitCount(CachePrefixKeyB, start, end, &count, 1); } if (s.ok()) { @@ -173,14 +175,14 @@ void BitCountCmd::ReadCache(std::shared_ptr slot) { } } -void BitCountCmd::DoThroughDB(std::shared_ptr slot) { +void BitCountCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void BitCountCmd::DoUpdateCache(std::shared_ptr slot) { +void BitCountCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, db_); } } @@ -224,15 +226,15 @@ void BitPosCmd::DoInitial() { } } -void BitPosCmd::Do(std::shared_ptr slot) { +void BitPosCmd::Do() { int64_t pos = 0; rocksdb::Status s; if (pos_all_) { - s_ = slot->db()->BitPos(key_, static_cast(bit_val_), &pos); + s_ = db_->storage()->BitPos(key_, static_cast(bit_val_), &pos); } else if (!pos_all_ && !endoffset_set_) { - s_ = slot->db()->BitPos(key_, static_cast(bit_val_), start_offset_, &pos); + s_ = db_->storage()->BitPos(key_, static_cast(bit_val_), start_offset_, &pos); } else if (!pos_all_ && endoffset_set_) { - s_ = slot->db()->BitPos(key_, static_cast(bit_val_), start_offset_, end_offset_, &pos); + s_ = db_->storage()->BitPos(key_, static_cast(bit_val_), start_offset_, end_offset_, &pos); } if (s_.ok()) { res_.AppendInteger(static_cast(pos)); @@ -241,7 +243,7 @@ void BitPosCmd::Do(std::shared_ptr slot) { } } -void BitPosCmd::ReadCache(std::shared_ptr slot) { +void BitPosCmd::ReadCache() { int64_t pos = 0; rocksdb::Status s; int64_t bit = static_cast(bit_val_); @@ -249,11 +251,11 @@ void BitPosCmd::ReadCache(std::shared_ptr slot) { int64_t end = static_cast(end_offset_);\ std::string CachePrefixKeyB = PCacheKeyPrefixB + key_; if (pos_all_) { - s = slot->cache()->BitPos(CachePrefixKeyB, bit, &pos); + s = db_->cache()->BitPos(CachePrefixKeyB, bit, &pos); } else if (!pos_all_ && !endoffset_set_) { - s = slot->cache()->BitPos(CachePrefixKeyB, bit, start, &pos); + s = db_->cache()->BitPos(CachePrefixKeyB, bit, start, &pos); } else if (!pos_all_ && endoffset_set_) { - s = slot->cache()->BitPos(CachePrefixKeyB, bit, start, end, &pos); + s = db_->cache()->BitPos(CachePrefixKeyB, bit, start, end, &pos); } if (s.ok()) { res_.AppendInteger(pos); @@ -264,14 +266,14 @@ void BitPosCmd::ReadCache(std::shared_ptr slot) { } } -void BitPosCmd::DoThroughDB(std::shared_ptr slot) { +void BitPosCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void BitPosCmd::DoUpdateCache(std::shared_ptr slot) { +void BitPosCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_KV, key_, db_); } } @@ -310,9 +312,9 @@ void BitOpCmd::DoInitial() { } } -void BitOpCmd::Do(std::shared_ptr slot) { +void BitOpCmd::Do() { int64_t result_length = 0; - s_ = slot->db()->BitOp(op_, dest_key_, src_keys_, value_to_dest_, &result_length); + s_ = db_->storage()->BitOp(op_, dest_key_, src_keys_, value_to_dest_, &result_length); if (s_.ok()) { res_.AppendInteger(result_length); } else { @@ -320,19 +322,19 @@ void BitOpCmd::Do(std::shared_ptr slot) { } } -void BitOpCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void BitOpCmd::DoThroughDB() { + Do(); } -void BitOpCmd::DoUpdateCache(std::shared_ptr slot) { +void BitOpCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixB + dest_key_); - slot->cache()->Del(v); + db_->cache()->Del(v); } } -void BitOpCmd::DoBinlog(const std::shared_ptr& slot) { +void BitOpCmd::DoBinlog() { PikaCmdArgsType set_args; //used "set" instead of "SET" to distinguish the binlog of SetCmd set_args.emplace_back("set"); @@ -342,5 +344,5 @@ void BitOpCmd::DoBinlog(const std::shared_ptr& slot) { set_cmd_->SetConn(GetConn()); set_cmd_->SetResp(resp_.lock()); //value of this binlog might be strange if you print it out(eg. set bitkey_out1 «ѦFODoBinlog(slot); + set_cmd_->DoBinlog(); } diff --git a/src/pika_cache.cc b/src/pika_cache.cc index 203a697212..3d54e68d8f 100644 --- a/src/pika_cache.cc +++ b/src/pika_cache.cc @@ -15,7 +15,7 @@ #include "cache/include/cache.h" #include "cache/include/config.h" -extern PikaServer *g_pika_server; +extern PikaServer* g_pika_server; #define EXTEND_CACHE_SIZE(N) (N * 12 / 10) using rocksdb::Status; @@ -94,17 +94,17 @@ void PikaCache::Info(CacheInfo &info) { } } -bool PikaCache::Exists(std::string &key) { +bool PikaCache::Exists(std::string& key) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Exists(key); } -void PikaCache::FlushSlot(void) { +void PikaCache::FlushCache(void) { std::lock_guard l(rwlock_); for (uint32_t i = 0; i < caches_.size(); ++i) { std::lock_guard lm(*cache_mutexs_[i]); - caches_[i]->FlushDb(); + caches_[i]->FlushCache(); } } @@ -118,19 +118,19 @@ Status PikaCache::Del(const std::vector &keys) { return s; } -Status PikaCache::Expire(std::string &key, int64_t ttl) { +Status PikaCache::Expire(std::string& key, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Expire(key, ttl); } -Status PikaCache::Expireat(std::string &key, int64_t ttl) { +Status PikaCache::Expireat(std::string& key, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Expireat(key, ttl); } -Status PikaCache::TTL(std::string &key, int64_t *ttl) { +Status PikaCache::TTL(std::string& key, int64_t *ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->TTL(key, ttl); @@ -199,7 +199,7 @@ Status PikaCache::Persist(std::string &key) { return caches_[cache_index]->Persist(key); } -Status PikaCache::Type(std::string &key, std::string *value) { +Status PikaCache::Type(std::string& key, std::string *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Type(key, value); @@ -295,37 +295,37 @@ Status PikaCache::GetType(const std::string& key, bool single, std::vectorSet(key, value, ttl); } -Status PikaCache::Setnx(std::string &key, std::string &value, int64_t ttl) { +Status PikaCache::Setnx(std::string& key, std::string &value, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Setnx(key, value, ttl); } -Status PikaCache::SetnxWithoutTTL(std::string &key, std::string &value) { +Status PikaCache::SetnxWithoutTTL(std::string& key, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SetnxWithoutTTL(key, value); } -Status PikaCache::Setxx(std::string &key, std::string &value, int64_t ttl) { +Status PikaCache::Setxx(std::string& key, std::string &value, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Setxx(key, value, ttl); } -Status PikaCache::SetxxWithoutTTL(std::string &key, std::string &value) { +Status PikaCache::SetxxWithoutTTL(std::string& key, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SetxxWithoutTTL(key, value); } -Status PikaCache::Get(std::string &key, std::string *value) { +Status PikaCache::Get(std::string& key, std::string *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Get(key, value); @@ -356,7 +356,7 @@ Status PikaCache::MGet(const std::vector &keys, std::vectorExists(key)) { @@ -365,7 +365,7 @@ Status PikaCache::Incrxx(std::string &key) { return Status::NotFound("key not exist"); } -Status PikaCache::Decrxx(std::string &key) { +Status PikaCache::Decrxx(std::string& key) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -374,7 +374,7 @@ Status PikaCache::Decrxx(std::string &key) { return Status::NotFound("key not exist"); } -Status PikaCache::IncrByxx(std::string &key, uint64_t incr) { +Status PikaCache::IncrByxx(std::string& key, uint64_t incr) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -383,7 +383,7 @@ Status PikaCache::IncrByxx(std::string &key, uint64_t incr) { return Status::NotFound("key not exist"); } -Status PikaCache::DecrByxx(std::string &key, uint64_t incr) { +Status PikaCache::DecrByxx(std::string& key, uint64_t incr) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -392,7 +392,7 @@ Status PikaCache::DecrByxx(std::string &key, uint64_t incr) { return Status::NotFound("key not exist"); } -Status PikaCache::Incrbyfloatxx(std::string &key, long double incr) { +Status PikaCache::Incrbyfloatxx(std::string& key, long double incr) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -401,7 +401,7 @@ Status PikaCache::Incrbyfloatxx(std::string &key, long double incr) { return Status::NotFound("key not exist"); } -Status PikaCache::Appendxx(std::string &key, std::string &value) { +Status PikaCache::Appendxx(std::string& key, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -410,13 +410,13 @@ Status PikaCache::Appendxx(std::string &key, std::string &value) { return Status::NotFound("key not exist"); } -Status PikaCache::GetRange(std::string &key, int64_t start, int64_t end, std::string *value) { +Status PikaCache::GetRange(std::string& key, int64_t start, int64_t end, std::string *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->GetRange(key, start, end, value); } -Status PikaCache::SetRangexx(std::string &key, int64_t start, std::string &value) { +Status PikaCache::SetRangexx(std::string& key, int64_t start, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -425,7 +425,7 @@ Status PikaCache::SetRangexx(std::string &key, int64_t start, std::string &value return Status::NotFound("key not exist"); } -Status PikaCache::Strlen(std::string &key, int32_t *len) { +Status PikaCache::Strlen(std::string& key, int32_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->Strlen(key, len); @@ -434,19 +434,19 @@ Status PikaCache::Strlen(std::string &key, int32_t *len) { /*----------------------------------------------------------------------------- * Hash Commands *----------------------------------------------------------------------------*/ -Status PikaCache::HDel(std::string &key, std::vector &fields) { +Status PikaCache::HDel(std::string& key, std::vector &fields) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HDel(key, fields); } -Status PikaCache::HSet(std::string &key, std::string &field, std::string &value) { +Status PikaCache::HSet(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HSet(key, field, value); } -Status PikaCache::HSetIfKeyExist(std::string &key, std::string &field, std::string &value) { +Status PikaCache::HSetIfKeyExist(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -455,7 +455,7 @@ Status PikaCache::HSetIfKeyExist(std::string &key, std::string &field, std::stri return Status::NotFound("key not exist"); } -Status PikaCache::HSetIfKeyExistAndFieldNotExist(std::string &key, std::string &field, std::string &value) { +Status PikaCache::HSetIfKeyExistAndFieldNotExist(std::string& key, std::string &field, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -464,13 +464,13 @@ Status PikaCache::HSetIfKeyExistAndFieldNotExist(std::string &key, std::string & return Status::NotFound("key not exist"); } -Status PikaCache::HMSet(std::string &key, std::vector &fvs) { +Status PikaCache::HMSet(std::string& key, std::vector &fvs) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HMSet(key, fvs); } -Status PikaCache::HMSetnx(std::string &key, std::vector &fvs, int64_t ttl) { +Status PikaCache::HMSetnx(std::string& key, std::vector &fvs, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -482,7 +482,7 @@ Status PikaCache::HMSetnx(std::string &key, std::vector &fv } } -Status PikaCache::HMSetnxWithoutTTL(std::string &key, std::vector &fvs) { +Status PikaCache::HMSetnxWithoutTTL(std::string& key, std::vector &fvs) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -493,7 +493,7 @@ Status PikaCache::HMSetnxWithoutTTL(std::string &key, std::vector &fvs) { +Status PikaCache::HMSetxx(std::string& key, std::vector &fvs) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -503,44 +503,44 @@ Status PikaCache::HMSetxx(std::string &key, std::vector &fv } } -Status PikaCache::HGet(std::string &key, std::string &field, std::string *value) { +Status PikaCache::HGet(std::string& key, std::string &field, std::string *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HGet(key, field, value); } -Status PikaCache::HMGet(std::string &key, std::vector &fields, std::vector *vss) { +Status PikaCache::HMGet(std::string& key, std::vector &fields, std::vector *vss) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HMGet(key, fields, vss); } -Status PikaCache::HGetall(std::string &key, std::vector *fvs) { +Status PikaCache::HGetall(std::string& key, std::vector *fvs) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HGetall(key, fvs); } -Status PikaCache::HKeys(std::string &key, std::vector *fields) { +Status PikaCache::HKeys(std::string& key, std::vector *fields) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HKeys(key, fields); } -Status PikaCache::HVals(std::string &key, std::vector *values) { +Status PikaCache::HVals(std::string& key, std::vector *values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HVals(key, values); } -Status PikaCache::HExists(std::string &key, std::string &field) { +Status PikaCache::HExists(std::string& key, std::string &field) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HExists(key, field); } -Status PikaCache::HIncrbyxx(std::string &key, std::string &field, int64_t value) { +Status PikaCache::HIncrbyxx(std::string& key, std::string &field, int64_t value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -549,7 +549,7 @@ Status PikaCache::HIncrbyxx(std::string &key, std::string &field, int64_t value) return Status::NotFound("key not exist"); } -Status PikaCache::HIncrbyfloatxx(std::string &key, std::string &field, long double value) { +Status PikaCache::HIncrbyfloatxx(std::string& key, std::string &field, long double value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -558,13 +558,13 @@ Status PikaCache::HIncrbyfloatxx(std::string &key, std::string &field, long doub return Status::NotFound("key not exist"); } -Status PikaCache::HLen(std::string &key, uint64_t *len) { +Status PikaCache::HLen(std::string& key, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HLen(key, len); } -Status PikaCache::HStrlen(std::string &key, std::string &field, uint64_t *len) { +Status PikaCache::HStrlen(std::string& key, std::string &field, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->HStrlen(key, field, len); @@ -573,86 +573,86 @@ Status PikaCache::HStrlen(std::string &key, std::string &field, uint64_t *len) { /*----------------------------------------------------------------------------- * List Commands *----------------------------------------------------------------------------*/ -Status PikaCache::LIndex(std::string &key, int64_t index, std::string *element) { +Status PikaCache::LIndex(std::string& key, int64_t index, std::string *element) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LIndex(key, index, element); } -Status PikaCache::LInsert(std::string &key, storage::BeforeOrAfter &before_or_after, std::string &pivot, +Status PikaCache::LInsert(std::string& key, storage::BeforeOrAfter &before_or_after, std::string &pivot, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LInsert(key, before_or_after, pivot, value); } -Status PikaCache::LLen(std::string &key, uint64_t *len) { +Status PikaCache::LLen(std::string& key, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LLen(key, len); } -Status PikaCache::LPop(std::string &key, std::string *element) { +Status PikaCache::LPop(std::string& key, std::string *element) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LPop(key, element); } -Status PikaCache::LPush(std::string &key, std::vector &values) { +Status PikaCache::LPush(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LPush(key, values); } -Status PikaCache::LPushx(std::string &key, std::vector &values) { +Status PikaCache::LPushx(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LPushx(key, values); } -Status PikaCache::LRange(std::string &key, int64_t start, int64_t stop, std::vector *values) { +Status PikaCache::LRange(std::string& key, int64_t start, int64_t stop, std::vector *values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LRange(key, start, stop, values); } -Status PikaCache::LRem(std::string &key, int64_t count, std::string &value) { +Status PikaCache::LRem(std::string& key, int64_t count, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LRem(key, count, value); } -Status PikaCache::LSet(std::string &key, int64_t index, std::string &value) { +Status PikaCache::LSet(std::string& key, int64_t index, std::string &value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LSet(key, index, value); } -Status PikaCache::LTrim(std::string &key, int64_t start, int64_t stop) { +Status PikaCache::LTrim(std::string& key, int64_t start, int64_t stop) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->LTrim(key, start, stop); } -Status PikaCache::RPop(std::string &key, std::string *element) { +Status PikaCache::RPop(std::string& key, std::string *element) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->RPop(key, element); } -Status PikaCache::RPush(std::string &key, std::vector &values) { +Status PikaCache::RPush(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->RPush(key, values); } -Status PikaCache::RPushx(std::string &key, std::vector &values) { +Status PikaCache::RPushx(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->RPushx(key, values); } -Status PikaCache::RPushnx(std::string &key, std::vector &values, int64_t ttl) { +Status PikaCache::RPushnx(std::string& key, std::vector &values, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -664,7 +664,7 @@ Status PikaCache::RPushnx(std::string &key, std::vector &values, in } } -Status PikaCache::RPushnxWithoutTTL(std::string &key, std::vector &values) { +Status PikaCache::RPushnxWithoutTTL(std::string& key, std::vector &values) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -678,13 +678,13 @@ Status PikaCache::RPushnxWithoutTTL(std::string &key, std::vector & /*----------------------------------------------------------------------------- * Set Commands *----------------------------------------------------------------------------*/ -Status PikaCache::SAdd(std::string &key, std::vector &members) { +Status PikaCache::SAdd(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SAdd(key, members); } -Status PikaCache::SAddIfKeyExist(std::string &key, std::vector &members) { +Status PikaCache::SAddIfKeyExist(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -693,7 +693,7 @@ Status PikaCache::SAddIfKeyExist(std::string &key, std::vector &mem return Status::NotFound("key not exist"); } -Status PikaCache::SAddnx(std::string &key, std::vector &members, int64_t ttl) { +Status PikaCache::SAddnx(std::string& key, std::vector &members, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -705,7 +705,7 @@ Status PikaCache::SAddnx(std::string &key, std::vector &members, in } } -Status PikaCache::SAddnxWithoutTTL(std::string &key, std::vector &members) { +Status PikaCache::SAddnxWithoutTTL(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -716,31 +716,31 @@ Status PikaCache::SAddnxWithoutTTL(std::string &key, std::vector &m } } -Status PikaCache::SCard(std::string &key, uint64_t *len) { +Status PikaCache::SCard(std::string& key, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SCard(key, len); } -Status PikaCache::SIsmember(std::string &key, std::string &member) { +Status PikaCache::SIsmember(std::string& key, std::string& member) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SIsmember(key, member); } -Status PikaCache::SMembers(std::string &key, std::vector *members) { +Status PikaCache::SMembers(std::string& key, std::vector *members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SMembers(key, members); } -Status PikaCache::SRem(std::string &key, std::vector &members) { +Status PikaCache::SRem(std::string& key, std::vector &members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SRem(key, members); } -Status PikaCache::SRandmember(std::string &key, int64_t count, std::vector *members) { +Status PikaCache::SRandmember(std::string& key, int64_t count, std::vector *members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SRandmember(key, count, members); @@ -749,7 +749,7 @@ Status PikaCache::SRandmember(std::string &key, int64_t count, std::vector &score_members) { +Status PikaCache::ZAdd(std::string& key, std::vector &score_members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->ZAdd(key, score_members); @@ -770,7 +770,7 @@ void PikaCache::GetMinMaxScore(std::vector &score_members, } } -bool PikaCache::GetCacheMinMaxSM(cache::RedisCache *cache_obj, std::string &key, storage::ScoreMember &min_m, +bool PikaCache::GetCacheMinMaxSM(cache::RedisCache *cache_obj, std::string& key, storage::ScoreMember &min_m, storage::ScoreMember &max_m) { if (cache_obj) { std::vector score_members; @@ -791,7 +791,7 @@ bool PikaCache::GetCacheMinMaxSM(cache::RedisCache *cache_obj, std::string &key, return false; } -Status PikaCache::ZAddIfKeyExist(std::string &key, std::vector &score_members) { +Status PikaCache::ZAddIfKeyExist(std::string& key, std::vector &score_members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); auto cache_obj = caches_[cache_index]; @@ -887,7 +887,7 @@ Status PikaCache::ZAddIfKeyExist(std::string &key, std::vectorZCard(key, &cache_len); if (cache_len > (unsigned long)zset_cache_field_num_per_key_) { @@ -907,7 +907,7 @@ Status PikaCache::CleanCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::strin return Status::OK(); } -Status PikaCache::ZAddnx(std::string &key, std::vector &score_members, int64_t ttl) { +Status PikaCache::ZAddnx(std::string& key, std::vector &score_members, int64_t ttl) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -919,7 +919,7 @@ Status PikaCache::ZAddnx(std::string &key, std::vector &sc } } -Status PikaCache::ZAddnxWithoutTTL(std::string &key, std::vector &score_members) { +Status PikaCache::ZAddnxWithoutTTL(std::string& key, std::vector &score_members) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (!caches_[cache_index]->Exists(key)) { @@ -930,14 +930,14 @@ Status PikaCache::ZAddnxWithoutTTL(std::string &key, std::vector &slot) { +Status PikaCache::ZCard(std::string& key, uint32_t *len, const std::shared_ptr& db) { int32_t db_len = 0; - slot->db()->ZCard(key, &db_len); + db->storage()->ZCard(key, &db_len); *len = db_len; return Status::OK(); } -Status PikaCache::CacheZCard(std::string &key, uint64_t *len) { +Status PikaCache::CacheZCard(std::string& key, uint64_t *len) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1003,7 +1003,7 @@ RangeStatus PikaCache::CheckCacheRangeByScore(uint64_t cache_len, double cache_m } } -Status PikaCache::ZCount(std::string &key, std::string &min, std::string &max, uint64_t *len, ZCountCmd *cmd) { +Status PikaCache::ZCount(std::string& key, std::string &min, std::string &max, uint64_t *len, ZCountCmd *cmd) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1031,14 +1031,14 @@ Status PikaCache::ZCount(std::string &key, std::string &min, std::string &max, u } } -Status PikaCache::ZIncrby(std::string &key, std::string &member, double increment) { +Status PikaCache::ZIncrby(std::string& key, std::string& member, double increment) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->ZIncrby(key, member, increment); } -bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string &key, int mem_len, int db_len, - const std::shared_ptr &slot) { +bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string& key, int mem_len, int db_len, + const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; if (mem_len == -1) { uint64_t cache_len = 0; @@ -1047,7 +1047,7 @@ bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string } if (db_len == -1) { db_len = 0; - slot->db()->ZCard(key, &db_len); + db->storage()->ZCard(key, &db_len); if (!db_len) { return false; } @@ -1055,7 +1055,7 @@ bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string if (db_len < zset_cache_field_num_per_key_) { if (mem_len * 2 < db_len) { cache_obj->Del(CachePrefixKeyZ); - PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key, slot); + PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key, db); return true; } else { return false; @@ -1063,7 +1063,7 @@ bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string } else { if (zset_cache_field_num_per_key_ && mem_len * 2 < zset_cache_field_num_per_key_) { cache_obj->Del(CachePrefixKeyZ); - PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key, slot); + PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key, db); return true; } else { return false; @@ -1071,8 +1071,7 @@ bool PikaCache::ReloadCacheKeyIfNeeded(cache::RedisCache *cache_obj, std::string } } -Status PikaCache::ZIncrbyIfKeyExist(std::string &key, std::string &member, double increment, ZIncrbyCmd *cmd, - const std::shared_ptr &slot) { +Status PikaCache::ZIncrbyIfKeyExist(std::string& key, std::string& member, double increment, ZIncrbyCmd *cmd, const std::shared_ptr& db) { auto eps = std::numeric_limits::epsilon(); if (-eps < increment && increment < eps) { return Status::NotFound("icrement is 0, nothing to be done"); @@ -1094,17 +1093,17 @@ Status PikaCache::ZIncrbyIfKeyExist(std::string &key, std::string &member, doubl } auto cache_min_score = cache_min_sm.score; auto cache_max_score = cache_max_sm.score; - auto RemCacheRangebyscoreAndCheck = [this, cache_obj, &key, cache_len, slot](double score) { + auto RemCacheRangebyscoreAndCheck = [this, cache_obj, &key, cache_len, db](double score) { auto score_rm = std::to_string(score); auto s = cache_obj->ZRemrangebyscore(key, score_rm, score_rm); - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, db); return s; }; - auto RemCacheKeyMember = [this, cache_obj, &key, cache_len, slot](const std::string &member, bool check = true) { + auto RemCacheKeyMember = [this, cache_obj, &key, cache_len, db](const std::string& member, bool check = true) { std::vector member_rm = {member}; auto s = cache_obj->ZRem(key, member_rm); if (check) { - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, db); } return s; }; @@ -1203,14 +1202,14 @@ RangeStatus PikaCache::CheckCacheRevRange(int32_t cache_len, int32_t db_len, int } } -Status PikaCache::ZRange(std::string &key, int64_t start, int64_t stop, std::vector *score_members, - const std::shared_ptr &slot) { +Status PikaCache::ZRange(std::string& key, int64_t start, int64_t stop, std::vector *score_members, + const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); auto cache_obj = caches_[cache_index]; - auto db_obj = slot->db(); + auto db_obj = db->storage(); Status s; if (cache_obj->Exists(CachePrefixKeyZ)) { uint64_t cache_len = 0; @@ -1223,7 +1222,7 @@ Status PikaCache::ZRange(std::string &key, int64_t start, int64_t stop, std::vec if (rs == RangeStatus::RangeHit) { return cache_obj->ZRange(CachePrefixKeyZ, out_start, out_stop, score_members); } else if (rs == RangeStatus::RangeMiss) { - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len, db); return Status::NotFound("key not in cache"); } else if (rs == RangeStatus::RangeError) { return Status::NotFound("error range"); @@ -1235,7 +1234,7 @@ Status PikaCache::ZRange(std::string &key, int64_t start, int64_t stop, std::vec } } -Status PikaCache::ZRangebyscore(std::string &key, std::string &min, std::string &max, +Status PikaCache::ZRangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, ZRangebyscoreCmd *cmd) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); @@ -1263,7 +1262,7 @@ Status PikaCache::ZRangebyscore(std::string &key, std::string &min, std::string } } -Status PikaCache::ZRank(std::string &key, std::string &member, int64_t *rank, const std::shared_ptr &slot) { +Status PikaCache::ZRank(std::string& key, std::string& member, int64_t *rank, const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1278,7 +1277,7 @@ Status PikaCache::ZRank(std::string &key, std::string &member, int64_t *rank, co if (s.ok()) { if (zset_cache_start_pos_ == cache::CACHE_START_FROM_END) { int32_t db_len = 0; - slot->db()->ZCard(key, &db_len); + db->storage()->ZCard(key, &db_len); *rank = db_len - cache_len + *rank; } return s; @@ -1288,17 +1287,17 @@ Status PikaCache::ZRank(std::string &key, std::string &member, int64_t *rank, co } } -Status PikaCache::ZRem(std::string &key, std::vector &members, std::shared_ptr slot) { +Status PikaCache::ZRem(std::string& key, std::vector &members, std::shared_ptr db) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); auto s = caches_[cache_index]->ZRem(key, members); - ReloadCacheKeyIfNeeded(caches_[cache_index], key, -1, -1, slot); + ReloadCacheKeyIfNeeded(caches_[cache_index], key, -1, -1, db); return s; } -Status PikaCache::ZRemrangebyrank(std::string &key, std::string &min, std::string &max, int32_t ele_deleted, - const std::shared_ptr &slot) { +Status PikaCache::ZRemrangebyrank(std::string& key, std::string &min, std::string &max, int32_t ele_deleted, + const std::shared_ptr& db) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); auto cache_obj = caches_[cache_index]; @@ -1307,7 +1306,7 @@ Status PikaCache::ZRemrangebyrank(std::string &key, std::string &min, std::strin if (cache_len <= 0) { return Status::NotFound("key not in cache"); } else { - auto db_obj = slot->db(); + auto db_obj = db->storage(); int32_t db_len = 0; db_obj->ZCard(key, &db_len); db_len += ele_deleted; @@ -1327,7 +1326,7 @@ Status PikaCache::ZRemrangebyrank(std::string &key, std::string &min, std::strin auto cache_min_str = std::to_string(start_index); auto cache_max_str = std::to_string(stop_index); auto s = cache_obj->ZRemrangebyrank(key, cache_min_str, cache_max_str); - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len - ele_deleted, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len - ele_deleted, db); return s; } else { return Status::NotFound("error range"); @@ -1343,7 +1342,7 @@ Status PikaCache::ZRemrangebyrank(std::string &key, std::string &min, std::strin auto cache_max_str = std::to_string(cache_max); auto s = cache_obj->ZRemrangebyrank(key, cache_min_str, cache_max_str); - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len - ele_deleted, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len - ele_deleted, db); return s; } else { return Status::NotFound("error range"); @@ -1354,23 +1353,23 @@ Status PikaCache::ZRemrangebyrank(std::string &key, std::string &min, std::strin } } -Status PikaCache::ZRemrangebyscore(std::string &key, std::string &min, std::string &max, - const std::shared_ptr &slot) { +Status PikaCache::ZRemrangebyscore(std::string& key, std::string &min, std::string &max, + const std::shared_ptr& db) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); auto s = caches_[cache_index]->ZRemrangebyscore(key, min, max); - ReloadCacheKeyIfNeeded(caches_[cache_index], key, -1, -1, slot); + ReloadCacheKeyIfNeeded(caches_[cache_index], key, -1, -1, db); return s; } -Status PikaCache::ZRevrange(std::string &key, int64_t start, int64_t stop, std::vector *score_members, - const std::shared_ptr &slot) { +Status PikaCache::ZRevrange(std::string& key, int64_t start, int64_t stop, std::vector *score_members, + const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); auto cache_obj = caches_[cache_index]; - auto db_obj = slot->db(); + auto db_obj = db->storage(); Status s; if (cache_obj->Exists(CachePrefixKeyZ)) { uint64_t cache_len = 0; @@ -1383,7 +1382,7 @@ Status PikaCache::ZRevrange(std::string &key, int64_t start, int64_t stop, std:: if (rs == RangeStatus::RangeHit) { return cache_obj->ZRevrange(CachePrefixKeyZ, out_start, out_stop, score_members); } else if (rs == RangeStatus::RangeMiss) { - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, db_len, db); return Status::NotFound("key not in cache"); } else if (rs == RangeStatus::RangeError) { return Status::NotFound("error revrange"); @@ -1395,9 +1394,9 @@ Status PikaCache::ZRevrange(std::string &key, int64_t start, int64_t stop, std:: } } -Status PikaCache::ZRevrangebyscore(std::string &key, std::string &min, std::string &max, +Status PikaCache::ZRevrangebyscore(std::string& key, std::string &min, std::string &max, std::vector *score_members, ZRevrangebyscoreCmd *cmd, - const std::shared_ptr &slot) { + const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1421,7 +1420,7 @@ Status PikaCache::ZRevrangebyscore(std::string &key, std::string &min, std::stri if (RangeStatus::RangeHit == rs) { return cache_obj->ZRevrangebyscore(CachePrefixKeyZ, min, max, score_members, cmd->Offset(), cmd->Count()); } else if (RangeStatus::RangeMiss == rs) { - ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, slot); + ReloadCacheKeyIfNeeded(cache_obj, key, cache_len, -1, db); return Status::NotFound("score range miss"); } else { return Status::NotFound("score range error"); @@ -1429,9 +1428,9 @@ Status PikaCache::ZRevrangebyscore(std::string &key, std::string &min, std::stri } } -bool PikaCache::CacheSizeEqsDB(std::string &key, const std::shared_ptr &slot) { +bool PikaCache::CacheSizeEqsDB(std::string& key, const std::shared_ptr& db) { int32_t db_len = 0; - slot->db()->ZCard(key, &db_len); + db->storage()->ZCard(key, &db_len); std::lock_guard l(rwlock_); std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; @@ -1442,9 +1441,9 @@ bool PikaCache::CacheSizeEqsDB(std::string &key, const std::shared_ptr &sl return (db_len == (int32_t)cache_len) && cache_len; } -Status PikaCache::ZRevrangebylex(std::string &key, std::string &min, std::string &max, - std::vector *members, const std::shared_ptr &slot) { - if (CacheSizeEqsDB(key, slot)) { +Status PikaCache::ZRevrangebylex(std::string& key, std::string &min, std::string &max, + std::vector *members, const std::shared_ptr& db) { + if (CacheSizeEqsDB(key, db)) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1454,7 +1453,7 @@ Status PikaCache::ZRevrangebylex(std::string &key, std::string &min, std::string } } -Status PikaCache::ZRevrank(std::string &key, std::string &member, int64_t *rank, const std::shared_ptr &slot) { +Status PikaCache::ZRevrank(std::string& key, std::string& member, int64_t *rank, const std::shared_ptr& db) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1468,7 +1467,7 @@ Status PikaCache::ZRevrank(std::string &key, std::string &member, int64_t *rank, if (s.ok()) { if (zset_cache_start_pos_ == cache::CACHE_START_FROM_BEGIN) { int32_t db_len = 0; - slot->db()->ZCard(key, &db_len); + db->storage()->ZCard(key, &db_len); *rank = db_len - cache_len + *rank; } return s; @@ -1477,7 +1476,7 @@ Status PikaCache::ZRevrank(std::string &key, std::string &member, int64_t *rank, } } } -Status PikaCache::ZScore(std::string &key, std::string &member, double *score, const std::shared_ptr &slot) { +Status PikaCache::ZScore(std::string& key, std::string& member, double *score, const std::shared_ptr& db) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); auto s = caches_[cache_index]->ZScore(key, member, score); @@ -1487,9 +1486,9 @@ Status PikaCache::ZScore(std::string &key, std::string &member, double *score, c return s; } -Status PikaCache::ZRangebylex(std::string &key, std::string &min, std::string &max, std::vector *members, - const std::shared_ptr &slot) { - if (CacheSizeEqsDB(key, slot)) { +Status PikaCache::ZRangebylex(std::string& key, std::string &min, std::string &max, std::vector *members, + const std::shared_ptr& db) { + if (CacheSizeEqsDB(key, db)) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1499,9 +1498,9 @@ Status PikaCache::ZRangebylex(std::string &key, std::string &min, std::string &m } } -Status PikaCache::ZLexcount(std::string &key, std::string &min, std::string &max, uint64_t *len, - const std::shared_ptr &slot) { - if (CacheSizeEqsDB(key, slot)) { +Status PikaCache::ZLexcount(std::string& key, std::string &min, std::string &max, uint64_t *len, + const std::shared_ptr& db) { + if (CacheSizeEqsDB(key, db)) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; int cache_index = CacheIndex(CachePrefixKeyZ); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1512,9 +1511,9 @@ Status PikaCache::ZLexcount(std::string &key, std::string &min, std::string &max } } -Status PikaCache::ZRemrangebylex(std::string &key, std::string &min, std::string &max, - const std::shared_ptr &slot) { - if (CacheSizeEqsDB(key, slot)) { +Status PikaCache::ZRemrangebylex(std::string& key, std::string &min, std::string &max, + const std::shared_ptr& db) { + if (CacheSizeEqsDB(key, db)) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); @@ -1527,13 +1526,13 @@ Status PikaCache::ZRemrangebylex(std::string &key, std::string &min, std::string /*----------------------------------------------------------------------------- * Bit Commands *----------------------------------------------------------------------------*/ -Status PikaCache::SetBit(std::string &key, size_t offset, int64_t value) { +Status PikaCache::SetBit(std::string& key, size_t offset, int64_t value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->SetBit(key, offset, value); } -Status PikaCache::SetBitIfKeyExist(std::string &key, size_t offset, int64_t value) { +Status PikaCache::SetBitIfKeyExist(std::string& key, size_t offset, int64_t value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); if (caches_[cache_index]->Exists(key)) { @@ -1542,31 +1541,31 @@ Status PikaCache::SetBitIfKeyExist(std::string &key, size_t offset, int64_t valu return Status::NotFound("key not exist"); } -Status PikaCache::GetBit(std::string &key, size_t offset, int64_t *value) { +Status PikaCache::GetBit(std::string& key, size_t offset, int64_t *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->GetBit(key, offset, value); } -Status PikaCache::BitCount(std::string &key, int64_t start, int64_t end, int64_t *value, bool have_offset) { +Status PikaCache::BitCount(std::string& key, int64_t start, int64_t end, int64_t *value, bool have_offset) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->BitCount(key, start, end, value, have_offset); } -Status PikaCache::BitPos(std::string &key, int64_t bit, int64_t *value) { +Status PikaCache::BitPos(std::string& key, int64_t bit, int64_t *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->BitPos(key, bit, value); } -Status PikaCache::BitPos(std::string &key, int64_t bit, int64_t start, int64_t *value) { +Status PikaCache::BitPos(std::string& key, int64_t bit, int64_t start, int64_t *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->BitPos(key, bit, start, value); } -Status PikaCache::BitPos(std::string &key, int64_t bit, int64_t start, int64_t end, int64_t *value) { +Status PikaCache::BitPos(std::string& key, int64_t bit, int64_t start, int64_t end, int64_t *value) { int cache_index = CacheIndex(key); std::lock_guard lm(*cache_mutexs_[cache_index]); return caches_[cache_index]->BitPos(key, bit, start, end, value); @@ -1607,12 +1606,12 @@ void PikaCache::DestroyWithoutLock(void) cache_mutexs_.clear(); } -int PikaCache::CacheIndex(const std::string &key) { +int PikaCache::CacheIndex(const std::string& key) { uint32_t crc = CRC32Update(0, key.data(), (int)key.size()); return (int)(crc % caches_.size()); } -Status PikaCache::WriteKVToCache(std::string &key, std::string &value, int64_t ttl) { +Status PikaCache::WriteKVToCache(std::string& key, std::string &value, int64_t ttl) { if (0 >= ttl) { if (PIKA_TTL_NONE == ttl) { return SetnxWithoutTTL(key, value); @@ -1625,7 +1624,7 @@ Status PikaCache::WriteKVToCache(std::string &key, std::string &value, int64_t t return Status::OK(); } -Status PikaCache::WriteHashToCache(std::string &key, std::vector &fvs, int64_t ttl) { +Status PikaCache::WriteHashToCache(std::string& key, std::vector &fvs, int64_t ttl) { if (0 >= ttl) { if (PIKA_TTL_NONE == ttl) { return HMSetnxWithoutTTL(key, fvs); @@ -1638,7 +1637,7 @@ Status PikaCache::WriteHashToCache(std::string &key, std::vector &values, int64_t ttl) { +Status PikaCache::WriteListToCache(std::string& key, std::vector &values, int64_t ttl) { if (0 >= ttl) { if (PIKA_TTL_NONE == ttl) { return RPushnxWithoutTTL(key, values); @@ -1651,7 +1650,7 @@ Status PikaCache::WriteListToCache(std::string &key, std::vector &v return Status::OK(); } -Status PikaCache::WriteSetToCache(std::string &key, std::vector &members, int64_t ttl) { +Status PikaCache::WriteSetToCache(std::string& key, std::vector &members, int64_t ttl) { if (0 >= ttl) { if (PIKA_TTL_NONE == ttl) { return SAddnxWithoutTTL(key, members); @@ -1664,7 +1663,7 @@ Status PikaCache::WriteSetToCache(std::string &key, std::vector &me return Status::OK(); } -Status PikaCache::WriteZSetToCache(std::string &key, std::vector &score_members, int64_t ttl) { +Status PikaCache::WriteZSetToCache(std::string& key, std::vector &score_members, int64_t ttl) { if (0 >= ttl) { if (PIKA_TTL_NONE == ttl) { return ZAddnxWithoutTTL(key, score_members); @@ -1677,8 +1676,8 @@ Status PikaCache::WriteZSetToCache(std::string &key, std::vector &slot) { - cache_load_thread_->Push(key_type, key, slot); +void PikaCache::PushKeyToAsyncLoadQueue(const char key_type, std::string& key, const std::shared_ptr& db) { + cache_load_thread_->Push(key_type, key, db); } void PikaCache::ClearHitRatio(void) { diff --git a/src/pika_cache_load_thread.cc b/src/pika_cache_load_thread.cc index 9b7f4ac7f9..c61e63ca53 100644 --- a/src/pika_cache_load_thread.cc +++ b/src/pika_cache_load_thread.cc @@ -9,7 +9,7 @@ #include "include/pika_cache.h" #include "pstd/include/scope_record_lock.h" -extern PikaServer *g_pika_server; +extern PikaServer* g_pika_server; PikaCacheLoadThread::PikaCacheLoadThread(int zset_cache_start_pos, int zset_cache_field_num_per_key) : should_exit_(false) @@ -32,7 +32,7 @@ PikaCacheLoadThread::~PikaCacheLoadThread() { StopThread(); } -void PikaCacheLoadThread::Push(const char key_type, std::string &key, const std::shared_ptr &slot) { +void PikaCacheLoadThread::Push(const char key_type, std::string& key, const std::shared_ptr& db) { std::unique_lock lq(loadkeys_mutex_); std::unique_lock lm(loadkeys_map_mutex_); if (CACHE_LOAD_QUEUE_MAX_SIZE < loadkeys_queue_.size()) { @@ -46,29 +46,29 @@ void PikaCacheLoadThread::Push(const char key_type, std::string &key, const std: } if (loadkeys_map_.find(key) == loadkeys_map_.end()) { - std::tuple> ktuple = std::make_tuple(key_type, key, slot); + std::tuple> ktuple = std::make_tuple(key_type, key, db); loadkeys_queue_.push_back(ktuple); loadkeys_map_[key] = std::string(""); loadkeys_cond_.notify_all(); } } -bool PikaCacheLoadThread::LoadKV(std::string &key, const std::shared_ptr& slot) { +bool PikaCacheLoadThread::LoadKV(std::string& key, const std::shared_ptr& db) { std::string value; int64_t ttl = -1; - rocksdb::Status s = slot->db()->GetWithTTL(key, &value, &ttl); + rocksdb::Status s = db->storage()->GetWithTTL(key, &value, &ttl); if (!s.ok()) { LOG(WARNING) << "load kv failed, key=" << key; return false; } std::string CachePrefixKeyK = PCacheKeyPrefixK + key; - slot->cache()->WriteKVToCache(CachePrefixKeyK, value, ttl); + db->cache()->WriteKVToCache(CachePrefixKeyK, value, ttl); return true; } -bool PikaCacheLoadThread::LoadHash(std::string &key, const std::shared_ptr& slot) { +bool PikaCacheLoadThread::LoadHash(std::string& key, const std::shared_ptr& db) { int32_t len = 0; - slot->db()->HLen(key, &len); + db->storage()->HLen(key, &len); if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; @@ -77,19 +77,19 @@ bool PikaCacheLoadThread::LoadHash(std::string &key, const std::shared_ptr std::vector fvs; int64_t ttl = -1; - rocksdb::Status s = slot->db()->HGetallWithTTL(key, &fvs, &ttl); + rocksdb::Status s = db->storage()->HGetallWithTTL(key, &fvs, &ttl); if (!s.ok()) { LOG(WARNING) << "load hash failed, key=" << key; return false; } std::string CachePrefixKeyH = PCacheKeyPrefixH + key; - slot->cache()->WriteHashToCache(CachePrefixKeyH, fvs, ttl); + db->cache()->WriteHashToCache(CachePrefixKeyH, fvs, ttl); return true; } -bool PikaCacheLoadThread::LoadList(std::string &key, const std::shared_ptr& slot) { +bool PikaCacheLoadThread::LoadList(std::string& key, const std::shared_ptr& db) { uint64_t len = 0; - slot->db()->LLen(key, &len); + db->storage()->LLen(key, &len); if (len <= 0 || CACHE_VALUE_ITEM_MAX_SIZE < len) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; @@ -98,19 +98,19 @@ bool PikaCacheLoadThread::LoadList(std::string &key, const std::shared_ptr std::vector values; int64_t ttl = -1; - rocksdb::Status s = slot->db()->LRangeWithTTL(key, 0, -1, &values, &ttl); + rocksdb::Status s = db->storage()->LRangeWithTTL(key, 0, -1, &values, &ttl); if (!s.ok()) { LOG(WARNING) << "load list failed, key=" << key; return false; } std::string CachePrefixKeyL = PCacheKeyPrefixL + key; - slot->cache()->WriteListToCache(CachePrefixKeyL, values, ttl); + db->cache()->WriteListToCache(CachePrefixKeyL, values, ttl); return true; } -bool PikaCacheLoadThread::LoadSet(std::string &key, const std::shared_ptr& slot) { +bool PikaCacheLoadThread::LoadSet(std::string& key, const std::shared_ptr& db) { int32_t len = 0; - slot->db()->SCard(key, &len); + db->storage()->SCard(key, &len); if (0 >= len || CACHE_VALUE_ITEM_MAX_SIZE < len) { LOG(WARNING) << "can not load key, because item size:" << len << " beyond max item size:" << CACHE_VALUE_ITEM_MAX_SIZE; @@ -119,28 +119,28 @@ bool PikaCacheLoadThread::LoadSet(std::string &key, const std::shared_ptr& std::vector values; int64_t ttl = -1; - rocksdb::Status s = slot->db()->SMembersWithTTL(key, &values, &ttl); + rocksdb::Status s = db->storage()->SMembersWithTTL(key, &values, &ttl); if (!s.ok()) { LOG(WARNING) << "load set failed, key=" << key; return false; } std::string CachePrefixKeyS = PCacheKeyPrefixS + key; - slot->cache()->WriteSetToCache(CachePrefixKeyS, values, ttl); + db->cache()->WriteSetToCache(CachePrefixKeyS, values, ttl); return true; } -bool PikaCacheLoadThread::LoadZset(std::string &key, const std::shared_ptr& slot) { +bool PikaCacheLoadThread::LoadZset(std::string& key, const std::shared_ptr& db) { int32_t len = 0; int start_index = 0; int stop_index = -1; - slot->db()->ZCard(key, &len); + db->storage()->ZCard(key, &len); if (0 >= len) { return false; } uint64_t cache_len = 0; std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key; - slot->cache()->CacheZCard(CachePrefixKeyZ, &cache_len); + db->cache()->CacheZCard(CachePrefixKeyZ, &cache_len); if (cache_len != 0) { return true; } @@ -156,28 +156,28 @@ bool PikaCacheLoadThread::LoadZset(std::string &key, const std::shared_ptr std::vector score_members; int64_t ttl = -1; - rocksdb::Status s = slot->db()->ZRangeWithTTL(key, start_index, stop_index, &score_members, &ttl); + rocksdb::Status s = db->storage()->ZRangeWithTTL(key, start_index, stop_index, &score_members, &ttl); if (!s.ok()) { LOG(WARNING) << "load zset failed, key=" << key; return false; } - slot->cache()->WriteZSetToCache(CachePrefixKeyZ, score_members, ttl); + db->cache()->WriteZSetToCache(CachePrefixKeyZ, score_members, ttl); return true; } -bool PikaCacheLoadThread::LoadKey(const char key_type, std::string &key, const std::shared_ptr& slot) { - pstd::lock::ScopeRecordLock record_lock(slot->LockMgr(), key); +bool PikaCacheLoadThread::LoadKey(const char key_type, std::string& key, const std::shared_ptr& db) { + pstd::lock::ScopeRecordLock record_lock(db->LockMgr(), key); switch (key_type) { case 'k': - return LoadKV(key, slot); + return LoadKV(key, db); case 'h': - return LoadHash(key, slot); + return LoadHash(key, db); case 'l': - return LoadList(key, slot); + return LoadList(key, db); case 's': - return LoadSet(key, slot); + return LoadSet(key, db); case 'z': - return LoadZset(key, slot); + return LoadZset(key, db); default: LOG(WARNING) << "PikaCacheLoadThread::LoadKey invalid key type : " << key_type; return false; @@ -188,7 +188,7 @@ void *PikaCacheLoadThread::ThreadMain() { LOG(INFO) << "PikaCacheLoadThread::ThreadMain Start"; while (!should_exit_) { - std::deque>> load_keys; + std::deque>> load_keys; { std::unique_lock lq(loadkeys_mutex_); waitting_load_keys_num_ = loadkeys_queue_.size(); diff --git a/src/pika_client_conn.cc b/src/pika_client_conn.cc index 439ad83b5c..2099281006 100644 --- a/src/pika_client_conn.cc +++ b/src/pika_client_conn.cc @@ -276,40 +276,6 @@ void PikaClientConn::DoBackgroundTask(void* arg) { conn_ptr->BatchExecRedisCmd(bg_arg->redis_cmds); } -void PikaClientConn::DoExecTask(void* arg) { - std::unique_ptr bg_arg(static_cast(arg)); - std::shared_ptr cmd_ptr = bg_arg->cmd_ptr; - std::shared_ptr conn_ptr = bg_arg->conn_ptr; - std::shared_ptr resp_ptr = bg_arg->resp_ptr; - LogOffset offset = bg_arg->offset; - std::string db_name = bg_arg->db_name; - uint32_t slot_id = bg_arg->slot_id; - bg_arg.reset(); - - cmd_ptr->SetStage(Cmd::kExecuteStage); - cmd_ptr->Execute(); - if (g_pika_conf->slowlog_slower_than() >= 0) { - conn_ptr->ProcessSlowlog(cmd_ptr->argv(), cmd_ptr->GetDoDuration()); - } - - std::shared_ptr slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!slot) { - LOG(WARNING) << "Sync Master Slot not exist " << db_name << slot_id; - return; - } - slot->ConsensusUpdateAppliedIndex(offset); - - if (!conn_ptr || !resp_ptr) { - return; - } - - *resp_ptr = std::move(cmd_ptr->res().message()); - // last step to update resp_num, early update may casue another therad may - // TryWriteResp success with resp_ptr not updated - conn_ptr->resp_num--; - conn_ptr->TryWriteResp(); -} - void PikaClientConn::BatchExecRedisCmd(const std::vector& argvs) { resp_num.store(static_cast(argvs.size())); for (const auto& argv : argvs) { @@ -343,11 +309,6 @@ bool PikaClientConn::IsInTxn() { return txn_state_[TxnStateBitMask::Start]; } -bool PikaClientConn::IsTxnFailed() { - std::lock_guard lg(txn_state_mu_); - return txn_state_[TxnStateBitMask::WatchFailed] | txn_state_[TxnStateBitMask::InitCmdFailed]; -} - bool PikaClientConn::IsTxnInitFailed() { std::lock_guard lg(txn_state_mu_); return txn_state_[TxnStateBitMask::InitCmdFailed]; diff --git a/src/pika_client_processor.cc b/src/pika_client_processor.cc index 6117c0b951..8a26ccd4a4 100644 --- a/src/pika_client_processor.cc +++ b/src/pika_client_processor.cc @@ -24,7 +24,7 @@ int PikaClientProcessor::Start() { if (res != net::kSuccess) { return res; } - for (auto & bg_thread : bg_threads_) { + for (auto& bg_thread : bg_threads_) { res = bg_thread->StartThread(); if (res != net::kSuccess) { return res; diff --git a/src/pika_cmd_table_manager.cc b/src/pika_cmd_table_manager.cc index 04d14aad62..567c120a18 100644 --- a/src/pika_cmd_table_manager.cc +++ b/src/pika_cmd_table_manager.cc @@ -21,7 +21,6 @@ PikaCmdTableManager::PikaCmdTableManager() { void PikaCmdTableManager::InitCmdTable(void) { ::InitCmdTable(cmds_.get()); - for (const auto& cmd : *cmds_) { if (cmd.second->flag() & kCmdFlagsWrite) { cmd.second->AddAclCategory(static_cast(AclCategory::WRITE)); @@ -85,16 +84,6 @@ void PikaCmdTableManager::InsertCurrentThreadDistributionMap() { thread_distribution_map_.emplace(tid, std::move(distribution)); } -uint32_t PikaCmdTableManager::DistributeKey(const std::string& key, uint32_t slot_num) { - auto tid = std::this_thread::get_id(); - if (!CheckCurrentThreadDistributionMapExist(tid)) { - InsertCurrentThreadDistributionMap(); - } - - std::shared_lock l(map_protector_); - return thread_distribution_map_[tid]->Distribute(key, slot_num); -} - bool PikaCmdTableManager::CmdExist(const std::string& cmd) const { return cmds_->find(cmd) != cmds_->end(); } std::vector PikaCmdTableManager::GetAclCategoryCmdNames(uint32_t flag) { diff --git a/src/pika_command.cc b/src/pika_command.cc index 425c2bac73..9d87d8d6f0 100644 --- a/src/pika_command.cc +++ b/src/pika_command.cc @@ -26,7 +26,7 @@ #include "include/pika_transaction.h" #include "include/pika_zset.h" #include "pstd_defer.h" -#include "include/pika_cache.h" +#include "src/pstd/include/scope_record_lock.h" using pstd::Status; @@ -134,8 +134,7 @@ void InitCmdTable(CmdTable* cmd_table) { std::make_unique(kCmdNamePKPatternMatchDel, 3, kCmdFlagsWrite | kCmdFlagsAdmin); cmd_table->insert( std::pair>(kCmdNamePKPatternMatchDel, std::move(pkpatternmatchdelptr))); - - std::unique_ptr dummyptr = std::make_unique(kCmdDummy, 0, kCmdFlagsWrite | kCmdFlagsSingleSlot); + std::unique_ptr dummyptr = std::make_unique(kCmdDummy, 0, kCmdFlagsWrite); cmd_table->insert(std::pair>(kCmdDummy, std::move(dummyptr))); std::unique_ptr quitptr = @@ -241,503 +240,501 @@ void InitCmdTable(CmdTable* cmd_table) { // Kv ////SetCmd std::unique_ptr setptr = - std::make_unique(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSet, std::move(setptr))); ////GetCmd std::unique_ptr getptr = - std::make_unique(kCmdNameGet, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsSlow); + std::make_unique(kCmdNameGet, 2, kCmdFlagsRead | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGet, std::move(getptr))); ////DelCmd std::unique_ptr delptr = - std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameDel, std::move(delptr))); std::unique_ptr Unlinkptr = - std::make_unique(kCmdNameUnlink, -2, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsFast); + std::make_unique(kCmdNameUnlink, -2, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameUnlink, std::move(Unlinkptr))); ////IncrCmd std::unique_ptr incrptr = - std::make_unique(kCmdNameIncr, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameIncr, 2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameIncr, std::move(incrptr))); ////IncrbyCmd std::unique_ptr incrbyptr = std::make_unique( - kCmdNameIncrby, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameIncrby, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameIncrby, std::move(incrbyptr))); ////IncrbyfloatCmd std::unique_ptr incrbyfloatptr = std::make_unique( - kCmdNameIncrbyfloat, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameIncrbyfloat, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameIncrbyfloat, std::move(incrbyfloatptr))); ////DecrCmd std::unique_ptr decrptr = - std::make_unique(kCmdNameDecr, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameDecr, 2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameDecr, std::move(decrptr))); ////DecrbyCmd std::unique_ptr decrbyptr = std::make_unique( - kCmdNameDecrby, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameDecrby, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameDecrby, std::move(decrbyptr))); ////GetsetCmd std::unique_ptr getsetptr = std::make_unique( - kCmdNameGetset, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameGetset, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameGetset, std::move(getsetptr))); ////AppendCmd std::unique_ptr appendptr = std::make_unique( - kCmdNameAppend, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameAppend, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameAppend, std::move(appendptr))); ////MgetCmd std::unique_ptr mgetptr = - std::make_unique(kCmdNameMget, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameMget, -2, kCmdFlagsRead | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameMget, std::move(mgetptr))); ////KeysCmd std::unique_ptr keysptr = - std::make_unique(kCmdNameKeys, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsSlow); + std::make_unique(kCmdNameKeys, -2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameKeys, std::move(keysptr))); ////SetnxCmd std::unique_ptr setnxptr = - std::make_unique(kCmdNameSetnx, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsFast); + std::make_unique(kCmdNameSetnx, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSetnx, std::move(setnxptr))); ////SetexCmd std::unique_ptr setexptr = - std::make_unique(kCmdNameSetex, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSetex, 4, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSetex, std::move(setexptr))); ////PsetexCmd std::unique_ptr psetexptr = - std::make_unique(kCmdNamePsetex, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNamePsetex, 4, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePsetex, std::move(psetexptr))); ////DelvxCmd std::unique_ptr delvxptr = - std::make_unique(kCmdNameDelvx, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsSlow); + std::make_unique(kCmdNameDelvx, 3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameDelvx, std::move(delvxptr))); ////MSetCmd std::unique_ptr msetptr = - std::make_unique(kCmdNameMset, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameMset, -3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameMset, std::move(msetptr))); ////MSetnxCmd std::unique_ptr msetnxptr = std::make_unique( - kCmdNameMsetnx, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsKv | kCmdFlagsSlow); + kCmdNameMsetnx, -3, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameMsetnx, std::move(msetnxptr))); ////GetrangeCmd std::unique_ptr getrangeptr = std::make_unique( - kCmdNameGetrange, 4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsSlow); + kCmdNameGetrange, 4, kCmdFlagsRead | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGetrange, std::move(getrangeptr))); ////SetrangeCmd std::unique_ptr setrangeptr = std::make_unique( - kCmdNameSetrange, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameSetrange, 4, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSetrange, std::move(setrangeptr))); ////StrlenCmd std::unique_ptr strlenptr = - std::make_unique(kCmdNameStrlen, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameStrlen, 2, kCmdFlagsRead | kCmdFlagsKv | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameStrlen, std::move(strlenptr))); ////ExistsCmd std::unique_ptr existsptr = - std::make_unique(kCmdNameExists, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameExists, -2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameExists, std::move(existsptr))); ////ExpireCmd std::unique_ptr expireptr = std::make_unique( - kCmdNameExpire, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameExpire, 3, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameExpire, std::move(expireptr))); ////PexpireCmd std::unique_ptr pexpireptr = std::make_unique( - kCmdNamePexpire, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNamePexpire, 3, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePexpire, std::move(pexpireptr))); ////ExpireatCmd std::unique_ptr expireatptr = - std::make_unique(kCmdNameExpireat, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameExpireat, 3, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameExpireat, std::move(expireatptr))); ////PexpireatCmd std::unique_ptr pexpireatptr = - std::make_unique(kCmdNamePexpireat, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNamePexpireat, 3, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePexpireat, std::move(pexpireatptr))); ////TtlCmd std::unique_ptr ttlptr = - std::make_unique(kCmdNameTtl, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameTtl, 2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameTtl, std::move(ttlptr))); ////PttlCmd std::unique_ptr pttlptr = - std::make_unique(kCmdNamePttl, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNamePttl, 2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePttl, std::move(pttlptr))); ////PersistCmd std::unique_ptr persistptr = - std::make_unique(kCmdNamePersist, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNamePersist, 2, kCmdFlagsWrite | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePersist, std::move(persistptr))); ////TypeCmd std::unique_ptr typeptr = - std::make_unique(kCmdNameType, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameType, 2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameType, std::move(typeptr))); ////PTypeCmd std::unique_ptr pTypeptr = - std::make_unique(kCmdNamePType, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsFast); + std::make_unique(kCmdNamePType, 2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePType, std::move(pTypeptr))); ////ScanCmd std::unique_ptr scanptr = - std::make_unique(kCmdNameScan, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsSlow); + std::make_unique(kCmdNameScan, -2, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameScan, std::move(scanptr))); ////ScanxCmd std::unique_ptr scanxptr = - std::make_unique(kCmdNameScanx, -3, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsOperateKey | kCmdFlagsSlow); + std::make_unique(kCmdNameScanx, -3, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameScanx, std::move(scanxptr))); ////PKSetexAtCmd std::unique_ptr pksetexatptr = std::make_unique( - kCmdNamePKSetexAt, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsKv | kCmdFlagsSlow); + kCmdNamePKSetexAt, 4, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePKSetexAt, std::move(pksetexatptr))); ////PKScanRange std::unique_ptr pkscanrangeptr = std::make_unique( - kCmdNamePKScanRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsSlow); + kCmdNamePKScanRange, -4, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePKScanRange, std::move(pkscanrangeptr))); ////PKRScanRange std::unique_ptr pkrscanrangeptr = std::make_unique( - kCmdNamePKRScanRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsOperateKey | kCmdFlagsSlow); + kCmdNamePKRScanRange, -4, kCmdFlagsRead | kCmdFlagsOperateKey | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePKRScanRange, std::move(pkrscanrangeptr))); // Hash ////HDelCmd std::unique_ptr hdelptr = - std::make_unique(kCmdNameHDel, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHDel, -3, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHDel, std::move(hdelptr))); ////HSetCmd std::unique_ptr hsetptr = - std::make_unique(kCmdNameHSet, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHSet, 4, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHSet, std::move(hsetptr))); ////HGetCmd std::unique_ptr hgetptr = - std::make_unique(kCmdNameHGet, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHGet, 3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHGet, std::move(hgetptr))); ////HGetallCmd std::unique_ptr hgetallptr = - std::make_unique(kCmdNameHGetall, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + std::make_unique(kCmdNameHGetall, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameHGetall, std::move(hgetallptr))); ////HExistsCmd std::unique_ptr hexistsptr = - std::make_unique(kCmdNameHExists, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHExists, 3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHExists, std::move(hexistsptr))); ////HIncrbyCmd std::unique_ptr hincrbyptr = - std::make_unique(kCmdNameHIncrby, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHIncrby, 4, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHIncrby, std::move(hincrbyptr))); ////HIncrbyfloatCmd std::unique_ptr hincrbyfloatptr = - std::make_unique(kCmdNameHIncrbyfloat, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHIncrbyfloat, 4, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHIncrbyfloat, std::move(hincrbyfloatptr))); ////HKeysCmd std::unique_ptr hkeysptr = - std::make_unique(kCmdNameHKeys, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHKeys, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHKeys, std::move(hkeysptr))); ////HLenCmd std::unique_ptr hlenptr = - std::make_unique(kCmdNameHLen, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHLen, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHLen, std::move(hlenptr))); ////HMgetCmd std::unique_ptr hmgetptr = - std::make_unique(kCmdNameHMget, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHMget, -3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHMget, std::move(hmgetptr))); ////HMsetCmd std::unique_ptr hmsetptr = - std::make_unique(kCmdNameHMset, -4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHMset, -4, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHMset, std::move(hmsetptr))); ////HSetnxCmd std::unique_ptr hsetnxptr = - std::make_unique(kCmdNameHSetnx, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); + std::make_unique(kCmdNameHSetnx, 4, kCmdFlagsWrite | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHSetnx, std::move(hsetnxptr))); ////HStrlenCmd std::unique_ptr hstrlenptr = - std::make_unique(kCmdNameHStrlen, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameHStrlen, 3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsUpdateCache | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameHStrlen, std::move(hstrlenptr))); ////HValsCmd std::unique_ptr hvalsptr = - std::make_unique(kCmdNameHVals, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + std::make_unique(kCmdNameHVals, 2, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameHVals, std::move(hvalsptr))); ////HScanCmd std::unique_ptr hscanptr = std::make_unique( - kCmdNameHScan, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + kCmdNameHScan, -3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameHScan, std::move(hscanptr))); ////HScanxCmd std::unique_ptr hscanxptr = std::make_unique( - kCmdNameHScanx, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + kCmdNameHScanx, -3, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameHScanx, std::move(hscanxptr))); ////PKHScanRange std::unique_ptr pkhscanrangeptr = std::make_unique( - kCmdNamePKHScanRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + kCmdNamePKHScanRange, -4, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePKHScanRange, std::move(pkhscanrangeptr))); ////PKHRScanRange std::unique_ptr pkhrscanrangeptr = std::make_unique( - kCmdNamePKHRScanRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsHash | kCmdFlagsSlow); + kCmdNamePKHRScanRange, -4, kCmdFlagsRead | kCmdFlagsHash | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePKHRScanRange, std::move(pkhrscanrangeptr))); // List std::unique_ptr lindexptr = - std::make_unique(kCmdNameLIndex, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLIndex, 3, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLIndex, std::move(lindexptr))); std::unique_ptr linsertptr = - std::make_unique(kCmdNameLInsert, 5, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLInsert, 5, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLInsert, std::move(linsertptr))); std::unique_ptr llenptr = - std::make_unique(kCmdNameLLen, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameLLen, 2, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameLLen, std::move(llenptr))); - std::unique_ptr blpopptr = std::make_unique( - kCmdNameBLPop, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsSlow); + kCmdNameBLPop, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBLPop, std::move(blpopptr))); std::unique_ptr lpopptr = - std::make_unique(kCmdNameLPop, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameLPop, -2, kCmdFlagsWrite | kCmdFlagsList |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameLPop, std::move(lpopptr))); std::unique_ptr lpushptr = std::make_unique( - kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameLPush, std::move(lpushptr))); - std::unique_ptr lpushxptr = std::make_unique(kCmdNameLPushx, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::unique_ptr lpushxptr = std::make_unique(kCmdNameLPushx, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameLPushx, std::move(lpushxptr))); std::unique_ptr lrangeptr = std::make_unique( - kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLRange, std::move(lrangeptr))); std::unique_ptr lremptr = - std::make_unique(kCmdNameLRem, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLRem, 4, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLRem, std::move(lremptr))); std::unique_ptr lsetptr = - std::make_unique(kCmdNameLSet, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLSet, 4, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLSet, std::move(lsetptr))); std::unique_ptr ltrimptr = - std::make_unique(kCmdNameLTrim, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameLTrim, 4, kCmdFlagsWrite | kCmdFlagsList |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameLTrim, std::move(ltrimptr))); std::unique_ptr brpopptr = std::make_unique( - kCmdNameBRpop, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsSlow); + kCmdNameBRpop, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBRpop, std::move(brpopptr))); std::unique_ptr rpopptr = - std::make_unique(kCmdNameRPop, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameRPop, -2, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameRPop, std::move(rpopptr))); - std::unique_ptr rpoplpushptr = std::make_unique( - kCmdNameRPopLPush, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsSlow); + kCmdNameRPopLPush, 3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameRPopLPush, std::move(rpoplpushptr))); std::unique_ptr rpushptr = - std::make_unique(kCmdNameRPush, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot |kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameRPush, -3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameRPush, std::move(rpushptr))); std::unique_ptr rpushxptr = - std::make_unique(kCmdNameRPushx, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); - std::make_unique(kCmdNameRPushx, 3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameRPushx, -3, kCmdFlagsWrite | kCmdFlagsList); + std::make_unique(kCmdNameRPushx, 3, kCmdFlagsWrite | kCmdFlagsList | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameRPushx, std::move(rpushxptr))); // Zset ////ZAddCmd std::unique_ptr zaddptr = - std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZAdd, std::move(zaddptr))); ////ZCardCmd std::unique_ptr zcardptr = - std::make_unique(kCmdNameZCard, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameZCard, 2, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZCard, std::move(zcardptr))); ////ZScanCmd std::unique_ptr zscanptr = std::make_unique( - kCmdNameZScan, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsSlow); + kCmdNameZScan, -3, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZScan, std::move(zscanptr))); ////ZIncrbyCmd std::unique_ptr zincrbyptr = - std::make_unique(kCmdNameZIncrby, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast) ; + std::make_unique(kCmdNameZIncrby, 4, kCmdFlagsWrite | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast) ; cmd_table->insert(std::pair>(kCmdNameZIncrby, std::move(zincrbyptr))); ////ZRangeCmd std::unique_ptr zrangeptr = - std::make_unique(kCmdNameZRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZRange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRange, std::move(zrangeptr))); ////ZRevrangeCmd std::unique_ptr zrevrangeptr = - std::make_unique(kCmdNameZRevrange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZRevrange, -4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRevrange, std::move(zrevrangeptr))); ////ZRangebyscoreCmd std::unique_ptr zrangebyscoreptr = std::make_unique( - kCmdNameZRangebyscore, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsSlow); + kCmdNameZRangebyscore, -4, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRangebyscore, std::move(zrangebyscoreptr))); ////ZRevrangebyscoreCmd std::unique_ptr zrevrangebyscoreptr = std::make_unique( - kCmdNameZRevrangebyscore, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsSlow); + kCmdNameZRevrangebyscore, -4, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsSlow); cmd_table->insert( std::pair>(kCmdNameZRevrangebyscore, std::move(zrevrangebyscoreptr))); ////ZCountCmd std::unique_ptr zcountptr = - std::make_unique(kCmdNameZCount, 4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZCount, 4, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZCount, std::move(zcountptr))); ////ZRemCmd std::unique_ptr zremptr = - std::make_unique(kCmdNameZRem, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZRem, -3, kCmdFlagsWrite | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZRem, std::move(zremptr))); ////ZUnionstoreCmd std::unique_ptr zunionstoreptr = - std::make_unique(kCmdNameZUnionstore, -4, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZUnionstore, -4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZUnionstore, std::move(zunionstoreptr))); ////ZInterstoreCmd std::unique_ptr zinterstoreptr = - std::make_unique(kCmdNameZInterstore, -4, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameZInterstore, -4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZInterstore, std::move(zinterstoreptr))); ////ZRankCmd std::unique_ptr zrankptr = - std::make_unique(kCmdNameZRank, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZRank, 3, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZRank, std::move(zrankptr))); ////ZRevrankCmd std::unique_ptr zrevrankptr = - std::make_unique(kCmdNameZRevrank, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameZRevrank, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZRevrank, std::move(zrevrankptr))); ////ZScoreCmd std::unique_ptr zscoreptr = - std::make_unique(kCmdNameZScore, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); + std::make_unique(kCmdNameZScore, 3, kCmdFlagsRead | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZScore, std::move(zscoreptr))); ////ZRangebylexCmd std::unique_ptr zrangebylexptr = - std::make_unique(kCmdNameZRangebylex, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsSlow); + std::make_unique(kCmdNameZRangebylex, -4, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRangebylex, std::move(zrangebylexptr))); ////ZRevrangebylexCmd std::unique_ptr zrevrangebylexptr = std::make_unique( - kCmdNameZRevrangebylex, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsSlow); + kCmdNameZRevrangebylex, -4, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRevrangebylex, std::move(zrevrangebylexptr))); ////ZLexcountCmd std::unique_ptr zlexcountptr = - std::make_unique(kCmdNameZLexcount, 4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsFast); + std::make_unique(kCmdNameZLexcount, 4, kCmdFlagsRead | kCmdFlagsZset | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZLexcount, std::move(zlexcountptr))); ////ZRemrangebyrankCmd std::unique_ptr zremrangebyrankptr = std::make_unique( - kCmdNameZRemrangebyrank, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameZRemrangebyrank, 4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert( std::pair>(kCmdNameZRemrangebyrank, std::move(zremrangebyrankptr))); ////ZRemrangebyscoreCmd std::unique_ptr zremrangebyscoreptr = std::make_unique( - kCmdNameZRemrangebyscore, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameZRemrangebyscore, 4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert( std::pair>(kCmdNameZRemrangebyscore, std::move(zremrangebyscoreptr))); ////ZRemrangebylexCmd std::unique_ptr zremrangebylexptr = std::make_unique( - kCmdNameZRemrangebylex, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + kCmdNameZRemrangebylex, 4, kCmdFlagsWrite | kCmdFlagsZset |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameZRemrangebylex, std::move(zremrangebylexptr))); ////ZPopmax std::unique_ptr zpopmaxptr = std::make_unique( - kCmdNameZPopmax, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsFast); + kCmdNameZPopmax, -2, kCmdFlagsWrite | kCmdFlagsZset | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZPopmax, std::move(zpopmaxptr))); ////ZPopmin std::unique_ptr zpopminptr = std::make_unique( - kCmdNameZPopmin, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsZset | kCmdFlagsFast); + kCmdNameZPopmin, -2, kCmdFlagsWrite | kCmdFlagsZset | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameZPopmin, std::move(zpopminptr))); // Set ////SAddCmd std::unique_ptr saddptr = - std::make_unique(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSAdd, std::move(saddptr))); ////SPopCmd std::unique_ptr spopptr = - std::make_unique(kCmdNameSPop, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSPop, -2, kCmdFlagsWrite | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSPop, std::move(spopptr))); ////SCardCmd std::unique_ptr scardptr = - std::make_unique(kCmdNameSCard, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSCard, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSCard, std::move(scardptr))); ////SMembersCmd std::unique_ptr smembersptr = - std::make_unique(kCmdNameSMembers, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSMembers, 2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSMembers, std::move(smembersptr))); ////SScanCmd std::unique_ptr sscanptr = - std::make_unique(kCmdNameSScan, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsSet | kCmdFlagsSlow); + std::make_unique(kCmdNameSScan, -3, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSScan, std::move(sscanptr))); ////SRemCmd std::unique_ptr sremptr = - std::make_unique(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSRem, std::move(sremptr))); ////SUnionCmd std::unique_ptr sunionptr = std::make_unique( - kCmdNameSUnion, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsSlow); + kCmdNameSUnion, -2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSUnion, std::move(sunionptr))); ////SUnionstoreCmd std::unique_ptr sunionstoreptr = - std::make_unique(kCmdNameSUnionstore, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSUnionstore, -3, kCmdFlagsWrite | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSUnionstore, std::move(sunionstoreptr))); ////SInterCmd std::unique_ptr sinterptr = std::make_unique( - kCmdNameSInter, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsSlow); + kCmdNameSInter, -2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSInter, std::move(sinterptr))); ////SInterstoreCmd std::unique_ptr sinterstoreptr = - std::make_unique(kCmdNameSInterstore, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSInterstore, -3, kCmdFlagsWrite | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSInterstore, std::move(sinterstoreptr))); ////SIsmemberCmd std::unique_ptr sismemberptr = - std::make_unique(kCmdNameSIsmember, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSIsmember, 3, kCmdFlagsRead | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSIsmember, std::move(sismemberptr))); ////SDiffCmd std::unique_ptr sdiffptr = - std::make_unique(kCmdNameSDiff, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsSlow); + std::make_unique(kCmdNameSDiff, -2, kCmdFlagsRead | kCmdFlagsSet | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSDiff, std::move(sdiffptr))); ////SDiffstoreCmd std::unique_ptr sdiffstoreptr = - std::make_unique(kCmdNameSDiffstore, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSDiffstore, -3, kCmdFlagsWrite | kCmdFlagsSet |kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSDiffstore, std::move(sdiffstoreptr))); ////SMoveCmd std::unique_ptr smoveptr = - std::make_unique(kCmdNameSMove, 4, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); + std::make_unique(kCmdNameSMove, 4, kCmdFlagsWrite | kCmdFlagsSet | kCmdFlagsDoThroughDB | kCmdFlagsUpdateCache | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameSMove, std::move(smoveptr))); ////SRandmemberCmd std::unique_ptr srandmemberptr = - std::make_unique(kCmdNameSRandmember, -2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsSet|kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); + std::make_unique(kCmdNameSRandmember, -2, kCmdFlagsRead | kCmdFlagsSet|kCmdFlagsDoThroughDB | kCmdFlagsReadCache | kCmdFlagsUpdateCache | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameSRandmember, std::move(srandmemberptr))); // BitMap ////bitsetCmd std::unique_ptr bitsetptr = - std::make_unique(kCmdNameBitSet, 4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsBit | kCmdFlagsSlow); + std::make_unique(kCmdNameBitSet, 4, kCmdFlagsWrite | kCmdFlagsBit | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBitSet, std::move(bitsetptr))); ////bitgetCmd std::unique_ptr bitgetptr = - std::make_unique(kCmdNameBitGet, 3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsBit | kCmdFlagsSlow); + std::make_unique(kCmdNameBitGet, 3, kCmdFlagsRead | kCmdFlagsBit | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBitGet, std::move(bitgetptr))); ////bitcountCmd std::unique_ptr bitcountptr = - std::make_unique(kCmdNameBitCount, -2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsBit | kCmdFlagsSlow); + std::make_unique(kCmdNameBitCount, -2, kCmdFlagsRead | kCmdFlagsBit | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBitCount, std::move(bitcountptr))); ////bitposCmd std::unique_ptr bitposptr = - std::make_unique(kCmdNameBitPos, -3, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsBit | kCmdFlagsSlow); + std::make_unique(kCmdNameBitPos, -3, kCmdFlagsRead | kCmdFlagsBit | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBitPos, std::move(bitposptr))); ////bitopCmd std::unique_ptr bitopptr = - std::make_unique(kCmdNameBitOp, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsBit | kCmdFlagsSlow); + std::make_unique(kCmdNameBitOp, -3, kCmdFlagsWrite | kCmdFlagsBit | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameBitOp, std::move(bitopptr))); // HyperLogLog ////pfaddCmd std::unique_ptr pfaddptr = std::make_unique( - kCmdNamePfAdd, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsHyperLogLog | kCmdFlagsFast); + kCmdNamePfAdd, -2, kCmdFlagsWrite | kCmdFlagsHyperLogLog | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNamePfAdd, std::move(pfaddptr))); ////pfcountCmd std::unique_ptr pfcountptr = std::make_unique( - kCmdNamePfCount, -2, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsHyperLogLog | kCmdFlagsSlow); + kCmdNamePfCount, -2, kCmdFlagsRead | kCmdFlagsHyperLogLog | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePfCount, std::move(pfcountptr))); ////pfmergeCmd std::unique_ptr pfmergeptr = std::make_unique( - kCmdNamePfMerge, -3, kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsHyperLogLog | kCmdFlagsSlow); + kCmdNamePfMerge, -3, kCmdFlagsWrite | kCmdFlagsHyperLogLog | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNamePfMerge, std::move(pfmergeptr))); // GEO ////GepAdd std::unique_ptr geoaddptr = std::make_unique( - kCmdNameGeoAdd, -5, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoAdd, -5, kCmdFlagsWrite | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGeoAdd, std::move(geoaddptr))); ////GeoPos std::unique_ptr geoposptr = std::make_unique( - kCmdNameGeoPos, -2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoPos, -2, kCmdFlagsRead | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGeoPos, std::move(geoposptr))); ////GeoDist std::unique_ptr geodistptr = std::make_unique( - kCmdNameGeoDist, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoDist, -4, kCmdFlagsRead | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGeoDist, std::move(geodistptr))); ////GeoHash std::unique_ptr geohashptr = std::make_unique( - kCmdNameGeoHash, -2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoHash, -2, kCmdFlagsRead | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGeoHash, std::move(geohashptr))); ////GeoRadius std::unique_ptr georadiusptr = std::make_unique( - kCmdNameGeoRadius, -6, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoRadius, -6, kCmdFlagsRead | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameGeoRadius, std::move(georadiusptr))); ////GeoRadiusByMember std::unique_ptr georadiusbymemberptr = std::make_unique( - kCmdNameGeoRadiusByMember, -5, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsGeo | kCmdFlagsSlow); + kCmdNameGeoRadiusByMember, -5, kCmdFlagsRead | kCmdFlagsGeo | kCmdFlagsSlow); cmd_table->insert( std::pair>(kCmdNameGeoRadiusByMember, std::move(georadiusbymemberptr))); @@ -774,11 +771,11 @@ void InitCmdTable(CmdTable* cmd_table) { // Transaction ////Multi std::unique_ptr multiptr = - std::make_unique(kCmdNameMulti, 1, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsFast); + std::make_unique(kCmdNameMulti, 1, kCmdFlagsRead | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameMulti, std::move(multiptr))); ////Exec std::unique_ptr execptr = std::make_unique( - kCmdNameExec, 1, kCmdFlagsRead | kCmdFlagsWrite | kCmdFlagsMultiSlot | kCmdFlagsSuspend | kCmdFlagsSlow); + kCmdNameExec, 1, kCmdFlagsRead | kCmdFlagsWrite | kCmdFlagsSuspend | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameExec, std::move(execptr))); ////Discard std::unique_ptr discardptr = std::make_unique(kCmdNameDiscard, 1, kCmdFlagsRead | kCmdFlagsFast); @@ -793,35 +790,35 @@ void InitCmdTable(CmdTable* cmd_table) { // Stream ////XAdd std::unique_ptr xaddptr = - std::make_unique(kCmdNameXAdd, -4, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsFast); + std::make_unique(kCmdNameXAdd, -4, kCmdFlagsWrite | kCmdFlagsStream | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameXAdd, std::move(xaddptr))); ////XLen std::unique_ptr xlenptr = - std::make_unique(kCmdNameXLen, 2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsFast); + std::make_unique(kCmdNameXLen, 2, kCmdFlagsRead | kCmdFlagsStream | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameXLen, std::move(xlenptr))); ////XRead std::unique_ptr xreadptr = - std::make_unique(kCmdNameXRead, -3, kCmdFlagsRead | kCmdFlagsMultiSlot | kCmdFlagsStream | kCmdFlagsSlow); + std::make_unique(kCmdNameXRead, -3, kCmdFlagsRead | kCmdFlagsStream | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameXRead, std::move(xreadptr))); ////XRange std::unique_ptr xrangeptr = - std::make_unique(kCmdNameXRange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsSlow); + std::make_unique(kCmdNameXRange, -4, kCmdFlagsRead | kCmdFlagsStream | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameXRange, std::move(xrangeptr))); ////XRerange std::unique_ptr xrerverangeptr = - std::make_unique(kCmdNameXRevrange, -4, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsSlow); + std::make_unique(kCmdNameXRevrange, -4, kCmdFlagsRead | kCmdFlagsStream | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameXRevrange, std::move(xrerverangeptr))); ////XTrim std::unique_ptr xtrimptr = - std::make_unique(kCmdNameXTrim, -2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsSlow); + std::make_unique(kCmdNameXTrim, -2, kCmdFlagsWrite | kCmdFlagsStream | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameXTrim, std::move(xtrimptr))); ////XDel std::unique_ptr xdelptr = - std::make_unique(kCmdNameXDel, -3, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsFast); + std::make_unique(kCmdNameXDel, -3, kCmdFlagsWrite | kCmdFlagsStream | kCmdFlagsFast); cmd_table->insert(std::pair>(kCmdNameXDel, std::move(xdelptr))); ////XINFO std::unique_ptr xinfoptr = - std::make_unique(kCmdNameXInfo, -2, kCmdFlagsRead | kCmdFlagsSingleSlot | kCmdFlagsStream | kCmdFlagsSlow); + std::make_unique(kCmdNameXInfo, -2, kCmdFlagsRead | kCmdFlagsStream | kCmdFlagsSlow); cmd_table->insert(std::pair>(kCmdNameXInfo, std::move(xinfoptr))); } @@ -833,6 +830,8 @@ Cmd* GetCmdFromDB(const std::string& opt, const CmdTable& cmd_table) { return nullptr; } +bool Cmd::CheckArg(uint64_t num) const { return !((arity_ > 0 && num != arity_) || (arity_ < 0 && num < -arity_)); } + Cmd::Cmd(std::string name, int arity, uint32_t flag, uint32_t aclCategory) : name_(std::move(name)), arity_(arity), flag_(flag), aclCategory_(aclCategory) { // assign cmd id @@ -843,119 +842,104 @@ void Cmd::Initial(const PikaCmdArgsType& argv, const std::string& db_name) { argv_ = argv; db_name_ = db_name; res_.clear(); // Clear res content + db_ = g_pika_server->GetDB(db_name_); + sync_db_ = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); Clear(); // Clear cmd, Derived class can has own implement DoInitial(); }; std::vector Cmd::current_key() const { return {""}; } -void Cmd::Execute() { ProcessSingleSlotCmd(); } - -void Cmd::ProcessSingleSlotCmd() { - std::shared_ptr slot; - slot = g_pika_server->GetSlotByDBName(db_name_); - - if (!slot) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - - std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); - if (!sync_slot) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - ProcessCommand(slot, sync_slot); +void Cmd::Execute() { + ProcessCommand(); } -void Cmd::ProcessCommand(const std::shared_ptr& slot, const std::shared_ptr& sync_slot, - const HintKeys& hint_keys) { +void Cmd::ProcessCommand(const HintKeys& hint_keys) { + LOG(INFO) << "lmz"; if (stage_ == kNone) { - InternalProcessCommand(slot, sync_slot, hint_keys); + InternalProcessCommand(hint_keys); } else { if (stage_ == kBinlogStage) { - DoBinlog(sync_slot); + DoBinlog(); } else if (stage_ == kExecuteStage) { - DoCommand(slot, hint_keys); + DoCommand(hint_keys); } } } -void Cmd::InternalProcessCommand(const std::shared_ptr& slot, const std::shared_ptr& sync_slot, - const HintKeys& hint_keys) { - pstd::lock::MultiRecordLock record_lock(slot->LockMgr()); +void Cmd::InternalProcessCommand(const HintKeys& hint_keys) { + LOG(INFO) << db_->GetDBName(); + pstd::lock::MultiRecordLock record_lock(db_->LockMgr()); if (is_write()) { record_lock.Lock(current_key()); } - uint64_t start_us = 0; if (g_pika_conf->slowlog_slower_than() >= 0) { start_us = pstd::NowMicros(); } - DoCommand(slot, hint_keys); + DoCommand(hint_keys); if (g_pika_conf->slowlog_slower_than() >= 0) { do_duration_ += pstd::NowMicros() - start_us; } - DoBinlog(sync_slot); + DoBinlog(); if (is_write()) { record_lock.Unlock(current_key()); } } -void Cmd::DoCommand(const std::shared_ptr& slot, const HintKeys& hint_keys) { +void Cmd::DoCommand(const HintKeys& hint_keys) { if (!IsSuspend()) { - slot->DbRWLockReader(); + db_->DbRWLockReader(); } DEFER { if (!IsSuspend()) { - slot->DbRWUnLock(); + db_->DbRWUnLock(); } }; if (IsNeedCacheDo() && PIKA_CACHE_NONE != g_pika_conf->cache_model() - && slot->cache()->CacheStatus() == PIKA_CACHE_STATUS_OK) { + && db_->cache()->CacheStatus() == PIKA_CACHE_STATUS_OK) { if (IsNeedReadCache()) { - ReadCache(slot); + ReadCache(); } if (is_read() && res().CacheMiss()) { - pstd::lock::MultiScopeRecordLock record_lock(slot->LockMgr(), current_key()); - DoThroughDB(slot); + pstd::lock::MultiScopeRecordLock record_lock(db_->LockMgr(), current_key()); + DoThroughDB(); if (IsNeedUpdateCache()) { - DoUpdateCache(slot); + DoUpdateCache(); } } else if (is_write()) { - DoThroughDB(slot); + DoThroughDB(); if (IsNeedUpdateCache()) { - DoUpdateCache(slot); + DoUpdateCache(); } } } else { - Do(slot); + Do(); } } -void Cmd::DoBinlog(const std::shared_ptr& slot) { +void Cmd::DoBinlog() { if (res().ok() && is_write() && g_pika_conf->write_binlog()) { std::shared_ptr conn_ptr = GetConn(); std::shared_ptr resp_ptr = GetResp(); // Consider that dummy cmd appended by system, both conn and resp are null. if ((!conn_ptr || !resp_ptr) && (name_ != kCmdDummy)) { if (!conn_ptr) { - LOG(WARNING) << slot->SyncSlotInfo().ToString() << " conn empty."; + LOG(WARNING) << sync_db_->SyncDBInfo().ToString() << " conn empty."; } if (!resp_ptr) { - LOG(WARNING) << slot->SyncSlotInfo().ToString() << " resp empty."; + LOG(WARNING) << sync_db_->SyncDBInfo().ToString() << " resp empty."; } res().SetRes(CmdRes::kErrOther); return; } - Status s = slot->ConsensusProposeLog(shared_from_this()); + Status s = sync_db_->ConsensusProposeLog(shared_from_this()); if (!s.ok()) { - LOG(WARNING) << slot->SyncSlotInfo().ToString() << " Writing binlog failed, maybe no space left on device " + LOG(WARNING) << sync_db_->SyncDBInfo().ToString() << " Writing binlog failed, maybe no space left on device " << s.ToString(); res().SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -963,61 +947,6 @@ void Cmd::DoBinlog(const std::shared_ptr& slot) { } } -void Cmd::ProcessMultiSlotCmd() { - std::shared_ptr slot; - std::vector cur_key = current_key(); - if (cur_key.empty()) { - res_.SetRes(CmdRes::kErrOther, "Internal Error"); - return; - } - - int hint = 0; - std::unordered_map process_map; - // split cur_key into slots - std::shared_ptr db = g_pika_server->GetDB(db_name_); - if (!db) { - res_.SetRes(CmdRes::kErrOther, "DB not found"); - return; - } - - CmdStage current_stage = stage_; - for (auto& key : cur_key) { - // in sharding mode we select slot by key - uint32_t slot_id = g_pika_cmd_table_manager->DistributeKey(key, db->SlotNum()); - auto iter = process_map.find(slot_id); - if (iter == process_map.end()) { - std::shared_ptr slot = db->GetSlotById(slot_id); - if (!slot) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); - if (!sync_slot) { - res_.SetRes(CmdRes::kErrOther, "Slot not found"); - return; - } - HintKeys hint_keys; - hint_keys.Push(key, hint); - process_map[slot_id] = ProcessArg(slot, sync_slot, hint_keys); - } else { - iter->second.hint_keys.Push(key, hint); - } - hint++; - } - for (auto& iter : process_map) { - ProcessArg& arg = iter.second; - ProcessCommand(arg.slot, arg.sync_slot, arg.hint_keys); - if (!res_.ok()) { - return; - } - } - if (current_stage == kNone || current_stage == kExecuteStage) { - Merge(); - } -} - -uint32_t Cmd::flag() const { return flag_; } bool Cmd::hasFlag(uint32_t flag) const { return (flag_ & flag); } bool Cmd::is_read() const { return (flag_ & kCmdFlagsRead); } bool Cmd::is_write() const { return (flag_ & kCmdFlagsWrite); } @@ -1037,9 +966,6 @@ int8_t Cmd::SubCmdIndex(const std::string& cmdName) { // Others need to be suspended when a suspend command run bool Cmd::IsSuspend() const { return (flag_ & kCmdFlagsSuspend); } -// Must with admin auth -bool Cmd::is_single_slot() const { return (flag_ & kCmdFlagsSingleSlot); } -bool Cmd::is_multi_slot() const { return (flag_ & kCmdFlagsMultiSlot); } // std::string Cmd::CurrentSubCommand() const { return ""; }; bool Cmd::HasSubCommand() const { return subCmdName_.size() > 0; }; std::vector Cmd::SubCommand() const { return subCmdName_; }; @@ -1092,6 +1018,7 @@ PikaCmdArgsType& Cmd::argv() { return argv_; } uint32_t Cmd::AclCategory() const { return aclCategory_; } void Cmd::AddAclCategory(uint32_t aclCategory) { aclCategory_ |= aclCategory; } +uint32_t Cmd::flag() const { return flag_; } std::string Cmd::ToRedisProtocol() { std::string content; @@ -1106,8 +1033,6 @@ std::string Cmd::ToRedisProtocol() { return content; } -bool Cmd::CheckArg(uint64_t num) const { return !((arity_ > 0 && num != arity_) || (arity_ < 0 && num < -arity_)); } - void Cmd::LogCommand() const { std::string command; for (const auto& item : argv_) { diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 9499b36972..69579998aa 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -22,119 +22,6 @@ using pstd::Status; PikaConf::PikaConf(const std::string& path) : pstd::BaseConf(path), conf_path_(path), local_meta_(std::make_unique()) {} -Status PikaConf::InternalGetTargetDB(const std::string& db_name, uint32_t* const target) { - int32_t db_index = -1; - for (size_t idx = 0; idx < db_structs_.size(); ++idx) { - if (db_structs_[idx].db_name == db_name) { - db_index = static_cast(idx); - break; - } - } - if (db_index == -1) { - return Status::NotFound("db : " + db_name + " not found"); - } - *target = db_index; - return Status::OK(); -} - -Status PikaConf::DBSlotsSanityCheck(const std::string& db_name, const std::set& slot_ids, bool is_add) { - std::shared_lock l(rwlock_); - uint32_t db_index = 0; - Status s = InternalGetTargetDB(db_name, &db_index); - if (!s.ok()) { - return s; - } - // Sanity Check - for (const auto& id : slot_ids) { - if (id >= db_structs_[db_index].slot_num) { - return Status::Corruption("slot index out of range"); - } else if (is_add && db_structs_[db_index].slot_ids.count(id) != 0) { - return Status::Corruption("slot : " + std::to_string(id) + " exist"); - } else if (!is_add && db_structs_[db_index].slot_ids.count(id) == 0) { - return Status::Corruption("slot : " + std::to_string(id) + " not exist"); - } - } - return Status::OK(); -} - -Status PikaConf::AddDBSlots(const std::string& db_name, const std::set& slot_ids) { - Status s = DBSlotsSanityCheck(db_name, slot_ids, true); - if (!s.ok()) { - return s; - } - - std::lock_guard l(rwlock_); - uint32_t index = 0; - s = InternalGetTargetDB(db_name, &index); - if (s.ok()) { - for (const auto& id : slot_ids) { - db_structs_[index].slot_ids.insert(id); - } - s = local_meta_->StableSave(db_structs_); - } - return s; -} - -Status PikaConf::RemoveDBSlots(const std::string& db_name, const std::set& slot_ids) { - Status s = DBSlotsSanityCheck(db_name, slot_ids, false); - if (!s.ok()) { - return s; - } - - std::lock_guard l(rwlock_); - uint32_t index = 0; - s = InternalGetTargetDB(db_name, &index); - if (s.ok()) { - for (const auto& id : slot_ids) { - db_structs_[index].slot_ids.erase(id); - } - s = local_meta_->StableSave(db_structs_); - } - return s; -} - -Status PikaConf::AddDB(const std::string& db_name, const uint32_t slot_num) { - Status s = AddDBSanityCheck(db_name); - if (!s.ok()) { - return s; - } - std::lock_guard l(rwlock_); - db_structs_.push_back({db_name, slot_num, {}}); - s = local_meta_->StableSave(db_structs_); - return s; -} - -Status PikaConf::DelDB(const std::string& db_name) { - Status s = DelDBSanityCheck(db_name); - if (!s.ok()) { - return s; - } - std::lock_guard l(rwlock_); - for (auto iter = db_structs_.begin(); iter != db_structs_.end(); iter++) { - if (iter->db_name == db_name) { - db_structs_.erase(iter); - break; - } - } - return local_meta_->StableSave(db_structs_); -} - -Status PikaConf::AddDBSanityCheck(const std::string& db_name) { - std::shared_lock l(rwlock_); - uint32_t db_index = 0; - Status s = InternalGetTargetDB(db_name, &db_index); - if (!s.IsNotFound()) { - return Status::Corruption("db: " + db_name + " already exist"); - } - return Status::OK(); -} - -Status PikaConf::DelDBSanityCheck(const std::string& db_name) { - std::shared_lock l(rwlock_); - uint32_t db_index = 0; - return InternalGetTargetDB(db_name, &db_index); -} - int PikaConf::Load() { int ret = LoadConf(); if (ret) { @@ -288,7 +175,7 @@ int PikaConf::Load() { LOG(FATAL) << "config databases error, limit [1 ~ 8], the actual is: " << databases_; } for (int idx = 0; idx < databases_; ++idx) { - db_structs_.push_back({"db" + std::to_string(idx), 1, {0}}); + db_structs_.push_back({"db" + std::to_string(idx)}); } } default_db_ = db_structs_[0].db_name; diff --git a/src/pika_consensus.cc b/src/pika_consensus.cc index c616036867..3d08a4a642 100644 --- a/src/pika_consensus.cc +++ b/src/pika_consensus.cc @@ -62,11 +62,6 @@ Status Context::Init() { } } -void Context::PrepareUpdateAppliedIndex(const LogOffset& offset) { - std::lock_guard l(rwlock_); - applied_win_.Push(SyncWinItem(offset)); -} - void Context::UpdateAppliedIndex(const LogOffset& offset) { std::lock_guard l(rwlock_); LogOffset cur_offset; @@ -100,13 +95,7 @@ std::unordered_map> SyncProgress::GetAll return slaves_; } -std::unordered_map SyncProgress::GetAllMatchIndex() { - std::shared_lock l(rwlock_); - return match_index_; -} - -Status SyncProgress::AddSlaveNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, - int session_id) { +Status SyncProgress::AddSlaveNode(const std::string& ip, int port, const std::string& db_name, int session_id) { std::string slave_key = ip + std::to_string(port); std::shared_ptr exist_ptr = GetSlaveNode(ip, port); if (exist_ptr) { @@ -114,7 +103,7 @@ Status SyncProgress::AddSlaveNode(const std::string& ip, int port, const std::st exist_ptr->SetSessionId(session_id); return Status::OK(); } - std::shared_ptr slave_ptr = std::make_shared(ip, port, db_name, slot_id, session_id); + std::shared_ptr slave_ptr = std::make_shared(ip, port, db_name, session_id); slave_ptr->SetLastSendTime(pstd::NowMicros()); slave_ptr->SetLastRecvTime(pstd::NowMicros()); @@ -172,18 +161,6 @@ MemLog::MemLog() = default; int MemLog::Size() { return static_cast(logs_.size()); } -// purge [begin, offset] -Status MemLog::PurgeLogs(const LogOffset& offset, std::vector* logs) { - std::lock_guard l_logs(logs_mu_); - int index = InternalFindLogByBinlogOffset(offset); - if (index < 0) { - return Status::NotFound("Cant find correct index"); - } - logs->assign(logs_.begin(), logs_.begin() + index + 1); - logs_.erase(logs_.begin(), logs_.begin() + index + 1); - return Status::OK(); -} - // keep mem_log [mem_log.begin, offset] Status MemLog::TruncateTo(const LogOffset& offset) { std::lock_guard l_logs(logs_mu_); @@ -202,16 +179,6 @@ void MemLog::Reset(const LogOffset& offset) { last_offset_ = offset; } -Status MemLog::GetRangeLogs(int start, int end, std::vector* logs) { - std::lock_guard l_logs(logs_mu_); - int log_size = static_cast(logs_.size()); - if (start > end || start >= log_size || end >= log_size) { - return Status::Corruption("Invalid index"); - } - logs->assign(logs_.begin() + start, logs_.begin() + end + 1); - return Status::OK(); -} - bool MemLog::FindLogItem(const LogOffset& offset, LogOffset* found_offset) { std::lock_guard l_logs(logs_mu_); int index = InternalFindLogByLogicIndex(offset); @@ -248,12 +215,12 @@ int MemLog::InternalFindLogByBinlogOffset(const LogOffset& offset) { /* ConsensusCoordinator */ -ConsensusCoordinator::ConsensusCoordinator(const std::string& db_name, uint32_t slot_id) - : db_name_(db_name), slot_id_(slot_id) { +ConsensusCoordinator::ConsensusCoordinator(const std::string& db_name) + : db_name_(db_name) { std::string db_log_path = g_pika_conf->log_path() + "log_" + db_name + "/"; std::string log_path = db_log_path; context_ = std::make_shared(log_path + kContext); - stable_logger_ = std::make_shared(db_name, slot_id, log_path); + stable_logger_ = std::make_shared(db_name, log_path); mem_logger_ = std::make_shared(); } @@ -268,7 +235,7 @@ void ConsensusCoordinator::Init() { // load term_ term_ = stable_logger_->Logger()->term(); - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "Restore applied index " + LOG(INFO) << DBInfo(db_name_).ToString() << "Restore applied index " << context_->applied_index_.ToString() << " current term " << term_; if (committed_index_ == LogOffset()) { return; @@ -283,7 +250,7 @@ void ConsensusCoordinator::Init() { int res = binlog_reader.Seek(stable_logger_->Logger(), committed_index_.b_offset.filenum, committed_index_.b_offset.offset); if (res != 0) { - LOG(FATAL) << SlotInfo(db_name_, slot_id_).ToString() << "Binlog reader init failed"; + LOG(FATAL) << DBInfo(db_name_).ToString() << "Binlog reader init failed"; } while (true) { @@ -293,11 +260,11 @@ void ConsensusCoordinator::Init() { if (s.IsEndFile()) { break; } else if (s.IsCorruption() || s.IsIOError()) { - LOG(FATAL) << SlotInfo(db_name_, slot_id_).ToString() << "Read Binlog error"; + LOG(FATAL) << DBInfo(db_name_).ToString() << "Read Binlog error"; } BinlogItem item; if (!PikaBinlogTransverter::BinlogItemWithoutContentDecode(TypeFirst, binlog, &item)) { - LOG(FATAL) << SlotInfo(db_name_, slot_id_).ToString() << "Binlog item decode failed"; + LOG(FATAL) << DBInfo(db_name_).ToString() << "Binlog item decode failed"; } offset.l_offset.term = item.term_id(); offset.l_offset.index = item.logic_id(); @@ -308,7 +275,7 @@ void ConsensusCoordinator::Init() { int processed_len = 0; net::RedisParserStatus ret = redis_parser.ProcessInputBuffer(redis_parser_start, redis_parser_len, &processed_len); if (ret != net::kRedisParserDone) { - LOG(FATAL) << SlotInfo(db_name_, slot_id_).ToString() << "Redis parser parse failed"; + LOG(FATAL) << DBInfo(db_name_).ToString() << "Redis parser parse failed"; return; } auto arg = static_cast(redis_parser.data); @@ -331,7 +298,7 @@ Status ConsensusCoordinator::Reset(const LogOffset& offset) { Status s = stable_logger_->Logger()->SetProducerStatus(offset.b_offset.filenum, offset.b_offset.offset, offset.l_offset.term, offset.l_offset.index); if (!s.ok()) { - LOG(WARNING) << SlotInfo(db_name_, slot_id_).ToString() << "Consensus reset status failed " + LOG(WARNING) << DBInfo(db_name_).ToString() << "Consensus reset status failed " << s.ToString(); return s; } @@ -370,7 +337,7 @@ Status ConsensusCoordinator::InternalAppendLog(const std::shared_ptr& cmd_p Status ConsensusCoordinator::ProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute) { LogOffset last_index = mem_logger_->last_offset(); if (attribute.logic_id() < last_index.l_offset.index) { - LOG(WARNING) << SlotInfo(db_name_, slot_id_).ToString() << "Drop log from leader logic_id " + LOG(WARNING) << DBInfo(db_name_).ToString() << "Drop log from leader logic_id " << attribute.logic_id() << " cur last index " << last_index.l_offset.index; return Status::OK(); } @@ -408,45 +375,8 @@ Status ConsensusCoordinator::InternalAppendBinlog(const std::shared_ptr& cm return stable_logger_->Logger()->GetProducerStatus(&filenum, &offset); } -Status ConsensusCoordinator::ScheduleApplyLog(const LogOffset& committed_index) { - // logs from PurgeLogs goes to InternalApply in order - std::lock_guard l(order_mu_); - std::vector logs; - Status s = mem_logger_->PurgeLogs(committed_index, &logs); - if (!s.ok()) { - return Status::NotFound("committed index not found " + committed_index.ToString()); - } - for (const auto& log : logs) { - context_->PrepareUpdateAppliedIndex(log.offset); - InternalApply(log); - } - return Status::OK(); -} - -Status ConsensusCoordinator::ScheduleApplyFollowerLog(const LogOffset& committed_index) { - // logs from PurgeLogs goes to InternalApply in order - std::lock_guard l(order_mu_); - std::vector logs; - Status s = mem_logger_->PurgeLogs(committed_index, &logs); - if (!s.ok()) { - return Status::NotFound("committed index not found " + committed_index.ToString()); - } - for (const auto& log : logs) { - context_->PrepareUpdateAppliedIndex(log.offset); - InternalApplyFollower(log); - } - return Status::OK(); -} - -Status ConsensusCoordinator::CheckEnoughFollower() { - if (!MatchConsensusLevel()) { - return Status::Incomplete("Not enough follower"); - } - return Status::OK(); -} - Status ConsensusCoordinator::AddSlaveNode(const std::string& ip, int port, int session_id) { - Status s = sync_pros_.AddSlaveNode(ip, port, db_name_, slot_id_, session_id); + Status s = sync_pros_.AddSlaveNode(ip, port, db_name_, session_id); if (!s.ok()) { return s; } @@ -474,23 +404,8 @@ uint32_t ConsensusCoordinator::term() { return term_; } -bool ConsensusCoordinator::MatchConsensusLevel() { - return sync_pros_.SlaveSize() >= static_cast(g_pika_conf->consensus_level()); -} - -void ConsensusCoordinator::InternalApply(const MemLog::LogItem& log) { - auto arg = new PikaClientConn::BgTaskArg(); - arg->cmd_ptr = log.cmd_ptr; - arg->conn_ptr = log.conn_ptr; - arg->resp_ptr = log.resp_ptr; - arg->offset = log.offset; - arg->db_name = db_name_; - arg->slot_id = slot_id_; - g_pika_server->ScheduleClientBgThreads(PikaClientConn::DoExecTask, arg, log.cmd_ptr->current_key().front()); -} - void ConsensusCoordinator::InternalApplyFollower(const MemLog::LogItem& log) { - g_pika_rm->ScheduleWriteDBTask(log.cmd_ptr, log.offset, db_name_, slot_id_); + g_pika_rm->ScheduleWriteDBTask(log.cmd_ptr, log.offset, db_name_); } int ConsensusCoordinator::InitCmd(net::RedisParser* parser, const net::RedisCmdArgsType& argv) { @@ -512,13 +427,13 @@ int ConsensusCoordinator::InitCmd(net::RedisParser* parser, const net::RedisCmdA } Status ConsensusCoordinator::TruncateTo(const LogOffset& offset) { - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "Truncate to " << offset.ToString(); + LOG(INFO) << DBInfo(db_name_).ToString() << "Truncate to " << offset.ToString(); LogOffset founded_offset; Status s = FindLogicOffset(offset.b_offset, offset.l_offset.index, &founded_offset); if (!s.ok()) { return s; } - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << " Founded truncate pos " + LOG(INFO) << DBInfo(db_name_).ToString() << " Founded truncate pos " << founded_offset.ToString(); LogOffset committed = committed_index(); stable_logger_->Logger()->Lock(); @@ -644,7 +559,7 @@ Status ConsensusCoordinator::FindBinlogFileNum(const std::map binlogs; @@ -666,7 +581,7 @@ Status ConsensusCoordinator::FindLogicOffsetBySearchingBinlog(const BinlogOffset return s; } - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "FindBinlogFilenum res " // NOLINT + LOG(INFO) << DBInfo(db_name_).ToString() << "FindBinlogFilenum res " // NOLINT << found_filenum; BinlogOffset traversal_start(found_filenum, 0); BinlogOffset traversal_end(found_filenum + 1, 0); @@ -677,7 +592,7 @@ Status ConsensusCoordinator::FindLogicOffsetBySearchingBinlog(const BinlogOffset } for (auto& offset : offsets) { if (offset.l_offset.index == target_index) { - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "Founded " << target_index << " " + LOG(INFO) << DBInfo(db_name_).ToString() << "Founded " << target_index << " " << offset.ToString(); *found_offset = offset; return Status::OK(); @@ -692,9 +607,9 @@ Status ConsensusCoordinator::FindLogicOffset(const BinlogOffset& start_offset, u Status s = GetBinlogOffset(start_offset, &possible_offset); if (!s.ok() || possible_offset.l_offset.index != target_index) { if (!s.ok()) { - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "GetBinlogOffset res: " << s.ToString(); + LOG(INFO) << DBInfo(db_name_).ToString() << "GetBinlogOffset res: " << s.ToString(); } else { - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "GetBInlogOffset res: " << s.ToString() + LOG(INFO) << DBInfo(db_name_).ToString() << "GetBInlogOffset res: " << s.ToString() << " possible_offset " << possible_offset.ToString() << " target_index " << target_index; } return FindLogicOffsetBySearchingBinlog(start_offset, target_index, found_offset); @@ -729,7 +644,7 @@ Status ConsensusCoordinator::GetLogsBefore(const BinlogOffset& start_offset, std Status ConsensusCoordinator::LeaderNegotiate(const LogOffset& f_last_offset, bool* reject, std::vector* hints) { uint64_t f_index = f_last_offset.l_offset.index; - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "LeaderNeotiate follower last offset " + LOG(INFO) << DBInfo(db_name_).ToString() << "LeaderNeotiate follower last offset " << f_last_offset.ToString() << " first_offsert " << stable_logger_->first_offset().ToString() << " last_offset " << mem_logger_->last_offset().ToString(); *reject = true; @@ -741,14 +656,14 @@ Status ConsensusCoordinator::LeaderNegotiate(const LogOffset& f_last_offset, boo << " get logs before last index failed " << s.ToString(); return s; } - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() + LOG(INFO) << DBInfo(db_name_).ToString() << "follower index larger then last_offset index, get logs before " << mem_logger_->last_offset().ToString(); return Status::OK(); } if (f_index < stable_logger_->first_offset().l_offset.index) { // need full sync - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << f_index << " not found current first index" + LOG(INFO) << DBInfo(db_name_).ToString() << f_index << " not found current first index" << stable_logger_->first_offset().ToString(); return Status::NotFound("logic index"); } @@ -761,11 +676,11 @@ Status ConsensusCoordinator::LeaderNegotiate(const LogOffset& f_last_offset, boo Status s = FindLogicOffset(f_last_offset.b_offset, f_index, &found_offset); if (!s.ok()) { if (s.IsNotFound()) { - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << f_last_offset.ToString() << " not found " + LOG(INFO) << DBInfo(db_name_).ToString() << f_last_offset.ToString() << " not found " << s.ToString(); return s; } else { - LOG(WARNING) << SlotInfo(db_name_, slot_id_).ToString() << "find logic offset failed" + LOG(WARNING) << DBInfo(db_name_).ToString() << "find logic offset failed" << s.ToString(); return s; } @@ -774,14 +689,14 @@ Status ConsensusCoordinator::LeaderNegotiate(const LogOffset& f_last_offset, boo if (found_offset.l_offset.term != f_last_offset.l_offset.term || !(f_last_offset.b_offset == found_offset.b_offset)) { Status s = GetLogsBefore(found_offset.b_offset, hints); if (!s.ok()) { - LOG(WARNING) << SlotInfo(db_name_, slot_id_).ToString() << "Try to get logs before " + LOG(WARNING) << DBInfo(db_name_).ToString() << "Try to get logs before " << found_offset.ToString() << " failed"; return s; } return Status::OK(); } - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "Found equal offset " << found_offset.ToString(); + LOG(INFO) << DBInfo(db_name_).ToString() << "Found equal offset " << found_offset.ToString(); *reject = false; return Status::OK(); } @@ -791,7 +706,7 @@ Status ConsensusCoordinator::FollowerNegotiate(const std::vector& hin if (hints.empty()) { return Status::Corruption("hints empty"); } - LOG(INFO) << SlotInfo(db_name_, slot_id_).ToString() << "FollowerNegotiate from " << hints[0].ToString() + LOG(INFO) << DBInfo(db_name_).ToString() << "FollowerNegotiate from " << hints[0].ToString() << " to " << hints[hints.size() - 1].ToString(); if (mem_logger_->last_offset().l_offset.index < hints[0].l_offset.index) { *reply_offset = mem_logger_->last_offset(); diff --git a/src/pika_data_distribution.cc b/src/pika_data_distribution.cc index dae40ee6f8..49d6af125e 100644 --- a/src/pika_data_distribution.cc +++ b/src/pika_data_distribution.cc @@ -5,60 +5,7 @@ #include "include/pika_data_distribution.h" -#include - -const std::string kTagBegin = "{"; -const std::string kTagEnd = "}"; - void HashModulo::Init() {} -uint32_t HashModulo::Distribute(const std::string& str, uint32_t slot_num) { - return std::hash()(str) % slot_num; -} - -void Crc32::Init() { Crc32TableInit(IEEE_POLY); } - -void Crc32::Crc32TableInit(uint32_t poly) { - int i; - int j; - for (i = 0; i < 256; i++) { - uint32_t crc = i; - for (j = 0; j < 8; j++) { - if ((crc & 1) != 0U) { - crc = (crc >> 1) ^ poly; - } else { - crc = (crc >> 1); - } - } - crc32tab[i] = crc; - } -} - -uint32_t Crc32::Distribute(const std::string& str, uint32_t slot_num) { - std::string key = GetHashkey(str); - uint32_t crc = Crc32Update(0, key.data(), static_cast(key.size())); - assert(slot_num != 0); - return crc % slot_num; -} -uint32_t Crc32::Crc32Update(uint32_t crc, const char* buf, int len) { - int i; - crc = ~crc; - for (i = 0; i < len; i++) { - crc = crc32tab[static_cast(static_cast(crc) ^ buf[i])] ^ (crc >> 8); - } - return ~crc; -} -std::string GetHashkey(const std::string& key) { - auto beg = key.find_first_of(kTagBegin); - if (beg == std::string::npos) { - return key; - } - auto end = key.find_first_of(kTagEnd, beg + 1); - if (end == std::string::npos) { - return key; - } else { - return key.substr(beg + 1, end - beg - 1); - } -} diff --git a/src/pika_db.cc b/src/pika_db.cc index 677b0bc8aa..f6d3cfbadd 100644 --- a/src/pika_db.cc +++ b/src/pika_db.cc @@ -3,6 +3,7 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. +#include #include #include "include/pika_db.h" @@ -10,6 +11,7 @@ #include "include/pika_cmd_table_manager.h" #include "include/pika_rm.h" #include "include/pika_server.h" +#include "mutex_impl.h" using pstd::Status; extern PikaServer* g_pika_server; @@ -22,108 +24,57 @@ std::string DBPath(const std::string& path, const std::string& db_name) { return path + buf; } -DB::DB(std::string db_name, uint32_t slot_num, const std::string& db_path, +std::string DbSyncPath(const std::string& sync_path, const std::string& db_name) { + char buf[256]; + snprintf(buf, sizeof(buf), "%s/", db_name.data()); + return sync_path + buf; +} + +DB::DB(std::string db_name, const std::string& db_path, const std::string& log_path) - : db_name_(std::move(db_name)), slot_num_(slot_num) { + : db_name_(db_name), bgsave_engine_(nullptr) { db_path_ = DBPath(db_path, db_name_); + bgsave_sub_path_ = db_name; + dbsync_path_ = DbSyncPath(g_pika_conf->db_sync_path(), db_name); log_path_ = DBPath(log_path, "log_" + db_name_); - + storage_ = std::make_shared(); + rocksdb::Status s = storage_->Open(g_pika_server->storage_options(), db_path_); pstd::CreatePath(db_path_); pstd::CreatePath(log_path_); - + lock_mgr_ = std::make_shared(1000, 0, std::make_shared()); binlog_io_error_.store(false); + opened_ = s.ok(); + assert(storage_); + assert(s.ok()); + LOG(INFO) << db_name_ << " DB Success"; } DB::~DB() { StopKeyScan(); - slots_.clear(); } std::string DB::GetDBName() { return db_name_; } void DB::BgSaveDB() { - std::shared_lock l(slots_rw_); - for (const auto& item : slots_) { - item.second->BgSaveSlot(); - } -} - -void DB::CompactDB(const storage::DataType& type) { - std::shared_lock l(slots_rw_); - for (const auto& item : slots_) { - item.second->Compact(type); - } -} - -bool DB::FlushSlotDB() { - std::shared_lock l(slots_rw_); - std::lock_guard ml(key_scan_protector_); - if (key_scan_info_.key_scaning_) { - return false; - } - for (const auto& item : slots_) { - item.second->FlushDB(); - } - return true; -} - -bool DB::FlushSlotSubDB(const std::string& db_name) { - std::shared_lock l(slots_rw_); - std::lock_guard ml(key_scan_protector_); - if (key_scan_info_.key_scaning_) { - return false; - } - for (const auto& item : slots_) { - item.second->FlushSubDB(db_name); + std::shared_lock l(dbs_rw_); + std::lock_guard ml(bgsave_protector_); + if (bgsave_info_.bgsaving) { + return; } - return true; + bgsave_info_.bgsaving = true; + auto bg_task_arg = new BgTaskArg(); + bg_task_arg->db = shared_from_this(); + g_pika_server->BGSaveTaskSchedule(&DoBgSave, static_cast(bg_task_arg)); } void DB::SetBinlogIoError() { return binlog_io_error_.store(true); } void DB::SetBinlogIoErrorrelieve() { return binlog_io_error_.store(false); } bool DB::IsBinlogIoError() { return binlog_io_error_.load(); } - -uint32_t DB::SlotNum() { return slot_num_; } - -Status DB::AddSlots(const std::set& slot_ids) { - std::lock_guard l(slots_rw_); - for (const uint32_t& id : slot_ids) { - if (id >= slot_num_) { - return Status::Corruption("slot index out of range[0, " + std::to_string(slot_num_ - 1) + "]"); - } else if (slots_.find(id) != slots_.end()) { - return Status::Corruption("slot " + std::to_string(id) + " already exist"); - } - } - - for (const uint32_t& id : slot_ids) { - slots_.emplace(id, std::make_shared(db_name_, id, db_path_)); - slots_[id]->Init(); - - } - return Status::OK(); -} - -Status DB::RemoveSlots(const std::set& slot_ids) { - std::lock_guard l(slots_rw_); - for (const uint32_t& id : slot_ids) { - if (slots_.find(id) == slots_.end()) { - return Status::Corruption("slot " + std::to_string(id) + " not found"); - } - } - - for (const uint32_t& id : slot_ids) { - slots_[id]->Leave(); - slots_.erase(id); - } - return Status::OK(); -} - -void DB::GetAllSlots(std::set& slot_ids) { - std::shared_lock l(slots_rw_); - for (const auto& iter : slots_) { - slot_ids.insert(iter.first); - } -} +std::shared_ptr DB::LockMgr() { return lock_mgr_; } +void DB::DbRWLockReader() { db_rwlock_.lock_shared(); } +void DB::DbRWUnLock() { db_rwlock_.unlock(); } +std::shared_ptr DB::cache() const { return cache_; } +std::shared_ptr DB::storage() const { return storage_; } void DB::KeyScan() { std::lock_guard ml(key_scan_protector_); @@ -149,19 +100,15 @@ void DB::RunKeyScan() { std::vector new_key_infos(5); InitKeyScan(); - std::shared_lock l(slots_rw_); - for (const auto& item : slots_) { - std::vector tmp_key_infos; - s = item.second->GetKeyNum(&tmp_key_infos); - if (s.ok()) { - for (size_t idx = 0; idx < tmp_key_infos.size(); ++idx) { - new_key_infos[idx].keys += tmp_key_infos[idx].keys; - new_key_infos[idx].expires += tmp_key_infos[idx].expires; - new_key_infos[idx].avg_ttl += tmp_key_infos[idx].avg_ttl; - new_key_infos[idx].invaild_keys += tmp_key_infos[idx].invaild_keys; - } - } else { - break; + std::shared_lock l(dbs_rw_); + std::vector tmp_key_infos; + s = GetKeyNum(&tmp_key_infos); + if (s.ok()) { + for (size_t idx = 0; idx < tmp_key_infos.size(); ++idx) { + new_key_infos[idx].keys += tmp_key_infos[idx].keys; + new_key_infos[idx].expires += tmp_key_infos[idx].expires; + new_key_infos[idx].avg_ttl += tmp_key_infos[idx].avg_ttl; + new_key_infos[idx].invaild_keys += tmp_key_infos[idx].invaild_keys; } } key_scan_info_.duration = static_cast(time(nullptr) - key_scan_info_.start_time); @@ -173,33 +120,40 @@ void DB::RunKeyScan() { key_scan_info_.key_scaning_ = false; } +Status DB::GetKeyNum(std::vector* key_info) { + std::lock_guard l(key_info_protector_); + if (key_scan_info_.key_scaning_) { + *key_info = key_scan_info_.key_infos; + return Status::OK(); + } + InitKeyScan(); + key_scan_info_.key_scaning_ = true; + key_scan_info_.duration = -2; // duration -2 mean the task in waiting status, + // has not been scheduled for exec + rocksdb::Status s = storage_->GetKeyNum(key_info); + key_scan_info_.key_scaning_ = false; + if (!s.ok()) { + return Status::Corruption(s.ToString()); + } + key_scan_info_.key_infos = *key_info; + key_scan_info_.duration = static_cast(time(nullptr) - key_scan_info_.start_time); + return Status::OK(); +} + void DB::StopKeyScan() { - std::shared_lock rwl(slots_rw_); + std::shared_lock rwl(dbs_rw_); std::lock_guard ml(key_scan_protector_); if (!key_scan_info_.key_scaning_) { return; } - for (const auto& item : slots_) { - item.second->db()->StopScanKeyNum(); - } + storage_->StopScanKeyNum(); key_scan_info_.key_scaning_ = false; } void DB::ScanDatabase(const storage::DataType& type) { - std::shared_lock l(slots_rw_); - for (const auto& item : slots_) { - printf("\n\nslot name : %s\n", item.second->GetSlotName().c_str()); - item.second->db()->ScanDatabase(type); - } -} - -Status DB::GetSlotsKeyScanInfo(std::map* infos) { - std::shared_lock l(slots_rw_); - for (const auto& [id, slot] : slots_) { - (*infos)[id] = slot->GetKeyScanInfo(); - } - return Status::OK(); + std::shared_lock l(dbs_rw_); + storage_->ScanDatabase(type); } KeyScanInfo DB::GetKeyScanInfo() { @@ -208,17 +162,19 @@ KeyScanInfo DB::GetKeyScanInfo() { } void DB::Compact(const storage::DataType& type) { - std::lock_guard rwl(slots_rw_); - for (const auto& item : slots_) { - item.second->Compact(type); + std::lock_guard rwl(dbs_rw_); + if (!opened_) { + return; } + storage_->Compact(type); } void DB::CompactRange(const storage::DataType& type, const std::string& start, const std::string& end) { - std::lock_guard rwl(slots_rw_); - for (const auto& item : slots_) { - item.second->CompactRange(type, start, end); + std::lock_guard rwl(dbs_rw_); + if (!opened_) { + return; } + storage_->CompactRange(type, start, end); } void DB::DoKeyScan(void* arg) { @@ -234,60 +190,442 @@ void DB::InitKeyScan() { key_scan_info_.duration = -1; // duration -1 mean the task in processing } -void DB::LeaveAllSlot() { - std::lock_guard l(slots_rw_); - for (const auto& item : slots_) { - item.second->Leave(); +void DB::DbRWLockWriter() { db_rwlock_.lock(); } + +DisplayCacheInfo DB::GetCacheInfo() { + std::lock_guard l(key_info_protector_); + return cache_info_; +} + +bool DB::FlushDBWithoutLock() { + if (bgsave_info_.bgsaving) { + return false; + } + + LOG(INFO) << db_name_ << " Delete old db..."; + storage_.reset(); + + std::string dbpath = db_path_; + if (dbpath[dbpath.length() - 1] == '/') { + dbpath.erase(dbpath.length() - 1); + } + dbpath.append("_deleting/"); + pstd::RenameFile(db_path_, dbpath); + + storage_ = std::make_shared(); + rocksdb::Status s = storage_->Open(g_pika_server->storage_options(), db_path_); + assert(storage_); + assert(s.ok()); + LOG(INFO) << db_name_ << " Open new db success"; + g_pika_server->PurgeDir(dbpath); + return true; +} + +bool DB::FlushSubDBWithoutLock(const std::string& db_name) { + std::lock_guard l(bgsave_protector_); + if (bgsave_info_.bgsaving) { + return false; + } + + LOG(INFO) << db_name_ << " Delete old " + db_name + " db..."; + storage_.reset(); + + std::string dbpath = db_path_; + if (dbpath[dbpath.length() - 1] != '/') { + dbpath.append("/"); + } + + std::string sub_dbpath = dbpath + db_name; + std::string del_dbpath = dbpath + db_name + "_deleting"; + pstd::RenameFile(sub_dbpath, del_dbpath); + + storage_ = std::make_shared(); + rocksdb::Status s = storage_->Open(g_pika_server->storage_options(), db_path_); + assert(storage_); + assert(s.ok()); + LOG(INFO) << db_name_ << " open new " + db_name + " db success"; + g_pika_server->PurgeDir(del_dbpath); + return true; +} + +void DB::DoBgSave(void* arg) { + std::unique_ptr bg_task_arg(static_cast(arg)); + + // Do BgSave + bool success = bg_task_arg->db->RunBgsaveEngine(); + + // Some output + BgSaveInfo info = bg_task_arg->db->bgsave_info(); + std::stringstream info_content; + std::ofstream out; + out.open(info.path + "/" + kBgsaveInfoFile, std::ios::in | std::ios::trunc); + if (out.is_open()) { + info_content << (time(nullptr) - info.start_time) << "s\n" + << g_pika_server->host() << "\n" + << g_pika_server->port() << "\n" + << info.offset.b_offset.filenum << "\n" + << info.offset.b_offset.offset << "\n"; + bg_task_arg->db->snapshot_uuid_ = md5(info_content.str()); + out << info_content.rdbuf(); + out.close(); + } + if (!success) { + std::string fail_path = info.path + "_FAILED"; + pstd::RenameFile(info.path, fail_path); } - slots_.clear(); + bg_task_arg->db->FinishBgsave(); } -std::set DB::GetSlotIDs() { - std::set ids; - std::shared_lock l(slots_rw_); - for (const auto& item : slots_) { - ids.insert(item.first); +bool DB::RunBgsaveEngine() { + // Prepare for Bgsaving + if (!InitBgsaveEnv() || !InitBgsaveEngine()) { + ClearBgsave(); + return false; + } + LOG(INFO) << db_name_ << " after prepare bgsave"; + + BgSaveInfo info = bgsave_info(); + LOG(INFO) << db_name_ << " bgsave_info: path=" << info.path << ", filenum=" << info.offset.b_offset.filenum + << ", offset=" << info.offset.b_offset.offset; + + // Backup to tmp dir + rocksdb::Status s = bgsave_engine_->CreateNewBackup(info.path); + + if (!s.ok()) { + LOG(WARNING) << db_name_ << " create new backup failed :" << s.ToString(); + return false; } - return ids; + LOG(INFO) << db_name_ << " create new backup finished."; + + return true; } -std::shared_ptr DB::GetSlotById(uint32_t slot_id) { - std::shared_lock l(slots_rw_); - auto iter = slots_.find(slot_id); - return (iter == slots_.end()) ? nullptr : iter->second; +BgSaveInfo DB::bgsave_info() { + std::lock_guard l(bgsave_protector_); + return bgsave_info_; } -std::shared_ptr DB::GetSlotByKey(const std::string& key) { - assert(slot_num_ != 0); - uint32_t index = g_pika_cmd_table_manager->DistributeKey(key, slot_num_); - std::shared_lock l(slots_rw_); - auto iter = slots_.find(index); - return (iter == slots_.end()) ? nullptr : iter->second; +void DB::FinishBgsave() { + std::lock_guard l(bgsave_protector_); + bgsave_info_.bgsaving = false; + g_pika_server->UpdateLastSave(time(nullptr)); } -bool DB::DBIsEmpty() { - std::shared_lock l(slots_rw_); - return slots_.empty(); +// Prepare engine, need bgsave_protector protect +bool DB::InitBgsaveEnv() { + std::lock_guard l(bgsave_protector_); + // Prepare for bgsave dir + bgsave_info_.start_time = time(nullptr); + char s_time[32]; + int len = static_cast(strftime(s_time, sizeof(s_time), "%Y%m%d%H%M%S", localtime(&bgsave_info_.start_time))); + bgsave_info_.s_start_time.assign(s_time, len); + std::string time_sub_path = g_pika_conf->bgsave_prefix() + std::string(s_time, 8); + bgsave_info_.path = g_pika_conf->bgsave_path() + time_sub_path + "/" + bgsave_sub_path_; + if (!pstd::DeleteDirIfExist(bgsave_info_.path)) { + LOG(WARNING) << db_name_ << " remove exist bgsave dir failed"; + return false; + } + pstd::CreatePath(bgsave_info_.path, 0755); + // Prepare for failed dir + if (!pstd::DeleteDirIfExist(bgsave_info_.path + "_FAILED")) { + LOG(WARNING) << db_name_ << " remove exist fail bgsave dir failed :"; + return false; + } + return true; } -Status DB::Leave() { - if (!DBIsEmpty()) { - return Status::Corruption("DB have slots!"); +// Prepare bgsave env, need bgsave_protector protect +bool DB::InitBgsaveEngine() { + bgsave_engine_.reset(); + rocksdb::Status s = storage::BackupEngine::Open(storage().get(), bgsave_engine_); + if (!s.ok()) { + LOG(WARNING) << db_name_ << " open backup engine failed " << s.ToString(); + return false; + } + + std::shared_ptr db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); + if (!db) { + LOG(WARNING) << db_name_ << " not found"; + return false; } - return MovetoToTrash(db_path_); + + { + std::lock_guard lock(db_rwlock_); + LogOffset bgsave_offset; + // term, index are 0 + db->Logger()->GetProducerStatus(&(bgsave_offset.b_offset.filenum), &(bgsave_offset.b_offset.offset)); + { + std::lock_guard l(bgsave_protector_); + bgsave_info_.offset = bgsave_offset; + } + s = bgsave_engine_->SetBackupContent(); + if (!s.ok()) { + LOG(WARNING) << db_name_ << " set backup content failed " << s.ToString(); + return false; + } + } + return true; } -Status DB::MovetoToTrash(const std::string& path) { - std::string path_tmp = path; - if (path_tmp[path_tmp.length() - 1] == '/') { - path_tmp.erase(path_tmp.length() - 1); +void DB::Init() { + cache_ = std::make_shared(g_pika_conf->zset_cache_start_pos(), g_pika_conf->zset_cache_field_num_per_key()); + // Create cache + cache::CacheConfig cache_cfg; + g_pika_server->CacheConfigInit(cache_cfg); + cache_->Init(g_pika_conf->GetCacheNum(), &cache_cfg); +} + +void DB::GetBgSaveMetaData(std::vector* fileNames, std::string* snapshot_uuid) { + const std::string dbPath = bgsave_info().path; + + std::string types[] = {storage::STRINGS_DB, storage::HASHES_DB, storage::LISTS_DB, storage::ZSETS_DB, storage::SETS_DB}; + for (const auto& type : types) { + std::string typePath = dbPath + ((dbPath.back() != '/') ? "/" : "") + type; + if (!pstd::FileExists(typePath)) { + continue ; + } + + std::vector tmpFileNames; + int ret = pstd::GetChildren(typePath, tmpFileNames); + if (ret) { + LOG(WARNING) << dbPath << " read dump meta files failed, path " << typePath; + return; + } + + for (const std::string fileName : tmpFileNames) { + fileNames -> push_back(type + "/" + fileName); + } } - path_tmp += "_deleting/"; - if (pstd::RenameFile(path, path_tmp) != 0) { - LOG(WARNING) << "Failed to move " << path << " to trash, error: " << strerror(errno); - return Status::Corruption("Failed to move %s to trash", path); + fileNames->push_back(kBgsaveInfoFile); + pstd::Status s = GetBgSaveUUID(snapshot_uuid); + if (!s.ok()) { + LOG(WARNING) << "read dump meta info failed! error:" << s.ToString(); + return; + } +} + +Status DB::GetBgSaveUUID(std::string* snapshot_uuid) { + if (snapshot_uuid_.empty()) { + std::string info_data; + const std::string infoPath = bgsave_info().path + "/info"; + //TODO: using file read function to replace rocksdb::ReadFileToString + rocksdb::Status s = rocksdb::ReadFileToString(rocksdb::Env::Default(), infoPath, &info_data); + if (!s.ok()) { + LOG(WARNING) << "read dump meta info failed! error:" << s.ToString(); + return Status::IOError("read dump meta info failed", infoPath); + } + pstd::MD5 md5 = pstd::MD5(info_data); + snapshot_uuid_ = md5.hexdigest(); } - g_pika_server->PurgeDir(path_tmp); - LOG(WARNING) << path << " move to trash success"; + *snapshot_uuid = snapshot_uuid_; return Status::OK(); } + +// Try to update master offset +// This may happend when dbsync from master finished +// Here we do: +// 1, Check dbsync finished, got the new binlog offset +// 2, Replace the old db +// 3, Update master offset, and the PikaAuxiliaryThread cron will connect and do slaveof task with master +bool DB::TryUpdateMasterOffset() { + std::string info_path = dbsync_path_ + kBgsaveInfoFile; + if (!pstd::FileExists(info_path)) { + LOG(WARNING) << "info path: " << info_path << " not exist"; + return false; + } + + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name_)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_name_ << " not exist"; + return false; + } + + // Got new binlog offset + std::ifstream is(info_path); + if (!is) { + LOG(WARNING) << "DB: " << db_name_ << ", Failed to open info file after db sync"; + slave_db->SetReplState(ReplState::kError); + return false; + } + std::string line; + std::string master_ip; + int lineno = 0; + int64_t filenum = 0; + int64_t offset = 0; + int64_t term = 0; + int64_t index = 0; + int64_t tmp = 0; + int64_t master_port = 0; + while (std::getline(is, line)) { + lineno++; + if (lineno == 2) { + master_ip = line; + } else if (lineno > 2 && lineno < 8) { + if ((pstd::string2int(line.data(), line.size(), &tmp) == 0) || tmp < 0) { + LOG(WARNING) << "DB: " << db_name_ + << ", Format of info file after db sync error, line : " << line; + is.close(); + slave_db->SetReplState(ReplState::kError); + return false; + } + if (lineno == 3) { + master_port = tmp; + } else if (lineno == 4) { + filenum = tmp; + } else if (lineno == 5) { + offset = tmp; + } else if (lineno == 6) { + term = tmp; + } else if (lineno == 7) { + index = tmp; + } + } else if (lineno > 8) { + LOG(WARNING) << "DB: " << db_name_ << ", Format of info file after db sync error, line : " << line; + is.close(); + slave_db->SetReplState(ReplState::kError); + return false; + } + } + is.close(); + + LOG(INFO) << "DB: " << db_name_ << " Information from dbsync info" + << ", master_ip: " << master_ip << ", master_port: " << master_port << ", filenum: " << filenum + << ", offset: " << offset << ", term: " << term << ", index: " << index; + + pstd::DeleteFile(info_path); + if (!ChangeDb(dbsync_path_)) { + LOG(WARNING) << "DB: " << db_name_ << ", Failed to change db"; + slave_db->SetReplState(ReplState::kError); + return false; + } + + // Update master offset + std::shared_ptr master_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); + if (!master_db) { + LOG(WARNING) << "Master DB: " << db_name_ << " not exist"; + return false; + } + master_db->Logger()->SetProducerStatus(filenum, offset); + slave_db->SetReplState(ReplState::kTryConnect); + return true; +} + +void DB::PrepareRsync() { + pstd::DeleteDirIfExist(dbsync_path_); + pstd::CreatePath(dbsync_path_ + "strings"); + pstd::CreatePath(dbsync_path_ + "hashes"); + pstd::CreatePath(dbsync_path_ + "lists"); + pstd::CreatePath(dbsync_path_ + "sets"); + pstd::CreatePath(dbsync_path_ + "zsets"); +} + +bool DB::IsBgSaving() { + std::lock_guard ml(bgsave_protector_); + return bgsave_info_.bgsaving; +} + +/* + * Change a new db locate in new_path + * return true when change success + * db remain the old one if return false + */ +bool DB::ChangeDb(const std::string& new_path) { + std::string tmp_path(db_path_); + if (tmp_path.back() == '/') { + tmp_path.resize(tmp_path.size() - 1); + } + tmp_path += "_bak"; + pstd::DeleteDirIfExist(tmp_path); + + std::lock_guard l(db_rwlock_); + LOG(INFO) << "DB: " << db_name_ << ", Prepare change db from: " << tmp_path; + storage_.reset(); + + if (0 != pstd::RenameFile(db_path_, tmp_path)) { + LOG(WARNING) << "DB: " << db_name_ + << ", Failed to rename db path when change db, error: " << strerror(errno); + return false; + } + + if (0 != pstd::RenameFile(new_path, db_path_)) { + LOG(WARNING) << "DB: " << db_name_ + << ", Failed to rename new db path when change db, error: " << strerror(errno); + return false; + } + + storage_ = std::make_shared(); + rocksdb::Status s = storage_->Open(g_pika_server->storage_options(), db_path_); + assert(storage_); + assert(s.ok()); + pstd::DeleteDirIfExist(tmp_path); + LOG(INFO) << "DB: " << db_name_ << ", Change db success"; + return true; +} + +void DB::ClearBgsave() { + std::lock_guard l(bgsave_protector_); + bgsave_info_.Clear(); +} + +bool DB::FlushSubDB(const std::string& db_name) { + std::lock_guard rwl(db_rwlock_); + return FlushSubDBWithoutLock(db_name); +} + +void DB::UpdateCacheInfo(CacheInfo& cache_info) { + std::unique_lock lock(cache_info_rwlock_); + + cache_info_.status = cache_info.status; + cache_info_.cache_num = cache_info.cache_num; + cache_info_.keys_num = cache_info.keys_num; + cache_info_.used_memory = cache_info.used_memory; + cache_info_.waitting_load_keys_num = cache_info.waitting_load_keys_num; + cache_usage_ = cache_info.used_memory; + + uint64_t all_cmds = cache_info.hits + cache_info.misses; + cache_info_.hitratio_all = (0 >= all_cmds) ? 0.0 : (cache_info.hits * 100.0) / all_cmds; + + uint64_t cur_time_us = pstd::NowMicros(); + uint64_t delta_time = cur_time_us - cache_info_.last_time_us + 1; + uint64_t delta_hits = cache_info.hits - cache_info_.hits; + cache_info_.hits_per_sec = delta_hits * 1000000 / delta_time; + + uint64_t delta_all_cmds = all_cmds - (cache_info_.hits + cache_info_.misses); + cache_info_.read_cmd_per_sec = delta_all_cmds * 1000000 / delta_time; + + cache_info_.hitratio_per_sec = (0 >= delta_all_cmds) ? 0.0 : (delta_hits * 100.0) / delta_all_cmds; + + uint64_t delta_load_keys = cache_info.async_load_keys_num - cache_info_.last_load_keys_num; + cache_info_.load_keys_per_sec = delta_load_keys * 1000000 / delta_time; + + cache_info_.hits = cache_info.hits; + cache_info_.misses = cache_info.misses; + cache_info_.last_time_us = cur_time_us; + cache_info_.last_load_keys_num = cache_info.async_load_keys_num; +} + +void DB::ResetDisplayCacheInfo(int status) { + std::unique_lock lock(cache_info_rwlock_); + cache_info_.status = status; + cache_info_.cache_num = 0; + cache_info_.keys_num = 0; + cache_info_.used_memory = 0; + cache_info_.hits = 0; + cache_info_.misses = 0; + cache_info_.hits_per_sec = 0; + cache_info_.read_cmd_per_sec = 0; + cache_info_.hitratio_per_sec = 0.0; + cache_info_.hitratio_all = 0.0; + cache_info_.load_keys_per_sec = 0; + cache_info_.waitting_load_keys_num = 0; + cache_usage_ = 0; +} + +bool DB::FlushDB() { + std::lock_guard rwl(db_rwlock_); + std::lock_guard l(bgsave_protector_); + return FlushDBWithoutLock(); +} \ No newline at end of file diff --git a/src/pika_geo.cc b/src/pika_geo.cc index d3a71ea4f5..82aafde8d8 100644 --- a/src/pika_geo.cc +++ b/src/pika_geo.cc @@ -42,7 +42,7 @@ void GeoAddCmd::DoInitial() { } } -void GeoAddCmd::Do(std::shared_ptr slot) { +void GeoAddCmd::Do() { std::vector score_members; for (const auto& geo_point : pos_) { // Convert coordinates to geohash @@ -56,13 +56,13 @@ void GeoAddCmd::Do(std::shared_ptr slot) { score_members.push_back({score, geo_point.member}); } int32_t count = 0; - rocksdb::Status s = slot->db()->ZAdd(key_, score_members, &count); + rocksdb::Status s = db_->storage()->ZAdd(key_, score_members, &count); if (s.ok()) { res_.AppendInteger(count); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } - } +} void GeoPosCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -77,11 +77,11 @@ void GeoPosCmd::DoInitial() { } } -void GeoPosCmd::Do(std::shared_ptr slot) { +void GeoPosCmd::Do() { double score = 0.0; res_.AppendArrayLenUint64(members_.size()); for (const auto& member : members_) { - rocksdb::Status s = slot->db()->ZScore(key_, member, &score); + rocksdb::Status s = db_->storage()->ZScore(key_, member, &score); if (s.ok()) { double xy[2]; GeoHashBits hash = {.bits = static_cast(score), .step = GEO_STEP_MAX}; @@ -152,12 +152,12 @@ void GeoDistCmd::DoInitial() { } } -void GeoDistCmd::Do(std::shared_ptr slot) { +void GeoDistCmd::Do() { double first_score = 0.0; double second_score = 0.0; double first_xy[2]; double second_xy[2]; - rocksdb::Status s = slot->db()->ZScore(key_, first_pos_, &first_score); + rocksdb::Status s = db_->storage()->ZScore(key_, first_pos_, &first_score); if (s.ok()) { GeoHashBits hash = {.bits = static_cast(first_score), .step = GEO_STEP_MAX}; geohashDecodeToLongLatWGS84(hash, first_xy); @@ -169,7 +169,7 @@ void GeoDistCmd::Do(std::shared_ptr slot) { return; } - s = slot->db()->ZScore(key_, second_pos_, &second_score); + s = db_->storage()->ZScore(key_, second_pos_, &second_score); if (s.ok()) { GeoHashBits hash = {.bits = static_cast(second_score), .step = GEO_STEP_MAX}; geohashDecodeToLongLatWGS84(hash, second_xy); @@ -202,12 +202,12 @@ void GeoHashCmd::DoInitial() { } } -void GeoHashCmd::Do(std::shared_ptr slot) { +void GeoHashCmd::Do() { const char* geoalphabet = "0123456789bcdefghjkmnpqrstuvwxyz"; res_.AppendArrayLenUint64(members_.size()); for (const auto& member : members_) { double score = 0.0; - rocksdb::Status s = slot->db()->ZScore(key_, member, &score); + rocksdb::Status s = db_->storage()->ZScore(key_, member, &score); if (s.ok()) { double xy[2]; GeoHashBits hash = {.bits = static_cast(score), .step = GEO_STEP_MAX}; @@ -248,7 +248,7 @@ static bool sort_distance_desc(const NeighborPoint& pos1, const NeighborPoint& p return pos1.distance > pos2.distance; } -static void GetAllNeighbors(const std::shared_ptr& slot, std::string& key, GeoRange& range, CmdRes& res) { +static void GetAllNeighbors(const std::shared_ptr& db, std::string& key, GeoRange& range, CmdRes& res) { rocksdb::Status s; double longitude = range.longitude; double latitude = range.latitude; @@ -299,7 +299,7 @@ static void GetAllNeighbors(const std::shared_ptr& slot, std::string& key, continue; } std::vector score_members; - s = slot->db()->ZRangebyscore(key, static_cast(min), static_cast(max), true, true, &score_members); + s = db->storage()->ZRangebyscore(key, static_cast(min), static_cast(max), true, true, &score_members); if (!s.ok() && !s.IsNotFound()) { res.SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -343,7 +343,7 @@ static void GetAllNeighbors(const std::shared_ptr& slot, std::string& key, score_members.push_back({score, result[i].member}); } int32_t count = 0; - s = slot->db()->ZAdd(range.storekey, score_members, &count); + s = db->storage()->ZAdd(range.storekey, score_members, &count); if (!s.ok()) { res.SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -470,7 +470,7 @@ void GeoRadiusCmd::DoInitial() { } } -void GeoRadiusCmd::Do(std::shared_ptr slot) { GetAllNeighbors(slot, key_, range_, this->res_); } +void GeoRadiusCmd::Do() { GetAllNeighbors(db_, key_, range_, this->res_); } void GeoRadiusByMemberCmd::DoInitial() { if (!CheckArg(argv_.size())) { @@ -541,9 +541,9 @@ void GeoRadiusByMemberCmd::DoInitial() { } } -void GeoRadiusByMemberCmd::Do(std::shared_ptr slot) { +void GeoRadiusByMemberCmd::Do() { double score = 0.0; - rocksdb::Status s = slot->db()->ZScore(key_, range_.member, &score); + rocksdb::Status s = db_->storage()->ZScore(key_, range_.member, &score); if (s.ok()) { double xy[2]; GeoHashBits hash = {.bits = static_cast(score), .step = GEO_STEP_MAX}; @@ -551,5 +551,5 @@ void GeoRadiusByMemberCmd::Do(std::shared_ptr slot) { range_.longitude = xy[0]; range_.latitude = xy[1]; } - GetAllNeighbors(slot, key_, range_, this->res_); + GetAllNeighbors(db_, key_, range_, this->res_); } diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 4e5cb1e362..976ed5ae5b 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -26,8 +26,8 @@ void HDelCmd::DoInitial() { fields_.assign(iter, argv_.end()); } -void HDelCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->HDel(key_, fields_, &deleted_); +void HDelCmd::Do() { + s_ = db_->storage()->HDel(key_, fields_, &deleted_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(deleted_); } else { @@ -35,14 +35,14 @@ void HDelCmd::Do(std::shared_ptr slot) { } } -void HDelCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HDelCmd::DoThroughDB() { + Do(); } -void HDelCmd::DoUpdateCache(std::shared_ptr slot) { +void HDelCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HDel(CachePrefixKeyH, fields_); + db_->cache()->HDel(CachePrefixKeyH, fields_); } } @@ -63,25 +63,25 @@ void HSetCmd::DoInitial() { } } -void HSetCmd::Do(std::shared_ptr slot) { +void HSetCmd::Do() { int32_t ret = 0; - s_ = slot->db()->HSet(key_, field_, value_, &ret); + s_ = db_->storage()->HSet(key_, field_, value_, &ret); if (s_.ok()) { res_.AppendContent(":" + std::to_string(ret)); - AddSlotKey("h", key_, slot); + AddSlotKey("h", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void HSetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HSetCmd::DoThroughDB() { + Do(); } -void HSetCmd::DoUpdateCache(std::shared_ptr slot) { +void HSetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); + db_->cache()->HSetIfKeyExist(CachePrefixKeyH, field_, value_); } } @@ -94,9 +94,9 @@ void HGetCmd::DoInitial() { field_ = argv_[2]; } -void HGetCmd::Do(std::shared_ptr slot) { +void HGetCmd::Do() { std::string value; - s_ = slot->db()->HGet(key_, field_, &value); + s_ = db_->storage()->HGet(key_, field_, &value); if (s_.ok()) { res_.AppendStringLenUint64(value.size()); res_.AppendContent(value); @@ -107,10 +107,10 @@ void HGetCmd::Do(std::shared_ptr slot) { } } -void HGetCmd::ReadCache(std::shared_ptr slot) { +void HGetCmd::ReadCache() { std::string value; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HGet(CachePrefixKeyH, field_, &value); + auto s = db_->cache()->HGet(CachePrefixKeyH, field_, &value); if (s.ok()) { res_.AppendStringLen(value.size()); res_.AppendContent(value); @@ -121,14 +121,14 @@ void HGetCmd::ReadCache(std::shared_ptr slot) { } } -void HGetCmd::DoThroughDB(std::shared_ptr slot) { +void HGetCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HGetCmd::DoUpdateCache(std::shared_ptr slot) { +void HGetCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -140,7 +140,7 @@ void HGetallCmd::DoInitial() { key_ = argv_[1]; } -void HGetallCmd::Do(std::shared_ptr slot) { +void HGetallCmd::Do() { int64_t total_fv = 0; int64_t cursor = 0; int64_t next_cursor = 0; @@ -150,7 +150,7 @@ void HGetallCmd::Do(std::shared_ptr slot) { do { fvs.clear(); - s_ = slot->db()->HScan(key_, cursor, "*", PIKA_SCAN_STEP_LENGTH, &fvs, &next_cursor); + s_ = db_->storage()->HScan(key_, cursor, "*", PIKA_SCAN_STEP_LENGTH, &fvs, &next_cursor); if (!s_.ok()) { raw.clear(); total_fv = 0; @@ -179,10 +179,10 @@ void HGetallCmd::Do(std::shared_ptr slot) { } } -void HGetallCmd::ReadCache(std::shared_ptr slot) { +void HGetallCmd::ReadCache() { std::vector fvs; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HGetall(CachePrefixKeyH, &fvs); + auto s = db_->cache()->HGetall(CachePrefixKeyH, &fvs); if (s.ok()) { res_.AppendArrayLen(fvs.size() * 2); for (const auto& fv : fvs) { @@ -198,14 +198,14 @@ void HGetallCmd::ReadCache(std::shared_ptr slot) { } } -void HGetallCmd::DoThroughDB(std::shared_ptr slot) { +void HGetallCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HGetallCmd::DoUpdateCache(std::shared_ptr slot) { +void HGetallCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -218,8 +218,8 @@ void HExistsCmd::DoInitial() { field_ = argv_[2]; } -void HExistsCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->HExists(key_, field_); +void HExistsCmd::Do() { + s_ = db_->storage()->HExists(key_, field_); if (s_.ok()) { res_.AppendContent(":1"); } else if (s_.IsNotFound()) { @@ -229,9 +229,9 @@ void HExistsCmd::Do(std::shared_ptr slot) { } } -void HExistsCmd::ReadCache(std::shared_ptr slot) { +void HExistsCmd::ReadCache() { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HExists(CachePrefixKeyH, field_); + auto s = db_->cache()->HExists(CachePrefixKeyH, field_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -241,14 +241,14 @@ void HExistsCmd::ReadCache(std::shared_ptr slot) { } } -void HExistsCmd::DoThroughDB(std::shared_ptr slot) { +void HExistsCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HExistsCmd::DoUpdateCache(std::shared_ptr slot) { +void HExistsCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -265,12 +265,12 @@ void HIncrbyCmd::DoInitial() { } } -void HIncrbyCmd::Do(std::shared_ptr slot) { +void HIncrbyCmd::Do() { int64_t new_value = 0; - s_ = slot->db()->HIncrby(key_, field_, by_, &new_value); + s_ = db_->storage()->HIncrby(key_, field_, by_, &new_value); if (s_.ok() || s_.IsNotFound()) { res_.AppendContent(":" + std::to_string(new_value)); - AddSlotKey("h", key_, slot); + AddSlotKey("h", key_, db_); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: hash value is not an integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s_.IsInvalidArgument()) { @@ -280,14 +280,14 @@ void HIncrbyCmd::Do(std::shared_ptr slot) { } } -void HIncrbyCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HIncrbyCmd::DoThroughDB() { + Do(); } -void HIncrbyCmd::DoUpdateCache(std::shared_ptr slot) { +void HIncrbyCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HIncrbyxx(CachePrefixKeyH, field_, by_); + db_->cache()->HIncrbyxx(CachePrefixKeyH, field_, by_); } } @@ -301,13 +301,13 @@ void HIncrbyfloatCmd::DoInitial() { by_ = argv_[3]; } -void HIncrbyfloatCmd::Do(std::shared_ptr slot) { +void HIncrbyfloatCmd::Do() { std::string new_value; - s_ = slot->db()->HIncrbyfloat(key_, field_, by_, &new_value); + s_ = db_->storage()->HIncrbyfloat(key_, field_, by_, &new_value); if (s_.ok()) { res_.AppendStringLenUint64(new_value.size()); res_.AppendContent(new_value); - AddSlotKey("h", key_, slot); + AddSlotKey("h", key_, db_); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: value is not a vaild float") { res_.SetRes(CmdRes::kInvalidFloat); } else if (s_.IsInvalidArgument()) { @@ -317,16 +317,16 @@ void HIncrbyfloatCmd::Do(std::shared_ptr slot) { } } -void HIncrbyfloatCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HIncrbyfloatCmd::DoThroughDB() { + Do(); } -void HIncrbyfloatCmd::DoUpdateCache(std::shared_ptr slot) { +void HIncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(by_.data(), by_.size(), &long_double_by) != -1) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HIncrbyfloatxx(CachePrefixKeyH, field_, long_double_by); + db_->cache()->HIncrbyfloatxx(CachePrefixKeyH, field_, long_double_by); } } } @@ -339,9 +339,9 @@ void HKeysCmd::DoInitial() { key_ = argv_[1]; } -void HKeysCmd::Do(std::shared_ptr slot) { +void HKeysCmd::Do() { std::vector fields; - s_ = slot->db()->HKeys(key_, &fields); + s_ = db_->storage()->HKeys(key_, &fields); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(fields.size()); for (const auto& field : fields) { @@ -352,10 +352,10 @@ void HKeysCmd::Do(std::shared_ptr slot) { } } -void HKeysCmd::ReadCache(std::shared_ptr slot) { +void HKeysCmd::ReadCache() { std::vector fields; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HKeys(CachePrefixKeyH, &fields); + auto s = db_->cache()->HKeys(CachePrefixKeyH, &fields); if (s.ok()) { res_.AppendArrayLen(fields.size()); for (const auto& field : fields) { @@ -368,14 +368,14 @@ void HKeysCmd::ReadCache(std::shared_ptr slot) { } } -void HKeysCmd::DoThroughDB(std::shared_ptr slot) { +void HKeysCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HKeysCmd::DoUpdateCache(std::shared_ptr slot) { +void HKeysCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -387,9 +387,9 @@ void HLenCmd::DoInitial() { key_ = argv_[1]; } -void HLenCmd::Do(std::shared_ptr slot) { +void HLenCmd::Do() { int32_t len = 0; - s_ = slot->db()->HLen(key_, &len); + s_ = db_->storage()->HLen(key_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); } else { @@ -397,10 +397,10 @@ void HLenCmd::Do(std::shared_ptr slot) { } } -void HLenCmd::ReadCache(std::shared_ptr slot) { +void HLenCmd::ReadCache() { uint64_t len = 0; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HLen(CachePrefixKeyH, &len); + auto s = db_->cache()->HLen(CachePrefixKeyH, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -410,14 +410,14 @@ void HLenCmd::ReadCache(std::shared_ptr slot) { } } -void HLenCmd::DoThroughDB(std::shared_ptr slot) { +void HLenCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HLenCmd::DoUpdateCache(std::shared_ptr slot) { +void HLenCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -433,9 +433,9 @@ void HMgetCmd::DoInitial() { fields_.assign(iter, argv_.end()); } -void HMgetCmd::Do(std::shared_ptr slot) { +void HMgetCmd::Do() { std::vector vss; - s_ = slot->db()->HMGet(key_, fields_, &vss); + s_ = db_->storage()->HMGet(key_, fields_, &vss); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(vss.size()); for (const auto& vs : vss) { @@ -451,10 +451,10 @@ void HMgetCmd::Do(std::shared_ptr slot) { } } -void HMgetCmd::ReadCache(std::shared_ptr slot) { +void HMgetCmd::ReadCache() { std::vector vss; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HMGet(CachePrefixKeyH, fields_, &vss); + auto s = db_->cache()->HMGet(CachePrefixKeyH, fields_, &vss); if (s.ok()) { res_.AppendArrayLen(vss.size()); for (const auto& vs : vss) { @@ -472,14 +472,14 @@ void HMgetCmd::ReadCache(std::shared_ptr slot) { } } -void HMgetCmd::DoThroughDB(std::shared_ptr slot) { +void HMgetCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HMgetCmd::DoUpdateCache(std::shared_ptr slot) { +void HMgetCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -501,24 +501,24 @@ void HMsetCmd::DoInitial() { } } -void HMsetCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->HMSet(key_, fvs_); +void HMsetCmd::Do() { + s_ = db_->storage()->HMSet(key_, fvs_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); - AddSlotKey("h", key_, slot); + AddSlotKey("h", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void HMsetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HMsetCmd::DoThroughDB() { + Do(); } -void HMsetCmd::DoUpdateCache(std::shared_ptr slot) { +void HMsetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HMSetxx(CachePrefixKeyH, fvs_); + db_->cache()->HMSetxx(CachePrefixKeyH, fvs_); } } @@ -532,25 +532,25 @@ void HSetnxCmd::DoInitial() { value_ = argv_[3]; } -void HSetnxCmd::Do(std::shared_ptr slot) { +void HSetnxCmd::Do() { int32_t ret = 0; - s_ = slot->db()->HSetnx(key_, field_, value_, &ret); + s_ = db_->storage()->HSetnx(key_, field_, value_, &ret); if (s_.ok()) { res_.AppendContent(":" + std::to_string(ret)); - AddSlotKey("h", key_, slot); + AddSlotKey("h", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void HSetnxCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void HSetnxCmd::DoThroughDB() { + Do(); } -void HSetnxCmd::DoUpdateCache(std::shared_ptr slot) { +void HSetnxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - slot->cache()->HSetIfKeyExistAndFieldNotExist(CachePrefixKeyH, field_, value_); + db_->cache()->HSetIfKeyExistAndFieldNotExist(CachePrefixKeyH, field_, value_); } } @@ -563,9 +563,9 @@ void HStrlenCmd::DoInitial() { field_ = argv_[2]; } -void HStrlenCmd::Do(std::shared_ptr slot) { +void HStrlenCmd::Do() { int32_t len = 0; - s_ = slot->db()->HStrlen(key_, field_, &len); + s_ = db_->storage()->HStrlen(key_, field_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); } else { @@ -573,10 +573,10 @@ void HStrlenCmd::Do(std::shared_ptr slot) { } } -void HStrlenCmd::ReadCache(std::shared_ptr slot) { +void HStrlenCmd::ReadCache() { uint64_t len = 0; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HStrlen(CachePrefixKeyH, field_, &len); + auto s = db_->cache()->HStrlen(CachePrefixKeyH, field_, &len); if (s.ok()) { res_.AppendInteger(len); } else if (s.IsNotFound()) { @@ -587,14 +587,14 @@ void HStrlenCmd::ReadCache(std::shared_ptr slot) { return; } -void HStrlenCmd::DoThroughDB(std::shared_ptr slot) { +void HStrlenCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HStrlenCmd::DoUpdateCache(std::shared_ptr slot) { +void HStrlenCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -606,9 +606,9 @@ void HValsCmd::DoInitial() { key_ = argv_[1]; } -void HValsCmd::Do(std::shared_ptr slot) { +void HValsCmd::Do() { std::vector values; - s_ = slot->db()->HVals(key_, &values); + s_ = db_->storage()->HVals(key_, &values); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(values.size()); for (const auto& value : values) { @@ -620,10 +620,10 @@ void HValsCmd::Do(std::shared_ptr slot) { } } -void HValsCmd::ReadCache(std::shared_ptr slot) { +void HValsCmd::ReadCache() { std::vector values; std::string CachePrefixKeyH = PCacheKeyPrefixH + key_; - auto s = slot->cache()->HVals(CachePrefixKeyH, &values); + auto s = db_->cache()->HVals(CachePrefixKeyH, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { @@ -637,14 +637,14 @@ void HValsCmd::ReadCache(std::shared_ptr slot) { } } -void HValsCmd::DoThroughDB(std::shared_ptr slot) { +void HValsCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void HValsCmd::DoUpdateCache(std::shared_ptr slot) { +void HValsCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_HASH, key_, db_); } } @@ -687,10 +687,10 @@ void HScanCmd::DoInitial() { } } -void HScanCmd::Do(std::shared_ptr slot) { +void HScanCmd::Do() { int64_t next_cursor = 0; std::vector field_values; - auto s = slot->db()->HScan(key_, cursor_, pattern_, count_, &field_values, &next_cursor); + auto s = db_->storage()->HScan(key_, cursor_, pattern_, count_, &field_values, &next_cursor); if (s.ok() || s.IsNotFound()) { res_.AppendContent("*2"); @@ -745,10 +745,10 @@ void HScanxCmd::DoInitial() { } } -void HScanxCmd::Do(std::shared_ptr slot) { +void HScanxCmd::Do() { std::string next_field; std::vector field_values; - rocksdb::Status s = slot->db()->HScanx(key_, start_field_, pattern_, count_, &field_values, &next_field); + rocksdb::Status s = db_->storage()->HScanx(key_, start_field_, pattern_, count_, &field_values, &next_field); if (s.ok() || s.IsNotFound()) { res_.AppendArrayLen(2); @@ -798,11 +798,11 @@ void PKHScanRangeCmd::DoInitial() { } } -void PKHScanRangeCmd::Do(std::shared_ptr slot) { +void PKHScanRangeCmd::Do() { std::string next_field; std::vector field_values; rocksdb::Status s = - slot->db()->PKHScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); + db_->storage()->PKHScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); if (s.ok() || s.IsNotFound()) { res_.AppendArrayLen(2); @@ -851,11 +851,11 @@ void PKHRScanRangeCmd::DoInitial() { } } -void PKHRScanRangeCmd::Do(std::shared_ptr slot) { +void PKHRScanRangeCmd::Do() { std::string next_field; std::vector field_values; rocksdb::Status s = - slot->db()->PKHRScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); + db_->storage()->PKHRScanRange(key_, field_start_, field_end_, pattern_, static_cast(limit_), &field_values, &next_field); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLen(2); diff --git a/src/pika_hyperloglog.cc b/src/pika_hyperloglog.cc index 0cb6615a73..ac3c0c12aa 100644 --- a/src/pika_hyperloglog.cc +++ b/src/pika_hyperloglog.cc @@ -19,9 +19,9 @@ void PfAddCmd::DoInitial() { } } -void PfAddCmd::Do(std::shared_ptr slot) { +void PfAddCmd::Do() { bool update = false; - rocksdb::Status s = slot->db()->PfAdd(key_, values_, &update); + rocksdb::Status s = db_->storage()->PfAdd(key_, values_, &update); if (s.ok() && update) { res_.AppendInteger(1); } else if (s.ok() && !update) { @@ -42,9 +42,9 @@ void PfCountCmd::DoInitial() { } } -void PfCountCmd::Do(std::shared_ptr slot) { +void PfCountCmd::Do() { int64_t value_ = 0; - rocksdb::Status s = slot->db()->PfCount(keys_, &value_); + rocksdb::Status s = db_->storage()->PfCount(keys_, &value_); if (s.ok()) { res_.AppendInteger(value_); } else { @@ -63,15 +63,15 @@ void PfMergeCmd::DoInitial() { } } -void PfMergeCmd::Do(std::shared_ptr slot) { - rocksdb::Status s = slot->db()->PfMerge(keys_, value_to_dest_); +void PfMergeCmd::Do() { + rocksdb::Status s = db_->storage()->PfMerge(keys_, value_to_dest_); if (s.ok()) { res_.SetRes(CmdRes::kOk); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } } -void PfMergeCmd::DoBinlog(const std::shared_ptr& slot) { +void PfMergeCmd::DoBinlog() { PikaCmdArgsType set_args; //used "set" instead of "SET" to distinguish the binlog of SetCmd set_args.emplace_back("set"); @@ -81,5 +81,5 @@ void PfMergeCmd::DoBinlog(const std::shared_ptr& slot) { set_cmd_->SetConn(GetConn()); set_cmd_->SetResp(resp_.lock()); //value of this binlog might be strange, it's an string with size of 128KB - set_cmd_->DoBinlog(slot); + set_cmd_->DoBinlog(); } diff --git a/src/pika_kv.cc b/src/pika_kv.cc index ab22ea625c..c294cfb214 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -65,23 +65,23 @@ void SetCmd::DoInitial() { } } -void SetCmd::Do(std::shared_ptr slot) { +void SetCmd::Do() { int32_t res = 1; switch (condition_) { case SetCmd::kXX: - s_ = slot->db()->Setxx(key_, value_, &res, static_cast(sec_)); + s_ = db_->storage()->Setxx(key_, value_, &res, static_cast(sec_)); break; case SetCmd::kNX: - s_ = slot->db()->Setnx(key_, value_, &res, static_cast(sec_)); + s_ = db_->storage()->Setnx(key_, value_, &res, static_cast(sec_)); break; case SetCmd::kVX: - s_ = slot->db()->Setvx(key_, target_, value_, &success_, static_cast(sec_)); + s_ = db_->storage()->Setvx(key_, target_, value_, &success_, static_cast(sec_)); break; case SetCmd::kEXORPX: - s_ = slot->db()->Setex(key_, value_, static_cast(sec_)); + s_ = db_->storage()->Setex(key_, value_, static_cast(sec_)); break; default: - s_ = slot->db()->Set(key_, value_); + s_ = db_->storage()->Set(key_, value_); break; } @@ -91,7 +91,7 @@ void SetCmd::Do(std::shared_ptr slot) { } else { if (res == 1) { res_.SetRes(CmdRes::kOk); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.AppendStringLen(-1); } @@ -101,20 +101,20 @@ void SetCmd::Do(std::shared_ptr slot) { } } -void SetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SetCmd::DoThroughDB() { + Do(); } -void SetCmd::DoUpdateCache(std::shared_ptr slot) { +void SetCmd::DoUpdateCache() { if (SetCmd::kNX == condition_) { return; } if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; if (has_ttl_) { - slot->cache()->Setxx(CachePrefixKeyK, value_, sec_); + db_->cache()->Setxx(CachePrefixKeyK, value_, sec_); } else { - slot->cache()->SetxxWithoutTTL(CachePrefixKeyK, value_); + db_->cache()->SetxxWithoutTTL(CachePrefixKeyK, value_); } } } @@ -156,8 +156,8 @@ void GetCmd::DoInitial() { key_ = argv_[1]; } -void GetCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->GetWithTTL(key_, &value_, &sec_); +void GetCmd::Do() { + s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); if (s_.ok()) { res_.AppendStringLenUint64(value_.size()); res_.AppendContent(value_); @@ -168,9 +168,9 @@ void GetCmd::Do(std::shared_ptr slot) { } } -void GetCmd::ReadCache(std::shared_ptr slot) { +void GetCmd::ReadCache() { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s = slot->cache()->Get(CachePrefixKeyK, &value_); + auto s = db_->cache()->Get(CachePrefixKeyK, &value_); if (s.ok()) { res_.AppendStringLen(value_.size()); res_.AppendContent(value_); @@ -179,15 +179,15 @@ void GetCmd::ReadCache(std::shared_ptr slot) { } } -void GetCmd::DoThroughDB(std::shared_ptr slot) { +void GetCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void GetCmd::DoUpdateCache(std::shared_ptr slot) { +void GetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -200,24 +200,22 @@ void DelCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void DelCmd::Do(std::shared_ptr slot) { +void DelCmd::Do() { std::map type_status; - - int64_t count = slot->db()->Del(keys_, &type_status); + int64_t count = db_->storage()->Del(keys_, &type_status); // stream's destory need to be treated specially - auto s = StreamStorage::DestoryStreams(keys_, slot.get()); + auto s = StreamStorage::DestoryStreams(keys_, db_.get()); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, "stream delete error: " + s.ToString()); return; } - if (count >= 0) { res_.AppendInteger(count); s_ = rocksdb::Status::OK(); std::vector::const_iterator it; for (it = keys_.begin(); it != keys_.end(); it++) { - RemSlotKey(*it, slot); + RemSlotKey(*it, db_); } } else { res_.SetRes(CmdRes::kErrOther, "delete error"); @@ -225,11 +223,11 @@ void DelCmd::Do(std::shared_ptr slot) { } } -void DelCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void DelCmd::DoThroughDB() { + Do(); } -void DelCmd::DoUpdateCache(std::shared_ptr slot) { +void DelCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; for (auto key : keys_) { @@ -239,13 +237,13 @@ void DelCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key); v.emplace_back(PCacheKeyPrefixH + key); } - slot->cache()->Del(v); + db_->cache()->Del(v); } } -void DelCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { +void DelCmd::Split(const HintKeys& hint_keys) { std::map type_status; - int64_t count = slot->db()->Del(hint_keys.keys, &type_status); + int64_t count = db_->storage()->Del(hint_keys.keys, &type_status); if (count >= 0) { split_res_ += count; } else { @@ -255,13 +253,13 @@ void DelCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { void DelCmd::Merge() { res_.AppendInteger(split_res_); } -void DelCmd::DoBinlog(const std::shared_ptr& slot) { +void DelCmd::DoBinlog() { std::string opt = argv_.at(0); for(auto& key: keys_) { argv_.clear(); argv_.emplace_back(opt); argv_.emplace_back(key); - Cmd::DoBinlog(slot); + Cmd::DoBinlog(); } } @@ -273,11 +271,11 @@ void IncrCmd::DoInitial() { key_ = argv_[1]; } -void IncrCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Incrby(key_, 1, &new_value_); +void IncrCmd::Do() { + s_ = db_->storage()->Incrby(key_, 1, &new_value_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: Value is not a integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s_.IsInvalidArgument()) { @@ -287,14 +285,14 @@ void IncrCmd::Do(std::shared_ptr slot) { } } -void IncrCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void IncrCmd::DoThroughDB() { + Do(); } -void IncrCmd::DoUpdateCache(std::shared_ptr slot) { +void IncrCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->Incrxx(CachePrefixKeyK); + db_->cache()->Incrxx(CachePrefixKeyK); } } @@ -310,11 +308,11 @@ void IncrbyCmd::DoInitial() { } } -void IncrbyCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Incrby(key_, by_, &new_value_); +void IncrbyCmd::Do() { + s_ = db_->storage()->Incrby(key_, by_, &new_value_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: Value is not a integer") { res_.SetRes(CmdRes::kInvalidInt); } else if (s_.IsInvalidArgument()) { @@ -324,14 +322,14 @@ void IncrbyCmd::Do(std::shared_ptr slot) { } } -void IncrbyCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void IncrbyCmd::DoThroughDB() { + Do(); } -void IncrbyCmd::DoUpdateCache(std::shared_ptr slot) { +void IncrbyCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->IncrByxx(CachePrefixKeyK, by_); + db_->cache()->IncrByxx(CachePrefixKeyK, by_); } } @@ -348,12 +346,12 @@ void IncrbyfloatCmd::DoInitial() { } } -void IncrbyfloatCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Incrbyfloat(key_, value_, &new_value_); +void IncrbyfloatCmd::Do() { + s_ = db_->storage()->Incrbyfloat(key_, value_, &new_value_); if (s_.ok()) { res_.AppendStringLenUint64(new_value_.size()); res_.AppendContent(new_value_); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: Value is not a vaild float") { res_.SetRes(CmdRes::kInvalidFloat); } else if (s_.IsInvalidArgument()) { @@ -363,16 +361,16 @@ void IncrbyfloatCmd::Do(std::shared_ptr slot) { } } -void IncrbyfloatCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void IncrbyfloatCmd::DoThroughDB() { + Do(); } -void IncrbyfloatCmd::DoUpdateCache(std::shared_ptr slot) { +void IncrbyfloatCmd::DoUpdateCache() { if (s_.ok()) { long double long_double_by; if (storage::StrToLongDouble(value_.data(), value_.size(), &long_double_by) != -1) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->Incrbyfloatxx(CachePrefixKeyK, long_double_by); + db_->cache()->Incrbyfloatxx(CachePrefixKeyK, long_double_by); } } } @@ -385,8 +383,8 @@ void DecrCmd::DoInitial() { key_ = argv_[1]; } -void DecrCmd::Do(std::shared_ptr slot) { - s_= slot->db()->Decrby(key_, 1, &new_value_); +void DecrCmd::Do() { + s_= db_->storage()->Decrby(key_, 1, &new_value_); if (s_.ok()) { res_.AppendContent(":" + std::to_string(new_value_)); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: Value is not a integer") { @@ -398,14 +396,14 @@ void DecrCmd::Do(std::shared_ptr slot) { } } -void DecrCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void DecrCmd::DoThroughDB() { + Do(); } -void DecrCmd::DoUpdateCache(std::shared_ptr slot) { +void DecrCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->Decrxx(CachePrefixKeyK); + db_->cache()->Decrxx(CachePrefixKeyK); } } @@ -421,10 +419,10 @@ void DecrbyCmd::DoInitial() { } } -void DecrbyCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Decrby(key_, by_, &new_value_); +void DecrbyCmd::Do() { + s_ = db_->storage()->Decrby(key_, by_, &new_value_); if (s_.ok()) { - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); res_.AppendContent(":" + std::to_string(new_value_)); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: Value is not a integer") { res_.SetRes(CmdRes::kInvalidInt); @@ -435,14 +433,14 @@ void DecrbyCmd::Do(std::shared_ptr slot) { } } -void DecrbyCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void DecrbyCmd::DoThroughDB() { + Do(); } -void DecrbyCmd::DoUpdateCache(std::shared_ptr slot) { +void DecrbyCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->DecrByxx(CachePrefixKeyK, by_); + db_->cache()->DecrByxx(CachePrefixKeyK, by_); } } @@ -455,9 +453,9 @@ void GetsetCmd::DoInitial() { new_value_ = argv_[2]; } -void GetsetCmd::Do(std::shared_ptr slot) { +void GetsetCmd::Do() { std::string old_value; - s_ = slot->db()->GetSet(key_, new_value_, &old_value); + s_ = db_->storage()->GetSet(key_, new_value_, &old_value); if (s_.ok()) { if (old_value.empty()) { res_.AppendContent("$-1"); @@ -465,20 +463,20 @@ void GetsetCmd::Do(std::shared_ptr slot) { res_.AppendStringLenUint64(old_value.size()); res_.AppendContent(old_value); } - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void GetsetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void GetsetCmd::DoThroughDB() { + Do(); } -void GetsetCmd::DoUpdateCache(std::shared_ptr slot) { +void GetsetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->SetxxWithoutTTL(CachePrefixKeyK, new_value_); + db_->cache()->SetxxWithoutTTL(CachePrefixKeyK, new_value_); } } @@ -491,25 +489,25 @@ void AppendCmd::DoInitial() { value_ = argv_[2]; } -void AppendCmd::Do(std::shared_ptr slot) { +void AppendCmd::Do() { int32_t new_len = 0; - s_ = slot->db()->Append(key_, value_, &new_len); + s_ = db_->storage()->Append(key_, value_, &new_len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(new_len); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void AppendCmd::DoThroughDB(std::shared_ptr slot){ - Do(slot); +void AppendCmd::DoThroughDB(){ + Do(); } -void AppendCmd::DoUpdateCache(std::shared_ptr slot) { +void AppendCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->Appendxx(CachePrefixKeyK, value_); + db_->cache()->Appendxx(CachePrefixKeyK, value_); } } @@ -523,9 +521,9 @@ void MgetCmd::DoInitial() { split_res_.resize(keys_.size()); } -void MgetCmd::Do(std::shared_ptr slot) { +void MgetCmd::Do() { db_value_status_array_.clear(); - s_ = slot->db()->MGetWithTTL(keys_, &db_value_status_array_); + s_ = db_->storage()->MGetWithTTL(keys_, &db_value_status_array_); if (s_.ok()) { res_.AppendArrayLenUint64(db_value_status_array_.size()); for (const auto& vs : db_value_status_array_) { @@ -541,10 +539,10 @@ void MgetCmd::Do(std::shared_ptr slot) { } } -void MgetCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { +void MgetCmd::Split(const HintKeys& hint_keys) { std::vector vss; const std::vector& keys = hint_keys.keys; - rocksdb::Status s = slot->db()->MGet(keys, &vss); + rocksdb::Status s = db_->storage()->MGet(keys, &vss); if (s.ok()) { if (hint_keys.hints.size() != vss.size()) { res_.SetRes(CmdRes::kErrOther, "internal Mget return size invalid"); @@ -570,13 +568,13 @@ void MgetCmd::Merge() { } } -void MgetCmd::ReadCache(std::shared_ptr slot) { +void MgetCmd::ReadCache() { if (1 < keys_.size()) { res_.SetRes(CmdRes::kCacheMiss); return; } std::string CachePrefixKeyK = PCacheKeyPrefixK + keys_[0]; - auto s = slot->cache()->Get(CachePrefixKeyK, &value_); + auto s = db_->cache()->Get(CachePrefixKeyK, &value_); if (s.ok()) { res_.AppendArrayLen(1); res_.AppendStringLen(value_.size()); @@ -586,17 +584,17 @@ void MgetCmd::ReadCache(std::shared_ptr slot) { } } -void MgetCmd::DoThroughDB(std::shared_ptr slot) { +void MgetCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void MgetCmd::DoUpdateCache(std::shared_ptr slot) { +void MgetCmd::DoUpdateCache() { for (size_t i = 0; i < keys_.size(); i++) { if (db_value_status_array_[i].status.ok()) { std::string CachePrefixKeyK; CachePrefixKeyK = PCacheKeyPrefixK + keys_[i]; - slot->cache()->WriteKVToCache(CachePrefixKeyK, db_value_status_array_[i].value, db_value_status_array_[i].ttl); + db_->cache()->WriteKVToCache(CachePrefixKeyK, db_value_status_array_[i].value, db_value_status_array_[i].ttl); } } } @@ -627,7 +625,7 @@ void KeysCmd::DoInitial() { } } -void KeysCmd::Do(std::shared_ptr slot) { +void KeysCmd::Do() { int64_t total_key = 0; int64_t cursor = 0; size_t raw_limit = g_pika_conf->max_client_response_size(); @@ -635,7 +633,7 @@ void KeysCmd::Do(std::shared_ptr slot) { std::vector keys; do { keys.clear(); - cursor = slot->db()->Scan(type_, cursor, pattern_, PIKA_SCAN_STEP_LENGTH, &keys); + cursor = db_->storage()->Scan(type_, cursor, pattern_, PIKA_SCAN_STEP_LENGTH, &keys); for (const auto& key : keys) { RedisAppendLenUint64(raw, key.size(), "$"); RedisAppendContent(raw, key); @@ -660,12 +658,12 @@ void SetnxCmd::DoInitial() { value_ = argv_[2]; } -void SetnxCmd::Do(std::shared_ptr slot) { +void SetnxCmd::Do() { success_ = 0; - s_ = slot->db()->Setnx(key_, value_, &success_); + s_ = db_->storage()->Setnx(key_, value_, &success_); if (s_.ok()) { res_.AppendInteger(success_); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } @@ -703,24 +701,24 @@ void SetexCmd::DoInitial() { value_ = argv_[3]; } -void SetexCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Setex(key_, value_, static_cast(sec_)); +void SetexCmd::Do() { + s_ = db_->storage()->Setex(key_, value_, static_cast(sec_)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void SetexCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SetexCmd::DoThroughDB() { + Do(); } -void SetexCmd::DoUpdateCache(std::shared_ptr slot) { +void SetexCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->Setxx(CachePrefixKeyK, value_, sec_); + db_->cache()->Setxx(CachePrefixKeyK, value_, sec_); } } @@ -762,8 +760,8 @@ void PsetexCmd::DoInitial() { value_ = argv_[3]; } -void PsetexCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->Setex(key_, value_, static_cast(usec_ / 1000)); +void PsetexCmd::Do() { + s_ = db_->storage()->Setex(key_, value_, static_cast(usec_ / 1000)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else { @@ -771,14 +769,14 @@ void PsetexCmd::Do(std::shared_ptr slot) { } } -void PsetexCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void PsetexCmd::DoThroughDB() { + Do(); } -void PsetexCmd::DoUpdateCache(std::shared_ptr slot) { +void PsetexCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->WriteKVToCache(CachePrefixKeyK, value_, static_cast(usec_ / 1000)); + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, static_cast(usec_ / 1000)); } } @@ -816,8 +814,8 @@ void DelvxCmd::DoInitial() { value_ = argv_[2]; } -void DelvxCmd::Do(std::shared_ptr slot) { - rocksdb::Status s = slot->db()->Delvx(key_, value_, &success_); +void DelvxCmd::Do() { + rocksdb::Status s = db_->storage()->Delvx(key_, value_, &success_); if (s.ok() || s.IsNotFound()) { res_.AppendInteger(success_); } else { @@ -841,34 +839,34 @@ void MsetCmd::DoInitial() { } } -void MsetCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->MSet(kvs_); +void MsetCmd::Do() { + s_ = db_->storage()->MSet(kvs_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); std::vector::const_iterator it; for (it = kvs_.begin(); it != kvs_.end(); it++) { - AddSlotKey("k", it->key, slot); + AddSlotKey("k", it->key, db_); } } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void MsetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void MsetCmd::DoThroughDB() { + Do(); } -void MsetCmd::DoUpdateCache(std::shared_ptr slot) { +void MsetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK; for (auto key : kvs_) { CachePrefixKeyK = PCacheKeyPrefixK + key.key; - slot->cache()->SetxxWithoutTTL(CachePrefixKeyK, key.value); + db_->cache()->SetxxWithoutTTL(CachePrefixKeyK, key.value); } } } -void MsetCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { +void MsetCmd::Split(const HintKeys& hint_keys) { std::vector kvs; const std::vector& keys = hint_keys.keys; const std::vector& hints = hint_keys.hints; @@ -883,7 +881,7 @@ void MsetCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { return; } } - storage::Status s = slot->db()->MSet(kvs); + storage::Status s = db_->storage()->MSet(kvs); if (s.ok()) { res_.SetRes(CmdRes::kOk); } else { @@ -894,7 +892,7 @@ void MsetCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { void MsetCmd::Merge() {} -void MsetCmd::DoBinlog(const std::shared_ptr& slot) { +void MsetCmd::DoBinlog() { PikaCmdArgsType set_argv; set_argv.resize(3); //used "set" instead of "SET" to distinguish the binlog of Set @@ -905,7 +903,7 @@ void MsetCmd::DoBinlog(const std::shared_ptr& slot) { set_argv[1] = kv.key; set_argv[2] = kv.value; set_cmd_->Initial(set_argv, db_name_); - set_cmd_->DoBinlog(slot); + set_cmd_->DoBinlog(); } } @@ -925,21 +923,21 @@ void MsetnxCmd::DoInitial() { } } -void MsetnxCmd::Do(std::shared_ptr slot) { +void MsetnxCmd::Do() { success_ = 0; - rocksdb::Status s = slot->db()->MSetnx(kvs_, &success_); + rocksdb::Status s = db_->storage()->MSetnx(kvs_, &success_); if (s.ok()) { res_.AppendInteger(success_); std::vector::const_iterator it; for (it = kvs_.begin(); it != kvs_.end(); it++) { - AddSlotKey("k", it->key, slot); + AddSlotKey("k", it->key, db_); } } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } } -void MsetnxCmd::DoBinlog(const std::shared_ptr& slot) { +void MsetnxCmd::DoBinlog() { if (!success_) { //some keys already exist, set operations aborted, no need of binlog return; @@ -954,7 +952,7 @@ void MsetnxCmd::DoBinlog(const std::shared_ptr& slot) { set_argv[1] = kv.key; set_argv[2] = kv.value; set_cmd_->Initial(set_argv, db_name_); - set_cmd_->DoBinlog(slot); + set_cmd_->DoBinlog(); } } @@ -974,9 +972,9 @@ void GetrangeCmd::DoInitial() { } } -void GetrangeCmd::Do(std::shared_ptr slot) { +void GetrangeCmd::Do() { std::string substr; - s_= slot->db()->Getrange(key_, start_, end_, &substr); + s_= db_->storage()->Getrange(key_, start_, end_, &substr); if (s_.ok() || s_.IsNotFound()) { res_.AppendStringLenUint64(substr.size()); res_.AppendContent(substr); @@ -985,10 +983,10 @@ void GetrangeCmd::Do(std::shared_ptr slot) { } } -void GetrangeCmd::ReadCache(std::shared_ptr slot) { +void GetrangeCmd::ReadCache() { std::string substr; std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s = slot->cache()->GetRange(CachePrefixKeyK, start_, end_, &substr); + auto s = db_->cache()->GetRange(CachePrefixKeyK, start_, end_, &substr); if (s.ok()) { res_.AppendStringLen(substr.size()); res_.AppendContent(substr); @@ -997,10 +995,10 @@ void GetrangeCmd::ReadCache(std::shared_ptr slot) { } } -void GetrangeCmd::DoThroughDB(std::shared_ptr slot) { +void GetrangeCmd::DoThroughDB() { res_.clear(); std::string substr; - s_ = slot->db()->GetrangeWithValue(key_, start_, end_, &substr, &value_, &sec_); + s_ = db_->storage()->GetrangeWithValue(key_, start_, end_, &substr, &value_, &sec_); if (s_.ok()) { res_.AppendStringLen(substr.size()); res_.AppendContent(substr); @@ -1012,10 +1010,10 @@ void GetrangeCmd::DoThroughDB(std::shared_ptr slot) { } } -void GetrangeCmd::DoUpdateCache(std::shared_ptr slot) { +void GetrangeCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -1032,25 +1030,25 @@ void SetrangeCmd::DoInitial() { value_ = argv_[3]; } -void SetrangeCmd::Do(std::shared_ptr slot) { +void SetrangeCmd::Do() { int32_t new_len = 0; - s_ = slot->db()->Setrange(key_, offset_, value_, &new_len); + s_ = db_->storage()->Setrange(key_, offset_, value_, &new_len); if (s_.ok()) { res_.AppendInteger(new_len); - AddSlotKey("k", key_, slot); + AddSlotKey("k", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void SetrangeCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SetrangeCmd::DoThroughDB() { + Do(); } -void SetrangeCmd::DoUpdateCache(std::shared_ptr slot) { +void SetrangeCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->SetRangexx(CachePrefixKeyK, offset_, value_); + db_->cache()->SetRangexx(CachePrefixKeyK, offset_, value_); } } @@ -1062,9 +1060,9 @@ void StrlenCmd::DoInitial() { key_ = argv_[1]; } -void StrlenCmd::Do(std::shared_ptr slot) { +void StrlenCmd::Do() { int32_t len = 0; - s_ = slot->db()->Strlen(key_, &len); + s_ = db_->storage()->Strlen(key_, &len); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(len); @@ -1073,10 +1071,10 @@ void StrlenCmd::Do(std::shared_ptr slot) { } } -void StrlenCmd::ReadCache(std::shared_ptr slot) { +void StrlenCmd::ReadCache() { int32_t len = 0; std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - auto s= slot->cache()->Strlen(CachePrefixKeyK, &len); + auto s= db_->cache()->Strlen(CachePrefixKeyK, &len); if (s.ok()) { res_.AppendInteger(len); } else { @@ -1084,9 +1082,9 @@ void StrlenCmd::ReadCache(std::shared_ptr slot) { } } -void StrlenCmd::DoThroughDB(std::shared_ptr slot) { +void StrlenCmd::DoThroughDB() { res_.clear(); - s_ = slot->db()->GetWithTTL(key_, &value_, &sec_); + s_ = db_->storage()->GetWithTTL(key_, &value_, &sec_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(value_.size()); } else { @@ -1094,10 +1092,10 @@ void StrlenCmd::DoThroughDB(std::shared_ptr slot) { } } -void StrlenCmd::DoUpdateCache(std::shared_ptr slot) { +void StrlenCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyK = PCacheKeyPrefixK + key_; - slot->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); + db_->cache()->WriteKVToCache(CachePrefixKeyK, value_, sec_); } } @@ -1110,9 +1108,9 @@ void ExistsCmd::DoInitial() { keys_.erase(keys_.begin()); } -void ExistsCmd::Do(std::shared_ptr slot) { +void ExistsCmd::Do() { std::map type_status; - int64_t res = slot->db()->Exists(keys_, &type_status); + int64_t res = db_->storage()->Exists(keys_, &type_status); if (res != -1) { res_.AppendInteger(res); } else { @@ -1120,9 +1118,9 @@ void ExistsCmd::Do(std::shared_ptr slot) { } } -void ExistsCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { +void ExistsCmd::Split(const HintKeys& hint_keys) { std::map type_status; - int64_t res = slot->db()->Exists(hint_keys.keys, &type_status); + int64_t res = db_->storage()->Exists(hint_keys.keys, &type_status); if (res != -1) { split_res_ += res; } else { @@ -1132,7 +1130,7 @@ void ExistsCmd::Split(std::shared_ptr slot, const HintKeys& hint_keys) { void ExistsCmd::Merge() { res_.AppendInteger(split_res_); } -void ExistsCmd::ReadCache(std::shared_ptr slot) { +void ExistsCmd::ReadCache() { if (1 < keys_.size()) { res_.SetRes(CmdRes::kCacheMiss); return; @@ -1145,7 +1143,7 @@ void ExistsCmd::ReadCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + keys_[0]); v.emplace_back(PCacheKeyPrefixH + keys_[0]); for (auto key : v) { - bool exist = slot->cache()->Exists(key); + bool exist = db_->cache()->Exists(key); if (exist) { nums++; } @@ -1157,9 +1155,9 @@ void ExistsCmd::ReadCache(std::shared_ptr slot) { } } -void ExistsCmd::DoThroughDB(std::shared_ptr slot) { +void ExistsCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } void ExpireCmd::DoInitial() { @@ -1174,9 +1172,9 @@ void ExpireCmd::DoInitial() { } } -void ExpireCmd::Do(std::shared_ptr slot) { +void ExpireCmd::Do() { std::map type_status; - int64_t res = slot->db()->Expire(key_, static_cast(sec_), &type_status); + int64_t res = db_->storage()->Expire(key_, static_cast(sec_), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1208,11 +1206,11 @@ std::string ExpireCmd::ToRedisProtocol() { return content; } -void ExpireCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ExpireCmd::DoThroughDB() { + Do(); } -void ExpireCmd::DoUpdateCache(std::shared_ptr slot) { +void ExpireCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixK + key_); @@ -1221,7 +1219,7 @@ void ExpireCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key_); v.emplace_back(PCacheKeyPrefixH + key_); for (auto key : v) { - slot->cache()->Expire(key, sec_); + db_->cache()->Expire(key, sec_); } } } @@ -1238,9 +1236,9 @@ void PexpireCmd::DoInitial() { } } -void PexpireCmd::Do(std::shared_ptr slot) { +void PexpireCmd::Do() { std::map type_status; - int64_t res = slot->db()->Expire(key_, static_cast(msec_ / 1000), &type_status); + int64_t res = db_->storage()->Expire(key_, static_cast(msec_ / 1000), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1272,11 +1270,11 @@ std::string PexpireCmd::ToRedisProtocol() { return content; } -void PexpireCmd::DoThroughDB(std::shared_ptr slot){ - Do(slot); +void PexpireCmd::DoThroughDB(){ + Do(); } -void PexpireCmd::DoUpdateCache(std::shared_ptr slot) { +void PexpireCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixK + key_); @@ -1285,7 +1283,7 @@ void PexpireCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key_); v.emplace_back(PCacheKeyPrefixH + key_); for (auto key : v){ - slot->cache()->Expire(key, msec_/1000); + db_->cache()->Expire(key, msec_/1000); } } } @@ -1302,9 +1300,9 @@ void ExpireatCmd::DoInitial() { } } -void ExpireatCmd::Do(std::shared_ptr slot) { +void ExpireatCmd::Do() { std::map type_status; - int32_t res = slot->db()->Expireat(key_, static_cast(time_stamp_), &type_status); + int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1315,11 +1313,11 @@ void ExpireatCmd::Do(std::shared_ptr slot) { } } -void ExpireatCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ExpireatCmd::DoThroughDB() { + Do(); } -void ExpireatCmd::DoUpdateCache(std::shared_ptr slot) { +void ExpireatCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixK + key_); @@ -1328,7 +1326,7 @@ void ExpireatCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key_); v.emplace_back(PCacheKeyPrefixH + key_); for (auto key : v) { - slot->cache()->Expireat(key, time_stamp_); + db_->cache()->Expireat(key, time_stamp_); } } } @@ -1367,9 +1365,9 @@ std::string PexpireatCmd::ToRedisProtocol() { return content; } -void PexpireatCmd::Do(std::shared_ptr slot) { +void PexpireatCmd::Do() { std::map type_status; - int32_t res = slot->db()->Expireat(key_, static_cast(time_stamp_ms_ / 1000), &type_status); + int32_t res = db_->storage()->Expireat(key_, static_cast(time_stamp_ms_ / 1000), &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1379,11 +1377,11 @@ void PexpireatCmd::Do(std::shared_ptr slot) { } } -void PexpireatCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void PexpireatCmd::DoThroughDB() { + Do(); } -void PexpireatCmd::DoUpdateCache(std::shared_ptr slot) { +void PexpireatCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixK + key_); @@ -1392,7 +1390,7 @@ void PexpireatCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key_); v.emplace_back(PCacheKeyPrefixH + key_); for (auto key : v) { - slot->cache()->Expireat(key, time_stamp_ms_ / 1000); + db_->cache()->Expireat(key, time_stamp_ms_ / 1000); } } } @@ -1405,10 +1403,10 @@ void TtlCmd::DoInitial() { key_ = argv_[1]; } -void TtlCmd::Do(std::shared_ptr slot) { +void TtlCmd::Do() { std::map type_timestamp; std::map type_status; - type_timestamp = slot->db()->TTL(key_, &type_status); + type_timestamp = db_->storage()->TTL(key_, &type_status); for (const auto& item : type_timestamp) { // mean operation exception errors happen in database if (item.second == -3) { @@ -1432,11 +1430,11 @@ void TtlCmd::Do(std::shared_ptr slot) { } } -void TtlCmd::ReadCache(std::shared_ptr slot) { +void TtlCmd::ReadCache() { rocksdb::Status s; std::map type_timestamp; std::map type_status; - type_timestamp = slot->cache()->TTL(key_, &type_status); + type_timestamp = db_->cache()->TTL(key_, &type_status); for (const auto& item : type_timestamp) { // mean operation exception errors happen in database if (item.second == -3) { @@ -1460,9 +1458,9 @@ void TtlCmd::ReadCache(std::shared_ptr slot) { } } -void TtlCmd::DoThroughDB(std::shared_ptr slot) { +void TtlCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } void PttlCmd::DoInitial() { @@ -1473,10 +1471,10 @@ void PttlCmd::DoInitial() { key_ = argv_[1]; } -void PttlCmd::Do(std::shared_ptr slot) { +void PttlCmd::Do() { std::map type_timestamp; std::map type_status; - type_timestamp = slot->db()->TTL(key_, &type_status); + type_timestamp = db_->storage()->TTL(key_, &type_status); for (const auto& item : type_timestamp) { // mean operation exception errors happen in database if (item.second == -3) { @@ -1520,10 +1518,10 @@ void PttlCmd::Do(std::shared_ptr slot) { } } -void PttlCmd::ReadCache(std::shared_ptr slot) { +void PttlCmd::ReadCache() { std::map type_timestamp; std::map type_status; - type_timestamp = slot->cache()->TTL(key_, &type_status); + type_timestamp = db_->cache()->TTL(key_, &type_status); for (const auto& item : type_timestamp) { // mean operation exception errors happen in database if (item.second == -3) { @@ -1567,9 +1565,9 @@ void PttlCmd::ReadCache(std::shared_ptr slot) { } } -void PttlCmd::DoThroughDB(std::shared_ptr slot) { +void PttlCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } void PersistCmd::DoInitial() { @@ -1580,9 +1578,9 @@ void PersistCmd::DoInitial() { key_ = argv_[1]; } -void PersistCmd::Do(std::shared_ptr slot) { +void PersistCmd::Do() { std::map type_status; - int32_t res = slot->db()->Persist(key_, &type_status); + int32_t res = db_->storage()->Persist(key_, &type_status); if (res != -1) { res_.AppendInteger(res); s_ = rocksdb::Status::OK(); @@ -1592,11 +1590,11 @@ void PersistCmd::Do(std::shared_ptr slot) { } } -void PersistCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void PersistCmd::DoThroughDB() { + Do(); } -void PersistCmd::DoUpdateCache(std::shared_ptr slot) { +void PersistCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixK + key_); @@ -1605,7 +1603,7 @@ void PersistCmd::DoUpdateCache(std::shared_ptr slot) { v.emplace_back(PCacheKeyPrefixS + key_); v.emplace_back(PCacheKeyPrefixH + key_); for (auto key : v) { - slot->cache()->Persist(key); + db_->cache()->Persist(key); } } } @@ -1618,9 +1616,9 @@ void TypeCmd::DoInitial() { key_ = argv_[1]; } -void TypeCmd::Do(std::shared_ptr slot) { +void TypeCmd::Do() { std::vector types(1); - rocksdb::Status s = slot->db()->GetType(key_, true, types); + rocksdb::Status s = db_->storage()->GetType(key_, true, types); if (s.ok()) { res_.AppendContent("+" + types[0]); } else { @@ -1628,9 +1626,9 @@ void TypeCmd::Do(std::shared_ptr slot) { } } -void TypeCmd::ReadCache(std::shared_ptr slot) { +void TypeCmd::ReadCache() { std::vector types(1); - rocksdb::Status s = slot->db()->GetType(key_, true, types); + rocksdb::Status s = db_->storage()->GetType(key_, true, types); if (s.ok()) { res_.AppendContent("+" + types[0]); } else { @@ -1638,9 +1636,9 @@ void TypeCmd::ReadCache(std::shared_ptr slot) { } } -void TypeCmd::DoThroughDB(std::shared_ptr slot) { +void TypeCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } void PTypeCmd::DoInitial() { @@ -1651,9 +1649,9 @@ void PTypeCmd::DoInitial() { key_ = argv_[1]; } -void PTypeCmd::Do(std::shared_ptr slot) { +void PTypeCmd::Do() { std::vector types(5); - rocksdb::Status s = slot->db()->GetType(key_, false, types); + rocksdb::Status s = db_->storage()->GetType(key_, false, types); if (s.ok()) { res_.AppendArrayLenUint64(types.size()); @@ -1716,7 +1714,7 @@ void ScanCmd::DoInitial() { } } -void ScanCmd::Do(std::shared_ptr slot) { +void ScanCmd::Do() { int64_t total_key = 0; int64_t batch_count = 0; int64_t left = count_; @@ -1729,7 +1727,7 @@ void ScanCmd::Do(std::shared_ptr slot) { keys.clear(); batch_count = left < PIKA_SCAN_STEP_LENGTH ? left : PIKA_SCAN_STEP_LENGTH; left = left > PIKA_SCAN_STEP_LENGTH ? left - PIKA_SCAN_STEP_LENGTH : 0; - cursor_ret = slot->db()->Scan(type_, cursor_ret, pattern_, batch_count, &keys); + cursor_ret = db_->storage()->Scan(type_, cursor_ret, pattern_, batch_count, &keys); for (const auto& key : keys) { RedisAppendLenUint64(raw, key.size(), "$"); RedisAppendContent(raw, key); @@ -1797,10 +1795,10 @@ void ScanxCmd::DoInitial() { } } -void ScanxCmd::Do(std::shared_ptr slot) { +void ScanxCmd::Do() { std::string next_key; std::vector keys; - rocksdb::Status s = slot->db()->Scanx(type_, start_key_, pattern_, count_, &keys, &next_key); + rocksdb::Status s = db_->storage()->Scanx(type_, start_key_, pattern_, count_, &keys, &next_key); if (s.ok()) { res_.AppendArrayLen(2); @@ -1830,8 +1828,8 @@ void PKSetexAtCmd::DoInitial() { } } -void PKSetexAtCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->PKSetexAt(key_, value_, static_cast(time_stamp_)); +void PKSetexAtCmd::Do() { + s_ = db_->storage()->PKSetexAt(key_, value_, static_cast(time_stamp_)); if (s_.ok()) { res_.SetRes(CmdRes::kOk); } else { @@ -1893,11 +1891,11 @@ void PKScanRangeCmd::DoInitial() { } } -void PKScanRangeCmd::Do(std::shared_ptr slot) { +void PKScanRangeCmd::Do() { std::string next_key; std::vector keys; std::vector kvs; - s_ = slot->db()->PKScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), &keys, &kvs, &next_key); + s_ = db_->storage()->PKScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), &keys, &kvs, &next_key); if (s_.ok()) { res_.AppendArrayLen(2); @@ -1977,11 +1975,11 @@ void PKRScanRangeCmd::DoInitial() { } } -void PKRScanRangeCmd::Do(std::shared_ptr slot) { +void PKRScanRangeCmd::Do() { std::string next_key; std::vector keys; std::vector kvs; - s_ = slot->db()->PKRScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), + s_ = db_->storage()->PKRScanRange(type_, key_start_, key_end_, pattern_, static_cast(limit_), &keys, &kvs, &next_key); if (s_.ok()) { diff --git a/src/pika_list.cc b/src/pika_list.cc index c5ee7de3fe..314dfb1cdf 100644 --- a/src/pika_list.cc +++ b/src/pika_list.cc @@ -5,12 +5,13 @@ #include "include/pika_list.h" #include +#include "include/pika_cache.h" #include "include/pika_data_distribution.h" #include "include/pika_rm.h" #include "include/pika_server.h" -#include "pstd/include/pstd_string.h" #include "include/pika_slot_command.h" -#include "include/pika_cache.h" +#include "pstd/include/pstd_string.h" +#include "scope_record_lock.h" extern PikaServer* g_pika_server; extern std::unique_ptr g_pika_rm; @@ -27,9 +28,9 @@ void LIndexCmd::DoInitial() { } } -void LIndexCmd::Do(std::shared_ptr slot) { +void LIndexCmd::Do() { std::string value; - s_ = slot->db()->LIndex(key_, index_, &value); + s_ = db_->storage()->LIndex(key_, index_, &value); if (s_.ok()) { res_.AppendString(value); } else if (s_.IsNotFound()) { @@ -39,10 +40,10 @@ void LIndexCmd::Do(std::shared_ptr slot) { } } -void LIndexCmd::ReadCache(std::shared_ptr slot) { +void LIndexCmd::ReadCache() { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; std::string value; - auto s = slot->cache()->LIndex(CachePrefixKeyL, index_, &value); + auto s = db_->cache()->LIndex(CachePrefixKeyL, index_, &value); if (s.ok()) { res_.AppendString(value); } else if (s.IsNotFound()) { @@ -52,14 +53,14 @@ void LIndexCmd::ReadCache(std::shared_ptr slot) { } } -void LIndexCmd::DoThroughDB(std::shared_ptr slot) { +void LIndexCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void LIndexCmd::DoUpdateCache(std::shared_ptr slot) { +void LIndexCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } @@ -82,25 +83,25 @@ void LInsertCmd::DoInitial() { value_ = argv_[4]; } -void LInsertCmd::Do(std::shared_ptr slot) { +void LInsertCmd::Do() { int64_t llen = 0; - s_ = slot->db()->LInsert(key_, dir_, pivot_, value_, &llen); + s_ = db_->storage()->LInsert(key_, dir_, pivot_, value_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(llen); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void LInsertCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LInsertCmd::DoThroughDB() { + Do(); } -void LInsertCmd::DoUpdateCache(std::shared_ptr slot) { +void LInsertCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->LInsert(CachePrefixKeyL, dir_, pivot_, value_); + db_->cache()->LInsert(CachePrefixKeyL, dir_, pivot_, value_); } } @@ -112,9 +113,9 @@ void LLenCmd::DoInitial() { key_ = argv_[1]; } -void LLenCmd::Do(std::shared_ptr slot) { +void LLenCmd::Do() { uint64_t llen = 0; - s_ = slot->db()->LLen(key_, &llen); + s_ = db_->storage()->LLen(key_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); } else { @@ -122,10 +123,10 @@ void LLenCmd::Do(std::shared_ptr slot) { } } -void LLenCmd::ReadCache(std::shared_ptr slot) { +void LLenCmd::ReadCache() { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; uint64_t llen = 0; - auto s = slot->cache()->LLen(CachePrefixKeyL, &llen); + auto s = db_->cache()->LLen(CachePrefixKeyL, &llen); if (s.ok()){ res_.AppendInteger(llen); } else if (s.IsNotFound()) { @@ -135,18 +136,18 @@ void LLenCmd::ReadCache(std::shared_ptr slot) { } } -void LLenCmd::DoThroughDB(std::shared_ptr slot) { +void LLenCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void LLenCmd::DoUpdateCache(std::shared_ptr slot) { +void LLenCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } -void BlockingBaseCmd::TryToServeBLrPopWithThisKey(const std::string& key, std::shared_ptr slot) { +void BlockingBaseCmd::TryToServeBLrPopWithThisKey(const std::string& key, std::shared_ptr db) { std::shared_ptr curr_conn = std::dynamic_pointer_cast(GetConn()); if (!curr_conn) { // current node is a slave and is applying a binlog of lpush/rpush/rpoplpush, just return @@ -165,7 +166,7 @@ void BlockingBaseCmd::TryToServeBLrPopWithThisKey(const std::string& key, std::s } } - auto* args = new UnblockTaskArgs(key, std::move(slot), dispatchThread); + auto* args = new UnblockTaskArgs(key, std::move(db), dispatchThread); bool is_slow_cmd = g_pika_conf->is_slow_cmd("LPOP") || g_pika_conf->is_slow_cmd("RPOP"); g_pika_server->ScheduleClientPool(&ServeAndUnblockConns, args, is_slow_cmd); } @@ -173,12 +174,12 @@ void BlockingBaseCmd::TryToServeBLrPopWithThisKey(const std::string& key, std::s void BlockingBaseCmd::ServeAndUnblockConns(void* args) { auto bg_args = std::unique_ptr(static_cast(args)); net::DispatchThread* dispatchThread = bg_args->dispatchThread; - std::shared_ptr slot = bg_args->slot; + std::shared_ptr db = bg_args->db; std::string key = std::move(bg_args->key); auto& key_to_conns_ = dispatchThread->GetMapFromKeyToConns(); - net::BlockKey blrPop_key{slot->GetDBName(), key}; + net::BlockKey blrPop_key{db->GetDBName(), key}; - pstd::lock::ScopeRecordLock record_lock(slot->LockMgr(), key);//It's a RAII Lock + pstd::lock::ScopeRecordLock record_lock(db->LockMgr(), key);//It's a RAII Lock std::unique_lock map_lock(dispatchThread->GetBlockMtx());// do not change the sequence of these two lock, or deadlock will happen auto it = key_to_conns_.find(blrPop_key); if (it == key_to_conns_.end()) { @@ -192,9 +193,9 @@ void BlockingBaseCmd::ServeAndUnblockConns(void* args) { // traverse this list from head to tail(in the order of adding sequence) ,means "first blocked, first get served“ for (auto conn_blocked = waitting_list->begin(); conn_blocked != waitting_list->end();) { if (conn_blocked->GetBlockType() == BlockKeyType::Blpop) { - s = slot->db()->LPop(key, 1, &values); + s = db->storage()->LPop(key, 1, &values); } else { // BlockKeyType is Brpop - s = slot->db()->RPop(key, 1, &values); + s = db->storage()->RPop(key, 1, &values); } if (s.ok()) { res.AppendArrayLen(2); @@ -211,7 +212,7 @@ void BlockingBaseCmd::ServeAndUnblockConns(void* args) { conn_ptr->WriteResp(res.message()); res.clear(); conn_ptr->NotifyEpoll(true); - pop_binlog_args.emplace_back(conn_blocked->GetBlockType(), key, slot, conn_ptr); + pop_binlog_args.emplace_back(conn_blocked->GetBlockType(), key, db, conn_ptr); conn_blocked = waitting_list->erase(conn_blocked); // remove this conn from current waiting list // erase all waiting info of this conn dispatchThread->CleanWaitNodeOfUnBlockedBlrConn(conn_ptr); @@ -228,26 +229,20 @@ void BlockingBaseCmd::WriteBinlogOfPop(std::vector& pop_ar std::string pop_type; if (pop_arg.block_type == BlockKeyType::Blpop) { pop_type = kCmdNameLPop; - pop_cmd = std::make_shared(kCmdNameLPop, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); + pop_cmd = std::make_shared(kCmdNameLPop, 2, kCmdFlagsWrite | kCmdFlagsList); } else if (pop_arg.block_type == BlockKeyType::Brpop) { pop_type = kCmdNameRPop; - pop_cmd = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsSingleSlot | kCmdFlagsList); + pop_cmd = std::make_shared(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsList); } PikaCmdArgsType args; args.push_back(std::move(pop_type)); args.push_back(pop_arg.key); - pop_cmd->Initial(args, pop_arg.slot->GetDBName()); - std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(pop_arg.slot->GetDBName(), pop_arg.slot->GetSlotID())); - if (!sync_slot) { - LOG(WARNING) << "Writing binlog of blpop/brpop failed: SyncMasterSlot not found"; - } else { - pop_cmd->SetConn(pop_arg.conn); - auto resp_ptr = std::make_shared("this resp won't be used for current code(consensus-level always be 0)"); - pop_cmd->SetResp(resp_ptr); - pop_cmd->DoBinlog(sync_slot); - } + pop_cmd->Initial(args, pop_arg.db->GetDBName()); + pop_cmd->SetConn(pop_arg.conn); + auto resp_ptr = std::make_shared("this resp won't be used for current code(consensus-level always be 0)"); + pop_cmd->SetResp(resp_ptr); + pop_cmd->DoBinlog(); } } @@ -263,12 +258,12 @@ void LPushCmd::DoInitial() { } } -void LPushCmd::Do(std::shared_ptr slot) { +void LPushCmd::Do() { uint64_t llen = 0; - s_ = slot->db()->LPush(key_, values_, &llen); + s_ = db_->storage()->LPush(key_, values_, &llen); if (s_.ok()) { res_.AppendInteger(static_cast(llen)); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } @@ -277,17 +272,17 @@ void LPushCmd::Do(std::shared_ptr slot) { return; } } - TryToServeBLrPopWithThisKey(key_, slot); + TryToServeBLrPopWithThisKey(key_, db_); } -void LPushCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LPushCmd::DoThroughDB() { + Do(); } -void LPushCmd::DoUpdateCache(std::shared_ptr slot) { +void LPushCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->LPushx(CachePrefixKeyL, values_); + db_->cache()->LPushx(CachePrefixKeyL, values_); } } @@ -354,18 +349,18 @@ void BLPopCmd::DoInitial() { } // else(timeout is 0): expire_time_ default value is 0, means never expire; } -void BLPopCmd::Do(std::shared_ptr slot) { +void BLPopCmd::Do() { for (auto& this_key : keys_) { std::vector values; - rocksdb::Status s = slot->db()->LPop(this_key, 1, &values); + rocksdb::Status s = db_->storage()->LPop(this_key, 1, &values); if (s.ok()) { res_.AppendArrayLen(2); res_.AppendString(this_key); res_.AppendString(values[0]); - // write an binlog of lpop + // write a binlog of lpop binlog_args_.block_type = BlockKeyType::Blpop; binlog_args_.key = this_key; - binlog_args_.slot = slot; + binlog_args_.db = db_; binlog_args_.conn = GetConn(); is_binlog_deferred_ = false; return; @@ -386,7 +381,7 @@ void BLPopCmd::Do(std::shared_ptr slot) { BlockThisClientToWaitLRPush(BlockKeyType::Blpop, keys_, expire_time_); } -void BLPopCmd::DoBinlog(const std::shared_ptr& slot) { +void BLPopCmd::DoBinlog() { if (is_binlog_deferred_) { return; } @@ -416,9 +411,9 @@ void LPopCmd::DoInitial() { } } -void LPopCmd::Do(std::shared_ptr slot) { +void LPopCmd::Do() { std::vector elements; - s_ = slot->db()->LPop(key_, count_, &elements); + s_ = db_->storage()->LPop(key_, count_, &elements); if (s_.ok()) { if (elements.size() > 1) { @@ -434,15 +429,15 @@ void LPopCmd::Do(std::shared_ptr slot) { } } -void LPopCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LPopCmd::DoThroughDB() { + Do(); } -void LPopCmd::DoUpdateCache(std::shared_ptr slot) { +void LPopCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; std::string value; - slot->cache()->LPop(CachePrefixKeyL, &value); + db_->cache()->LPop(CachePrefixKeyL, &value); } } @@ -458,27 +453,27 @@ void LPushxCmd::DoInitial() { } } -void LPushxCmd::Do(std::shared_ptr slot) { +void LPushxCmd::Do() { uint64_t llen = 0; - s_ = slot->db()->LPushx(key_, values_, &llen); + s_ = db_->storage()->LPushx(key_, values_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void LPushxCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LPushxCmd::DoThroughDB() { + Do(); } -void LPushxCmd::DoUpdateCache(std::shared_ptr slot) { +void LPushxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; std::vector values; values.push_back(value_); - slot->cache()->LPushx(CachePrefixKeyL, values); + db_->cache()->LPushx(CachePrefixKeyL, values); } } @@ -499,9 +494,9 @@ void LRangeCmd::DoInitial() { } } -void LRangeCmd::Do(std::shared_ptr slot) { +void LRangeCmd::Do() { std::vector values; - s_ = slot->db()->LRange(key_, left_, right_, &values); + s_ = db_->storage()->LRange(key_, left_, right_, &values); if (s_.ok()) { res_.AppendArrayLenUint64(values.size()); for (const auto& value : values) { @@ -514,10 +509,10 @@ void LRangeCmd::Do(std::shared_ptr slot) { } } -void LRangeCmd::ReadCache(std::shared_ptr slot) { +void LRangeCmd::ReadCache() { std::vector values; std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - auto s = slot->cache()->LRange(CachePrefixKeyL, left_, right_, &values); + auto s = db_->cache()->LRange(CachePrefixKeyL, left_, right_, &values); if (s.ok()) { res_.AppendArrayLen(values.size()); for (const auto& value : values) { @@ -530,14 +525,14 @@ void LRangeCmd::ReadCache(std::shared_ptr slot) { } } -void LRangeCmd::DoThroughDB(std::shared_ptr slot) { +void LRangeCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void LRangeCmd::DoUpdateCache(std::shared_ptr slot) { +void LRangeCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_LIST, key_, db_); } } @@ -555,9 +550,9 @@ void LRemCmd::DoInitial() { value_ = argv_[3]; } -void LRemCmd::Do(std::shared_ptr slot) { +void LRemCmd::Do() { uint64_t res = 0; - s_ = slot->db()->LRem(key_, count_, value_, &res); + s_ = db_->storage()->LRem(key_, count_, value_, &res); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(res)); } else { @@ -565,14 +560,14 @@ void LRemCmd::Do(std::shared_ptr slot) { } } -void LRemCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LRemCmd::DoThroughDB() { + Do(); } -void LRemCmd::DoUpdateCache(std::shared_ptr slot) { +void LRemCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->LRem(CachePrefixKeyL, count_, value_); + db_->cache()->LRem(CachePrefixKeyL, count_, value_); } } @@ -590,11 +585,11 @@ void LSetCmd::DoInitial() { value_ = argv_[3]; } -void LSetCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->LSet(key_, index_, value_); +void LSetCmd::Do() { + s_ = db_->storage()->LSet(key_, index_, value_); if (s_.ok()) { res_.SetRes(CmdRes::kOk); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else if (s_.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); } else if (s_.IsCorruption() && s_.ToString() == "Corruption: index out of range") { @@ -605,14 +600,14 @@ void LSetCmd::Do(std::shared_ptr slot) { } } -void LSetCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LSetCmd::DoThroughDB() { + Do(); } -void LSetCmd::DoUpdateCache(std::shared_ptr slot) { +void LSetCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->LSet(CachePrefixKeyL, index_, value_); + db_->cache()->LSet(CachePrefixKeyL, index_, value_); } } @@ -633,8 +628,8 @@ void LTrimCmd::DoInitial() { } } -void LTrimCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->LTrim(key_, start_, stop_); +void LTrimCmd::Do() { + s_ = db_->storage()->LTrim(key_, start_, stop_); if (s_.ok() || s_.IsNotFound()) { res_.SetRes(CmdRes::kOk); } else { @@ -642,21 +637,21 @@ void LTrimCmd::Do(std::shared_ptr slot) { } } -void LTrimCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void LTrimCmd::DoThroughDB() { + Do(); } -void LTrimCmd::DoUpdateCache(std::shared_ptr slot) { +void LTrimCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->LTrim(CachePrefixKeyL, start_, stop_); + db_->cache()->LTrim(CachePrefixKeyL, start_, stop_); } } -void BRPopCmd::Do(std::shared_ptr slot) { +void BRPopCmd::Do() { for (auto& this_key : keys_) { std::vector values; - s_ = slot->db()->RPop(this_key, 1, &values); + s_ = db_->storage()->RPop(this_key, 1, &values); if (s_.ok()) { res_.AppendArrayLen(2); res_.AppendString(this_key); @@ -664,7 +659,7 @@ void BRPopCmd::Do(std::shared_ptr slot) { // write an binlog of rpop binlog_args_.block_type = BlockKeyType::Brpop; binlog_args_.key = this_key; - binlog_args_.slot = slot; + binlog_args_.db = db_; binlog_args_.conn = GetConn(); is_binlog_deferred_ = false; return; @@ -713,7 +708,7 @@ void BRPopCmd::DoInitial() { } // else(timeout is 0): expire_time_ default value is 0, means never expire; } -void BRPopCmd::DoBinlog(const std::shared_ptr& slot) { +void BRPopCmd::DoBinlog() { if (is_binlog_deferred_) { return; } @@ -744,9 +739,9 @@ void RPopCmd::DoInitial() { } } -void RPopCmd::Do(std::shared_ptr slot) { +void RPopCmd::Do() { std::vector elements; - s_ = slot->db()->RPop(key_, count_, &elements); + s_ = db_->storage()->RPop(key_, count_, &elements); if (s_.ok()) { if (elements.size() > 1) { res_.AppendArrayLenUint64(elements.size()); @@ -761,15 +756,15 @@ void RPopCmd::Do(std::shared_ptr slot) { } } -void RPopCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void RPopCmd::DoThroughDB() { + Do(); } -void RPopCmd::DoUpdateCache(std::shared_ptr slot) { +void RPopCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; std::string value; - slot->cache()->RPop(CachePrefixKeyL, &value); + db_->cache()->RPop(CachePrefixKeyL, &value); } } @@ -785,11 +780,11 @@ void RPopLPushCmd::DoInitial() { } } -void RPopLPushCmd::Do(std::shared_ptr slot) { +void RPopLPushCmd::Do() { std::string value; - s_ = slot->db()->RPoplpush(source_, receiver_, &value); + s_ = db_->storage()->RPoplpush(source_, receiver_, &value); if (s_.ok()) { - AddSlotKey("k", receiver_, slot); + AddSlotKey("k", receiver_, db_); res_.AppendString(value); value_poped_from_source_ = value; is_write_binlog_ = true; @@ -802,14 +797,14 @@ void RPopLPushCmd::Do(std::shared_ptr slot) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; } - TryToServeBLrPopWithThisKey(receiver_, slot); + TryToServeBLrPopWithThisKey(receiver_, db_); } -void RPopLPushCmd::ReadCache(std::shared_ptr slot) { +void RPopLPushCmd::ReadCache() { res_.SetRes(CmdRes::kErrOther, "the command is not support in cache mode"); } -void RPopLPushCmd::DoBinlog(const std::shared_ptr& slot) { +void RPopLPushCmd::DoBinlog() { if (!is_write_binlog_) { return; } @@ -829,8 +824,8 @@ void RPopLPushCmd::DoBinlog(const std::shared_ptr& slot) { lpush_cmd_->SetConn(GetConn()); lpush_cmd_->SetResp(resp_.lock()); - rpop_cmd_->DoBinlog(slot); - lpush_cmd_->DoBinlog(slot); + rpop_cmd_->DoBinlog(); + lpush_cmd_->DoBinlog(); } void RPushCmd::DoInitial() { @@ -845,12 +840,12 @@ void RPushCmd::DoInitial() { } } -void RPushCmd::Do(std::shared_ptr slot) { +void RPushCmd::Do() { uint64_t llen = 0; - s_ = slot->db()->RPush(key_, values_, &llen); + s_ = db_->storage()->RPush(key_, values_, &llen); if (s_.ok()) { res_.AppendInteger(static_cast(llen)); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } @@ -859,17 +854,17 @@ void RPushCmd::Do(std::shared_ptr slot) { return; } } - TryToServeBLrPopWithThisKey(key_, slot); + TryToServeBLrPopWithThisKey(key_, db_); } -void RPushCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void RPushCmd::DoThroughDB() { + Do(); } -void RPushCmd::DoUpdateCache(std::shared_ptr slot) { +void RPushCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; - slot->cache()->RPushx(CachePrefixKeyL, values_); + db_->cache()->RPushx(CachePrefixKeyL, values_); } } @@ -885,26 +880,26 @@ void RPushxCmd::DoInitial() { } } -void RPushxCmd::Do(std::shared_ptr slot) { +void RPushxCmd::Do() { uint64_t llen = 0; - s_ = slot->db()->RPushx(key_, values_, &llen); + s_ = db_->storage()->RPushx(key_, values_, &llen); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(static_cast(llen)); - AddSlotKey("l", key_, slot); + AddSlotKey("l", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void RPushxCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void RPushxCmd::DoThroughDB() { + Do(); } -void RPushxCmd::DoUpdateCache(std::shared_ptr slot) { +void RPushxCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyL = PCacheKeyPrefixL + key_; std::vector values; values.push_back(value_); - slot->cache()->RPushx(CachePrefixKeyL, values); + db_->cache()->RPushx(CachePrefixKeyL, values); } } \ No newline at end of file diff --git a/src/pika_meta.cc b/src/pika_meta.cc index 664b43e662..87e11fbe7d 100644 --- a/src/pika_meta.cc +++ b/src/pika_meta.cc @@ -39,10 +39,6 @@ Status PikaMeta::StableSave(const std::vector& db_structs) { for (const auto& ts : db_structs) { InnerMessage::DBInfo* db_info = meta.add_db_infos(); db_info->set_db_name(ts.db_name); - db_info->set_slot_num(ts.slot_num); - for (const auto& id : ts.slot_ids) { - db_info->add_slot_ids(id); - } } std::string meta_str; @@ -104,11 +100,7 @@ Status PikaMeta::ParseMeta(std::vector* const db_structs) { db_structs->clear(); for (int idx = 0; idx < meta.db_infos_size(); ++idx) { const InnerMessage::DBInfo& ti = meta.db_infos(idx); - std::set slot_ids; - for (int sidx = 0; sidx < ti.slot_ids_size(); ++sidx) { - slot_ids.insert(ti.slot_ids(sidx)); - } - db_structs->emplace_back(ti.db_name(), ti.slot_num(), slot_ids); + db_structs->emplace_back(ti.db_name()); } return Status::OK(); } diff --git a/src/pika_migrate_thread.cc b/src/pika_migrate_thread.cc index 5dc5884608..fce5c6886c 100644 --- a/src/pika_migrate_thread.cc +++ b/src/pika_migrate_thread.cc @@ -28,7 +28,7 @@ static int doMigrate(net::NetCli *cli, std::string send_str) { pstd::Status s; s = cli->Send(&send_str); if (!s.ok()) { - LOG(WARNING) << "Slot Migrate Send error: " << s.ToString(); + LOG(WARNING) << "DB Migrate Send error: " << s.ToString(); return -1; } return 1; @@ -50,30 +50,30 @@ static int doAuth(net::NetCli *cli) { pstd::Status s; s = cli->Send(&wbuf_str); if (!s.ok()) { - LOG(WARNING) << "Slot Migrate auth Send error: " << s.ToString(); + LOG(WARNING) << "DB Migrate auth Send error: " << s.ToString(); return -1; } // Recv s = cli->Recv(&argv); if (!s.ok()) { - LOG(WARNING) << "Slot Migrate auth Recv error: " << s.ToString(); + LOG(WARNING) << "DB Migrate auth Recv error: " << s.ToString(); return -1; } pstd::StringToLower(argv[0]); if (argv[0] != "ok" && argv[0] != "pong" && argv[0].find("no password") == std::string::npos) { - LOG(WARNING) << "Slot Migrate auth error: " << argv[0]; + LOG(WARNING) << "DB Migrate auth error: " << argv[0]; return -1; } return 0; } static int migrateKeyTTl(net::NetCli *cli, const std::string& key, storage::DataType data_type, - const std::shared_ptr& slot) { + const std::shared_ptr& db) { net::RedisCmdArgsType argv; std::string send_str; std::map type_timestamp; std::map type_status; - type_timestamp = slot->db()->TTL(key, &type_status); + type_timestamp = db->storage()->TTL(key, &type_status); if (PIKA_TTL_ZERO == type_timestamp[data_type] || PIKA_TTL_STALE == type_timestamp[data_type]) { argv.emplace_back("del"); argv.emplace_back(key); @@ -96,8 +96,8 @@ static int migrateKeyTTl(net::NetCli *cli, const std::string& key, storage::Data } // get set key all values -static int setGetall(const std::string& key, std::vector *members, const std::shared_ptr& slot) { - rocksdb::Status s = slot->db()->SMembers(key, members); +static int setGetall(const std::string& key, std::vector *members, const std::shared_ptr& db) { + rocksdb::Status s = db->storage()->SMembers(key, members); if (!s.ok()) { if (s.IsNotFound()) { LOG(WARNING) << "Set get key: " << key << " value not found "; @@ -110,9 +110,9 @@ static int setGetall(const std::string& key, std::vector *members, return 1; } -static int MigrateKv(net::NetCli *cli, const std::string& key, const std::shared_ptr& slot) { +static int MigrateKv(net::NetCli *cli, const std::string& key, const std::shared_ptr& db) { std::string value; - rocksdb::Status s = slot->db()->Get(key, &value); + rocksdb::Status s = db->storage()->Get(key, &value); if (!s.ok()) { if (s.IsNotFound()) { LOG(WARNING) << "Get kv key: " << key << " not found "; @@ -138,7 +138,7 @@ static int MigrateKv(net::NetCli *cli, const std::string& key, const std::shared } int r; - if (0 > (r = migrateKeyTTl(cli, key, storage::kStrings, slot))) { + if (0 > (r = migrateKeyTTl(cli, key, storage::kStrings, db))) { return -1; } else { send_num += r; @@ -147,14 +147,14 @@ static int MigrateKv(net::NetCli *cli, const std::string& key, const std::shared return send_num; } -static int MigrateHash(net::NetCli *cli, const std::string& key, const std::shared_ptr& slot) { +static int MigrateHash(net::NetCli *cli, const std::string& key, const std::shared_ptr& db) { int send_num = 0; int64_t cursor = 0; std::vector field_values; rocksdb::Status s; do { - s = slot->db()->HScan(key, cursor, "*", MAX_MEMBERS_NUM, &field_values, &cursor); + s = db->storage()->HScan(key, cursor, "*", MAX_MEMBERS_NUM, &field_values, &cursor); if (s.ok() && field_values.size() > 0) { net::RedisCmdArgsType argv; std::string send_str; @@ -175,7 +175,7 @@ static int MigrateHash(net::NetCli *cli, const std::string& key, const std::shar if (send_num > 0) { int r; - if ((r = migrateKeyTTl(cli, key, storage::kHashes, slot)) < 0) { + if ((r = migrateKeyTTl(cli, key, storage::kHashes, db)) < 0) { return -1; } else { send_num += r; @@ -185,7 +185,7 @@ static int MigrateHash(net::NetCli *cli, const std::string& key, const std::shar return send_num; } -static int MigrateList(net::NetCli *cli, const std::string& key, const std::shared_ptr& slot) { +static int MigrateList(net::NetCli *cli, const std::string& key, const std::shared_ptr& db) { // del old key, before migrate list; prevent redo when failed int send_num = 0; net::RedisCmdArgsType argv; @@ -200,7 +200,7 @@ static int MigrateList(net::NetCli *cli, const std::string& key, const std::shar } std::vector values; - rocksdb::Status s = slot->db()->LRange(key, 0, -1, &values); + rocksdb::Status s = db->storage()->LRange(key, 0, -1, &values); if (s.ok()) { auto iter = values.begin(); while (iter != values.end()) { @@ -225,7 +225,7 @@ static int MigrateList(net::NetCli *cli, const std::string& key, const std::shar // has send del key command if (send_num > 1) { int r; - if (0 > (r = migrateKeyTTl(cli, key, storage::kLists, slot))) { + if (0 > (r = migrateKeyTTl(cli, key, storage::kLists, db))) { return -1; } else { send_num += r; @@ -235,14 +235,14 @@ static int MigrateList(net::NetCli *cli, const std::string& key, const std::shar return send_num; } -static int MigrateSet(net::NetCli *cli, const std::string& key, const std::shared_ptr& slot) { +static int MigrateSet(net::NetCli *cli, const std::string& key, const std::shared_ptr& db) { int send_num = 0; int64_t cursor = 0; std::vector members; rocksdb::Status s; do { - s = slot->db()->SScan(key, cursor, "*", MAX_MEMBERS_NUM, &members, &cursor); + s = db->storage()->SScan(key, cursor, "*", MAX_MEMBERS_NUM, &members, &cursor); if (s.ok() && members.size() > 0) { net::RedisCmdArgsType argv; std::string send_str; @@ -263,7 +263,7 @@ static int MigrateSet(net::NetCli *cli, const std::string& key, const std::share if (0 < send_num) { int r; - if (0 > (r = migrateKeyTTl(cli, key, storage::kSets, slot))) { + if (0 > (r = migrateKeyTTl(cli, key, storage::kSets, db))) { return -1; } else { send_num += r; @@ -273,14 +273,14 @@ static int MigrateSet(net::NetCli *cli, const std::string& key, const std::share return send_num; } -static int MigrateZset(net::NetCli *cli, const std::string& key, const std::shared_ptr& slot) { +static int MigrateZset(net::NetCli *cli, const std::string& key, const std::shared_ptr& db) { int send_num = 0; int64_t cursor = 0; std::vector score_members; rocksdb::Status s; do { - s = slot->db()->ZScan(key, cursor, "*", MAX_MEMBERS_NUM, &score_members, &cursor); + s = db->storage()->ZScan(key, cursor, "*", MAX_MEMBERS_NUM, &score_members, &cursor); if (s.ok() && score_members.size() > 0) { net::RedisCmdArgsType argv; std::string send_str; @@ -302,7 +302,7 @@ static int MigrateZset(net::NetCli *cli, const std::string& key, const std::shar if (send_num > 0) { int r; - if ((r = migrateKeyTTl(cli, key, storage::kZSets, slot)) < 0) { + if ((r = migrateKeyTTl(cli, key, storage::kZSets, db)) < 0) { return -1; } else { send_num += r; @@ -313,8 +313,8 @@ static int MigrateZset(net::NetCli *cli, const std::string& key, const std::shar } // get list key all values -static int listGetall(const std::string& key, std::vector *values, const std::shared_ptr& slot) { - rocksdb::Status s = slot->db()->LRange(key, 0, -1, values); +static int listGetall(const std::string& key, std::vector *values, const std::shared_ptr& db) { + rocksdb::Status s = db->storage()->LRange(key, 0, -1, values); if (!s.ok()) { if (s.IsNotFound()) { LOG(WARNING) << "List get key: " << key << " value not found "; @@ -327,14 +327,13 @@ static int listGetall(const std::string& key, std::vector *values, return 1; } -PikaParseSendThread::PikaParseSendThread(PikaMigrateThread *migrate_thread, const std::shared_ptr& slot) +PikaParseSendThread::PikaParseSendThread(PikaMigrateThread *migrate_thread, const std::shared_ptr& db) : dest_ip_("none"), dest_port_(-1), timeout_ms_(3000), mgrtkeys_num_(64), should_exit_(false), - migrate_thread_(migrate_thread), - slot_(slot) {} + migrate_thread_(migrate_thread) {} PikaParseSendThread::~PikaParseSendThread() { if (is_running()) { @@ -382,27 +381,27 @@ int PikaParseSendThread::MigrateOneKey(net::NetCli *cli, const std::string& key, int send_num; switch (key_type) { case 'k': - if (0 > (send_num = MigrateKv(cli_, key, slot_))) { + if (0 > (send_num = MigrateKv(cli_, key, db_))) { return -1; } break; case 'h': - if (0 > (send_num = MigrateHash(cli_, key, slot_))) { + if (0 > (send_num = MigrateHash(cli_, key, db_))) { return -1; } break; case 'l': - if (0 > (send_num = MigrateList(cli_, key, slot_))) { + if (0 > (send_num = MigrateList(cli_, key, db_))) { return -1; } break; case 's': - if (0 > (send_num = MigrateSet(cli_, key, slot_))) { + if (0 > (send_num = MigrateSet(cli_, key, db_))) { return -1; } break; case 'z': - if (0 > (send_num = MigrateZset(cli_, key, slot_))) { + if (0 > (send_num = MigrateZset(cli_, key, db_))) { return -1; } break; @@ -414,24 +413,24 @@ int PikaParseSendThread::MigrateOneKey(net::NetCli *cli, const std::string& key, } void PikaParseSendThread::DelKeysAndWriteBinlog(std::deque> &send_keys, - const std::shared_ptr& slot) { + const std::shared_ptr& db) { for (const auto& send_key : send_keys) { - DeleteKey(send_key.second, send_key.first, slot); - WriteDelKeyToBinlog(send_key.second, slot); + DeleteKey(send_key.second, send_key.first, db_); + WriteDelKeyToBinlog(send_key.second, db_); } } // write del key to binlog for slave -void WriteDelKeyToBinlog(const std::string &key, const std::shared_ptr& slot) { +void WriteDelKeyToBinlog(const std::string& key, const std::shared_ptr& db) { std::shared_ptr cmd_ptr = g_pika_cmd_table_manager->GetCmd("del"); std::unique_ptr args = std::make_unique(); args->emplace_back("DEL"); args->emplace_back(key); - cmd_ptr->Initial(*args, slot->GetDBName()); + cmd_ptr->Initial(*args, db->GetDBName()); - std::shared_ptr sync_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(slot->GetDBName(), slot->GetSlotID())); - pstd::Status s = sync_slot->ConsensusProposeLog(cmd_ptr); + std::shared_ptr sync_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db->GetDBName())); + pstd::Status s = sync_db->ConsensusProposeLog(cmd_ptr); if (!s.ok()) { LOG(ERROR) << "write delete key to binlog failed, key: " << key; } @@ -512,7 +511,7 @@ void *PikaParseSendThread::ThreadMain() { migrate_thread_->DecWorkingThreadNum(); return nullptr; } else { - DelKeysAndWriteBinlog(send_keys, slot_); + DelKeysAndWriteBinlog(send_keys, db_); } migrate_thread_->AddResponseNum(migrate_keys_num); @@ -527,8 +526,8 @@ PikaMigrateThread::PikaMigrateThread() dest_ip_("none"), dest_port_(-1), timeout_ms_(3000), - slot_id_(-1), keys_num_(-1), + slot_id_(-1), is_migrating_(false), should_exit_(false), is_task_success_(true), @@ -552,17 +551,16 @@ PikaMigrateThread::~PikaMigrateThread() { } bool PikaMigrateThread::ReqMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_id, - int64_t keys_num, const std::shared_ptr& slot) { + int64_t keys_num, const std::shared_ptr& db) { if (migrator_mutex_.try_lock()) { if (is_migrating_) { - if (dest_ip_ != ip || dest_port_ != port || slot_id_ != slot_id) { + if (dest_ip_ != ip || dest_port_ != port || slot_id != slot_id_) { LOG(INFO) << "PikaMigrateThread::ReqMigrate current: " << dest_ip_ << ":" << dest_port_ << " slot[" << slot_id_ - << "]" - << "request: " << ip << ":" << port << " slot[" << slot << "]"; + << "] request: " << ip << ":" << port << "db[" << db << "]";; migrator_mutex_.unlock(); return false; } - slot_ = slot; + db_ = db; timeout_ms_ = time_out; keys_num_ = keys_num; NotifyRequestMigrate(); @@ -572,10 +570,10 @@ bool PikaMigrateThread::ReqMigrateBatch(const std::string &ip, int64_t port, int dest_ip_ = ip; dest_port_ = port; timeout_ms_ = time_out; - slot_id_ = slot_id; keys_num_ = keys_num; + slot_id_ = slot_id; should_exit_ = false; - slot_ = slot; + db_ = db; ResetThread(); int ret = StartThread(); @@ -585,7 +583,7 @@ bool PikaMigrateThread::ReqMigrateBatch(const std::string &ip, int64_t port, int is_migrating_ = false; StopThread(); } else { - LOG(INFO) << "PikaMigrateThread::ReqMigrateBatch slot: " << slot_id; + LOG(INFO) << "PikaMigrateThread::ReqMigrateBatch DB" << db; is_migrating_ = true; NotifyRequestMigrate(); } @@ -596,13 +594,13 @@ bool PikaMigrateThread::ReqMigrateBatch(const std::string &ip, int64_t port, int return false; } -int PikaMigrateThread::ReqMigrateOne(const std::string &key, const std::shared_ptr& slot) { +int PikaMigrateThread::ReqMigrateOne(const std::string& key, const std::shared_ptr& db) { std::unique_lock lm(migrator_mutex_); int slot_id = GetSlotID(key); std::vector type_str(1); char key_type; - rocksdb::Status s = slot->db()->GetType(key, true, type_str); + rocksdb::Status s = db->storage()->GetType(key, true, type_str); if (!s.ok()) { if (s.IsNotFound()) { LOG(INFO) << "PikaMigrateThread::ReqMigrateOne key: " << key << " not found"; @@ -665,7 +663,7 @@ int PikaMigrateThread::ReqMigrateOne(const std::string &key, const std::shared_p return 1; } -void PikaMigrateThread::GetMigrateStatus(std::string *ip, int64_t *port, int64_t *slot, bool *migrating, int64_t *moved, +void PikaMigrateThread::GetMigrateStatus(std::string *ip, int64_t* port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained) { std::unique_lock lm(migrator_mutex_); // todo for sure @@ -676,14 +674,14 @@ void PikaMigrateThread::GetMigrateStatus(std::string *ip, int64_t *port, int64_t *ip = dest_ip_; *port = dest_port_; - *slot = slot_id_; *migrating = is_migrating_; *moved = moved_num_; + *slot = slot_id_; std::unique_lock lq(mgrtkeys_queue_mutex_); int64_t migrating_keys_num = static_cast(mgrtkeys_queue_.size()); std::string slotKey = GetSlotKey(static_cast(slot_id_)); int32_t slot_size = 0; - rocksdb::Status s = slot_->db()->SCard(slotKey, &slot_size); + rocksdb::Status s = db_->storage()->SCard(slotKey, &slot_size); if (s.ok()) { *remained = slot_size + migrating_keys_num; } else { @@ -779,10 +777,10 @@ void PikaMigrateThread::ReadSlotKeys(const std::string &slotKey, int64_t need_re int32_t is_member = 0; std::vector members; - rocksdb::Status s = slot_->db()->SScan(slotKey, cursor_, "*", need_read_num, &members, &cursor_); + rocksdb::Status s = db_->storage()->SScan(slotKey, cursor_, "*", need_read_num, &members, &cursor_); if (s.ok() && 0 < members.size()) { for (const auto &member : members) { - slot_->db()->SIsmember(slotKey, member, &is_member); + db_->storage()->SIsmember(slotKey, member, &is_member); if (is_member) { key = member; key_type = key.at(0); @@ -805,7 +803,7 @@ void PikaMigrateThread::ReadSlotKeys(const std::string &slotKey, int64_t need_re bool PikaMigrateThread::CreateParseSendThreads(int32_t dispatch_num) { workers_num_ = static_cast(g_pika_conf->slotmigrate_thread_num()); for (int32_t i = 0; i < workers_num_; ++i) { - auto worker = new PikaParseSendThread(this, slot_); + auto worker = new PikaParseSendThread(this, db_); if (!worker->Init(dest_ip_, dest_port_, timeout_ms_, dispatch_num)) { delete worker; DestroyParseSendThreads(); @@ -856,7 +854,7 @@ void *PikaMigrateThread::ThreadMain() { std::string slotKey = GetSlotKey(static_cast(slot_id_)); int32_t slot_size = 0; - slot_->db()->SCard(slotKey, &slot_size); + db_->storage()->SCard(slotKey, &slot_size); while (!should_exit_) { // Waiting migrate task @@ -933,7 +931,7 @@ void *PikaMigrateThread::ThreadMain() { // check slot migrate finish int32_t slot_remained_keys = 0; - slot_->db()->SCard(slotKey, &slot_remained_keys); + db_->storage()->SCard(slotKey, &slot_remained_keys); if (0 == slot_remained_keys) { LOG(INFO) << "PikaMigrateThread::ThreadMain slot_size:" << slot_size << " moved_num:" << moved_num_; if (slot_size != moved_num_) { diff --git a/src/pika_monitor_thread.cc b/src/pika_monitor_thread.cc deleted file mode 100644 index 672f9cd94a..0000000000 --- a/src/pika_monitor_thread.cc +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "include/pika_monitor_thread.h" - -#include - -PikaMonitorThread::PikaMonitorThread() { - set_thread_name("MonitorThread"); - has_monitor_clients_.store(false); -} - -PikaMonitorThread::~PikaMonitorThread() { - set_should_stop(); - if (is_running()) { - monitor_cond_.notify_all(); - StopThread(); - } - for (auto & monitor_client : monitor_clients_) { - close(monitor_client.fd); - } - LOG(INFO) << "PikaMonitorThread " << pthread_self() << " exit!!!"; -} - -void PikaMonitorThread::AddMonitorClient(const std::shared_ptr& client_ptr) { - StartThread(); - std::lock_guard lm(monitor_mutex_protector_); - monitor_clients_.push_back(ClientInfo{client_ptr->fd(), client_ptr->ip_port(), 0, client_ptr}); - has_monitor_clients_.store(true); -} - -void PikaMonitorThread::RemoveMonitorClient(const std::string& ip_port) { - auto iter = monitor_clients_.begin(); - for (; iter != monitor_clients_.end(); ++iter) { - if (ip_port == "all") { - close(iter->fd); - continue; - } - if (iter->ip_port == ip_port) { - close(iter->fd); - break; - } - } - if (ip_port == "all") { - monitor_clients_.clear(); - } else if (iter != monitor_clients_.end()) { - monitor_clients_.erase(iter); - } - has_monitor_clients_.store(!monitor_clients_.empty()); -} - -void PikaMonitorThread::AddMonitorMessage(const std::string& monitor_message) { - std::lock_guard lm(monitor_mutex_protector_); - if (monitor_messages_.empty() && cron_tasks_.empty()) { - monitor_messages_.push_back(monitor_message); - monitor_cond_.notify_one(); - } else { - monitor_messages_.push_back(monitor_message); - } -} - -int32_t PikaMonitorThread::ThreadClientList(std::vector* clients_ptr) { - if (clients_ptr) { - for (auto & monitor_client : monitor_clients_) { - clients_ptr->push_back(monitor_client); - } - } - return static_cast(monitor_clients_.size()); -} - -void PikaMonitorThread::AddCronTask(const MonitorCronTask& task) { - std::lock_guard lm(monitor_mutex_protector_); - if (monitor_messages_.empty() && cron_tasks_.empty()) { - cron_tasks_.push(task); - monitor_cond_.notify_one(); - } else { - cron_tasks_.push(task); - } -} - -bool PikaMonitorThread::FindClient(const std::string& ip_port) { - std::lock_guard lm(monitor_mutex_protector_); - for (auto & monitor_client : monitor_clients_) { - if (monitor_client.ip_port == ip_port) { - return true; - } - } - return false; -} - -bool PikaMonitorThread::ThreadClientKill(const std::string& ip_port) { - if (is_running()) { - if (ip_port == "all") { - AddCronTask({TASK_KILLALL, "all"}); - } else if (FindClient(ip_port)) { - AddCronTask({TASK_KILL, ip_port}); - } else { - return false; - } - } - return true; -} - -bool PikaMonitorThread::HasMonitorClients() { return has_monitor_clients_.load(); } - -net::WriteStatus PikaMonitorThread::SendMessage(int32_t fd, std::string& message) { - size_t retry = 0; - size_t nwritten = 0; - size_t message_len_sended = 0; - size_t message_len_left = message.size(); - while (message_len_left > 0) { - nwritten = write(fd, message.data() + message_len_sended, message_len_left); - if (nwritten == -1 && (errno == EAGAIN || errno == EWOULDBLOCK)) { - // If the write buffer is full, but the client no longer consumes, it will - // get stuck in the loop and cause the entire Pika to block becase of monitor_mutex_protector_. - // So we put a limit on the number of retries - if (++retry >= 10) { - return net::kWriteError; - } else { - // Sleep one second wait for client consume message - sleep(1); - continue; - } - } else if (nwritten == -1) { - return net::kWriteError; - } - if (retry > 0) { - retry = 0; - } - message_len_sended += nwritten; - message_len_left -= nwritten; - } - return net::kWriteAll; -} - -void* PikaMonitorThread::ThreadMain() { - std::deque messages_deque; - std::string messages_transfer; - MonitorCronTask task; - net::WriteStatus write_status; - while (!should_stop()) { - { - std::unique_lock lm(monitor_mutex_protector_); - monitor_cond_.wait(lm, [this]() { return !monitor_messages_.empty() || !cron_tasks_.empty() || should_stop(); }); - } - if (should_stop()) { - break; - } - { - std::lock_guard lm(monitor_mutex_protector_); - while (!cron_tasks_.empty()) { - task = cron_tasks_.front(); - cron_tasks_.pop(); - RemoveMonitorClient(task.ip_port); - if (task.task == TASK_KILLALL) { - std::queue empty_queue; - cron_tasks_.swap(empty_queue); - } - } - } - - messages_deque.clear(); - { - std::lock_guard lm(monitor_mutex_protector_); - messages_deque.swap(monitor_messages_); - if (monitor_clients_.empty() || messages_deque.empty()) { - continue; - } - } - messages_transfer = "+"; - for (const auto& msg : messages_deque) { - messages_transfer.append(msg.data(), msg.size()); - messages_transfer.append(" ", 1); - } - if (messages_transfer == "+") { - continue; - } - - messages_transfer.pop_back(); // no space follow last param - messages_transfer.append("\r\n", 2); - - std::lock_guard lm(monitor_mutex_protector_); - for (auto & monitor_client : monitor_clients_) { - write_status = SendMessage(monitor_client.fd, messages_transfer); - if (write_status == net::kWriteError) { - cron_tasks_.push({TASK_KILL, monitor_client.ip_port}); - } - } - } - return nullptr; -} diff --git a/src/pika_monotonic_time.cc b/src/pika_monotonic_time.cc index 51aeba1f94..e1c8c51496 100644 --- a/src/pika_monotonic_time.cc +++ b/src/pika_monotonic_time.cc @@ -49,4 +49,4 @@ uint64_t getMonotonicUs() { #else #error "Unsupported platform" -#endif // __APPLE__, __linux__ +#endif // __APPLE__, __linux__ \ No newline at end of file diff --git a/src/pika_pubsub.cc b/src/pika_pubsub.cc index e3e2b567e4..935015ae7c 100644 --- a/src/pika_pubsub.cc +++ b/src/pika_pubsub.cc @@ -38,7 +38,7 @@ void PublishCmd::DoInitial() { msg_ = argv_[2]; } -void PublishCmd::Do(std::shared_ptr slot) { +void PublishCmd::Do() { int receivers = g_pika_server->Publish(channel_, msg_); res_.AppendInteger(receivers); } @@ -53,7 +53,7 @@ void SubscribeCmd::DoInitial() { } } -void SubscribeCmd::Do(std::shared_ptr slot) { +void SubscribeCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNameSubscribe); @@ -88,7 +88,7 @@ void UnSubscribeCmd::DoInitial() { } } -void UnSubscribeCmd::Do(std::shared_ptr slot) { +void UnSubscribeCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNameUnSubscribe); @@ -127,7 +127,7 @@ void PSubscribeCmd::DoInitial() { } } -void PSubscribeCmd::Do(std::shared_ptr slot) { +void PSubscribeCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNamePSubscribe); @@ -163,7 +163,7 @@ void PUnSubscribeCmd::DoInitial() { } -void PUnSubscribeCmd::Do(std::shared_ptr slot) { +void PUnSubscribeCmd::Do() { std::shared_ptr conn = GetConn(); if (!conn) { res_.SetRes(CmdRes::kErrOther, kCmdNamePUnSubscribe); @@ -207,7 +207,7 @@ void PubSubCmd::DoInitial() { } } -void PubSubCmd::Do(std::shared_ptr slot) { +void PubSubCmd::Do() { if (strcasecmp(subcommand_.data(), "channels") == 0) { std::string pattern; std::vector result; diff --git a/src/pika_repl_bgworker.cc b/src/pika_repl_bgworker.cc index dc8c067e08..d25a9ba4c7 100644 --- a/src/pika_repl_bgworker.cc +++ b/src/pika_repl_bgworker.cc @@ -24,7 +24,6 @@ PikaReplBgWorker::PikaReplBgWorker(int queue_size) : bg_thread_(queue_size) { redis_parser_.RedisParserInit(REDIS_PARSER_REQUEST, settings); redis_parser_.data = this; db_name_ = g_pika_conf->default_db(); - slot_id_ = 0; } int PikaReplBgWorker::StartThread() { return bg_thread_.StartThread(); } @@ -56,7 +55,7 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { }; std::string db_name; - uint32_t slot_id = 0; + LogOffset pb_begin; LogOffset pb_end; bool only_keepalive = false; @@ -66,7 +65,6 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync((*index)[i]); if (i == 0) { db_name = binlog_res.slot().db_name(); - slot_id = binlog_res.slot().slot_id(); } if (!binlog_res.binlog().empty()) { ParseBinlogOffset(binlog_res.binlog_offset(), &pb_begin); @@ -94,45 +92,43 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { ack_start = pb_begin; } - // db_name and slot_id in the vector are same in the bgworker, // because DispatchBinlogRes() have been order them. worker->db_name_ = db_name; - worker->slot_id_ = slot_id; - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!slot) { - LOG(WARNING) << "Slot " << db_name << "_" << slot_id << " Not Found"; + std::shared_ptr db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!db) { + LOG(WARNING) << "DB " << db_name << " Not Found"; return; } - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot " << db_name << "_" << slot_id << " Not Found"; + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "Slave DB " << db_name << " Not Found"; return; } if (res->has_consensus_meta()) { const InnerMessage::ConsensusMeta& meta = res->consensus_meta(); - if (meta.term() > slot->ConsensusTerm()) { - LOG(INFO) << "Update " << db_name << "_" << slot_id << " term from " << slot->ConsensusTerm() + if (meta.term() > db->ConsensusTerm()) { + LOG(INFO) << "Update " << db_name << " term from " << db->ConsensusTerm() << " to " << meta.term(); - slot->ConsensusUpdateTerm(meta.term()); - } else if (meta.term() < slot->ConsensusTerm()) /*outdated pb*/ { - LOG(WARNING) << "Drop outdated binlog sync response " << db_name << "_" << slot_id - << " recv term: " << meta.term() << " local term: " << slot->ConsensusTerm(); + db->ConsensusUpdateTerm(meta.term()); + } else if (meta.term() < db->ConsensusTerm()) /*outdated pb*/ { + LOG(WARNING) << "Drop outdated binlog sync response " << db_name + << " recv term: " << meta.term() << " local term: " << db->ConsensusTerm(); return; } if (!only_keepalive) { - LogOffset last_offset = slot->ConsensusLastIndex(); + LogOffset last_offset = db->ConsensusLastIndex(); LogOffset prev_offset; ParseBinlogOffset(res->consensus_meta().log_offset(), &prev_offset); if (last_offset.l_offset.index != 0 && (last_offset.l_offset != prev_offset.l_offset || last_offset.b_offset != prev_offset.b_offset)) { LOG(WARNING) << "last_offset " << last_offset.ToString() << " NOT equal to pb prev_offset " << prev_offset.ToString(); - slave_slot->SetReplState(ReplState::kTryConnect); + slave_db->SetReplState(ReplState::kTryConnect); return; } } @@ -140,21 +136,20 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { for (int i : *index) { const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync(i); - // if pika are not current a slave or Slot not in + // if pika are not current a slave or DB not in // BinlogSync state, we drop remain write binlog task if (((g_pika_server->role() & PIKA_ROLE_SLAVE) == 0) || - ((slave_slot->State() != ReplState::kConnected) && (slave_slot->State() != ReplState::kWaitDBSync))) { + ((slave_db->State() != ReplState::kConnected) && (slave_db->State() != ReplState::kWaitDBSync))) { return; } - if (slave_slot->MasterSessionId() != binlog_res.session_id()) { - LOG(WARNING) << "Check SessionId Mismatch: " << slave_slot->MasterIp() << ":" - << slave_slot->MasterPort() << ", " << slave_slot->SyncSlotInfo().ToString() + if (slave_db->MasterSessionId() != binlog_res.session_id()) { + LOG(WARNING) << "Check SessionId Mismatch: " << slave_db->MasterIp() << ":" + << slave_db->MasterPort() << ", " << slave_db->SyncDBInfo().ToString() << " expected_session: " << binlog_res.session_id() - << ", actual_session:" << slave_slot->MasterSessionId(); - LOG(WARNING) << "Check Session failed " << binlog_res.slot().db_name() << "_" - << binlog_res.slot().slot_id(); - slave_slot->SetReplState(ReplState::kTryConnect); + << ", actual_session:" << slave_db->MasterSessionId(); + LOG(WARNING) << "Check Session failed " << binlog_res.slot().db_name(); + slave_db->SetReplState(ReplState::kTryConnect); return; } @@ -164,7 +159,7 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { } if (!PikaBinlogTransverter::BinlogItemWithoutContentDecode(TypeFirst, binlog_res.binlog(), &worker->binlog_item_)) { LOG(WARNING) << "Binlog item decode failed"; - slave_slot->SetReplState(ReplState::kTryConnect); + slave_db->SetReplState(ReplState::kTryConnect); return; } const char* redis_parser_start = binlog_res.binlog().data() + BINLOG_ENCODE_LEN; @@ -174,7 +169,7 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { worker->redis_parser_.ProcessInputBuffer(redis_parser_start, redis_parser_len, &processed_len); if (ret != net::kRedisParserDone) { LOG(WARNING) << "Redis parser failed"; - slave_slot->SetReplState(ReplState::kTryConnect); + slave_db->SetReplState(ReplState::kTryConnect); return; } } @@ -192,14 +187,14 @@ void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { } else { LogOffset productor_status; // Reply Ack to master immediately - std::shared_ptr logger = slot->Logger(); + std::shared_ptr logger = db->Logger(); logger->GetProducerStatus(&productor_status.b_offset.filenum, &productor_status.b_offset.offset, &productor_status.l_offset.term, &productor_status.l_offset.index); ack_end = productor_status; ack_end.l_offset.term = pb_end.l_offset.term; } - g_pika_rm->SendSlotBinlogSyncAckRequest(db_name, slot_id, ack_start, ack_end); + g_pika_rm->SendBinlogSyncAckRequest(db_name, ack_start, ack_end); } int PikaReplBgWorker::HandleWriteBinlog(net::RedisParser* parser, const net::RedisCmdArgsType& argv) { @@ -231,13 +226,13 @@ int PikaReplBgWorker::HandleWriteBinlog(net::RedisParser* parser, const net::Red g_pika_server->UpdateQueryNumAndExecCountDB(worker->db_name_, opt, c_ptr->is_write()); - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(worker->db_name_, worker->slot_id_)); - if (!slot) { - LOG(WARNING) << worker->db_name_ << worker->slot_id_ << "Not found."; + std::shared_ptr db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(worker->db_name_)); + if (!db) { + LOG(WARNING) << worker->db_name_ << "Not found."; } - slot->ConsensusProcessLeaderLog(c_ptr, worker->binlog_item_); + db->ConsensusProcessLeaderLog(c_ptr, worker->binlog_item_); return 0; } @@ -247,33 +242,31 @@ void PikaReplBgWorker::HandleBGWorkerWriteDB(void* arg) { const PikaCmdArgsType& argv = c_ptr->argv(); LogOffset offset = task_arg->offset; std::string db_name = task_arg->db_name; - uint32_t slot_id = task_arg->slot_id; uint64_t start_us = 0; if (g_pika_conf->slowlog_slower_than() >= 0) { start_us = pstd::NowMicros(); } - std::shared_ptr slot = g_pika_server->GetDBSlotById(db_name, slot_id); // Add read lock for no suspend command if (!c_ptr->IsSuspend()) { - slot->DbRWLockReader(); + c_ptr->GetDB()->DbRWLockReader(); } if (c_ptr->IsNeedCacheDo() && PIKA_CACHE_NONE != g_pika_conf->cache_model() - && slot->cache()->CacheStatus() == PIKA_CACHE_STATUS_OK) { + && c_ptr->GetDB()->cache()->CacheStatus() == PIKA_CACHE_STATUS_OK) { if (c_ptr->is_write()) { - c_ptr->DoThroughDB(slot); + c_ptr->DoThroughDB(); if (c_ptr->IsNeedUpdateCache()) { - c_ptr->DoUpdateCache(slot); + c_ptr->DoUpdateCache(); } } else { LOG(WARNING) << "This branch is not impossible reach"; } } else { - c_ptr->Do(slot); + c_ptr->Do(); } if (!c_ptr->IsSuspend()) { - slot->DbRWUnLock(); + c_ptr->GetDB()->DbRWUnLock(); } if (g_pika_conf->slowlog_slower_than() >= 0) { diff --git a/src/pika_repl_client.cc b/src/pika_repl_client.cc index f27d43abb8..2754eb2f6e 100644 --- a/src/pika_repl_client.cc +++ b/src/pika_repl_client.cc @@ -66,20 +66,20 @@ void PikaReplClient::Schedule(net::TaskFunc func, void* arg) { UpdateNextAvail(); } -void PikaReplClient::ScheduleWriteBinlogTask(const std::string& db_slot, +void PikaReplClient::ScheduleWriteBinlogTask(const std::string& db_name, const std::shared_ptr& res, const std::shared_ptr& conn, void* res_private_data) { - size_t index = GetHashIndex(db_slot, true); + size_t index = GetHashIndex(db_name, true); auto task_arg = new ReplClientWriteBinlogTaskArg(res, conn, res_private_data, bg_workers_[index].get()); bg_workers_[index]->Schedule(&PikaReplBgWorker::HandleBGWorkerWriteBinlog, static_cast(task_arg)); } void PikaReplClient::ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, - const std::string& db_name, uint32_t slot_id) { + const std::string& db_name) { const PikaCmdArgsType& argv = cmd_ptr->argv(); std::string dispatch_key = argv.size() >= 2 ? argv[1] : argv[0]; size_t index = GetHashIndex(dispatch_key, false); - auto task_arg = new ReplClientWriteDBTaskArg(cmd_ptr, offset, db_name, slot_id); + auto task_arg = new ReplClientWriteDBTaskArg(cmd_ptr, offset, db_name); bg_workers_[index]->Schedule(&PikaReplBgWorker::HandleBGWorkerWriteDB, static_cast(task_arg)); } @@ -139,18 +139,22 @@ Status PikaReplClient::SendMetaSync() { return client_thread_->Write(master_ip, master_port + kPortShiftReplServer, to_send); } -Status PikaReplClient::SendSlotDBSync(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const BinlogOffset& boffset, - const std::string& local_ip) { +Status PikaReplClient::SendDBSync(const std::string& ip, uint32_t port, const std::string& db_name, + const BinlogOffset& boffset, const std::string& local_ip) { InnerMessage::InnerRequest request; request.set_type(InnerMessage::kDBSync); InnerMessage::InnerRequest::DBSync* db_sync = request.mutable_db_sync(); InnerMessage::Node* node = db_sync->mutable_node(); node->set_ip(local_ip); node->set_port(g_pika_server->port()); - InnerMessage::Slot* slot = db_sync->mutable_slot(); - slot->set_db_name(db_name); - slot->set_slot_id(slot_id); + InnerMessage::Slot* db = db_sync->mutable_slot(); + db->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db->set_slot_id(0); InnerMessage::BinlogOffset* binlog_offset = db_sync->mutable_binlog_offset(); binlog_offset->set_filenum(boffset.filenum); @@ -158,24 +162,28 @@ Status PikaReplClient::SendSlotDBSync(const std::string& ip, uint32_t port, cons std::string to_send; if (!request.SerializeToString(&to_send)) { - LOG(WARNING) << "Serialize Slot DBSync Request Failed, to Master (" << ip << ":" << port << ")"; + LOG(WARNING) << "Serialize DB DBSync Request Failed, to Master (" << ip << ":" << port << ")"; return Status::Corruption("Serialize Failed"); } return client_thread_->Write(ip, static_cast(port) + kPortShiftReplServer, to_send); } -Status PikaReplClient::SendSlotTrySync(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const BinlogOffset& boffset, - const std::string& local_ip) { +Status PikaReplClient::SendTrySync(const std::string& ip, uint32_t port, const std::string& db_name, + const BinlogOffset& boffset, const std::string& local_ip) { InnerMessage::InnerRequest request; request.set_type(InnerMessage::kTrySync); InnerMessage::InnerRequest::TrySync* try_sync = request.mutable_try_sync(); InnerMessage::Node* node = try_sync->mutable_node(); node->set_ip(local_ip); node->set_port(g_pika_server->port()); - InnerMessage::Slot* slot = try_sync->mutable_slot(); - slot->set_db_name(db_name); - slot->set_slot_id(slot_id); + InnerMessage::Slot* db = try_sync->mutable_slot(); + db->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db->set_slot_id(0); InnerMessage::BinlogOffset* binlog_offset = try_sync->mutable_binlog_offset(); binlog_offset->set_filenum(boffset.filenum); @@ -183,16 +191,15 @@ Status PikaReplClient::SendSlotTrySync(const std::string& ip, uint32_t port, con std::string to_send; if (!request.SerializeToString(&to_send)) { - LOG(WARNING) << "Serialize Slot TrySync Request Failed, to Master (" << ip << ":" << port << ")"; + LOG(WARNING) << "Serialize DB TrySync Request Failed, to Master (" << ip << ":" << port << ")"; return Status::Corruption("Serialize Failed"); } return client_thread_->Write(ip, static_cast(port + kPortShiftReplServer), to_send); } -Status PikaReplClient::SendSlotBinlogSync(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const LogOffset& ack_start, - const LogOffset& ack_end, const std::string& local_ip, - bool is_first_send) { +Status PikaReplClient::SendBinlogSync(const std::string& ip, uint32_t port, const std::string& db_name, + const LogOffset& ack_start, const LogOffset& ack_end, + const std::string& local_ip, bool is_first_send) { InnerMessage::InnerRequest request; request.set_type(InnerMessage::kBinlogSync); InnerMessage::InnerRequest::BinlogSync* binlog_sync = request.mutable_binlog_sync(); @@ -200,7 +207,12 @@ Status PikaReplClient::SendSlotBinlogSync(const std::string& ip, uint32_t port, node->set_ip(local_ip); node->set_port(g_pika_server->port()); binlog_sync->set_db_name(db_name); - binlog_sync->set_slot_id(slot_id); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + binlog_sync->set_slot_id(0); binlog_sync->set_first_send(is_first_send); InnerMessage::BinlogOffset* ack_range_start = binlog_sync->mutable_ack_range_start(); @@ -215,25 +227,25 @@ Status PikaReplClient::SendSlotBinlogSync(const std::string& ip, uint32_t port, ack_range_end->set_term(ack_end.l_offset.term); ack_range_end->set_index(ack_end.l_offset.index); - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_name << "_" << slot_id << " not exist"; - return Status::NotFound("SyncSlaveSlot NotFound"); + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_name << " not exist"; + return Status::NotFound("SyncSlaveDB NotFound"); } - int32_t session_id = slave_slot->MasterSessionId(); + int32_t session_id = slave_db->MasterSessionId(); binlog_sync->set_session_id(session_id); std::string to_send; if (!request.SerializeToString(&to_send)) { - LOG(WARNING) << "Serialize Slot BinlogSync Request Failed, to Master (" << ip << ":" << port << ")"; + LOG(WARNING) << "Serialize DB BinlogSync Request Failed, to Master (" << ip << ":" << port << ")"; return Status::Corruption("Serialize Failed"); } return client_thread_->Write(ip, static_cast(port + kPortShiftReplServer), to_send); } Status PikaReplClient::SendRemoveSlaveNode(const std::string& ip, uint32_t port, const std::string& db_name, - uint32_t slot_id, const std::string& local_ip) { + const std::string& local_ip) { InnerMessage::InnerRequest request; request.set_type(InnerMessage::kRemoveSlaveNode); InnerMessage::InnerRequest::RemoveSlaveNode* remove_slave_node = request.add_remove_slave_node(); @@ -241,14 +253,18 @@ Status PikaReplClient::SendRemoveSlaveNode(const std::string& ip, uint32_t port, node->set_ip(local_ip); node->set_port(g_pika_server->port()); - InnerMessage::Slot* slot = remove_slave_node->mutable_slot(); - slot->set_db_name(db_name); - slot->set_slot_id(slot_id); + InnerMessage::Slot* db = remove_slave_node->mutable_slot(); + db->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db->set_slot_id(0); std::string to_send; if (!request.SerializeToString(&to_send)) { - LOG(WARNING) << "Serialize Remove Slave Node Failed, to Master (" << ip << ":" << port << "), " << db_name << "_" - << slot_id; + LOG(WARNING) << "Serialize Remove Slave Node Failed, to Master (" << ip << ":" << port << "), " << db_name; return Status::Corruption("Serialize Failed"); } return client_thread_->Write(ip, static_cast(port + kPortShiftReplServer), to_send); diff --git a/src/pika_repl_client_conn.cc b/src/pika_repl_client_conn.cc index 9f04b48a5d..10ef5e2089 100644 --- a/src/pika_repl_client_conn.cc +++ b/src/pika_repl_client_conn.cc @@ -109,7 +109,7 @@ void PikaReplClientConn::HandleMetaSyncResponse(void* arg) { std::vector master_db_structs; for (int idx = 0; idx < meta_sync.dbs_info_size(); ++idx) { const InnerMessage::InnerResponse_MetaSync_DBInfo& db_info = meta_sync.dbs_info(idx); - master_db_structs.push_back({db_info.db_name(), static_cast(db_info.slot_num()), {0}}); + master_db_structs.push_back({db_info.db_name()}); } std::vector self_db_structs = g_pika_conf->db_structs(); @@ -146,7 +146,7 @@ void PikaReplClientConn::HandleMetaSyncResponse(void* arg) { } g_pika_conf->SetWriteBinlog("yes"); - g_pika_server->PrepareSlotTrySync(); + g_pika_server->PrepareDBTrySync(); g_pika_server->FinishMetaSync(); LOG(INFO) << "Finish to handle meta sync response"; } @@ -158,30 +158,28 @@ void PikaReplClientConn::HandleDBSyncResponse(void* arg) { const InnerMessage::InnerResponse_DBSync db_sync_response = response->db_sync(); int32_t session_id = db_sync_response.session_id(); - const InnerMessage::Slot& slot_response = db_sync_response.slot(); - const std::string& db_name = slot_response.db_name(); - uint32_t slot_id = slot_response.slot_id(); - - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_name << ":" << slot_id << " Not Found"; + const InnerMessage::Slot& db_response = db_sync_response.slot(); + const std::string& db_name = db_response.db_name(); + + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_name << " Not Found"; return; } if (response->code() != InnerMessage::kOk) { - slave_slot->SetReplState(ReplState::kError); + slave_db->SetReplState(ReplState::kError); std::string reply = response->has_reply() ? response->reply() : ""; LOG(WARNING) << "DBSync Failed: " << reply; return; } - slave_slot->SetMasterSessionId(session_id); + slave_db->SetMasterSessionId(session_id); - std::string slot_name = slave_slot->SlotName(); - slave_slot->StopRsync(); - slave_slot->SetReplState(ReplState::kWaitDBSync); - LOG(INFO) << "Slot: " << slot_name << " Need Wait To Sync"; + slave_db->StopRsync(); + slave_db->SetReplState(ReplState::kWaitDBSync); + LOG(INFO) << "DB: " << db_name << " Need Wait To Sync"; } void PikaReplClientConn::HandleTrySyncResponse(void* arg) { @@ -196,76 +194,74 @@ void PikaReplClientConn::HandleTrySyncResponse(void* arg) { } const InnerMessage::InnerResponse_TrySync& try_sync_response = response->try_sync(); - const InnerMessage::Slot& slot_response = try_sync_response.slot(); - std::string db_name = slot_response.db_name(); - uint32_t slot_id = slot_response.slot_id(); - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!slot) { - LOG(WARNING) << "Slot: " << db_name << ":" << slot_id << " Not Found"; + const InnerMessage::Slot& db_response = try_sync_response.slot(); + std::string db_name = db_response.db_name(); + std::shared_ptr db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!db) { + LOG(WARNING) << "DB: " << db_name << " Not Found"; return; } - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_name << ":" << slot_id << " Not Found"; + std::shared_ptr slave_db = + g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "DB: " << db_name << "Not Found"; return; } LogicOffset logic_last_offset; if (response->has_consensus_meta()) { const InnerMessage::ConsensusMeta& meta = response->consensus_meta(); - if (meta.term() > slot->ConsensusTerm()) { - LOG(INFO) << "Update " << db_name << ":" << slot_id << " term from " << slot->ConsensusTerm() + if (meta.term() > db->ConsensusTerm()) { + LOG(INFO) << "Update " << db_name << " term from " << db->ConsensusTerm() << " to " << meta.term(); - slot->ConsensusUpdateTerm(meta.term()); - } else if (meta.term() < slot->ConsensusTerm()) /*outdated pb*/ { - LOG(WARNING) << "Drop outdated trysync response " << db_name << ":" << slot_id - << " recv term: " << meta.term() << " local term: " << slot->ConsensusTerm(); + db->ConsensusUpdateTerm(meta.term()); + } else if (meta.term() < db->ConsensusTerm()) /*outdated pb*/ { + LOG(WARNING) << "Drop outdated trysync response " << db_name + << " recv term: " << meta.term() << " local term: " << db->ConsensusTerm(); return; } if (response->consensus_meta().reject()) { - Status s = TrySyncConsensusCheck(response->consensus_meta(), slot, slave_slot); + Status s = TrySyncConsensusCheck(response->consensus_meta(), db, slave_db); if (!s.ok()) { - slave_slot->SetReplState(ReplState::kError); + slave_db->SetReplState(ReplState::kError); LOG(WARNING) << "Consensus Check failed " << s.ToString(); } return; } - logic_last_offset = slot->ConsensusLastIndex().l_offset; + logic_last_offset = db->ConsensusLastIndex().l_offset; } - std::string slot_name = slot->SlotName(); if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kOk) { BinlogOffset boffset; int32_t session_id = try_sync_response.session_id(); - slot->Logger()->GetProducerStatus(&boffset.filenum, &boffset.offset); - slave_slot->SetMasterSessionId(session_id); + db->Logger()->GetProducerStatus(&boffset.filenum, &boffset.offset); + slave_db->SetMasterSessionId(session_id); LogOffset offset(boffset, logic_last_offset); - g_pika_rm->SendSlotBinlogSyncAckRequest(db_name, slot_id, offset, offset, true); - slave_slot->SetReplState(ReplState::kConnected); + g_pika_rm->SendBinlogSyncAckRequest(db_name, offset, offset, true); + slave_db->SetReplState(ReplState::kConnected); // after connected, update receive time first to avoid connection timeout - slave_slot->SetLastRecvTime(pstd::NowMicros()); + slave_db->SetLastRecvTime(pstd::NowMicros()); - LOG(INFO) << "Slot: " << slot_name << " TrySync Ok"; + LOG(INFO) << "DB: " << db_name << " TrySync Ok"; } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kSyncPointBePurged) { - slave_slot->SetReplState(ReplState::kTryDBSync); - LOG(INFO) << "Slot: " << slot_name << " Need To Try DBSync"; + slave_db->SetReplState(ReplState::kTryDBSync); + LOG(INFO) << "DB: " << db_name << " Need To Try DBSync"; } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kSyncPointLarger) { - slave_slot->SetReplState(ReplState::kError); - LOG(WARNING) << "Slot: " << slot_name << " TrySync Error, Because the invalid filenum and offset"; + slave_db->SetReplState(ReplState::kError); + LOG(WARNING) << "DB: " << db_name << " TrySync Error, Because the invalid filenum and offset"; } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kError) { - slave_slot->SetReplState(ReplState::kError); - LOG(WARNING) << "Slot: " << slot_name << " TrySync Error"; + slave_db->SetReplState(ReplState::kError); + LOG(WARNING) << "DB: " << db_name << " TrySync Error"; } } Status PikaReplClientConn::TrySyncConsensusCheck(const InnerMessage::ConsensusMeta& consensus_meta, - const std::shared_ptr& slot, - const std::shared_ptr& slave_slot) { + const std::shared_ptr& db, + const std::shared_ptr& slave_db) { std::vector hints; for (int i = 0; i < consensus_meta.hint_size(); ++i) { const InnerMessage::BinlogOffset& pb_offset = consensus_meta.hint(i); @@ -277,41 +273,40 @@ Status PikaReplClientConn::TrySyncConsensusCheck(const InnerMessage::ConsensusMe hints.push_back(offset); } LogOffset reply_offset; - Status s = slot->ConsensusFollowerNegotiate(hints, &reply_offset); + Status s = db->ConsensusFollowerNegotiate(hints, &reply_offset); if (!s.ok()) { return s; } - slave_slot->SetReplState(ReplState::kTryConnect); + slave_db->SetReplState(ReplState::kTryConnect); return s; } void PikaReplClientConn::DispatchBinlogRes(const std::shared_ptr& res) { - // slot to a bunch of binlog chips - std::unordered_map*, hash_slot_info> par_binlog; + // db to a bunch of binlog chips + std::unordered_map*, hash_db_info> par_binlog; for (int i = 0; i < res->binlog_sync_size(); ++i) { const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync(i); - // hash key: db + slot_id - SlotInfo p_info(binlog_res.slot().db_name(), binlog_res.slot().slot_id()); + // hash key: db + DBInfo p_info(binlog_res.slot().db_name()); if (par_binlog.find(p_info) == par_binlog.end()) { par_binlog[p_info] = new std::vector(); } par_binlog[p_info]->push_back(i); } - std::shared_ptr slave_slot = nullptr; + std::shared_ptr slave_db; for (auto& binlog_nums : par_binlog) { - RmNode node(binlog_nums.first.db_name_, binlog_nums.first.slot_id_); - slave_slot = g_pika_rm->GetSyncSlaveSlotByName( - SlotInfo(binlog_nums.first.db_name_, binlog_nums.first.slot_id_)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << binlog_nums.first.db_name_ << "_" << binlog_nums.first.slot_id_ - << " not exist"; + RmNode node(binlog_nums.first.db_name_); + slave_db = g_pika_rm->GetSyncSlaveDBByName( + DBInfo(binlog_nums.first.db_name_)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << binlog_nums.first.db_name_ << " not exist"; break; } - slave_slot->SetLastRecvTime(pstd::NowMicros()); - g_pika_rm->ScheduleWriteBinlogTask(binlog_nums.first.db_name_ + std::to_string(binlog_nums.first.slot_id_), - res, std::dynamic_pointer_cast(shared_from_this()), + slave_db->SetLastRecvTime(pstd::NowMicros()); + g_pika_rm->ScheduleWriteBinlogTask(binlog_nums.first.db_name_, res, + std::dynamic_pointer_cast(shared_from_this()), reinterpret_cast(binlog_nums.second)); } } diff --git a/src/pika_repl_client_thread.cc b/src/pika_repl_client_thread.cc index 2a165b5648..355b373490 100644 --- a/src/pika_repl_client_thread.cc +++ b/src/pika_repl_client_thread.cc @@ -42,7 +42,7 @@ void PikaReplClientThread::ReplClientHandle::FdTimeoutHandle(int fd, const std:: } if (ip == g_pika_server->master_ip() && port == g_pika_server->master_port() + kPortShiftReplServer && PIKA_REPL_ERROR != g_pika_server->repl_state() && - PikaReplicaManager::CheckSlaveSlotState(ip, port)) { // if state machine in error state, no retry + PikaReplicaManager::CheckSlaveDBState(ip, port)) { // if state machine in error state, no retry LOG(WARNING) << "Master conn timeout : " << ip_port << " try reconnect"; g_pika_server->ResetMetaSyncStatus(); } diff --git a/src/pika_repl_server.cc b/src/pika_repl_server.cc index d977957bcf..a99fc18047 100644 --- a/src/pika_repl_server.cc +++ b/src/pika_repl_server.cc @@ -92,9 +92,14 @@ void PikaReplServer::BuildBinlogSyncResp(const std::vector& tasks, In for (const auto& task : tasks) { InnerMessage::InnerResponse::BinlogSync* binlog_sync = response->add_binlog_sync(); binlog_sync->set_session_id(task.rm_node_.SessionId()); - InnerMessage::Slot* slot = binlog_sync->mutable_slot(); - slot->set_db_name(task.rm_node_.DBName()); - slot->set_slot_id(task.rm_node_.SlotId()); + InnerMessage::Slot* db = binlog_sync->mutable_slot(); + db->set_db_name(task.rm_node_.DBName()); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db->set_slot_id(0); InnerMessage::BinlogOffset* boffset = binlog_sync->mutable_binlog_offset(); BuildBinlogOffset(task.binlog_chip_.offset_, boffset); binlog_sync->set_binlog(task.binlog_chip_.binlog_); diff --git a/src/pika_repl_server_conn.cc b/src/pika_repl_server_conn.cc index 94c0a140bc..37f45f63b7 100644 --- a/src/pika_repl_server_conn.cc +++ b/src/pika_repl_server_conn.cc @@ -58,7 +58,12 @@ void PikaReplServerConn::HandleMetaSyncRequest(void* arg) { for (const auto& db_struct : db_structs) { InnerMessage::InnerResponse_MetaSync_DBInfo* db_info = meta_sync->add_dbs_info(); db_info->set_db_name(db_struct.db_name); - db_info->set_slot_num(static_cast(db_struct.slot_num)); + /* + * Since the slot field is written in protobuffer, + * slot_num is set to the default value 1 for compatibility + * with older versions, but slot_num is not used + */ + db_info->set_slot_num(1); } } } @@ -78,62 +83,64 @@ void PikaReplServerConn::HandleTrySyncRequest(void* arg) { std::shared_ptr conn = task_arg->conn; InnerMessage::InnerRequest::TrySync try_sync_request = req->try_sync(); - const InnerMessage::Slot& slot_request = try_sync_request.slot(); + const InnerMessage::Slot& db_request = try_sync_request.slot(); const InnerMessage::BinlogOffset& slave_boffset = try_sync_request.binlog_offset(); const InnerMessage::Node& node = try_sync_request.node(); - std::string db_name = slot_request.db_name(); - uint32_t slot_id = slot_request.slot_id(); - std::string slot_name; + std::string db_name = db_request.db_name(); InnerMessage::InnerResponse response; InnerMessage::InnerResponse::TrySync* try_sync_response = response.mutable_try_sync(); try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); - InnerMessage::Slot* slot_response = try_sync_response->mutable_slot(); - slot_response->set_db_name(db_name); - slot_response->set_slot_id(slot_id); + InnerMessage::Slot* db_response = try_sync_response->mutable_slot(); + db_response->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db_response->set_slot_id(0); bool pre_success = true; response.set_type(InnerMessage::Type::kTrySync); - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!slot) { + std::shared_ptr db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!db) { response.set_code(InnerMessage::kError); - response.set_reply("Slot not found"); - LOG(WARNING) << "DB Name: " << db_name << " Slot ID: " << slot_id << " Not Found, TrySync Error"; + response.set_reply("DB not found"); + LOG(WARNING) << "DB Name: " << db_name << "Not Found, TrySync Error"; pre_success = false; } else { - slot_name = slot->SlotName(); LOG(INFO) << "Receive Trysync, Slave ip: " << node.ip() << ", Slave port:" << node.port() - << ", Slot: " << slot_name << ", filenum: " << slave_boffset.filenum() + << ", DB: " << db_name << ", filenum: " << slave_boffset.filenum() << ", pro_offset: " << slave_boffset.offset(); response.set_code(InnerMessage::kOk); } if (pre_success && req->has_consensus_meta()) { - if (slot->GetNumberOfSlaveNode() >= g_pika_conf->replication_num() && - !slot->CheckSlaveNodeExist(node.ip(), node.port())) { - LOG(WARNING) << "Current replication num: " << slot->GetNumberOfSlaveNode() + if (db->GetNumberOfSlaveNode() >= g_pika_conf->replication_num() && + !db->CheckSlaveNodeExist(node.ip(), node.port())) { + LOG(WARNING) << "Current replication num: " << db->GetNumberOfSlaveNode() << " hits configuration replication-num " << g_pika_conf->replication_num() << " stop trysync."; pre_success = false; } if (pre_success) { const InnerMessage::ConsensusMeta& meta = req->consensus_meta(); // need to response to outdated pb, new follower count on this response to update term - if (meta.term() > slot->ConsensusTerm()) { - LOG(INFO) << "Update " << slot_name << " term from " << slot->ConsensusTerm() << " to " + if (meta.term() > db->ConsensusTerm()) { + LOG(INFO) << "Update " << db_name << " term from " << db->ConsensusTerm() << " to " << meta.term(); - slot->ConsensusUpdateTerm(meta.term()); + db->ConsensusUpdateTerm(meta.term()); } } if (pre_success) { - pre_success = TrySyncConsensusOffsetCheck(slot, req->consensus_meta(), &response, try_sync_response); + pre_success = TrySyncConsensusOffsetCheck(db, req->consensus_meta(), &response, try_sync_response); } } else if (pre_success) { - pre_success = TrySyncOffsetCheck(slot, try_sync_request, try_sync_response); + pre_success = TrySyncOffsetCheck(db, try_sync_request, try_sync_response); } if (pre_success) { - pre_success = TrySyncUpdateSlaveNode(slot, try_sync_request, conn, try_sync_response); + pre_success = TrySyncUpdateSlaveNode(db, try_sync_request, conn, try_sync_response); } std::string reply_str; @@ -145,48 +152,46 @@ void PikaReplServerConn::HandleTrySyncRequest(void* arg) { conn->NotifyWrite(); } -bool PikaReplServerConn::TrySyncUpdateSlaveNode(const std::shared_ptr& slot, +bool PikaReplServerConn::TrySyncUpdateSlaveNode(const std::shared_ptr& db, const InnerMessage::InnerRequest::TrySync& try_sync_request, const std::shared_ptr& conn, InnerMessage::InnerResponse::TrySync* try_sync_response) { const InnerMessage::Node& node = try_sync_request.node(); - std::string slot_name = slot->SlotName(); - - if (!slot->CheckSlaveNodeExist(node.ip(), node.port())) { - int32_t session_id = slot->GenSessionId(); + if (!db->CheckSlaveNodeExist(node.ip(), node.port())) { + int32_t session_id = db->GenSessionId(); if (session_id == -1) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); - LOG(WARNING) << "Slot: " << slot_name << ", Gen Session id Failed"; + LOG(WARNING) << "DB: " << db->DBName() << ", Gen Session id Failed"; return false; } try_sync_response->set_session_id(session_id); // incremental sync - Status s = slot->AddSlaveNode(node.ip(), node.port(), session_id); + Status s = db->AddSlaveNode(node.ip(), node.port(), session_id); if (!s.ok()) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); - LOG(WARNING) << "Slot: " << slot_name << " TrySync Failed, " << s.ToString(); + LOG(WARNING) << "DB: " << db->DBName() << " TrySync Failed, " << s.ToString(); return false; } const std::string ip_port = pstd::IpPortString(node.ip(), node.port()); g_pika_rm->ReplServerUpdateClientConnMap(ip_port, conn->fd()); try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kOk); - LOG(INFO) << "Slot: " << slot_name << " TrySync Success, Session: " << session_id; + LOG(INFO) << "DB: " << db->DBName() << " TrySync Success, Session: " << session_id; } else { int32_t session_id; - Status s = slot->GetSlaveNodeSession(node.ip(), node.port(), &session_id); + Status s = db->GetSlaveNodeSession(node.ip(), node.port(), &session_id); if (!s.ok()) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); - LOG(WARNING) << "Slot: " << slot_name << ", Get Session id Failed" << s.ToString(); + LOG(WARNING) << "DB: " << db->DBName() << " Get Session id Failed" << s.ToString(); return false; } try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kOk); try_sync_response->set_session_id(session_id); - LOG(INFO) << "Slot: " << slot_name << " TrySync Success, Session: " << session_id; + LOG(INFO) << "DB: " << db->DBName() << " TrySync Success, Session: " << session_id; } return true; } -bool PikaReplServerConn::TrySyncConsensusOffsetCheck(const std::shared_ptr& slot, +bool PikaReplServerConn::TrySyncConsensusOffsetCheck(const std::shared_ptr& db, const InnerMessage::ConsensusMeta& meta, InnerMessage::InnerResponse* response, InnerMessage::InnerResponse::TrySync* try_sync_response) { @@ -195,70 +200,69 @@ bool PikaReplServerConn::TrySyncConsensusOffsetCheck(const std::shared_ptrSlotName(); + bool reject = false; std::vector hints; - Status s = slot->ConsensusLeaderNegotiate(last_log_offset, &reject, &hints); + Status s = db->ConsensusLeaderNegotiate(last_log_offset, &reject, &hints); if (!s.ok()) { if (s.IsNotFound()) { - LOG(INFO) << "Slot: " << slot_name << " need full sync"; + LOG(INFO) << "DB: " << db->DBName() << " need full sync"; try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kSyncPointBePurged); return false; } else { - LOG(WARNING) << "Slot:" << slot_name << " error " << s.ToString(); + LOG(WARNING) << "DB: " << db->DBName() << " error " << s.ToString(); try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); return false; } } try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kOk); - uint32_t term = slot->ConsensusTerm(); + uint32_t term = db->ConsensusTerm(); BuildConsensusMeta(reject, hints, term, response); return !reject; } -bool PikaReplServerConn::TrySyncOffsetCheck(const std::shared_ptr& slot, +bool PikaReplServerConn::TrySyncOffsetCheck(const std::shared_ptr& db, const InnerMessage::InnerRequest::TrySync& try_sync_request, InnerMessage::InnerResponse::TrySync* try_sync_response) { const InnerMessage::Node& node = try_sync_request.node(); const InnerMessage::BinlogOffset& slave_boffset = try_sync_request.binlog_offset(); - std::string slot_name = slot->SlotName(); - + std::string db_name = db->DBName(); BinlogOffset boffset; - Status s = slot->Logger()->GetProducerStatus(&(boffset.filenum), &(boffset.offset)); + Status s = db->Logger()->GetProducerStatus(&(boffset.filenum), &(boffset.offset)); if (!s.ok()) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); - LOG(WARNING) << "Handle TrySync, Slot: " << slot_name << " Get binlog offset error, TrySync failed"; + LOG(WARNING) << "Handle TrySync, DB: " << db_name << " Get binlog offset error, TrySync failed"; return false; } - InnerMessage::BinlogOffset* master_slot_boffset = try_sync_response->mutable_binlog_offset(); - master_slot_boffset->set_filenum(boffset.filenum); - master_slot_boffset->set_offset(boffset.offset); + InnerMessage::BinlogOffset* master_db_boffset = try_sync_response->mutable_binlog_offset(); + master_db_boffset->set_filenum(boffset.filenum); + master_db_boffset->set_offset(boffset.offset); if (boffset.filenum < slave_boffset.filenum() || (boffset.filenum == slave_boffset.filenum() && boffset.offset < slave_boffset.offset())) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kSyncPointLarger); LOG(WARNING) << "Slave offset is larger than mine, Slave ip: " << node.ip() << ", Slave port: " << node.port() - << ", Slot: " << slot_name << ", slave filenum: " << slave_boffset.filenum() + << ", DB: " << db_name << ", slave filenum: " << slave_boffset.filenum() << ", slave pro_offset_: " << slave_boffset.offset() << ", local filenum: " << boffset.filenum << ", local pro_offset_: " << boffset.offset; return false; } - std::string confile = NewFileName(slot->Logger()->filename(), slave_boffset.filenum()); + std::string confile = NewFileName(db->Logger()->filename(), slave_boffset.filenum()); if (!pstd::FileExists(confile)) { - LOG(INFO) << "Slot: " << slot_name << " binlog has been purged, may need full sync"; + LOG(INFO) << "DB: " << db_name << " binlog has been purged, may need full sync"; try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kSyncPointBePurged); return false; } PikaBinlogReader reader; - reader.Seek(slot->Logger(), slave_boffset.filenum(), slave_boffset.offset()); + reader.Seek(db->Logger(), slave_boffset.filenum(), slave_boffset.offset()); BinlogOffset seeked_offset; reader.GetReaderStatus(&(seeked_offset.filenum), &(seeked_offset.offset)); if (seeked_offset.filenum != slave_boffset.filenum() || seeked_offset.offset != slave_boffset.offset()) { try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); LOG(WARNING) << "Slave offset is not a start point of cur log, Slave ip: " << node.ip() - << ", Slave port: " << node.port() << ", Slot: " << slot_name - << ", closest start point, filenum: " << seeked_offset.filenum << ", offset: " << seeked_offset.offset; + << ", Slave port: " << node.port() << ", DB: " << db_name << " closest start point, filenum: " + << seeked_offset.filenum << ", offset: " << seeked_offset.offset; return false; } return true; @@ -287,48 +291,51 @@ void PikaReplServerConn::HandleDBSyncRequest(void* arg) { std::shared_ptr conn = task_arg->conn; InnerMessage::InnerRequest::DBSync db_sync_request = req->db_sync(); - const InnerMessage::Slot& slot_request = db_sync_request.slot(); + const InnerMessage::Slot& db_request = db_sync_request.slot(); const InnerMessage::Node& node = db_sync_request.node(); const InnerMessage::BinlogOffset& slave_boffset = db_sync_request.binlog_offset(); - std::string db_name = slot_request.db_name(); - uint32_t slot_id = slot_request.slot_id(); - std::string slot_name = db_name + "_" + std::to_string(slot_id); + std::string db_name = db_request.db_name(); InnerMessage::InnerResponse response; response.set_code(InnerMessage::kOk); response.set_type(InnerMessage::Type::kDBSync); InnerMessage::InnerResponse::DBSync* db_sync_response = response.mutable_db_sync(); - InnerMessage::Slot* slot_response = db_sync_response->mutable_slot(); - slot_response->set_db_name(db_name); - slot_response->set_slot_id(slot_id); - - LOG(INFO) << "Handle Slot DBSync Request"; + InnerMessage::Slot* db_response = db_sync_response->mutable_slot(); + db_response->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db_response->set_slot_id(0); + + LOG(INFO) << "Handle DBSync Request"; bool prior_success = true; - std::shared_ptr master_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!master_slot) { - LOG(WARNING) << "Sync Master Slot: " << db_name << ":" << slot_id << ", NotFound"; + std::shared_ptr master_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!master_db) { + LOG(WARNING) << "Sync Master DB: " << db_name << ", NotFound"; prior_success = false; response.set_code(InnerMessage::kError); } if (prior_success) { - if (!master_slot->CheckSlaveNodeExist(node.ip(), node.port())) { - int32_t session_id = master_slot->GenSessionId(); + if (!master_db->CheckSlaveNodeExist(node.ip(), node.port())) { + int32_t session_id = master_db->GenSessionId(); if (session_id == -1) { response.set_code(InnerMessage::kError); - LOG(WARNING) << "Slot: " << slot_name << ", Gen Session id Failed"; + LOG(WARNING) << "DB: " << db_name << ", Gen Session id Failed"; prior_success = false; } if (prior_success) { db_sync_response->set_session_id(session_id); - Status s = master_slot->AddSlaveNode(node.ip(), node.port(), session_id); + Status s = master_db->AddSlaveNode(node.ip(), node.port(), session_id); if (s.ok()) { const std::string ip_port = pstd::IpPortString(node.ip(), node.port()); g_pika_rm->ReplServerUpdateClientConnMap(ip_port, conn->fd()); - LOG(INFO) << "Slot: " << slot_name << " Handle DBSync Request Success, Session: " << session_id; + LOG(INFO) << "DB: " << db_name << " Handle DBSync Request Success, Session: " << session_id; } else { response.set_code(InnerMessage::kError); - LOG(WARNING) << "Slot: " << slot_name << " Handle DBSync Request Failed, " << s.ToString(); + LOG(WARNING) << "DB: " << db_name << " Handle DBSync Request Failed, " << s.ToString(); prior_success = false; } } else { @@ -336,26 +343,24 @@ void PikaReplServerConn::HandleDBSyncRequest(void* arg) { } } else { int32_t session_id; - Status s = master_slot->GetSlaveNodeSession(node.ip(), node.port(), &session_id); + Status s = master_db->GetSlaveNodeSession(node.ip(), node.port(), &session_id); if (!s.ok()) { response.set_code(InnerMessage::kError); - LOG(WARNING) << "Slot: " << slot_name << ", Get Session id Failed" << s.ToString(); + LOG(WARNING) << "DB: " << db_name << ", Get Session id Failed" << s.ToString(); prior_success = false; db_sync_response->set_session_id(-1); } else { db_sync_response->set_session_id(session_id); - LOG(INFO) << "Slot: " << slot_name << " Handle DBSync Request Success, Session: " << session_id; + LOG(INFO) << "DB: " << db_name << " Handle DBSync Request Success, Session: " << session_id; } } } - g_pika_server->DoBgSaveSlot(node.ip(), node.port() + kPortShiftRSync, db_name, slot_id, + g_pika_server->TryDBSync(node.ip(), node.port() + kPortShiftRSync, db_name, static_cast(slave_boffset.filenum())); // Change slave node's state to kSlaveDbSync so that the binlog will perserved. // See details in SyncMasterSlot::BinlogCloudPurge. - if (master_slot) { - master_slot->ActivateSlaveDbSync(node.ip(), node.port()); - } + master_db->ActivateSlaveDbSync(node.ip(), node.port()); std::string reply_str; if (!response.SerializeToString(&reply_str) || (conn->WriteResp(reply_str) != 0)) { @@ -372,13 +377,11 @@ void PikaReplServerConn::HandleBinlogSyncRequest(void* arg) { std::shared_ptr conn = task_arg->conn; if (!req->has_binlog_sync()) { LOG(WARNING) << "Pb parse error"; - // conn->NotifyClose(); return; } const InnerMessage::InnerRequest::BinlogSync& binlog_req = req->binlog_sync(); const InnerMessage::Node& node = binlog_req.node(); const std::string& db_name = binlog_req.db_name(); - uint32_t slot_id = binlog_req.slot_id(); bool is_first_send = binlog_req.first_send(); int32_t session_id = binlog_req.session_id(); @@ -391,40 +394,37 @@ void PikaReplServerConn::HandleBinlogSyncRequest(void* arg) { LogOffset range_start(b_range_start, l_range_start); LogOffset range_end(b_range_end, l_range_end); - std::shared_ptr master_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!master_slot) { - LOG(WARNING) << "Sync Master Slot: " << db_name << ":" << slot_id << ", NotFound"; + std::shared_ptr master_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!master_db) { + LOG(WARNING) << "Sync Master DB: " << db_name << ", NotFound"; return; } if (req->has_consensus_meta()) { const InnerMessage::ConsensusMeta& meta = req->consensus_meta(); - if (meta.term() > master_slot->ConsensusTerm()) { - LOG(INFO) << "Update " << db_name << ":" << slot_id << " term from " << master_slot->ConsensusTerm() + if (meta.term() > master_db->ConsensusTerm()) { + LOG(INFO) << "Update " << db_name << ": term from " << master_db->ConsensusTerm() << " to " << meta.term(); - master_slot->ConsensusUpdateTerm(meta.term()); - } else if (meta.term() < master_slot->ConsensusTerm()) /*outdated pb*/ { - LOG(WARNING) << "Drop outdated binlog sync req " << db_name << ":" << slot_id - << " recv term: " << meta.term() << " local term: " << master_slot->ConsensusTerm(); + master_db->ConsensusUpdateTerm(meta.term()); + } else if (meta.term() < master_db->ConsensusTerm()) /*outdated pb*/ { + LOG(WARNING) << "Drop outdated binlog sync req " << db_name << ":" + << " recv term: " << meta.term() << " local term: " << master_db->ConsensusTerm(); return; } } - if (!master_slot->CheckSessionId(node.ip(), node.port(), db_name, slot_id, session_id)) { - LOG(WARNING) << "Check Session failed " << node.ip() << ":" << node.port() << ", " << db_name << "_" - << slot_id; - // conn->NotifyClose(); + if (!master_db->CheckSessionId(node.ip(), node.port(), db_name, session_id)) { + LOG(WARNING) << "Check Session failed " << node.ip() << ":" << node.port() << ", " << db_name; return; } // Set ack info from slave - RmNode slave_node = RmNode(node.ip(), node.port(), db_name, slot_id); + RmNode slave_node = RmNode(node.ip(), node.port(), db_name); - Status s = master_slot->SetLastRecvTime(node.ip(), node.port(), pstd::NowMicros()); + Status s = master_db->SetLastRecvTime(node.ip(), node.port(), pstd::NowMicros()); if (!s.ok()) { - LOG(WARNING) << "SetMasterLastRecvTime failed " << node.ip() << ":" << node.port() << ", " << db_name << "_" - << slot_id << " " << s.ToString(); + LOG(WARNING) << "SetMasterLastRecvTime failed " << node.ip() << ":" << node.port() << ", " << db_name << " " << s.ToString(); conn->NotifyClose(); return; } @@ -436,7 +436,7 @@ void PikaReplServerConn::HandleBinlogSyncRequest(void* arg) { return; } - Status s = master_slot->ActivateSlaveBinlogSync(node.ip(), node.port(), range_start); + Status s = master_db->ActivateSlaveBinlogSync(node.ip(), node.port(), range_start); if (!s.ok()) { LOG(WARNING) << "Activate Binlog Sync failed " << slave_node.ToString() << " " << s.ToString(); conn->NotifyClose(); @@ -452,7 +452,7 @@ void PikaReplServerConn::HandleBinlogSyncRequest(void* arg) { } s = g_pika_rm->UpdateSyncBinlogStatus(slave_node, range_start, range_end); if (!s.ok()) { - LOG(WARNING) << "Update binlog ack failed " << db_name << " " << slot_id << " " << s.ToString(); + LOG(WARNING) << "Update binlog ack failed " << db_name << " " << s.ToString(); conn->NotifyClose(); return; } @@ -474,21 +474,25 @@ void PikaReplServerConn::HandleRemoveSlaveNodeRequest(void* arg) { const InnerMessage::Slot& slot = remove_slave_node_req.slot(); std::string db_name = slot.db_name(); - uint32_t slot_id = slot.slot_id(); - std::shared_ptr master_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!master_slot) { - LOG(WARNING) << "Sync Master Slot: " << db_name << ":" << slot_id << ", NotFound"; + std::shared_ptr master_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!master_db) { + LOG(WARNING) << "Sync Master DB: " << db_name << ", NotFound"; } - Status s = master_slot->RemoveSlaveNode(node.ip(), node.port()); + Status s = master_db->RemoveSlaveNode(node.ip(), node.port()); InnerMessage::InnerResponse response; response.set_code(InnerMessage::kOk); response.set_type(InnerMessage::Type::kRemoveSlaveNode); InnerMessage::InnerResponse::RemoveSlaveNode* remove_slave_node_response = response.add_remove_slave_node(); - InnerMessage::Slot* slot_response = remove_slave_node_response->mutable_slot(); - slot_response->set_db_name(db_name); - slot_response->set_slot_id(slot_id); + InnerMessage::Slot* db_response = remove_slave_node_response->mutable_slot (); + db_response->set_db_name(db_name); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + db_response->set_slot_id(0); InnerMessage::Node* node_response = remove_slave_node_response->mutable_node(); node_response->set_ip(g_pika_server->host()); node_response->set_port(g_pika_server->port()); diff --git a/src/pika_rm.cc b/src/pika_rm.cc index e6d6bbb0a1..d49fa877d5 100644 --- a/src/pika_rm.cc +++ b/src/pika_rm.cc @@ -25,28 +25,28 @@ using pstd::Status; extern std::unique_ptr g_pika_rm; extern PikaServer* g_pika_server; -/* SyncSlot */ +/* SyncDB */ -SyncSlot::SyncSlot(const std::string& db_name, uint32_t slot_id) - : slot_info_(db_name, slot_id) {} +SyncDB::SyncDB(const std::string& db_name) + : db_info_(db_name) {} -std::string SyncSlot::SlotName() { - return slot_info_.db_name_; +std::string SyncDB::DBName() { + return db_info_.db_name_; } -/* SyncMasterSlot*/ +/* SyncMasterDB*/ -SyncMasterSlot::SyncMasterSlot(const std::string& db_name, uint32_t slot_id) - : SyncSlot(db_name, slot_id), coordinator_(db_name, slot_id) {} +SyncMasterDB::SyncMasterDB(const std::string& db_name) + : SyncDB(db_name), coordinator_(db_name) {} -int SyncMasterSlot::GetNumberOfSlaveNode() { return coordinator_.SyncPros().SlaveSize(); } +int SyncMasterDB::GetNumberOfSlaveNode() { return coordinator_.SyncPros().SlaveSize(); } -bool SyncMasterSlot::CheckSlaveNodeExist(const std::string& ip, int port) { +bool SyncMasterDB::CheckSlaveNodeExist(const std::string& ip, int port) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); return static_cast(slave_ptr); } -Status SyncMasterSlot::GetSlaveNodeSession(const std::string& ip, int port, int32_t* session) { +Status SyncMasterDB::GetSlaveNodeSession(const std::string& ip, int port, int32_t* session) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("slave " + ip + ":" + std::to_string(port) + " not found"); @@ -59,29 +59,29 @@ Status SyncMasterSlot::GetSlaveNodeSession(const std::string& ip, int port, int3 return Status::OK(); } -Status SyncMasterSlot::AddSlaveNode(const std::string& ip, int port, int session_id) { +Status SyncMasterDB::AddSlaveNode(const std::string& ip, int port, int session_id) { Status s = coordinator_.AddSlaveNode(ip, port, session_id); if (!s.ok()) { - LOG(WARNING) << "Add Slave Node Failed, slot: " << SyncSlotInfo().ToString() << ", ip_port: " << ip << ":" + LOG(WARNING) << "Add Slave Node Failed, db: " << SyncDBInfo().ToString() << ", ip_port: " << ip << ":" << port; return s; } - LOG(INFO) << "Add Slave Node, slot: " << SyncSlotInfo().ToString() << ", ip_port: " << ip << ":" << port; + LOG(INFO) << "Add Slave Node, db: " << SyncDBInfo().ToString() << ", ip_port: " << ip << ":" << port; return Status::OK(); } -Status SyncMasterSlot::RemoveSlaveNode(const std::string& ip, int port) { +Status SyncMasterDB::RemoveSlaveNode(const std::string& ip, int port) { Status s = coordinator_.RemoveSlaveNode(ip, port); if (!s.ok()) { - LOG(WARNING) << "Remove Slave Node Failed, Slot: " << SyncSlotInfo().ToString() << ", ip_port: " << ip + LOG(WARNING) << "Remove Slave Node Failed, db: " << SyncDBInfo().ToString() << ", ip_port: " << ip << ":" << port; return s; } - LOG(INFO) << "Remove Slave Node, Slot: " << SyncSlotInfo().ToString() << ", ip_port: " << ip << ":" << port; + LOG(INFO) << "Remove Slave Node, DB: " << SyncDBInfo().ToString() << ", ip_port: " << ip << ":" << port; return Status::OK(); } -Status SyncMasterSlot::ActivateSlaveBinlogSync(const std::string& ip, int port, const LogOffset& offset) { +Status SyncMasterDB::ActivateSlaveBinlogSync(const std::string& ip, int port, const LogOffset& offset) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -111,7 +111,7 @@ Status SyncMasterSlot::ActivateSlaveBinlogSync(const std::string& ip, int port, return Status::OK(); } -Status SyncMasterSlot::SyncBinlogToWq(const std::string& ip, int port) { +Status SyncMasterDB::SyncBinlogToWq(const std::string& ip, int port) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -126,7 +126,7 @@ Status SyncMasterSlot::SyncBinlogToWq(const std::string& ip, int port) { return Status::OK(); } -Status SyncMasterSlot::ActivateSlaveDbSync(const std::string& ip, int port) { +Status SyncMasterDB::ActivateSlaveDbSync(const std::string& ip, int port) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -140,7 +140,7 @@ Status SyncMasterSlot::ActivateSlaveDbSync(const std::string& ip, int port) { return Status::OK(); } -Status SyncMasterSlot::ReadBinlogFileToWq(const std::shared_ptr& slave_ptr) { +Status SyncMasterDB::ReadBinlogFileToWq(const std::shared_ptr& slave_ptr) { int cnt = slave_ptr->sync_win.Remaining(); std::shared_ptr reader = slave_ptr->binlog_reader; if (!reader) { @@ -160,7 +160,7 @@ Status SyncMasterSlot::ReadBinlogFileToWq(const std::shared_ptr& slav if (s.IsEndFile()) { break; } else if (s.IsCorruption() || s.IsIOError()) { - LOG(WARNING) << SyncSlotInfo().ToString() << " Read Binlog error : " << s.ToString(); + LOG(WARNING) << SyncDBInfo().ToString() << " Read Binlog error : " << s.ToString(); return s; } BinlogItem item; @@ -174,30 +174,28 @@ Status SyncMasterSlot::ReadBinlogFileToWq(const std::shared_ptr& slav slave_ptr->sync_win.Push(SyncWinItem(sent_offset, msg.size())); slave_ptr->SetLastSendTime(pstd::NowMicros()); - RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->DBName(), slave_ptr->SlotId(), - slave_ptr->SessionId()); + RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->DBName(), slave_ptr->SessionId()); WriteTask task(rm_node, BinlogChip(sent_offset, msg), slave_ptr->sent_offset); tasks.push_back(task); slave_ptr->sent_offset = sent_offset; } if (!tasks.empty()) { - g_pika_rm->ProduceWriteQueue(slave_ptr->Ip(), slave_ptr->Port(), slot_info_.slot_id_, tasks); + g_pika_rm->ProduceWriteQueue(slave_ptr->Ip(), slave_ptr->Port(), db_info_.db_name_, tasks); } return Status::OK(); } -Status SyncMasterSlot::ConsensusUpdateSlave(const std::string& ip, int port, const LogOffset& start, - const LogOffset& end) { +Status SyncMasterDB::ConsensusUpdateSlave(const std::string& ip, int port, const LogOffset& start, const LogOffset& end) { Status s = coordinator_.UpdateSlave(ip, port, start, end); if (!s.ok()) { - LOG(WARNING) << SyncSlotInfo().ToString() << s.ToString(); + LOG(WARNING) << SyncDBInfo().ToString() << s.ToString(); return s; } return Status::OK(); } -Status SyncMasterSlot::ConsensusUpdateAppliedIndex(const LogOffset& offset) { +Status SyncMasterDB::ConsensusUpdateAppliedIndex(const LogOffset& offset) { std::shared_ptr context = coordinator_.context(); if (!context) { LOG(WARNING) << "Coordinator context empty."; @@ -207,10 +205,9 @@ Status SyncMasterSlot::ConsensusUpdateAppliedIndex(const LogOffset& offset) { return Status::OK(); } -LogOffset SyncMasterSlot::ConsensusAppliedIndex() { return coordinator_.applied_index(); } -Status SyncMasterSlot::GetSlaveSyncBinlogInfo(const std::string& ip, int port, BinlogOffset* sent_offset, - BinlogOffset* acked_offset) { +Status SyncMasterDB::GetSlaveSyncBinlogInfo(const std::string& ip, int port, BinlogOffset* sent_offset, + BinlogOffset* acked_offset) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -224,7 +221,7 @@ Status SyncMasterSlot::GetSlaveSyncBinlogInfo(const std::string& ip, int port, B return Status::OK(); } -Status SyncMasterSlot::GetSlaveState(const std::string& ip, int port, SlaveState* const slave_state) { +Status SyncMasterDB::GetSlaveState(const std::string& ip, int port, SlaveState* const slave_state) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -237,7 +234,7 @@ Status SyncMasterSlot::GetSlaveState(const std::string& ip, int port, SlaveState return Status::OK(); } -Status SyncMasterSlot::WakeUpSlaveBinlogSync() { +Status SyncMasterDB::WakeUpSlaveBinlogSync() { std::unordered_map> slaves = GetAllSlaveNodes(); std::vector> to_del; for (auto& slave_iter : slaves) { @@ -258,33 +255,7 @@ Status SyncMasterSlot::WakeUpSlaveBinlogSync() { return Status::OK(); } -Status SyncMasterSlot::SetLastSendTime(const std::string& ip, int port, uint64_t time) { - std::shared_ptr slave_ptr = GetSlaveNode(ip, port); - if (!slave_ptr) { - return Status::NotFound("ip " + ip + " port " + std::to_string(port)); - } - - slave_ptr->Lock(); - slave_ptr->SetLastSendTime(time); - slave_ptr->Unlock(); - - return Status::OK(); -} - -Status SyncMasterSlot::GetLastSendTime(const std::string& ip, int port, uint64_t* time) { - std::shared_ptr slave_ptr = GetSlaveNode(ip, port); - if (!slave_ptr) { - return Status::NotFound("ip " + ip + " port " + std::to_string(port)); - } - - slave_ptr->Lock(); - *time = slave_ptr->LastSendTime(); - slave_ptr->Unlock(); - - return Status::OK(); -} - -Status SyncMasterSlot::SetLastRecvTime(const std::string& ip, int port, uint64_t time) { +Status SyncMasterDB::SetLastRecvTime(const std::string& ip, int port, uint64_t time) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { return Status::NotFound("ip " + ip + " port " + std::to_string(port)); @@ -297,20 +268,7 @@ Status SyncMasterSlot::SetLastRecvTime(const std::string& ip, int port, uint64_t return Status::OK(); } -Status SyncMasterSlot::GetLastRecvTime(const std::string& ip, int port, uint64_t* time) { - std::shared_ptr slave_ptr = GetSlaveNode(ip, port); - if (!slave_ptr) { - return Status::NotFound("ip " + ip + " port " + std::to_string(port)); - } - - slave_ptr->Lock(); - *time = slave_ptr->LastRecvTime(); - slave_ptr->Unlock(); - - return Status::OK(); -} - -Status SyncMasterSlot::GetSafetyPurgeBinlog(std::string* safety_purge) { +Status SyncMasterDB::GetSafetyPurgeBinlog(std::string* safety_purge) { BinlogOffset boffset; Status s = Logger()->GetProducerStatus(&(boffset.filenum), &(boffset.offset)); if (!s.ok()) { @@ -337,7 +295,7 @@ Status SyncMasterSlot::GetSafetyPurgeBinlog(std::string* safety_purge) { return Status::OK(); } -bool SyncMasterSlot::BinlogCloudPurge(uint32_t index) { +bool SyncMasterDB::BinlogCloudPurge(uint32_t index) { BinlogOffset boffset; Status s = Logger()->GetProducerStatus(&(boffset.filenum), &(boffset.offset)); if (!s.ok()) { @@ -362,7 +320,7 @@ bool SyncMasterSlot::BinlogCloudPurge(uint32_t index) { return true; } -Status SyncMasterSlot::CheckSyncTimeout(uint64_t now) { +Status SyncMasterDB::CheckSyncTimeout(uint64_t now) { std::unordered_map> slaves = GetAllSlaveNodes(); std::vector to_del; @@ -374,8 +332,7 @@ Status SyncMasterSlot::CheckSyncTimeout(uint64_t now) { } else if (slave_ptr->LastSendTime() + kSendKeepAliveTimeout < now && slave_ptr->sent_offset == slave_ptr->acked_offset) { std::vector task; - RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->DBName(), slave_ptr->SlotId(), - slave_ptr->SessionId()); + RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->DBName(), slave_ptr->SessionId()); WriteTask empty_task(rm_node, BinlogChip(LogOffset(), ""), LogOffset()); task.push_back(empty_task); Status s = g_pika_rm->SendSlaveBinlogChipsRequest(slave_ptr->Ip(), slave_ptr->Port(), task); @@ -390,12 +347,12 @@ Status SyncMasterSlot::CheckSyncTimeout(uint64_t now) { for (auto& node : to_del) { coordinator_.SyncPros().RemoveSlaveNode(node.Ip(), node.Port()); g_pika_rm->DropItemInWriteQueue(node.Ip(), node.Port()); - LOG(WARNING) << SyncSlotInfo().ToString() << " Master del Recv Timeout slave success " << node.ToString(); + LOG(WARNING) << SyncDBInfo().ToString() << " Master del Recv Timeout slave success " << node.ToString(); } return Status::OK(); } -std::string SyncMasterSlot::ToStringStatus() { +std::string SyncMasterDB::ToStringStatus() { std::stringstream tmp_stream; tmp_stream << " Current Master Session: " << session_id_ << "\r\n"; tmp_stream << " Consensus: " @@ -412,7 +369,7 @@ std::string SyncMasterSlot::ToStringStatus() { return tmp_stream.str(); } -void SyncMasterSlot::GetValidSlaveNames(std::vector* slavenames) { +void SyncMasterDB::GetValidSlaveNames(std::vector* slavenames) { std::unordered_map> slaves = GetAllSlaveNodes(); for (const auto& slave_iter : slaves) { std::shared_ptr slave_ptr = slave_iter.second; @@ -425,7 +382,7 @@ void SyncMasterSlot::GetValidSlaveNames(std::vector* slavenames) { } } -Status SyncMasterSlot::GetInfo(std::string* info) { +Status SyncMasterDB::GetInfo(std::string* info) { std::unordered_map> slaves = GetAllSlaveNodes(); std::stringstream tmp_stream; @@ -454,115 +411,107 @@ Status SyncMasterSlot::GetInfo(std::string* info) { return Status::OK(); } -int32_t SyncMasterSlot::GenSessionId() { +int32_t SyncMasterDB::GenSessionId() { std::lock_guard ml(session_mu_); return session_id_++; } -bool SyncMasterSlot::CheckSessionId(const std::string& ip, int port, const std::string& db_name, - uint64_t slot_id, int session_id) { +bool SyncMasterDB::CheckSessionId(const std::string& ip, int port, const std::string& db_name, + int session_id) { std::shared_ptr slave_ptr = GetSlaveNode(ip, port); if (!slave_ptr) { - LOG(WARNING) << "Check SessionId Get Slave Node Error: " << ip << ":" << port << "," << db_name << "_" - << slot_id; + LOG(WARNING) << "Check SessionId Get Slave Node Error: " << ip << ":" << port << "," << db_name; return false; } std::lock_guard l(slave_ptr->slave_mu); if (session_id != slave_ptr->SessionId()) { - LOG(WARNING) << "Check SessionId Mismatch: " << ip << ":" << port << ", " << db_name << "_" << slot_id + LOG(WARNING) << "Check SessionId Mismatch: " << ip << ":" << port << ", " << db_name << "_" << " expected_session: " << session_id << ", actual_session:" << slave_ptr->SessionId(); return false; } return true; } -Status SyncMasterSlot::ConsensusProposeLog(const std::shared_ptr& cmd_ptr) { +Status SyncMasterDB::ConsensusProposeLog(const std::shared_ptr& cmd_ptr) { return coordinator_.ProposeLog(cmd_ptr); } -Status SyncMasterSlot::ConsensusProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute) { +Status SyncMasterDB::ConsensusProcessLeaderLog(const std::shared_ptr& cmd_ptr, const BinlogItem& attribute) { return coordinator_.ProcessLeaderLog(cmd_ptr, attribute); } -LogOffset SyncMasterSlot::ConsensusCommittedIndex() { return coordinator_.committed_index(); } +LogOffset SyncMasterDB::ConsensusCommittedIndex() { return coordinator_.committed_index(); } -LogOffset SyncMasterSlot::ConsensusLastIndex() { return coordinator_.MemLogger()->last_offset(); } +LogOffset SyncMasterDB::ConsensusLastIndex() { return coordinator_.MemLogger()->last_offset(); } -uint32_t SyncMasterSlot::ConsensusTerm() { return coordinator_.term(); } +uint32_t SyncMasterDB::ConsensusTerm() { return coordinator_.term(); } -void SyncMasterSlot::ConsensusUpdateTerm(uint32_t term) { +void SyncMasterDB::ConsensusUpdateTerm(uint32_t term) { coordinator_.UpdateTerm(term); if ((g_pika_server->role() & PIKA_ROLE_MASTER) != 0) { CommitPreviousLogs(term); } } -void SyncMasterSlot::CommitPreviousLogs(const uint32_t& term) { +void SyncMasterDB::CommitPreviousLogs(const uint32_t& term) { // Append dummy cmd - std::shared_ptr dummy_ptr = std::make_shared(kCmdDummy, 0, kCmdFlagsWrite | kCmdFlagsSingleSlot); + std::shared_ptr dummy_ptr = std::make_shared(kCmdDummy, 0, kCmdFlagsWrite); PikaCmdArgsType args; args.push_back(kCmdDummy); - dummy_ptr->Initial(args, SyncSlotInfo().db_name_); + dummy_ptr->Initial(args, SyncDBInfo().db_name_); dummy_ptr->SetStage(Cmd::kBinlogStage); dummy_ptr->Execute(); dummy_ptr->SetStage(Cmd::kExecuteStage); } -std::shared_ptr SyncMasterSlot::GetSlaveNode(const std::string& ip, int port) { +std::shared_ptr SyncMasterDB::GetSlaveNode(const std::string& ip, int port) { return coordinator_.SyncPros().GetSlaveNode(ip, port); } -std::unordered_map> SyncMasterSlot::GetAllSlaveNodes() { +std::unordered_map> SyncMasterDB::GetAllSlaveNodes() { return coordinator_.SyncPros().GetAllSlaveNodes(); } -Status SyncMasterSlot::ConsensusLeaderNegotiate(const LogOffset& f_last_offset, bool* reject, - std::vector* hints) { +Status SyncMasterDB::ConsensusLeaderNegotiate(const LogOffset& f_last_offset, bool* reject, + std::vector* hints) { return coordinator_.LeaderNegotiate(f_last_offset, reject, hints); } -Status SyncMasterSlot::ConsensusFollowerNegotiate(const std::vector& hints, LogOffset* reply_offset) { +Status SyncMasterDB::ConsensusFollowerNegotiate(const std::vector& hints, LogOffset* reply_offset) { return coordinator_.FollowerNegotiate(hints, reply_offset); } -Status SyncMasterSlot::ConsensusReset(const LogOffset& applied_offset) { return coordinator_.Reset(applied_offset); } - -/* SyncSlaveSlot */ -SyncSlaveSlot::SyncSlaveSlot(const std::string& db_name, uint32_t slot_id) - : SyncSlot(db_name, slot_id) { +/* SyncSlaveDB */ +SyncSlaveDB::SyncSlaveDB(const std::string& db_name) + : SyncDB(db_name) { std::string dbsync_path = g_pika_conf->db_sync_path() + "/" + db_name; - rsync_cli_.reset(new rsync::RsyncClient(dbsync_path, db_name, slot_id)); + rsync_cli_.reset(new rsync::RsyncClient(dbsync_path, db_name)); m_info_.SetLastRecvTime(pstd::NowMicros()); } -void SyncSlaveSlot::SetReplState(const ReplState& repl_state) { +void SyncSlaveDB::SetReplState(const ReplState& repl_state) { if (repl_state == ReplState::kNoConnect) { // deactivate Deactivate(); return; } - std::lock_guard l(slot_mu_); + std::lock_guard l(db_mu_); repl_state_ = repl_state; } -ReplState SyncSlaveSlot::State() { - std::lock_guard l(slot_mu_); +ReplState SyncSlaveDB::State() { + std::lock_guard l(db_mu_); return repl_state_; } -void SyncSlaveSlot::SetLastRecvTime(uint64_t time) { - std::lock_guard l(slot_mu_); +void SyncSlaveDB::SetLastRecvTime(uint64_t time) { + std::lock_guard l(db_mu_); m_info_.SetLastRecvTime(time); } -uint64_t SyncSlaveSlot::LastRecvTime() { - std::lock_guard l(slot_mu_); - return m_info_.LastRecvTime(); -} - -Status SyncSlaveSlot::CheckSyncTimeout(uint64_t now) { - std::lock_guard l(slot_mu_); +Status SyncSlaveDB::CheckSyncTimeout(uint64_t now) { + std::lock_guard l(db_mu_); // no need to do session keepalive return ok if (repl_state_ != ReplState::kWaitDBSync && repl_state_ != ReplState::kConnected) { return Status::OK(); @@ -570,12 +519,11 @@ Status SyncSlaveSlot::CheckSyncTimeout(uint64_t now) { if (m_info_.LastRecvTime() + kRecvKeepAliveTimeout < now) { // update slave state to kTryConnect, and try reconnect to master node repl_state_ = ReplState::kTryConnect; - g_pika_server->SetLoopSlotStateMachine(true); } return Status::OK(); } -Status SyncSlaveSlot::GetInfo(std::string* info) { +Status SyncSlaveDB::GetInfo(std::string* info) { std::string tmp_str = " Role: Slave\r\n"; tmp_str += " master: " + MasterIp() + ":" + std::to_string(MasterPort()) + "\r\n"; tmp_str += " slave status: " + ReplStateMsg[repl_state_] + "\r\n"; @@ -583,61 +531,61 @@ Status SyncSlaveSlot::GetInfo(std::string* info) { return Status::OK(); } -void SyncSlaveSlot::Activate(const RmNode& master, const ReplState& repl_state) { - std::lock_guard l(slot_mu_); +void SyncSlaveDB::Activate(const RmNode& master, const ReplState& repl_state) { + std::lock_guard l(db_mu_); m_info_ = master; repl_state_ = repl_state; m_info_.SetLastRecvTime(pstd::NowMicros()); } -void SyncSlaveSlot::Deactivate() { - std::lock_guard l(slot_mu_); +void SyncSlaveDB::Deactivate() { + std::lock_guard l(db_mu_); m_info_ = RmNode(); repl_state_ = ReplState::kNoConnect; rsync_cli_->Stop(); } -std::string SyncSlaveSlot::ToStringStatus() { +std::string SyncSlaveDB::ToStringStatus() { return " Master: " + MasterIp() + ":" + std::to_string(MasterPort()) + "\r\n" + " SessionId: " + std::to_string(MasterSessionId()) + "\r\n" + " SyncStatus " + ReplStateMsg[repl_state_] + "\r\n"; } -const std::string& SyncSlaveSlot::MasterIp() { - std::lock_guard l(slot_mu_); +const std::string& SyncSlaveDB::MasterIp() { + std::lock_guard l(db_mu_); return m_info_.Ip(); } -int SyncSlaveSlot::MasterPort() { - std::lock_guard l(slot_mu_); +int SyncSlaveDB::MasterPort() { + std::lock_guard l(db_mu_); return m_info_.Port(); } -void SyncSlaveSlot::SetMasterSessionId(int32_t session_id) { - std::lock_guard l(slot_mu_); +void SyncSlaveDB::SetMasterSessionId(int32_t session_id) { + std::lock_guard l(db_mu_); m_info_.SetSessionId(session_id); } -int32_t SyncSlaveSlot::MasterSessionId() { - std::lock_guard l(slot_mu_); +int32_t SyncSlaveDB::MasterSessionId() { + std::lock_guard l(db_mu_); return m_info_.SessionId(); } -void SyncSlaveSlot::SetLocalIp(const std::string& local_ip) { - std::lock_guard l(slot_mu_); +void SyncSlaveDB::SetLocalIp(const std::string& local_ip) { + std::lock_guard l(db_mu_); local_ip_ = local_ip; } -std::string SyncSlaveSlot::LocalIp() { - std::lock_guard l(slot_mu_); +std::string SyncSlaveDB::LocalIp() { + std::lock_guard l(db_mu_); return local_ip_; } -void SyncSlaveSlot::StopRsync() { +void SyncSlaveDB::StopRsync() { rsync_cli_->Stop(); } -void SyncSlaveSlot::ActivateRsync() { +void SyncSlaveDB::ActivateRsync() { if (!rsync_cli_->IsIdle()) { return; } @@ -657,7 +605,7 @@ PikaReplicaManager::PikaReplicaManager() { int port = g_pika_conf->port() + kPortShiftReplServer; pika_repl_client_ = std::make_unique(3000, 60); pika_repl_server_ = std::make_unique(ips, port, 3000); - InitSlot(); + InitDB(); } void PikaReplicaManager::Start() { @@ -681,11 +629,11 @@ void PikaReplicaManager::Stop() { } bool PikaReplicaManager::CheckMasterSyncFinished() { - for (auto& iter : sync_master_slots_) { - std::shared_ptr slot = iter.second; - LogOffset commit = slot->ConsensusCommittedIndex(); + for (auto& iter : sync_master_dbs_) { + std::shared_ptr db = iter.second; + LogOffset commit = db->ConsensusCommittedIndex(); BinlogOffset binlog; - Status s = slot->StableLogger()->Logger()->GetProducerStatus(&binlog.filenum, &binlog.offset); + Status s = db->StableLogger()->Logger()->GetProducerStatus(&binlog.filenum, &binlog.offset); if (!s.ok()) { return false; } @@ -696,25 +644,21 @@ bool PikaReplicaManager::CheckMasterSyncFinished() { return true; } -void PikaReplicaManager::InitSlot() { +void PikaReplicaManager::InitDB() { std::vector db_structs = g_pika_conf->db_structs(); for (const auto& db : db_structs) { const std::string& db_name = db.db_name; - for (const auto& slot_id : db.slot_ids) { - sync_master_slots_[SlotInfo(db_name, slot_id)] = - std::make_shared(db_name, slot_id); - sync_slave_slots_[SlotInfo(db_name, slot_id)] = - std::make_shared(db_name, slot_id); - } + sync_master_dbs_[DBInfo(db_name)] = std::make_shared(db_name); + sync_slave_dbs_[DBInfo(db_name)] = std::make_shared(db_name); } } -void PikaReplicaManager::ProduceWriteQueue(const std::string& ip, int port, uint32_t slot_id, +void PikaReplicaManager::ProduceWriteQueue(const std::string& ip, int port, std::string db_name, const std::vector& tasks) { std::lock_guard l(write_queue_mu_); std::string index = ip + ":" + std::to_string(port); for (auto& task : tasks) { - write_queues_[index][slot_id].push(task); + write_queues_[index][db_name].push(task); } } @@ -725,9 +669,9 @@ int PikaReplicaManager::ConsumeWriteQueue() { std::lock_guard l(write_queue_mu_); for (auto& iter : write_queues_) { const std::string& ip_port = iter.first; - std::unordered_map>& p_map = iter.second; - for (auto& slot_queue : p_map) { - std::queue& queue = slot_queue.second; + std::unordered_map>& p_map = iter.second; + for (auto& db_queue : p_map) { + std::queue& queue = db_queue.second; for (int i = 0; i < kBinlogSendPacketNum; ++i) { if (queue.empty()) { break; @@ -797,15 +741,15 @@ void PikaReplicaManager::ScheduleReplClientBGTask(net::TaskFunc func, void* arg) pika_repl_client_->Schedule(func, arg); } -void PikaReplicaManager::ScheduleWriteBinlogTask(const std::string& db_slot, +void PikaReplicaManager::ScheduleWriteBinlogTask(const std::string& db, const std::shared_ptr& res, const std::shared_ptr& conn, void* res_private_data) { - pika_repl_client_->ScheduleWriteBinlogTask(db_slot, res, conn, res_private_data); + pika_repl_client_->ScheduleWriteBinlogTask(db, res, conn, res_private_data); } void PikaReplicaManager::ScheduleWriteDBTask(const std::shared_ptr& cmd_ptr, const LogOffset& offset, - const std::string& db_name, uint32_t slot_id) { - pika_repl_client_->ScheduleWriteDBTask(cmd_ptr, offset, db_name, slot_id); + const std::string& db_name) { + pika_repl_client_->ScheduleWriteDBTask(cmd_ptr, offset, db_name); } void PikaReplicaManager::ReplServerRemoveClientConn(int fd) { pika_repl_server_->RemoveClientConn(fd); } @@ -816,29 +760,29 @@ void PikaReplicaManager::ReplServerUpdateClientConnMap(const std::string& ip_por Status PikaReplicaManager::UpdateSyncBinlogStatus(const RmNode& slave, const LogOffset& offset_start, const LogOffset& offset_end) { - std::shared_lock l(slots_rw_); - if (sync_master_slots_.find(slave.NodeSlotInfo()) == sync_master_slots_.end()) { + std::shared_lock l(dbs_rw_); + if (sync_master_dbs_.find(slave.NodeDBInfo()) == sync_master_dbs_.end()) { return Status::NotFound(slave.ToString() + " not found"); } - std::shared_ptr slot = sync_master_slots_[slave.NodeSlotInfo()]; - Status s = slot->ConsensusUpdateSlave(slave.Ip(), slave.Port(), offset_start, offset_end); + std::shared_ptr db = sync_master_dbs_[slave.NodeDBInfo()]; + Status s = db->ConsensusUpdateSlave(slave.Ip(), slave.Port(), offset_start, offset_end); if (!s.ok()) { return s; } - s = slot->SyncBinlogToWq(slave.Ip(), slave.Port()); + s = db->SyncBinlogToWq(slave.Ip(), slave.Port()); if (!s.ok()) { return s; } return Status::OK(); } -bool PikaReplicaManager::CheckSlaveSlotState(const std::string& ip, const int port) { - std::shared_ptr slot = nullptr; - for (const auto& iter : g_pika_rm->sync_slave_slots_) { - slot = iter.second; - if (slot->State() == ReplState::kDBNoConnect && slot->MasterIp() == ip && - slot->MasterPort() + kPortShiftReplServer == port) { - LOG(INFO) << "DB: " << slot->SyncSlotInfo().ToString() +bool PikaReplicaManager::CheckSlaveDBState(const std::string& ip, const int port) { + std::shared_ptr db = nullptr; + for (const auto& iter : g_pika_rm->sync_slave_dbs_) { + db = iter.second; + if (db->State() == ReplState::kDBNoConnect && db->MasterIp() == ip && + db->MasterPort() + kPortShiftReplServer == port) { + LOG(INFO) << "DB: " << db->SyncDBInfo().ToString() << " has been dbslaveof no one, then will not try reconnect."; return false; } @@ -847,29 +791,29 @@ bool PikaReplicaManager::CheckSlaveSlotState(const std::string& ip, const int po } Status PikaReplicaManager::LostConnection(const std::string& ip, int port) { - std::shared_lock l(slots_rw_); - for (auto& iter : sync_master_slots_) { - std::shared_ptr slot = iter.second; - Status s = slot->RemoveSlaveNode(ip, port); + std::shared_lock l(dbs_rw_); + for (auto& iter : sync_master_dbs_) { + std::shared_ptr db = iter.second; + Status s = db->RemoveSlaveNode(ip, port); if (!s.ok() && !s.IsNotFound()) { LOG(WARNING) << "Lost Connection failed " << s.ToString(); } } - for (auto& iter : sync_slave_slots_) { - std::shared_ptr slot = iter.second; - if (slot->MasterIp() == ip && slot->MasterPort() == port) { - slot->Deactivate(); + for (auto& iter : sync_slave_dbs_) { + std::shared_ptr db = iter.second; + if (db->MasterIp() == ip && db->MasterPort() == port) { + db->Deactivate(); } } return Status::OK(); } Status PikaReplicaManager::WakeUpBinlogSync() { - std::shared_lock l(slots_rw_); - for (auto& iter : sync_master_slots_) { - std::shared_ptr slot = iter.second; - Status s = slot->WakeUpSlaveBinlogSync(); + std::shared_lock l(dbs_rw_); + for (auto& iter : sync_master_dbs_) { + std::shared_ptr db = iter.second; + Status s = db->WakeUpSlaveBinlogSync(); if (!s.ok()) { return s; } @@ -878,18 +822,18 @@ Status PikaReplicaManager::WakeUpBinlogSync() { } Status PikaReplicaManager::CheckSyncTimeout(uint64_t now) { - std::shared_lock l(slots_rw_); + std::shared_lock l(dbs_rw_); - for (auto& iter : sync_master_slots_) { - std::shared_ptr slot = iter.second; - Status s = slot->CheckSyncTimeout(now); + for (auto& iter : sync_master_dbs_) { + std::shared_ptr db = iter.second; + Status s = db->CheckSyncTimeout(now); if (!s.ok()) { LOG(WARNING) << "CheckSyncTimeout Failed " << s.ToString(); } } - for (auto& iter : sync_slave_slots_) { - std::shared_ptr slot = iter.second; - Status s = slot->CheckSyncTimeout(now); + for (auto& iter : sync_slave_dbs_) { + std::shared_ptr db = iter.second; + Status s = db->CheckSyncTimeout(now); if (!s.ok()) { LOG(WARNING) << "CheckSyncTimeout Failed " << s.ToString(); } @@ -897,64 +841,28 @@ Status PikaReplicaManager::CheckSyncTimeout(uint64_t now) { return Status::OK(); } -Status PikaReplicaManager::CheckSlotRole(const std::string& db, uint32_t slot_id, int* role) { - std::shared_lock l(slots_rw_); +Status PikaReplicaManager::CheckDBRole(const std::string& db, int* role) { + std::shared_lock l(dbs_rw_); *role = 0; - SlotInfo p_info(db, slot_id); - if (sync_master_slots_.find(p_info) == sync_master_slots_.end()) { - return Status::NotFound(db + std::to_string(slot_id) + " not found"); + DBInfo p_info(db); + if (sync_master_dbs_.find(p_info) == sync_master_dbs_.end()) { + return Status::NotFound(db + " not found"); } - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - return Status::NotFound(db + std::to_string(slot_id) + " not found"); + if (sync_slave_dbs_.find(p_info) == sync_slave_dbs_.end()) { + return Status::NotFound(db + " not found"); } - if (sync_master_slots_[p_info]->GetNumberOfSlaveNode() != 0 || - (sync_master_slots_[p_info]->GetNumberOfSlaveNode() == 0 && - sync_slave_slots_[p_info]->State() == kNoConnect)) { + if (sync_master_dbs_[p_info]->GetNumberOfSlaveNode() != 0 || + (sync_master_dbs_[p_info]->GetNumberOfSlaveNode() == 0 && + sync_slave_dbs_[p_info]->State() == kNoConnect)) { *role |= PIKA_ROLE_MASTER; } - if (sync_slave_slots_[p_info]->State() != ReplState::kNoConnect) { + if (sync_slave_dbs_[p_info]->State() != ReplState::kNoConnect) { *role |= PIKA_ROLE_SLAVE; } // if role is not master or slave, the rest situations are all single return Status::OK(); } -Status PikaReplicaManager::GetSlotInfo(const std::string& db, uint32_t slot_id, std::string* info) { - int role = 0; - std::string tmp_res; - Status s = CheckSlotRole(db, slot_id, &role); - if (!s.ok()) { - return s; - } - - bool add_divider_line = (((role & PIKA_ROLE_MASTER) != 0) && ((role & PIKA_ROLE_SLAVE) != 0)); - std::shared_lock l(slots_rw_); - SlotInfo p_info(db, slot_id); - if ((role & PIKA_ROLE_MASTER) != 0) { - if (sync_master_slots_.find(p_info) == sync_master_slots_.end()) { - return Status::NotFound(db + std::to_string(slot_id) + " not found"); - } - Status s = sync_master_slots_[p_info]->GetInfo(info); - if (!s.ok()) { - return s; - } - } - if (add_divider_line) { - info->append(" -----------\r\n"); - } - if ((role & PIKA_ROLE_SLAVE) != 0) { - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - return Status::NotFound(db + std::to_string(slot_id) + " not found"); - } - Status s = sync_slave_slots_[p_info]->GetInfo(info); - if (!s.ok()) { - return s; - } - } - info->append("\r\n"); - return Status::OK(); -} - Status PikaReplicaManager::SelectLocalIp(const std::string& remote_ip, const int remote_port, std::string* const local_ip) { std::unique_ptr cli(net::NewRedisCli()); @@ -973,34 +881,25 @@ Status PikaReplicaManager::SelectLocalIp(const std::string& remote_ip, const int return Status::OK(); } -Status PikaReplicaManager::ActivateSyncSlaveSlot(const RmNode& node, const ReplState& repl_state) { - std::shared_lock l(slots_rw_); - const SlotInfo& p_info = node.NodeSlotInfo(); - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - return Status::NotFound("Sync Slave Slot " + node.ToString() + " not found"); +Status PikaReplicaManager::ActivateSyncSlaveDB(const RmNode& node, const ReplState& repl_state) { + std::shared_lock l(dbs_rw_); + const DBInfo& p_info = node.NodeDBInfo(); + if (sync_slave_dbs_.find(p_info) == sync_slave_dbs_.end()) { + return Status::NotFound("Sync Slave DB " + node.ToString() + " not found"); } - ReplState ssp_state = sync_slave_slots_[p_info]->State(); + ReplState ssp_state = sync_slave_dbs_[p_info]->State(); if (ssp_state != ReplState::kNoConnect && ssp_state != ReplState::kDBNoConnect) { - return Status::Corruption("Sync Slave Slot in " + ReplStateMsg[ssp_state]); + return Status::Corruption("Sync Slave DB in " + ReplStateMsg[ssp_state]); } std::string local_ip; Status s = SelectLocalIp(node.Ip(), node.Port(), &local_ip); if (s.ok()) { - sync_slave_slots_[p_info]->SetLocalIp(local_ip); - sync_slave_slots_[p_info]->Activate(node, repl_state); + sync_slave_dbs_[p_info]->SetLocalIp(local_ip); + sync_slave_dbs_[p_info]->Activate(node, repl_state); } return s; } -Status PikaReplicaManager::DeactivateSyncSlaveSlot(const SlotInfo& p_info) { - std::shared_lock l(slots_rw_); - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - return Status::NotFound("Sync Slave Slot " + p_info.ToString()); - } - sync_slave_slots_[p_info]->Deactivate(); - return Status::OK(); -} - Status PikaReplicaManager::SendMetaSyncRequest() { Status s; if (time(nullptr) - g_pika_server->GetMetaSyncTimestamp() >= PIKA_META_SYNC_MAX_WAIT_TIME || @@ -1014,86 +913,83 @@ Status PikaReplicaManager::SendMetaSyncRequest() { return s; } -Status PikaReplicaManager::SendRemoveSlaveNodeRequest(const std::string& db, uint32_t slot_id) { +Status PikaReplicaManager::SendRemoveSlaveNodeRequest(const std::string& db) { pstd::Status s; - std::shared_lock l(slots_rw_); - SlotInfo p_info(db, slot_id); - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - return Status::NotFound("Sync Slave Slot " + p_info.ToString()); + std::shared_lock l(dbs_rw_); + DBInfo p_info(db); + if (sync_slave_dbs_.find(p_info) == sync_slave_dbs_.end()) { + return Status::NotFound("Sync Slave DB " + p_info.ToString()); } else { - std::shared_ptr s_slot = sync_slave_slots_[p_info]; - s = pika_repl_client_->SendRemoveSlaveNode(s_slot->MasterIp(), s_slot->MasterPort(), db, slot_id, - s_slot->LocalIp()); + std::shared_ptr s_db = sync_slave_dbs_[p_info]; + s = pika_repl_client_->SendRemoveSlaveNode(s_db->MasterIp(), s_db->MasterPort(), db, s_db->LocalIp()); if (s.ok()) { - s_slot->SetReplState(ReplState::kDBNoConnect); + s_db->SetReplState(ReplState::kDBNoConnect); } } if (s.ok()) { - LOG(INFO) << "SlaveNode (" << db << ":" << slot_id << "), stop sync success"; + LOG(INFO) << "SlaveNode (" << db << ", stop sync success"; } else { - LOG(WARNING) << "SlaveNode (" << db << ":" << slot_id << "), stop sync faild, " << s.ToString(); + LOG(WARNING) << "SlaveNode (" << db << ", stop sync faild, " << s.ToString(); } return s; } -Status PikaReplicaManager::SendSlotTrySyncRequest(const std::string& db_name, size_t slot_id) { +Status PikaReplicaManager::SendTrySyncRequest(const std::string& db_name) { BinlogOffset boffset; - if (!g_pika_server->GetDBSlotBinlogOffset(db_name, slot_id, &boffset)) { - LOG(WARNING) << "Slot: " << db_name << ":" << slot_id << ", Get Slot binlog offset failed"; - return Status::Corruption("Slot get binlog offset error"); + if (!g_pika_server->GetDBBinlogOffset(db_name, &boffset)) { + LOG(WARNING) << "DB: " << db_name << ", Get DB binlog offset failed"; + return Status::Corruption("DB get binlog offset error"); } - std::shared_ptr slave_slot = - GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_name << ":" << slot_id << ", NotFound"; - return Status::Corruption("Slave Slot not found"); + std::shared_ptr slave_db = GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_name << ", NotFound"; + return Status::Corruption("Slave DB not found"); } Status status = - pika_repl_client_->SendSlotTrySync(slave_slot->MasterIp(), slave_slot->MasterPort(), db_name, - slot_id, boffset, slave_slot->LocalIp()); + pika_repl_client_->SendTrySync(slave_db->MasterIp(), slave_db->MasterPort(), db_name, + boffset, slave_db->LocalIp()); if (status.ok()) { - slave_slot->SetReplState(ReplState::kWaitReply); + slave_db->SetReplState(ReplState::kWaitReply); } else { - slave_slot->SetReplState(ReplState::kError); - LOG(WARNING) << "SendSlotTrySyncRequest failed " << status.ToString(); + slave_db->SetReplState(ReplState::kError); + LOG(WARNING) << "SendDBTrySyncRequest failed " << status.ToString(); } return status; } -Status PikaReplicaManager::SendSlotDBSyncRequest(const std::string& db_name, size_t slot_id) { +Status PikaReplicaManager::SendDBSyncRequest(const std::string& db_name) { BinlogOffset boffset; - if (!g_pika_server->GetDBSlotBinlogOffset(db_name, slot_id, &boffset)) { - LOG(WARNING) << "Slot: " << db_name << ":" << slot_id << ", Get slot binlog offset failed"; - return Status::Corruption("Slot get binlog offset error"); + if (!g_pika_server->GetDBBinlogOffset(db_name, &boffset)) { + LOG(WARNING) << "DB: " << db_name << ", Get DB binlog offset failed"; + return Status::Corruption("DB get binlog offset error"); } - std::shared_ptr slot = g_pika_server->GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "Slot: " << db_name << ":" << slot_id << ", NotFound"; - return Status::Corruption("Slot not found"); + std::shared_ptr db = g_pika_server->GetDB(db_name); + if (!db) { + LOG(WARNING) << "DB: " << db_name << " NotFound"; + return Status::Corruption("DB not found"); } - slot->PrepareRsync(); + db->PrepareRsync(); - std::shared_ptr slave_slot = - GetSyncSlaveSlotByName(SlotInfo(db_name, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_name << ":" << slot_id << ", NotFound"; - return Status::Corruption("Slave Slot not found"); + std::shared_ptr slave_db = GetSyncSlaveDBByName(DBInfo(db_name)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_name << ", NotFound"; + return Status::Corruption("Slave DB not found"); } - Status status = pika_repl_client_->SendSlotDBSync(slave_slot->MasterIp(), slave_slot->MasterPort(), - db_name, slot_id, boffset, slave_slot->LocalIp()); + Status status = pika_repl_client_->SendDBSync(slave_db->MasterIp(), slave_db->MasterPort(), + db_name, boffset, slave_db->LocalIp()); Status s; if (status.ok()) { - slave_slot->SetReplState(ReplState::kWaitReply); + slave_db->SetReplState(ReplState::kWaitReply); } else { - slave_slot->SetReplState(ReplState::kError); - LOG(WARNING) << "SendSlotDBSync failed " << status.ToString(); + slave_db->SetReplState(ReplState::kError); + LOG(WARNING) << "SendDBSync failed " << status.ToString(); } if (!s.ok()) { LOG(WARNING) << s.ToString(); @@ -1101,17 +997,15 @@ Status PikaReplicaManager::SendSlotDBSyncRequest(const std::string& db_name, siz return status; } -Status PikaReplicaManager::SendSlotBinlogSyncAckRequest(const std::string& db, uint32_t slot_id, - const LogOffset& ack_start, const LogOffset& ack_end, - bool is_first_send) { - std::shared_ptr slave_slot = GetSyncSlaveSlotByName(SlotInfo(db, slot_id)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db << ":" << slot_id << ", NotFound"; - return Status::Corruption("Slave Slot not found"); +Status PikaReplicaManager::SendBinlogSyncAckRequest(const std::string& db, const LogOffset& ack_start, + const LogOffset& ack_end, bool is_first_send) { + std::shared_ptr slave_db = GetSyncSlaveDBByName(DBInfo(db)); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db << ":, NotFound"; + return Status::Corruption("Slave DB not found"); } - return pika_repl_client_->SendSlotBinlogSync(slave_slot->MasterIp(), slave_slot->MasterPort(), db, - slot_id, ack_start, ack_end, slave_slot->LocalIp(), - is_first_send); + return pika_repl_client_->SendBinlogSync(slave_db->MasterIp(), slave_db->MasterPort(), db, + ack_start, ack_end, slave_db->LocalIp(), is_first_send); } Status PikaReplicaManager::CloseReplClientConn(const std::string& ip, int32_t port) { @@ -1123,177 +1017,57 @@ Status PikaReplicaManager::SendSlaveBinlogChipsRequest(const std::string& ip, in return pika_repl_server_->SendSlaveBinlogChips(ip, port, tasks); } -std::shared_ptr PikaReplicaManager::GetSyncMasterSlotByName(const SlotInfo& p_info) { - std::shared_lock l(slots_rw_); - if (sync_master_slots_.find(p_info) == sync_master_slots_.end()) { +std::shared_ptr PikaReplicaManager::GetSyncMasterDBByName(const DBInfo& p_info) { + std::shared_lock l(dbs_rw_); + if (sync_master_dbs_.find(p_info) == sync_master_dbs_.end()) { return nullptr; } - return sync_master_slots_[p_info]; + return sync_master_dbs_[p_info]; } -std::shared_ptr PikaReplicaManager::GetSyncSlaveSlotByName(const SlotInfo& p_info) { - std::shared_lock l(slots_rw_); - if (sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { +std::shared_ptr PikaReplicaManager::GetSyncSlaveDBByName(const DBInfo& p_info) { + std::shared_lock l(dbs_rw_); + if (sync_slave_dbs_.find(p_info) == sync_slave_dbs_.end()) { return nullptr; } - return sync_slave_slots_[p_info]; + return sync_slave_dbs_[p_info]; } -Status PikaReplicaManager::RunSyncSlaveSlotStateMachine() { - std::shared_lock l(slots_rw_); - for (const auto& item : sync_slave_slots_) { - SlotInfo p_info = item.first; - std::shared_ptr s_slot = item.second; - if (s_slot->State() == ReplState::kTryConnect) { - SendSlotTrySyncRequest(p_info.db_name_, p_info.slot_id_); - } else if (s_slot->State() == ReplState::kTryDBSync) { - SendSlotDBSyncRequest(p_info.db_name_, p_info.slot_id_); - } else if (s_slot->State() == ReplState::kWaitReply) { +Status PikaReplicaManager::RunSyncSlaveDBStateMachine() { + std::shared_lock l(dbs_rw_); + for (const auto& item : sync_slave_dbs_) { + DBInfo p_info = item.first; + std::shared_ptr s_db = item.second; + if (s_db->State() == ReplState::kTryConnect) { + SendTrySyncRequest(p_info.db_name_); + } else if (s_db->State() == ReplState::kTryDBSync) { + SendDBSyncRequest(p_info.db_name_); + } else if (s_db->State() == ReplState::kWaitReply) { continue; - } else if (s_slot->State() == ReplState::kWaitDBSync) { - s_slot->ActivateRsync(); - std::shared_ptr slot = - g_pika_server->GetDBSlotById(p_info.db_name_, p_info.slot_id_); - if (slot) { - if (!s_slot->IsRsyncRunning()) { - slot->TryUpdateMasterOffset(); + } else if (s_db->State() == ReplState::kWaitDBSync) { + s_db->ActivateRsync(); + std::shared_ptr db = + g_pika_server->GetDB(p_info.db_name_); + if (db) { + if (!s_db->IsRsyncRunning()) { + db->TryUpdateMasterOffset(); } } else { - LOG(WARNING) << "Slot not found, DB Name: " << p_info.db_name_ - << " Slot Id: " << p_info.slot_id_; + LOG(WARNING) << "DB not found, DB Name: " << p_info.db_name_; } - } else if (s_slot->State() == ReplState::kConnected || s_slot->State() == ReplState::kNoConnect || - s_slot->State() == ReplState::kDBNoConnect) { + } else if (s_db->State() == ReplState::kConnected || s_db->State() == ReplState::kNoConnect || + s_db->State() == ReplState::kDBNoConnect) { continue; } } return Status::OK(); } -Status PikaReplicaManager::AddSyncSlotSanityCheck(const std::set& p_infos) { - std::shared_lock l(slots_rw_); - for (const auto& p_info : p_infos) { - if (sync_master_slots_.find(p_info) != sync_master_slots_.end() || - sync_slave_slots_.find(p_info) != sync_slave_slots_.end()) { - LOG(WARNING) << "sync slot: " << p_info.ToString() << " exist"; - return Status::Corruption("sync slot " + p_info.ToString() + " exist"); - } - } - return Status::OK(); -} - -Status PikaReplicaManager::AddSyncSlot(const std::set& p_infos) { - Status s = AddSyncSlotSanityCheck(p_infos); - if (!s.ok()) { - return s; - } - - std::lock_guard l(slots_rw_); - for (const auto& p_info : p_infos) { - sync_master_slots_[p_info] = std::make_shared(p_info.db_name_, p_info.slot_id_); - sync_slave_slots_[p_info] = std::make_shared(p_info.db_name_, p_info.slot_id_); - } - return Status::OK(); -} - -Status PikaReplicaManager::RemoveSyncSlotSanityCheck(const std::set& p_infos) { - std::shared_lock l(slots_rw_); - for (const auto& p_info : p_infos) { - if (sync_master_slots_.find(p_info) == sync_master_slots_.end() || - sync_slave_slots_.find(p_info) == sync_slave_slots_.end()) { - LOG(WARNING) << "sync slot: " << p_info.ToString() << " not found"; - return Status::Corruption("sync slot " + p_info.ToString() + " not found"); - } - - if (sync_master_slots_[p_info]->GetNumberOfSlaveNode() != 0) { - LOG(WARNING) << "sync Master Slot: " << p_info.ToString() << " in syncing"; - return Status::Corruption("sync Master Slot " + p_info.ToString() + " in syncing"); - } - - ReplState state = sync_slave_slots_[p_info]->State(); - if (state != kNoConnect && state != kError) { - LOG(WARNING) << "sync Slave Slot: " << p_info.ToString() << " in " << ReplStateMsg[state] + " state"; - return Status::Corruption("sync Slave Slot " + p_info.ToString() + " in " + ReplStateMsg[state] + " state"); - } - } - return Status::OK(); -} - -Status PikaReplicaManager::RemoveSyncSlot(const std::set& p_infos) { - Status s = RemoveSyncSlotSanityCheck(p_infos); - if (!s.ok()) { - return s; - } - - std::lock_guard l(slots_rw_); - for (const auto& p_info : p_infos) { - if (sync_master_slots_.find(p_info) != sync_master_slots_.end()) { - sync_master_slots_[p_info]->StableLogger()->Leave(); - } - sync_master_slots_.erase(p_info); - sync_slave_slots_.erase(p_info); - } - return Status::OK(); -} - -Status PikaReplicaManager::SyncDBSanityCheck(const std::string& db_name) { - std::shared_lock l(slots_rw_); - for (const auto& master_slot : sync_master_slots_) { - if (master_slot.first.db_name_ == db_name) { - LOG(WARNING) << "sync slot: " << master_slot.first.ToString() << " exist"; - return Status::Corruption("sync slot " + master_slot.first.ToString() + " exist"); - } - } - for (const auto& slave_slot : sync_slave_slots_) { - if (slave_slot.first.db_name_ == db_name) { - LOG(WARNING) << "sync slot: " << slave_slot.first.ToString() << " exist"; - return Status::Corruption("sync slot " + slave_slot.first.ToString() + " exist"); - } - } - return Status::OK(); -} - -Status PikaReplicaManager::DelSyncDB(const std::string& db_name) { - Status s = SyncDBSanityCheck(db_name); - if (!s.ok()) { - return s; - } - std::string db_log_path = g_pika_conf->log_path() + "log_" + db_name; - std::string db_log_path_tmp = db_log_path + "_deleting/"; - if (pstd::RenameFile(db_log_path, db_log_path_tmp) != 0) { - LOG(WARNING) << "Failed to move log to trash, error: " << strerror(errno); - return Status::Corruption("Failed to move log to trash"); - } - g_pika_server->PurgeDir(db_log_path_tmp); - LOG(WARNING) << "Slot StableLog: " << db_name << " move to trash success"; - return Status::OK(); -} - -void PikaReplicaManager::FindCompleteReplica(std::vector* replica) { - std::unordered_map replica_slotnum; - std::shared_lock l(slots_rw_); - for (auto& iter : sync_master_slots_) { - std::vector names; - iter.second->GetValidSlaveNames(&names); - for (auto& name : names) { - if (replica_slotnum.find(name) == replica_slotnum.end()) { - replica_slotnum[name] = 0; - } - replica_slotnum[name]++; - } - } - for (const auto& item : replica_slotnum) { - if (item.second == sync_master_slots_.size()) { - replica->push_back(item.first); - } - } -} - void PikaReplicaManager::FindCommonMaster(std::string* master) { - std::shared_lock l(slots_rw_); + std::shared_lock l(dbs_rw_); std::string common_master_ip; int common_master_port = 0; - for (auto& iter : sync_slave_slots_) { + for (auto& iter : sync_slave_dbs_) { if (iter.second->State() != kConnected) { return; } @@ -1313,18 +1087,18 @@ void PikaReplicaManager::FindCommonMaster(std::string* master) { } void PikaReplicaManager::RmStatus(std::string* info) { - std::shared_lock l(slots_rw_); + std::shared_lock l(dbs_rw_); std::stringstream tmp_stream; - tmp_stream << "Master Slot(" << sync_master_slots_.size() << "):" + tmp_stream << "Master DB(" << sync_master_dbs_.size() << "):" << "\r\n"; - for (auto& iter : sync_master_slots_) { - tmp_stream << " Slot " << iter.second->SyncSlotInfo().ToString() << "\r\n" + for (auto& iter : sync_master_dbs_) { + tmp_stream << " DB " << iter.second->SyncDBInfo().ToString() << "\r\n" << iter.second->ToStringStatus() << "\r\n"; } - tmp_stream << "Slave Slot(" << sync_slave_slots_.size() << "):" + tmp_stream << "Slave DB(" << sync_slave_dbs_.size() << "):" << "\r\n"; - for (auto& iter : sync_slave_slots_) { - tmp_stream << " Slot " << iter.second->SyncSlotInfo().ToString() << "\r\n" + for (auto& iter : sync_slave_dbs_) { + tmp_stream << " DB " << iter.second->SyncDBInfo().ToString() << "\r\n" << iter.second->ToStringStatus() << "\r\n"; } info->append(tmp_stream.str()); diff --git a/src/pika_server.cc b/src/pika_server.cc index 369aaddac8..a01da37a44 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -4,17 +4,13 @@ // of patent rights can be found in the PATENTS file in the same directory. #include -#include #include #include -#include #include #include #include #include #include - -#include "net/include/bg_thread.h" #include "net/include/net_cli.h" #include "net/include/net_interfaces.h" #include "net/include/net_stats.h" @@ -47,12 +43,11 @@ void DoPurgeDir(void* arg) { void DoDBSync(void* arg) { std::unique_ptr dbsa(static_cast(arg)); PikaServer* const ps = dbsa->p; - ps->DbSyncSendFile(dbsa->ip, dbsa->port, dbsa->db_name, dbsa->slot_id); + ps->DbSyncSendFile(dbsa->ip, dbsa->port, dbsa->db_name); } PikaServer::PikaServer() : exit_(false), - slot_state_(INFREE), last_check_compact_time_({0, 0}), last_check_resume_time_({0, 0}), repl_state_(PIKA_REPL_NO_CONNECT), @@ -321,63 +316,23 @@ void PikaServer::InitDBStruct() { std::lock_guard rwl(dbs_rw_); for (const auto& db : db_structs) { std::string name = db.db_name; - uint32_t num = db.slot_num; - std::shared_ptr db_ptr = std::make_shared(name, num, db_path, log_path); - db_ptr->AddSlots(db.slot_ids); + std::shared_ptr db_ptr = std::make_shared(name, db_path, log_path); + db_ptr->Init(); dbs_.emplace(name, db_ptr); } } -Status PikaServer::AddDBStruct(const std::string& db_name, uint32_t num) { - std::shared_ptr db = g_pika_server->GetDB(db_name); - if (db) { - return Status::Corruption("db already exist"); - } - std::string db_path = g_pika_conf->db_path(); - std::string log_path = g_pika_conf->log_path(); - std::shared_ptr db_ptr = std::make_shared(db_name, num, db_path, log_path); - std::lock_guard rwl(dbs_rw_); - dbs_.emplace(db_name, db_ptr); - return Status::OK(); -} - -Status PikaServer::DelDBStruct(const std::string& db_name) { - std::shared_ptr db = g_pika_server->GetDB(db_name); - if (!db) { - return Status::Corruption("db not found"); - } - if (!db->DBIsEmpty()) { - return Status::Corruption("db have slots"); - } - Status s = db->Leave(); - if (!s.ok()) { - return s; - } - dbs_.erase(db_name); - return Status::OK(); -} - std::shared_ptr PikaServer::GetDB(const std::string& db_name) { std::shared_lock l(dbs_rw_); auto iter = dbs_.find(db_name); return (iter == dbs_.end()) ? nullptr : iter->second; } -std::set PikaServer::GetDBSlotIds(const std::string& db_name) { - std::set empty; - std::shared_lock l(dbs_rw_); - auto iter = dbs_.find(db_name); - return (iter == dbs_.end()) ? empty : iter->second->GetSlotIDs(); -} - bool PikaServer::IsBgSaving() { std::shared_lock l(dbs_rw_); for (const auto& db_item : dbs_) { - std::shared_lock slot_rwl(db_item.second->slots_rw_); - for (const auto& slot_item : db_item.second->slots_) { - if (slot_item.second->IsBgSaving()) { - return true; - } + if (db_item.second->IsBgSaving()) { + return true; } } return false; @@ -396,14 +351,11 @@ bool PikaServer::IsKeyScaning() { bool PikaServer::IsCompacting() { std::shared_lock db_rwl(dbs_rw_); for (const auto& db_item : dbs_) { - std::shared_lock slot_rwl(db_item.second->slots_rw_); - for (const auto& slot_item : db_item.second->slots_) { - slot_item.second->DbRWLockReader(); - std::string task_type = slot_item.second->db()->GetCurrentTaskType(); - slot_item.second->DbRWUnLock(); - if (strcasecmp(task_type.data(), "no") != 0) { - return true; - } + db_item.second->DbRWLockReader(); + std::string task_type = db_item.second->storage()->GetCurrentTaskType(); + db_item.second->DbRWUnLock(); + if (strcasecmp(task_type.data(), "no") != 0) { + return true; } } return false; @@ -411,15 +363,6 @@ bool PikaServer::IsCompacting() { bool PikaServer::IsDBExist(const std::string& db_name) { return static_cast(GetDB(db_name)); } -bool PikaServer::IsDBSlotExist(const std::string& db_name, uint32_t slot_id) { - std::shared_ptr db_ptr = GetDB(db_name); - if (!db_ptr) { - return false; - } else { - return static_cast(db_ptr->GetSlotById(slot_id)); - } -} - bool PikaServer::IsDBBinlogIoError(const std::string& db_name) { std::shared_ptr db = GetDB(db_name); return db ? db->IsBinlogIoError() : true; @@ -490,116 +433,86 @@ Status PikaServer::DoSameThingSpecificDB(const std::set& dbs, const return Status::OK(); } -void PikaServer::PrepareSlotTrySync() { +void PikaServer::PrepareDBTrySync() { std::shared_lock rwl(dbs_rw_); ReplState state = force_full_sync_ ? ReplState::kTryDBSync : ReplState::kTryConnect; for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - Status s = g_pika_rm->ActivateSyncSlaveSlot(RmNode(g_pika_server->master_ip(), g_pika_server->master_port(), - db_item.second->GetDBName(), slot_item.second->GetSlotID()), - state); - if (!s.ok()) { - LOG(WARNING) << s.ToString(); - } + Status s = g_pika_rm->ActivateSyncSlaveDB( + RmNode(g_pika_server->master_ip(), g_pika_server->master_port(), db_item.second->GetDBName()), state); + if (!s.ok()) { + LOG(WARNING) << s.ToString(); } } force_full_sync_ = false; - loop_slot_state_machine_ = true; LOG(INFO) << "Mark try connect finish"; } -void PikaServer::SlotSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys) { +void PikaServer::DBSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys) { std::shared_lock rwl(dbs_rw_); for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - slot_item.second->DbRWLockReader(); - slot_item.second->db()->SetMaxCacheStatisticKeys(max_cache_statistic_keys); - slot_item.second->DbRWUnLock(); - } + db_item.second->DbRWLockReader(); + db_item.second->storage()->SetMaxCacheStatisticKeys(max_cache_statistic_keys); + db_item.second->DbRWUnLock(); } } -void PikaServer::SlotSetSmallCompactionThreshold(uint32_t small_compaction_threshold) { +void PikaServer::DBSetSmallCompactionThreshold(uint32_t small_compaction_threshold) { std::shared_lock rwl(dbs_rw_); for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - slot_item.second->DbRWLockReader(); - slot_item.second->db()->SetSmallCompactionThreshold(small_compaction_threshold); - slot_item.second->DbRWUnLock(); - } + db_item.second->DbRWLockReader(); + db_item.second->storage()->SetSmallCompactionThreshold(small_compaction_threshold); + db_item.second->DbRWUnLock(); } } -void PikaServer::SlotSetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold) { +void PikaServer::DBSetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold) { std::shared_lock rwl(dbs_rw_); for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - slot_item.second->DbRWLockReader(); - slot_item.second->db()->SetSmallCompactionDurationThreshold(small_compaction_duration_threshold); - slot_item.second->DbRWUnLock(); - } + db_item.second->DbRWLockReader(); + db_item.second->storage()->SetSmallCompactionDurationThreshold(small_compaction_duration_threshold); + db_item.second->DbRWUnLock(); } } -bool PikaServer::GetDBSlotBinlogOffset(const std::string& db_name, uint32_t slot_id, - BinlogOffset* const boffset) { - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!slot) { +bool PikaServer::GetDBBinlogOffset(const std::string& db_name, BinlogOffset* const boffset) { + std::shared_ptr db = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!db) { return false; } - Status s = slot->Logger()->GetProducerStatus(&(boffset->filenum), &(boffset->offset)); + Status s = db->Logger()->GetProducerStatus(&(boffset->filenum), &(boffset->offset)); return s.ok(); } -std::shared_ptr PikaServer::GetSlotByDBName(const std::string& db_name) { - std::shared_ptr db = GetDB(db_name); - return db ? db->GetSlotById(0) : nullptr; -} - -std::shared_ptr PikaServer::GetDBSlotById(const std::string& db_name, uint32_t slot_id) { - std::shared_ptr db = GetDB(db_name); - return db ? db->GetSlotById(slot_id) : nullptr; -} - -std::shared_ptr PikaServer::GetDBSlotByKey(const std::string& db_name, const std::string& key) { - std::shared_ptr db = GetDB(db_name); - return db ? db->GetSlotByKey(key) : nullptr; -} - -Status PikaServer::DoSameThingEverySlot(const TaskType& type) { +Status PikaServer::DoSameThingEveryDB(const TaskType& type) { std::shared_lock rwl(dbs_rw_); - std::shared_ptr slave_slot = nullptr; + std::shared_ptr slave_db = nullptr; for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - switch (type) { - case TaskType::kResetReplState: { - slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_item.second->GetDBName() << ":" << slot_item.second->GetSlotID() - << " Not Found"; - } - slave_slot->SetReplState(ReplState::kNoConnect); - break; + switch (type) { + case TaskType::kResetReplState: { + slave_db = g_pika_rm->GetSyncSlaveDBByName(DBInfo(db_item.second->GetDBName())); + if (!slave_db) { + LOG(WARNING) << "Slave DB: " << db_item.second->GetDBName() << ":" + << " Not Found"; } - case TaskType::kPurgeLog: { - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); - if (!slot) { - LOG(WARNING) << "Slot: " << db_item.second->GetDBName() << ":" << slot_item.second->GetSlotID() - << " Not Found."; - break; - } - slot->StableLogger()->PurgeStableLogs(); + slave_db->SetReplState(ReplState::kNoConnect); + break; + } + case TaskType::kPurgeLog: { + std::shared_ptr db = g_pika_rm->GetSyncMasterDBByName( + DBInfo(db_item.second->GetDBName())); + if (!db) { + LOG(WARNING) << "DB: " << db_item.second->GetDBName() << ":" + << " Not Found."; break; } - case TaskType::kCompactAll: - slot_item.second->Compact(storage::kAll); - break; - default: - break; + db->StableLogger()->PurgeStableLogs(); + break; } + case TaskType::kCompactAll: + db_item.second->Compact(storage::kAll); + break; + default: + break; } } return Status::OK(); @@ -649,23 +562,6 @@ int32_t PikaServer::CountSyncSlaves() { return static_cast(db_sync_slaves_.size()); } -int32_t PikaServer::GetShardingSlaveListString(std::string& slave_list_str) { - std::vector complete_replica; - g_pika_rm->FindCompleteReplica(&complete_replica); - std::stringstream tmp_stream; - size_t index = 0; - for (const auto& replica : complete_replica) { - std::string ip; - int port; - if (!pstd::ParseIpPortString(replica, ip, port)) { - continue; - } - tmp_stream << "slave" << index++ << ":ip=" << ip << ",port=" << port << "\r\n"; - } - slave_list_str.assign(tmp_stream.str()); - return static_cast(index); -} - int32_t PikaServer::GetSlaveListString(std::string& slave_list_str) { size_t index = 0; SlaveState slave_state; @@ -674,32 +570,30 @@ int32_t PikaServer::GetSlaveListString(std::string& slave_list_str) { BinlogOffset acked_slave_boffset; std::stringstream tmp_stream; std::lock_guard l(slave_mutex_); - std::shared_ptr master_slot = nullptr; + std::shared_ptr master_db = nullptr; for (const auto& slave : slaves_) { tmp_stream << "slave" << index++ << ":ip=" << slave.ip << ",port=" << slave.port << ",conn_fd=" << slave.conn_fd << ",lag="; for (const auto& ts : slave.db_structs) { - for (size_t idx = 0; idx < ts.slot_num; ++idx) { - std::shared_ptr slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(ts.db_name, idx)); - if (!slot) { - LOG(WARNING) << "Sync Master Slot: " << ts.db_name << ":" << idx << ", NotFound"; + std::shared_ptr db = g_pika_rm->GetSyncMasterDBByName(DBInfo(ts.db_name)); + if (!db) { + LOG(WARNING) << "Sync Master DB: " << ts.db_name << ", NotFound"; + continue; + } + Status s = db->GetSlaveState(slave.ip, slave.port, &slave_state); + if (s.ok() && slave_state == SlaveState::kSlaveBinlogSync && + db->GetSlaveSyncBinlogInfo(slave.ip, slave.port, &sent_slave_boffset, &acked_slave_boffset).ok()) { + Status s = db->Logger()->GetProducerStatus(&(master_boffset.filenum), &(master_boffset.offset)); + if (!s.ok()) { continue; - } - Status s = slot->GetSlaveState(slave.ip, slave.port, &slave_state); - if (s.ok() && slave_state == SlaveState::kSlaveBinlogSync && - slot->GetSlaveSyncBinlogInfo(slave.ip, slave.port, &sent_slave_boffset, &acked_slave_boffset).ok()) { - Status s = slot->Logger()->GetProducerStatus(&(master_boffset.filenum), &(master_boffset.offset)); - if (!s.ok()) { - continue; - } else { - uint64_t lag = static_cast((master_boffset.filenum - sent_slave_boffset.filenum)) * - g_pika_conf->binlog_file_size() + - master_boffset.offset - sent_slave_boffset.offset; - tmp_stream << "(" << slot->SlotName() << ":" << lag << ")"; - } } else { - tmp_stream << "(" << slot->SlotName() << ":not syncing)"; + uint64_t lag = + static_cast((master_boffset.filenum - sent_slave_boffset.filenum)) * g_pika_conf->binlog_file_size() + + master_boffset.offset - sent_slave_boffset.offset; + tmp_stream << "(" << db->DBName() << ":" << lag << ")"; } + } else { + tmp_stream << "(" << db->DBName() << ":not syncing)"; } } tmp_stream << "\r\n"; @@ -752,14 +646,13 @@ void PikaServer::RemoveMaster() { if (!master_ip_.empty() && master_port_ != -1) { g_pika_rm->CloseReplClientConn(master_ip_, master_port_ + kPortShiftReplServer); g_pika_rm->LostConnection(master_ip_, master_port_); - loop_slot_state_machine_ = false; UpdateMetaSyncTimestampWithoutLock(); LOG(INFO) << "Remove Master Success, ip_port: " << master_ip_ << ":" << master_port_; } master_ip_ = ""; master_port_ = -1; - DoSameThingEverySlot(TaskType::kResetReplState); + DoSameThingEveryDB(TaskType::kResetReplState); } } @@ -800,44 +693,8 @@ void PikaServer::ResetMetaSyncStatus() { // not change by slaveof no one, so set repl_state = PIKA_REPL_SHOULD_META_SYNC, // continue to connect master repl_state_ = PIKA_REPL_SHOULD_META_SYNC; - loop_slot_state_machine_ = false; - DoSameThingEverySlot(TaskType::kResetReplState); - } -} - -bool PikaServer::AllSlotConnectSuccess() { - bool all_slot_connect_success = true; - std::shared_lock rwl(dbs_rw_); - std::shared_ptr slave_slot = nullptr; - for (const auto& db_item : dbs_) { - for (const auto& slot_item : db_item.second->slots_) { - slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_item.second->GetDBName(), slot_item.second->GetSlotID())); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << db_item.second->GetDBName() << ":" << slot_item.second->GetSlotID() - << ", NotFound"; - return false; - } - - ReplState repl_state = slave_slot->State(); - if (repl_state != ReplState::kConnected) { - all_slot_connect_success = false; - break; - } - } + DoSameThingEveryDB(TaskType::kResetReplState); } - return all_slot_connect_success; -} - -bool PikaServer::LoopSlotStateMachine() { - std::shared_lock sp_l(state_protector_); - return loop_slot_state_machine_; -} - -void PikaServer::SetLoopSlotStateMachine(bool need_loop) { - std::lock_guard sp_l(state_protector_); - assert(repl_state_ == PIKA_REPL_META_SYNC_DONE); - loop_slot_state_machine_ = need_loop; } int PikaServer::GetMetaSyncTimestamp() { @@ -926,9 +783,9 @@ void PikaServer::PurgeDirTaskSchedule(void (*function)(void*), void* arg) { purge_thread_.Schedule(function, arg); } -void PikaServer::DBSync(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id) { +void PikaServer::DBSync(const std::string& ip, int port, const std::string& db_name) { { - std::string task_index = DbSyncTaskIndex(ip, port, db_name, slot_id); + std::string task_index = DbSyncTaskIndex(ip, port, db_name); std::lock_guard ml(db_sync_protector_); if (db_sync_slaves_.find(task_index) != db_sync_slaves_.end()) { return; @@ -938,60 +795,64 @@ void PikaServer::DBSync(const std::string& ip, int port, const std::string& db_n // Reuse the bgsave_thread_ // Since we expect BgSave and DBSync execute serially bgsave_thread_.StartThread(); - auto arg = new DBSyncArg(this, ip, port, db_name, slot_id); + auto arg = new DBSyncArg(this, ip, port, db_name); bgsave_thread_.Schedule(&DoDBSync, reinterpret_cast(arg)); } -pstd::Status PikaServer::GetDumpUUID(const std::string& db_name, const uint32_t slot_id, std::string* snapshot_uuid) { - std::shared_ptr slot = GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "cannot find slot for db_name " << db_name << " slot_id: " << slot_id; - return pstd::Status::NotFound("slot no found"); +pstd::Status PikaServer::GetDumpUUID(const std::string& db_name, std::string* snapshot_uuid) { + std::shared_ptr db = GetDB(db_name); + if (!db) { + LOG(WARNING) << "cannot find db for db_name " << db_name; + return pstd::Status::NotFound("db no found"); } - slot->GetBgSaveUUID(snapshot_uuid); + db->GetBgSaveUUID(snapshot_uuid); return pstd::Status::OK(); } -pstd::Status PikaServer::GetDumpMeta(const std::string& db_name, const uint32_t slot_id, - std::vector* fileNames, std::string* snapshot_uuid) { - std::shared_ptr slot = GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "cannot find slot for db_name " << db_name << " slot_id: " << slot_id; - return pstd::Status::NotFound("slot no found"); +pstd::Status PikaServer::GetDumpMeta(const std::string& db_name, std::vector* fileNames, std::string* snapshot_uuid) { + std::shared_ptr db = GetDB(db_name); + if (!db) { + LOG(WARNING) << "cannot find db for db_name " << db_name; + return pstd::Status::NotFound("db no found"); } - slot->GetBgSaveMetaData(fileNames, snapshot_uuid); + db->GetBgSaveMetaData(fileNames, snapshot_uuid); return pstd::Status::OK(); } -void PikaServer::DoBgSaveSlot(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, int32_t top) { - std::shared_ptr slot = GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "can not find Slot whose id is " << slot_id << " in db " << db_name << ", DoBgSaveSlot Failed"; +void PikaServer::TryDBSync(const std::string& ip, int port, const std::string& db_name, + int32_t top) { + std::shared_ptr db = GetDB(db_name); + if (!db) { + LOG(WARNING) << "can not find DB : " << db_name + << ", TryDBSync Failed"; return; } - std::shared_ptr sync_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name, slot_id)); - if (!sync_slot) { - LOG(WARNING) << "can not find Slot whose id is " << slot_id << " in db " << db_name << ", DoBgSaveSlot Failed"; + std::shared_ptr sync_db = + g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name)); + if (!sync_db) { + LOG(WARNING) << "can not find DB: " << db_name + << ", TryDBSync Failed"; return; } - BgSaveInfo bgsave_info = slot->bgsave_info(); - std::string logger_filename = sync_slot->Logger()->filename(); + BgSaveInfo bgsave_info = db->bgsave_info(); + std::string logger_filename = sync_db->Logger()->filename(); if (pstd::IsDir(bgsave_info.path) != 0 || !pstd::FileExists(NewFileName(logger_filename, bgsave_info.offset.b_offset.filenum)) || top - bgsave_info.offset.b_offset.filenum > kDBSyncMaxGap) { // Need Bgsave first - slot->BgSaveSlot(); + db->BgSaveDB(); } } -void PikaServer::DbSyncSendFile(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id) { - std::shared_ptr slot = GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "can not find Slot whose id is " << slot_id << " in db " << db_name << ", DbSync send file Failed"; +void PikaServer::DbSyncSendFile(const std::string& ip, int port, const std::string& db_name) { + std::shared_ptr db = GetDB(db_name); + if (!db) { + LOG(WARNING) << "can not find DB: " << db_name + << ", DbSync send file Failed"; return; } - BgSaveInfo bgsave_info = slot->bgsave_info(); + BgSaveInfo bgsave_info = db->bgsave_info(); std::string bg_path = bgsave_info.path; uint32_t binlog_filenum = bgsave_info.offset.b_offset.filenum; uint64_t binlog_offset = bgsave_info.offset.b_offset.offset; @@ -1001,13 +862,13 @@ void PikaServer::DbSyncSendFile(const std::string& ip, int port, const std::stri // Get all files need to send std::vector descendant; int ret = 0; - LOG(INFO) << "Slot: " << slot->GetSlotName() << " Start Send files in " << bg_path << " to " << ip; + LOG(INFO) << "DB: " << db->GetDBName() << " Start Send files in " << bg_path << " to " << ip; ret = pstd::GetChildren(bg_path, descendant); if (ret) { std::string ip_port = pstd::IpPortString(ip, port); std::lock_guard ldb(db_sync_protector_); db_sync_slaves_.erase(ip_port); - LOG(WARNING) << "Slot: " << slot->GetSlotName() + LOG(WARNING) << "DB: " << db->GetDBName() << " Get child directory when try to do sync failed, error: " << strerror(ret); return; } @@ -1039,7 +900,7 @@ void PikaServer::DbSyncSendFile(const std::string& ip, int port, const std::stri // We need specify the speed limit for every single file ret = pstd::RsyncSendFile(local_path, target_path, secret_file_path, remote); if (0 != ret) { - LOG(WARNING) << "Slot: " << slot->GetSlotName() << " RSync send file failed! From: " << *iter + LOG(WARNING) << "DB: " << db->GetDBName() << " RSync send file failed! From: " << *iter << ", To: " << target_path << ", At: " << ip << ":" << port << ", Error: " << ret; break; } @@ -1078,28 +939,28 @@ void PikaServer::DbSyncSendFile(const std::string& ip, int port, const std::stri ret = pstd::RsyncSendFile(fn, remote_path + "/" + kBgsaveInfoFile, secret_file_path, remote); pstd::DeleteFile(fn); if (ret) { - LOG(WARNING) << "Slot: " << slot->GetSlotName() << " Send Modified Info File Failed"; + LOG(WARNING) << "DB: " << db->GetDBName() << " Send Modified Info File Failed"; } } else if (0 != (ret = pstd::RsyncSendFile(bg_path + "/" + kBgsaveInfoFile, remote_path + "/" + kBgsaveInfoFile, secret_file_path, remote))) { - LOG(WARNING) << "Slot: " << slot->GetSlotName() << " Send Info File Failed"; + LOG(WARNING) << "DB: " << db->GetDBName() << " Send Info File Failed"; } } // remove slave { - std::string task_index = DbSyncTaskIndex(ip, port, db_name, slot_id); + std::string task_index = DbSyncTaskIndex(ip, port, db_name); std::lock_guard ml(db_sync_protector_); db_sync_slaves_.erase(task_index); } if (0 == ret) { - LOG(INFO) << "Slot: " << slot->GetSlotName() << " RSync Send Files Success"; + LOG(INFO) << "DB: " << db->GetDBName() << " RSync Send Files Success"; } } -std::string PikaServer::DbSyncTaskIndex(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id) { +std::string PikaServer::DbSyncTaskIndex(const std::string& ip, int port, const std::string& db_name) { char buf[256]; - snprintf(buf, sizeof(buf), "%s:%d_%s:%d", ip.data(), port, db_name.data(), slot_id); + snprintf(buf, sizeof(buf), "%s:%d_%s:%d", ip.data(), port, db_name.data()); return buf; } @@ -1289,8 +1150,6 @@ std::unordered_map PikaServer::ServerExecCountDB() { return res; } -QpsStatistic PikaServer::ServerDBStat(const std::string& db_name) { return statistic_.DBStat(db_name); } - std::unordered_map PikaServer::ServerAllDBStat() { return statistic_.AllDBStat(); } int PikaServer::SendToPeer() { return g_pika_rm->ConsumeWriteQueue(); } @@ -1435,7 +1294,7 @@ void PikaServer::AutoCompactRange() { if (!have_scheduled_crontask_ && in_window) { if ((static_cast(free_size) / static_cast(total_size)) * 100 >= usage) { - Status s = DoSameThingEverySlot(TaskType::kCompactAll); + Status s = DoSameThingEveryDB(TaskType::kCompactAll); if (s.ok()) { LOG(INFO) << "[Cron]schedule compactRange, freesize: " << free_size / 1048576 << "MB, disksize: " << total_size / 1048576 << "MB"; @@ -1452,7 +1311,7 @@ void PikaServer::AutoCompactRange() { } } -void PikaServer::AutoPurge() { DoSameThingEverySlot(TaskType::kPurgeLog); } +void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } void PikaServer::AutoDeleteExpiredDump() { std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); @@ -1532,13 +1391,6 @@ void PikaServer::AutoDeleteExpiredDump() { } } -void PikaServer::AutoKeepAliveRSync() { - if (!pika_rsync_service_->CheckRsyncAlive()) { - LOG(WARNING) << "The Rsync service is down, Try to restart"; - pika_rsync_service_->StartRsync(); - } -} - void PikaServer::AutoUpdateNetworkMetric() { monotime current_time = getMonotonicUs(); size_t factor = 5e6; // us, 5s @@ -1652,15 +1504,13 @@ void PikaServer::InitStorageOptions() { storage::Status PikaServer::RewriteStorageOptions(const storage::OptionType& option_type, const std::unordered_map& options_map) { storage::Status s; + std::shared_lock db_rwl(dbs_rw_); for (const auto& db_item : dbs_) { - std::lock_guard slot_rwl(db_item.second->slots_rw_); - for (const auto& slot_item : db_item.second->slots_) { - slot_item.second->DbRWLockWriter(); - s = slot_item.second->db()->SetOptions(option_type, storage::ALL_DB, options_map); - slot_item.second->DbRWUnLock(); - if (!s.ok()) { - return s; - } + db_item.second->DbRWLockWriter(); + s = db_item.second->storage()->SetOptions(option_type, storage::ALL_DB, options_map); + db_item.second->DbRWUnLock(); + if (!s.ok()) { + return s; } } std::lock_guard rwl(storage_options_rw_); @@ -1683,18 +1533,16 @@ void PikaServer::ServerStatus(std::string* info) { info->append(tmp_stream.str()); } -bool PikaServer::SlotsMigrateBatch(const std::string& ip, int64_t port, int64_t time_out, int64_t slot_num, - int64_t keys_num, const std::shared_ptr& slot) { - return pika_migrate_thread_->ReqMigrateBatch(ip, port, time_out, slot_num, keys_num, slot); +bool PikaServer::SlotsMigrateBatch(const std::string &ip, int64_t port, int64_t time_out, int64_t slot_num,int64_t keys_num, const std::shared_ptr& db) { + return pika_migrate_thread_->ReqMigrateBatch(ip, port, time_out, slot_num, keys_num, db); } -void PikaServer::GetSlotsMgrtSenderStatus(std::string* ip, int64_t* port, int64_t* slot, bool* migrating, - int64_t* moved, int64_t* remained) { +void PikaServer::GetSlotsMgrtSenderStatus(std::string *ip, int64_t* port, int64_t *slot, bool *migrating, int64_t *moved, int64_t *remained) { return pika_migrate_thread_->GetMigrateStatus(ip, port, slot, migrating, moved, remained); } -int PikaServer::SlotsMigrateOne(const std::string& key, const std::shared_ptr& slot) { - return pika_migrate_thread_->ReqMigrateOne(key, slot); +int PikaServer::SlotsMigrateOne(const std::string& key, const std::shared_ptr& db) { + return pika_migrate_thread_->ReqMigrateOne(key, db); } bool PikaServer::SlotsMigrateAsyncCancel() { @@ -1702,7 +1550,7 @@ bool PikaServer::SlotsMigrateAsyncCancel() { return true; } -void PikaServer::Bgslotsreload(const std::shared_ptr& slot) { +void PikaServer::Bgslotsreload(const std::shared_ptr& db) { // Only one thread can go through { std::lock_guard ml(bgsave_protector_); @@ -1719,7 +1567,7 @@ void PikaServer::Bgslotsreload(const std::shared_ptr& slot) { bgslots_reload_.cursor = 0; bgslots_reload_.pattern = "*"; bgslots_reload_.count = 100; - bgslots_reload_.slot = slot; + bgslots_reload_.db = db; LOG(INFO) << "Start slot reloading"; @@ -1736,22 +1584,20 @@ void DoBgslotsreload(void* arg) { rocksdb::Status s; std::vector keys; int64_t cursor_ret = -1; - while (cursor_ret != 0 && p->GetSlotsreloading()) { - cursor_ret = reload.slot->db()->Scan(storage::DataType::kAll, reload.cursor, reload.pattern, reload.count, &keys); + while(cursor_ret != 0 && p->GetSlotsreloading()){ + cursor_ret = reload.db->storage()->Scan(storage::DataType::kAll, reload.cursor, reload.pattern, reload.count, &keys); std::vector::const_iterator iter; for (iter = keys.begin(); iter != keys.end(); iter++) { std::string key_type; - - int s = GetKeyType(*iter, key_type, reload.slot); - // if key is slotkey, can't add to SlotKey - if (s > 0) { - if (key_type == "s" && - ((*iter).find(SlotKeyPrefix) != std::string::npos || (*iter).find(SlotTagPrefix) != std::string::npos)) { + int s = GetKeyType(*iter, key_type, reload.db); + //if key is slotkey, can't add to SlotKey + if (s > 0){ + if (key_type == "s" && ((*iter).find(SlotKeyPrefix) != std::string::npos || (*iter).find(SlotTagPrefix) != std::string::npos)){ continue; } - AddSlotKey(key_type, *iter, reload.slot); + AddSlotKey(key_type, *iter, reload.db); } } @@ -1768,7 +1614,7 @@ void DoBgslotsreload(void* arg) { } } -void PikaServer::Bgslotscleanup(std::vector cleanupSlots, const std::shared_ptr& slot) { +void PikaServer::Bgslotscleanup(std::vector cleanupSlots, const std::shared_ptr& db) { // Only one thread can go through { std::lock_guard ml(bgsave_protector_); @@ -1785,7 +1631,7 @@ void PikaServer::Bgslotscleanup(std::vector cleanupSlots, const std::shared bgslots_cleanup_.cursor = 0; bgslots_cleanup_.pattern = "*"; bgslots_cleanup_.count = 100; - bgslots_cleanup_.slot = slot; + bgslots_cleanup_.db = db; bgslots_cleanup_.cleanup_slots.swap(cleanupSlots); std::string slotsStr; @@ -1836,9 +1682,8 @@ void DoBgslotscleanup(void* arg) { std::vector keys; int64_t cursor_ret = -1; std::vector cleanupSlots(cleanup.cleanup_slots); - while (cursor_ret != 0 && p->GetSlotscleaningup()) { - cursor_ret = g_pika_server->bgslots_cleanup_.slot->db()->Scan(storage::DataType::kAll, cleanup.cursor, - cleanup.pattern, cleanup.count, &keys); + while (cursor_ret != 0 && p->GetSlotscleaningup()){ + cursor_ret = g_pika_server->bgslots_cleanup_.db->storage()->Scan(storage::DataType::kAll, cleanup.cursor, cleanup.pattern, cleanup.count, &keys); std::string key_type; std::vector::const_iterator iter; @@ -1846,14 +1691,13 @@ void DoBgslotscleanup(void* arg) { if ((*iter).find(SlotKeyPrefix) != std::string::npos || (*iter).find(SlotTagPrefix) != std::string::npos) { continue; } - if (std::find(cleanupSlots.begin(), cleanupSlots.end(), GetSlotID(*iter)) != cleanupSlots.end()) { - if (GetKeyType(*iter, key_type, g_pika_server->bgslots_cleanup_.slot) <= 0) { + if (std::find(cleanupSlots.begin(), cleanupSlots.end(), GetSlotID(*iter)) != cleanupSlots.end()){ + if (GetKeyType(*iter, key_type, g_pika_server->bgslots_cleanup_.db) <= 0) { LOG(WARNING) << "slots clean get key type for slot " << GetSlotID(*iter) << " key " << *iter << " error"; continue; } - - if (DeleteKey(*iter, key_type[0], g_pika_server->bgslots_cleanup_.slot) <= 0) { - LOG(WARNING) << "slots clean del for slot " << GetSlotID(*iter) << " key " << *iter << " error"; + if (DeleteKey(*iter, key_type[0], g_pika_server->bgslots_cleanup_.db) <= 0){ + LOG(WARNING) << "slots clean del for slot " << GetSlotID(*iter) << " key "<< *iter << " error"; } } } @@ -1863,9 +1707,9 @@ void DoBgslotscleanup(void* arg) { keys.clear(); } - for (int cleanupSlot : cleanupSlots) { - WriteDelKeyToBinlog(GetSlotKey(cleanupSlot), g_pika_server->bgslots_cleanup_.slot); - WriteDelKeyToBinlog(GetSlotsTagKey(cleanupSlot), g_pika_server->bgslots_cleanup_.slot); + for (int cleanupSlot : cleanupSlots){ + WriteDelKeyToBinlog(GetSlotKey(cleanupSlot), g_pika_server->bgslots_cleanup_.db); + WriteDelKeyToBinlog(GetSlotsTagKey(cleanupSlot), g_pika_server->bgslots_cleanup_.db); } p->SetSlotscleaningup(false); @@ -1877,13 +1721,13 @@ void DoBgslotscleanup(void* arg) { LOG(INFO) << "Finish slots cleanup, slots " << slotsStr; } -void PikaServer::ResetCacheAsync(uint32_t cache_num, std::shared_ptr slot, cache::CacheConfig *cache_cfg) { - if (PIKA_CACHE_STATUS_OK == slot->cache()->CacheStatus() - || PIKA_CACHE_STATUS_NONE == slot->cache()->CacheStatus()) { +void PikaServer::ResetCacheAsync(uint32_t cache_num, std::shared_ptr db, cache::CacheConfig *cache_cfg) { + if (PIKA_CACHE_STATUS_OK == db->cache()->CacheStatus() + || PIKA_CACHE_STATUS_NONE == db->cache()->CacheStatus()) { common_bg_thread_.StartThread(); BGCacheTaskArg *arg = new BGCacheTaskArg(); - arg->slot = slot; + arg->db = db; arg->cache_num = cache_num; if (cache_cfg == nullptr) { arg->task_type = CACHE_BGTASK_RESET_NUM; @@ -1893,60 +1737,60 @@ void PikaServer::ResetCacheAsync(uint32_t cache_num, std::shared_ptr slot, } common_bg_thread_.Schedule(&DoCacheBGTask, static_cast(arg)); } else { - LOG(WARNING) << "can not reset cache in status: " << slot->cache()->CacheStatus(); + LOG(WARNING) << "can not reset cache in status: " << db->cache()->CacheStatus(); } } -void PikaServer::ClearCacheDbAsync(std::shared_ptr slot) { - if (PIKA_CACHE_STATUS_OK != slot->cache()->CacheStatus()) { - LOG(WARNING) << "can not clear cache in status: " << slot->cache()->CacheStatus(); +void PikaServer::ClearCacheDbAsync(std::shared_ptr db) { + if (PIKA_CACHE_STATUS_OK != db->cache()->CacheStatus()) { + LOG(WARNING) << "can not clear cache in status: " << db->cache()->CacheStatus(); return; } common_bg_thread_.StartThread(); BGCacheTaskArg *arg = new BGCacheTaskArg(); - arg->slot = slot; + arg->db = db; arg->task_type = CACHE_BGTASK_CLEAR; common_bg_thread_.Schedule(&DoCacheBGTask, static_cast(arg)); } void PikaServer::DoCacheBGTask(void* arg) { std::unique_ptr pCacheTaskArg(static_cast(arg)); - std::shared_ptr slot = pCacheTaskArg->slot; + std::shared_ptr db = pCacheTaskArg->db; switch (pCacheTaskArg->task_type) { case CACHE_BGTASK_CLEAR: LOG(INFO) << "clear cache start..."; - slot->cache()->SetCacheStatus(PIKA_CACHE_STATUS_CLEAR); - g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_CLEAR, slot); - slot->cache()->FlushSlot(); + db->cache()->SetCacheStatus(PIKA_CACHE_STATUS_CLEAR); + g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_CLEAR, db); + db->cache()->FlushCache(); LOG(INFO) << "clear cache finish"; break; case CACHE_BGTASK_RESET_NUM: LOG(INFO) << "reset cache num start..."; - slot->cache()->SetCacheStatus(PIKA_CACHE_STATUS_RESET); - g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_RESET, slot); - slot->cache()->Reset(pCacheTaskArg->cache_num); + db->cache()->SetCacheStatus(PIKA_CACHE_STATUS_RESET); + g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_RESET, db); + db->cache()->Reset(pCacheTaskArg->cache_num); LOG(INFO) << "reset cache num finish"; break; case CACHE_BGTASK_RESET_CFG: LOG(INFO) << "reset cache config start..."; - slot->cache()->SetCacheStatus(PIKA_CACHE_STATUS_RESET); - g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_RESET, slot); - slot->cache()->Reset(pCacheTaskArg->cache_num); + db->cache()->SetCacheStatus(PIKA_CACHE_STATUS_RESET); + g_pika_server->ResetDisplayCacheInfo(PIKA_CACHE_STATUS_RESET, db); + db->cache()->Reset(pCacheTaskArg->cache_num); LOG(INFO) << "reset cache config finish"; break; default: LOG(WARNING) << "invalid cache task type: " << pCacheTaskArg->task_type; break; } - slot->cache()->SetCacheStatus(PIKA_CACHE_STATUS_OK); + db->cache()->SetCacheStatus(PIKA_CACHE_STATUS_OK); if (pCacheTaskArg->reenable_cache && pCacheTaskArg->conf) { pCacheTaskArg->conf->UnsetCacheDisableFlag(); } } -void PikaServer::ResetCacheConfig(std::shared_ptr slot) { +void PikaServer::ResetCacheConfig(std::shared_ptr db) { cache::CacheConfig cache_cfg; cache_cfg.maxmemory = g_pika_conf->cache_maxmemory(); cache_cfg.maxmemory_policy = g_pika_conf->cache_maxmemory_policy(); @@ -1954,29 +1798,29 @@ void PikaServer::ResetCacheConfig(std::shared_ptr slot) { cache_cfg.lfu_decay_time = g_pika_conf->cache_lfu_decay_time(); cache_cfg.zset_cache_start_pos = g_pika_conf->zset_cache_start_pos(); cache_cfg.zset_cache_field_num_per_key = g_pika_conf->zset_cache_field_num_per_key(); - slot->cache()->ResetConfig(&cache_cfg); + db->cache()->ResetConfig(&cache_cfg); } -void PikaServer::ClearHitRatio(std::shared_ptr slot) { - slot->cache()->ClearHitRatio(); +void PikaServer::ClearHitRatio(std::shared_ptr db) { + db->cache()->ClearHitRatio(); } -void PikaServer::OnCacheStartPosChanged(int zset_cache_start_pos, std::shared_ptr slot) { +void PikaServer::OnCacheStartPosChanged(int zset_cache_start_pos, std::shared_ptr db) { // disable cache temporarily, and restore it after cache cleared g_pika_conf->SetCacheDisableFlag(); - ResetCacheConfig(slot); - ClearCacheDbAsyncV2(slot); + ResetCacheConfig(db); + ClearCacheDbAsyncV2(db); } -void PikaServer::ClearCacheDbAsyncV2(std::shared_ptr slot) { - if (PIKA_CACHE_STATUS_OK != slot->cache()->CacheStatus()) { - LOG(WARNING) << "can not clear cache in status: " << slot->cache()->CacheStatus(); +void PikaServer::ClearCacheDbAsyncV2(std::shared_ptr db) { + if (PIKA_CACHE_STATUS_OK != db->cache()->CacheStatus()) { + LOG(WARNING) << "can not clear cache in status: " << db->cache()->CacheStatus(); return; } common_bg_thread_.StartThread(); BGCacheTaskArg *arg = new BGCacheTaskArg(); - arg->slot = slot; + arg->db = db; arg->task_type = CACHE_BGTASK_CLEAR; arg->conf = std::move(g_pika_conf); arg->reenable_cache = true; @@ -1985,12 +1829,8 @@ void PikaServer::ClearCacheDbAsyncV2(std::shared_ptr slot) { void PikaServer::ProcessCronTask() { for (auto& dbs : dbs_) { - auto db = dbs.second; - auto slots = db->GetSlots(); - for (size_t i = 0; i < slots.size(); ++i) { - auto cache = slots[i]->cache(); - cache->ProcessCronTask(); - } + auto cache = dbs.second->cache(); + cache->ProcessCronTask(); } LOG(INFO) << "hit rate:" << HitRatio() << std::endl; } @@ -2009,22 +1849,18 @@ double PikaServer::HitRatio(void) { void PikaServer::UpdateCacheInfo(void) { for (auto& dbs : dbs_) { - auto db = dbs.second; - auto slots = db->GetSlots(); - for (size_t i = 0; i < slots.size(); ++i) { - if (PIKA_CACHE_STATUS_OK != slots[i]->cache()->CacheStatus()) { - return; - } - // get cache info from redis cache - CacheInfo cache_info; - slots[i]->cache()->Info(cache_info); - slots[i]->UpdateCacheInfo(cache_info); + if (PIKA_CACHE_STATUS_OK != dbs.second->cache()->CacheStatus()) { + return; } + // get cache info from redis cache + CacheInfo cache_info; + dbs.second->cache()->Info(cache_info); + dbs.second->UpdateCacheInfo(cache_info); } } -void PikaServer::ResetDisplayCacheInfo(int status, std::shared_ptr slot) { - slot->ResetDisplayCacheInfo(status); +void PikaServer::ResetDisplayCacheInfo(int status, std::shared_ptr db) { + db->ResetDisplayCacheInfo(status); } void PikaServer::CacheConfigInit(cache::CacheConfig& cache_cfg) { diff --git a/src/pika_set.cc b/src/pika_set.cc index b50e20887b..0c16624b76 100644 --- a/src/pika_set.cc +++ b/src/pika_set.cc @@ -22,25 +22,25 @@ void SAddCmd::DoInitial() { members_.assign(iter, argv_.end()); } -void SAddCmd::Do(std::shared_ptr slot) { +void SAddCmd::Do() { int32_t count = 0; - s_ = slot->db()->SAdd(key_, members_, &count); + s_ = db_->storage()->SAdd(key_, members_, &count); if (!s_.ok()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; } - AddSlotKey("s", key_, slot); + AddSlotKey("s", key_, db_); res_.AppendInteger(count); } -void SAddCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SAddCmd::DoThroughDB() { + Do(); } -void SAddCmd::DoUpdateCache(std::shared_ptr slot) { +void SAddCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - slot->cache()->SAddIfKeyExist(CachePrefixKeyS, members_); + db_->cache()->SAddIfKeyExist(CachePrefixKeyS, members_); } } @@ -66,8 +66,8 @@ void SPopCmd::DoInitial() { } } -void SPopCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->SPop(key_, &members_, count_); +void SPopCmd::Do() { + s_ = db_->storage()->SPop(key_, &members_, count_); if (s_.ok()) { res_.AppendArrayLenUint64(members_.size()); for (const auto& member : members_) { @@ -81,14 +81,14 @@ void SPopCmd::Do(std::shared_ptr slot) { } } -void SPopCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SPopCmd::DoThroughDB() { + Do(); } -void SPopCmd::DoUpdateCache(std::shared_ptr slot) { +void SPopCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - slot->cache()->SRem(CachePrefixKeyS, members_); + db_->cache()->SRem(CachePrefixKeyS, members_); } } @@ -100,9 +100,9 @@ void SCardCmd::DoInitial() { key_ = argv_[1]; } -void SCardCmd::Do(std::shared_ptr slot) { +void SCardCmd::Do() { int32_t card = 0; - s_ = slot->db()->SCard(key_, &card); + s_ = db_->storage()->SCard(key_, &card); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(card); } else { @@ -110,10 +110,10 @@ void SCardCmd::Do(std::shared_ptr slot) { } } -void SCardCmd::ReadCache(std::shared_ptr slot) { +void SCardCmd::ReadCache() { uint64_t card = 0; std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - auto s = slot->cache()->SCard(CachePrefixKeyS, &card); + auto s = db_->cache()->SCard(CachePrefixKeyS, &card); if (s.ok()) { res_.AppendInteger(card); } else if (s.IsNotFound()) { @@ -123,14 +123,14 @@ void SCardCmd::ReadCache(std::shared_ptr slot) { } } -void SCardCmd::DoThroughDB(std::shared_ptr slot) { +void SCardCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void SCardCmd::DoUpdateCache(std::shared_ptr slot) { +void SCardCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -142,9 +142,9 @@ void SMembersCmd::DoInitial() { key_ = argv_[1]; } -void SMembersCmd::Do(std::shared_ptr slot) { +void SMembersCmd::Do() { std::vector members; - s_ = slot->db()->SMembers(key_, &members); + s_ = db_->storage()->SMembers(key_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); for (const auto& member : members) { @@ -156,10 +156,10 @@ void SMembersCmd::Do(std::shared_ptr slot) { } } -void SMembersCmd::ReadCache(std::shared_ptr slot) { +void SMembersCmd::ReadCache() { std::vector members; std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - auto s = slot->cache()->SMembers(CachePrefixKeyS, &members); + auto s = db_->cache()->SMembers(CachePrefixKeyS, &members); if (s.ok()) { res_.AppendArrayLen(members.size()); for (const auto& member : members) { @@ -173,14 +173,14 @@ void SMembersCmd::ReadCache(std::shared_ptr slot) { } } -void SMembersCmd::DoThroughDB(std::shared_ptr slot) { +void SMembersCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void SMembersCmd::DoUpdateCache(std::shared_ptr slot) { +void SMembersCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -222,10 +222,10 @@ void SScanCmd::DoInitial() { } } -void SScanCmd::Do(std::shared_ptr slot) { +void SScanCmd::Do() { int64_t next_cursor = 0; std::vector members; - rocksdb::Status s = slot->db()->SScan(key_, cursor_, pattern_, count_, &members, &next_cursor); + rocksdb::Status s = db_->storage()->SScan(key_, cursor_, pattern_, count_, &members, &next_cursor); if (s.ok() || s.IsNotFound()) { res_.AppendContent("*2"); @@ -254,8 +254,8 @@ void SRemCmd::DoInitial() { members_.assign(++iter, argv_.end()); } -void SRemCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->SRem(key_, members_, &deleted_); +void SRemCmd::Do() { + s_ = db_->storage()->SRem(key_, members_, &deleted_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(deleted_); } else { @@ -263,14 +263,14 @@ void SRemCmd::Do(std::shared_ptr slot) { } } -void SRemCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SRemCmd::DoThroughDB() { + Do(); } -void SRemCmd::DoUpdateCache(std::shared_ptr slot) { +void SRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - slot->cache()->SRem(CachePrefixKeyS, members_); + db_->cache()->SRem(CachePrefixKeyS, members_); } } @@ -283,9 +283,9 @@ void SUnionCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SUnionCmd::Do(std::shared_ptr slot) { +void SUnionCmd::Do() { std::vector members; - s_ = slot->db()->SUnion(keys_, &members); + s_ = db_->storage()->SUnion(keys_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); for (const auto& member : members) { @@ -308,9 +308,9 @@ void SUnionstoreCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SUnionstoreCmd::Do(std::shared_ptr slot) { +void SUnionstoreCmd::Do() { int32_t count = 0; - s_ = slot->db()->SUnionstore(dest_key_, keys_, value_to_dest_, &count); + s_ = db_->storage()->SUnionstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); } else { @@ -318,26 +318,26 @@ void SUnionstoreCmd::Do(std::shared_ptr slot) { } } -void SUnionstoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SUnionstoreCmd::DoThroughDB() { + Do(); } -void SUnionstoreCmd::DoUpdateCache(std::shared_ptr slot) { +void SUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixS + dest_key_); - slot->cache()->Del(v); + db_->cache()->Del(v); } } -void SetOperationCmd::DoBinlog(const std::shared_ptr& slot) { +void SetOperationCmd::DoBinlog() { PikaCmdArgsType del_args; del_args.emplace_back("del"); del_args.emplace_back(dest_key_); del_cmd_->Initial(del_args, db_name_); del_cmd_->SetConn(GetConn()); del_cmd_->SetResp(resp_.lock()); - del_cmd_->DoBinlog(slot); + del_cmd_->DoBinlog(); if (value_to_dest_.size() == 0) { //The union/diff/inter operation got an empty set, just exec del to simulate overwrite an empty set to dest_key @@ -358,7 +358,7 @@ void SetOperationCmd::DoBinlog(const std::shared_ptr& slot) { for (size_t i = 1; i < value_to_dest_.size(); i++) { if (data_size >= 131072) { // If the binlog has reached the size of 128KB. (131,072 bytes = 128KB) - sadd_cmd_->DoBinlog(slot); + sadd_cmd_->DoBinlog(); sadd_argv.clear(); sadd_argv.emplace_back("sadd"); sadd_argv.emplace_back(dest_key_); @@ -367,7 +367,7 @@ void SetOperationCmd::DoBinlog(const std::shared_ptr& slot) { sadd_argv.emplace_back(value_to_dest_[i]); data_size += value_to_dest_[i].size(); } - sadd_cmd_->DoBinlog(slot); + sadd_cmd_->DoBinlog(); } void SInterCmd::DoInitial() { @@ -379,9 +379,9 @@ void SInterCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SInterCmd::Do(std::shared_ptr slot) { +void SInterCmd::Do() { std::vector members; - s_ = slot->db()->SInter(keys_, &members); + s_ = db_->storage()->SInter(keys_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); for (const auto& member : members) { @@ -404,9 +404,9 @@ void SInterstoreCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SInterstoreCmd::Do(std::shared_ptr slot) { +void SInterstoreCmd::Do() { int32_t count = 0; - s_ = slot->db()->SInterstore(dest_key_, keys_, value_to_dest_, &count); + s_ = db_->storage()->SInterstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); } else { @@ -414,15 +414,15 @@ void SInterstoreCmd::Do(std::shared_ptr slot) { } } -void SInterstoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SInterstoreCmd::DoThroughDB() { + Do(); } -void SInterstoreCmd::DoUpdateCache(std::shared_ptr slot) { +void SInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixS + dest_key_); - slot->cache()->Del(v); + db_->cache()->Del(v); } } @@ -435,9 +435,9 @@ void SIsmemberCmd::DoInitial() { member_ = argv_[2]; } -void SIsmemberCmd::Do(std::shared_ptr slot) { +void SIsmemberCmd::Do() { int32_t is_member = 0; - s_ = slot->db()->SIsmember(key_, member_, &is_member); + s_ = db_->storage()->SIsmember(key_, member_, &is_member); if (is_member != 0) { res_.AppendContent(":1"); } else { @@ -445,9 +445,9 @@ void SIsmemberCmd::Do(std::shared_ptr slot) { } } -void SIsmemberCmd::ReadCache(std::shared_ptr slot) { +void SIsmemberCmd::ReadCache() { std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - auto s = slot->cache()->SIsmember(CachePrefixKeyS, member_); + auto s = db_->cache()->SIsmember(CachePrefixKeyS, member_); if (s.ok()) { res_.AppendContent(":1"); } else if (s.IsNotFound()) { @@ -458,14 +458,14 @@ void SIsmemberCmd::ReadCache(std::shared_ptr slot) { } -void SIsmemberCmd::DoThroughDB(std::shared_ptr slot) { +void SIsmemberCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void SIsmemberCmd::DoUpdateCache(std::shared_ptr slot) { +void SIsmemberCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } @@ -478,9 +478,9 @@ void SDiffCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SDiffCmd::Do(std::shared_ptr slot) { +void SDiffCmd::Do() { std::vector members; - s_ = slot->db()->SDiff(keys_, &members); + s_ = db_->storage()->SDiff(keys_, &members); if (s_.ok() || s_.IsNotFound()) { res_.AppendArrayLenUint64(members.size()); for (const auto& member : members) { @@ -503,9 +503,9 @@ void SDiffstoreCmd::DoInitial() { keys_.assign(++iter, argv_.end()); } -void SDiffstoreCmd::Do(std::shared_ptr slot) { +void SDiffstoreCmd::Do() { int32_t count = 0; - s_ = slot->db()->SDiffstore(dest_key_, keys_, value_to_dest_, &count); + s_ = db_->storage()->SDiffstore(dest_key_, keys_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); } else { @@ -513,15 +513,15 @@ void SDiffstoreCmd::Do(std::shared_ptr slot) { } } -void SDiffstoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SDiffstoreCmd::DoThroughDB() { + Do(); } -void SDiffstoreCmd::DoUpdateCache(std::shared_ptr slot) { +void SDiffstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixS + dest_key_); - slot->cache()->Del(v); + db_->cache()->Del(v); } } @@ -535,9 +535,9 @@ void SMoveCmd::DoInitial() { member_ = argv_[3]; } -void SMoveCmd::Do(std::shared_ptr slot) { +void SMoveCmd::Do() { int32_t res = 0; - s_ = slot->db()->SMove(src_key_, dest_key_, member_, &res); + s_ = db_->storage()->SMove(src_key_, dest_key_, member_, &res); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(res); move_success_ = res; @@ -546,22 +546,22 @@ void SMoveCmd::Do(std::shared_ptr slot) { } } -void SMoveCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void SMoveCmd::DoThroughDB() { + Do(); } -void SMoveCmd::DoUpdateCache(std::shared_ptr slot) { +void SMoveCmd::DoUpdateCache() { if (s_.ok()) { std::vector members; members.emplace_back(member_); std::string CachePrefixKeyS = PCacheKeyPrefixS + src_key_; std::string CachePrefixKeyD = PCacheKeyPrefixS + dest_key_; - slot->cache()->SRem(CachePrefixKeyS, members); - slot->cache()->SAddIfKeyExist(CachePrefixKeyD, members); + db_->cache()->SRem(CachePrefixKeyS, members); + db_->cache()->SAddIfKeyExist(CachePrefixKeyD, members); } } -void SMoveCmd::DoBinlog(const std::shared_ptr& slot) { +void SMoveCmd::DoBinlog() { if (!move_success_) { //the member is not in the source set, nothing changed return; @@ -585,8 +585,8 @@ void SMoveCmd::DoBinlog(const std::shared_ptr& slot) { sadd_cmd_->SetConn(GetConn()); sadd_cmd_->SetResp(resp_.lock()); - srem_cmd_->DoBinlog(slot); - sadd_cmd_->DoBinlog(slot); + srem_cmd_->DoBinlog(); + sadd_cmd_->DoBinlog(); } void SRandmemberCmd::DoInitial() { @@ -607,9 +607,9 @@ void SRandmemberCmd::DoInitial() { } } -void SRandmemberCmd::Do(std::shared_ptr slot) { +void SRandmemberCmd::Do() { std::vector members; - s_ = slot->db()->SRandmember(key_, static_cast(count_), &members); + s_ = db_->storage()->SRandmember(key_, static_cast(count_), &members); if (s_.ok() || s_.IsNotFound()) { if (!reply_arr && (static_cast(!members.empty()) != 0U)) { res_.AppendStringLenUint64(members[0].size()); @@ -626,10 +626,10 @@ void SRandmemberCmd::Do(std::shared_ptr slot) { } } -void SRandmemberCmd::ReadCache(std::shared_ptr slot) { +void SRandmemberCmd::ReadCache() { std::vector members; std::string CachePrefixKeyS = PCacheKeyPrefixS + key_; - auto s = slot->cache()->SRandmember(CachePrefixKeyS, count_, &members); + auto s = db_->cache()->SRandmember(CachePrefixKeyS, count_, &members); if (s.ok()) { if (!reply_arr && members.size()) { res_.AppendStringLen(members[0].size()); @@ -648,14 +648,14 @@ void SRandmemberCmd::ReadCache(std::shared_ptr slot) { } } -void SRandmemberCmd::DoThroughDB(std::shared_ptr slot) { +void SRandmemberCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void SRandmemberCmd::DoUpdateCache(std::shared_ptr slot) { +void SRandmemberCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_SET, key_, db_); } } diff --git a/src/pika_slave_node.cc b/src/pika_slave_node.cc index b3faa6f067..dc36ac38e4 100644 --- a/src/pika_slave_node.cc +++ b/src/pika_slave_node.cc @@ -59,9 +59,8 @@ int SyncWindow::Remaining() { /* SlaveNode */ -SlaveNode::SlaveNode(const std::string& ip, int port, const std::string& db_name, uint32_t slot_id, - int session_id) - : RmNode(ip, port, db_name, slot_id, session_id) +SlaveNode::SlaveNode(const std::string& ip, int port, const std::string& db_name, int session_id) + : RmNode(ip, port, db_name, session_id) {} diff --git a/src/pika_slot.cc b/src/pika_slot.cc deleted file mode 100644 index 72a1e04314..0000000000 --- a/src/pika_slot.cc +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include -#include - -#include "include/pika_conf.h" -#include "include/pika_rm.h" -#include "include/pika_server.h" -#include "include/pika_slot.h" -#include "include/pika_command.h" - -#include "pstd/include/mutex_impl.h" -#include "pstd/include/pstd_hash.h" - -using pstd::Status; - -extern PikaServer* g_pika_server; -extern std::unique_ptr g_pika_rm; - -std::string SlotName(const std::string& db_name, uint32_t slot_id) { - char buf[256]; - snprintf(buf, sizeof(buf), "(%s:%u)", db_name.data(), slot_id); - return {buf}; -} - -std::string BgsaveSubPath(const std::string& db_name, uint32_t slot_id) { - char buf[256]; - std::string slot_id_str = std::to_string(slot_id); - snprintf(buf, sizeof(buf), "%s/%s", db_name.data(), slot_id_str.data()); - return {buf}; -} - -std::string DbSyncPath(const std::string& sync_path, const std::string& db_name, const uint32_t slot_id) { - char buf[256]; - std::string slot_id_str = std::to_string(slot_id); - snprintf(buf, sizeof(buf), "%s/", db_name.data()); - return sync_path + buf; -} - -Slot::Slot(const std::string& db_name, uint32_t slot_id, const std::string& table_db_path) - : db_name_(db_name), slot_id_(slot_id), bgsave_engine_(nullptr) { - db_path_ = table_db_path; - bgsave_sub_path_ = db_name; - dbsync_path_ = DbSyncPath(g_pika_conf->db_sync_path(), db_name, slot_id_); - slot_name_ = db_name; - - db_ = std::make_shared(); - rocksdb::Status s = db_->Open(g_pika_server->storage_options(), db_path_); - - lock_mgr_ = std::make_shared(1000, 0, std::make_shared()); - - opened_ = s.ok(); - assert(db_); - assert(s.ok()); - LOG(INFO) << slot_name_ << " DB Success"; -} - -Slot::~Slot() { - Close(); -} - -void Slot::Leave() { - Close(); - MoveToTrash(); -} - -void Slot::Close() { - if (!opened_) { - return; - } - std::lock_guard lock(db_rwlock_); - db_.reset(); - lock_mgr_.reset(); - opened_ = false; -} - -// Before call this function, you should close db and log firstly -void Slot::MoveToTrash() { - if (opened_) { - return; - } - - std::string dbpath = db_path_; - if (dbpath[dbpath.length() - 1] == '/') { - dbpath.erase(dbpath.length() - 1); - } - dbpath.append("_deleting/"); - if (pstd::RenameFile(db_path_, dbpath) != 0) { - LOG(WARNING) << "Failed to move db to trash, error: " << strerror(errno); - return; - } - g_pika_server->PurgeDir(dbpath); - - LOG(WARNING) << "Slot DB: " << slot_name_ << " move to trash success"; -} - -std::string Slot::GetDBName() const { return db_name_; } - -uint32_t Slot::GetSlotID() const { return slot_id_; } - -std::string Slot::GetSlotName() const { return slot_name_; } - -std::shared_ptr Slot::db() const { return db_; } - -std::shared_ptr Slot::cache() const { return cache_; } - -void Slot::Init() { - cache_ = std::make_shared(g_pika_conf->zset_cache_start_pos(), g_pika_conf->zset_cache_field_num_per_key()); - // Create cache - cache::CacheConfig cache_cfg; - g_pika_server->CacheConfigInit(cache_cfg); - cache_->Init(g_pika_conf->GetCacheNum(), &cache_cfg); -} - -void Slot::Compact(const storage::DataType& type) { - if (!opened_) { - return; - } - db_->Compact(type); -} - -void Slot::CompactRange(const storage::DataType& type, const std::string& start, const std::string& end) { - if (!opened_) { - return; - } - db_->CompactRange(type, start, end); -} - -void Slot::DbRWLockWriter() { db_rwlock_.lock(); } - -void Slot::DbRWLockReader() { db_rwlock_.lock_shared(); } - -void Slot::DbRWUnLock() { db_rwlock_.unlock(); } - -std::shared_ptr Slot::LockMgr() { return lock_mgr_; } - -void Slot::PrepareRsync() { - pstd::DeleteDirIfExist(dbsync_path_); - pstd::CreatePath(dbsync_path_ + "strings"); - pstd::CreatePath(dbsync_path_ + "hashes"); - pstd::CreatePath(dbsync_path_ + "lists"); - pstd::CreatePath(dbsync_path_ + "sets"); - pstd::CreatePath(dbsync_path_ + "zsets"); -} - -// Try to update master offset -// This may happend when dbsync from master finished -// Here we do: -// 1, Check dbsync finished, got the new binlog offset -// 2, Replace the old db -// 3, Update master offset, and the PikaAuxiliaryThread cron will connect and do slaveof task with master -bool Slot::TryUpdateMasterOffset() { - std::string info_path = dbsync_path_ + kBgsaveInfoFile; - if (!pstd::FileExists(info_path)) { - LOG(WARNING) << "info path: " << info_path << " not exist"; - return false; - } - - std::shared_ptr slave_slot = - g_pika_rm->GetSyncSlaveSlotByName(SlotInfo(db_name_, slot_id_)); - if (!slave_slot) { - LOG(WARNING) << "Slave Slot: " << slot_name_ << " not exist"; - return false; - } - - // Got new binlog offset - std::ifstream is(info_path); - if (!is) { - LOG(WARNING) << "Slot: " << slot_name_ << ", Failed to open info file after db sync"; - slave_slot->SetReplState(ReplState::kError); - return false; - } - std::string line; - std::string master_ip; - int lineno = 0; - int64_t filenum = 0; - int64_t offset = 0; - int64_t term = 0; - int64_t index = 0; - int64_t tmp = 0; - int64_t master_port = 0; - while (std::getline(is, line)) { - lineno++; - if (lineno == 2) { - master_ip = line; - } else if (lineno > 2 && lineno < 8) { - if ((pstd::string2int(line.data(), line.size(), &tmp) == 0) || tmp < 0) { - LOG(WARNING) << "Slot: " << slot_name_ - << ", Format of info file after db sync error, line : " << line; - is.close(); - slave_slot->SetReplState(ReplState::kError); - return false; - } - if (lineno == 3) { - master_port = tmp; - } else if (lineno == 4) { - filenum = tmp; - } else if (lineno == 5) { - offset = tmp; - } else if (lineno == 6) { - term = tmp; - } else if (lineno == 7) { - index = tmp; - } - } else if (lineno > 8) { - LOG(WARNING) << "Slot: " << slot_name_ << ", Format of info file after db sync error, line : " << line; - is.close(); - slave_slot->SetReplState(ReplState::kError); - return false; - } - } - is.close(); - - LOG(INFO) << "Slot: " << slot_name_ << " Information from dbsync info" - << ", master_ip: " << master_ip << ", master_port: " << master_port << ", filenum: " << filenum - << ", offset: " << offset << ", term: " << term << ", index: " << index; - - pstd::DeleteFile(info_path); - if (!ChangeDb(dbsync_path_)) { - LOG(WARNING) << "Slot: " << slot_name_ << ", Failed to change db"; - slave_slot->SetReplState(ReplState::kError); - return false; - } - - // Update master offset - std::shared_ptr master_slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name_, slot_id_)); - if (!master_slot) { - LOG(WARNING) << "Master Slot: " << slot_name_ << " not exist"; - return false; - } - master_slot->Logger()->SetProducerStatus(filenum, offset); - slave_slot->SetReplState(ReplState::kTryConnect); - return true; -} - -/* - * Change a new db locate in new_path - * return true when change success - * db remain the old one if return false - */ -bool Slot::ChangeDb(const std::string& new_path) { - std::string tmp_path(db_path_); - if (tmp_path.back() == '/') { - tmp_path.resize(tmp_path.size() - 1); - } - tmp_path += "_bak"; - pstd::DeleteDirIfExist(tmp_path); - - std::lock_guard l(db_rwlock_); - LOG(INFO) << "Slot: " << slot_name_ << ", Prepare change db from: " << tmp_path; - db_.reset(); - - if (0 != pstd::RenameFile(db_path_, tmp_path)) { - LOG(WARNING) << "Slot: " << slot_name_ - << ", Failed to rename db path when change db, error: " << strerror(errno); - return false; - } - - if (0 != pstd::RenameFile(new_path, db_path_)) { - LOG(WARNING) << "Slot: " << slot_name_ - << ", Failed to rename new db path when change db, error: " << strerror(errno); - return false; - } - - db_ = std::make_shared(); - rocksdb::Status s = db_->Open(g_pika_server->storage_options(), db_path_); - assert(db_); - assert(s.ok()); - pstd::DeleteDirIfExist(tmp_path); - LOG(INFO) << "Slot: " << slot_name_ << ", Change db success"; - return true; -} - -bool Slot::IsBgSaving() { - std::lock_guard ml(bgsave_protector_); - return bgsave_info_.bgsaving; -} - -void Slot::BgSaveSlot() { - std::lock_guard l(bgsave_protector_); - if (bgsave_info_.bgsaving) { - return; - } - bgsave_info_.bgsaving = true; - auto bg_task_arg = new BgTaskArg(); - bg_task_arg->slot = shared_from_this(); - g_pika_server->BGSaveTaskSchedule(&DoBgSave, static_cast(bg_task_arg)); -} - -BgSaveInfo Slot::bgsave_info() { - std::lock_guard l(bgsave_protector_); - return bgsave_info_; -} - -void Slot::GetBgSaveMetaData(std::vector* fileNames, std::string* snapshot_uuid) { - const std::string slotPath = bgsave_info().path; - - std::string types[] = {storage::STRINGS_DB, storage::HASHES_DB, storage::LISTS_DB, storage::ZSETS_DB, storage::SETS_DB}; - for (const auto& type : types) { - std::string typePath = slotPath + ((slotPath.back() != '/') ? "/" : "") + type; - if (!pstd::FileExists(typePath)) { - continue ; - } - - std::vector tmpFileNames; - int ret = pstd::GetChildren(typePath, tmpFileNames); - if (ret) { - LOG(WARNING) << slotPath << " read dump meta files failed, path " << typePath; - return; - } - - for (const std::string fileName : tmpFileNames) { - fileNames -> push_back(type + "/" + fileName); - } - } - fileNames->push_back(kBgsaveInfoFile); - pstd::Status s = GetBgSaveUUID(snapshot_uuid); - if (!s.ok()) { - LOG(WARNING) << "read dump meta info failed! error:" << s.ToString(); - return; - } -} - -Status Slot::GetBgSaveUUID(std::string* snapshot_uuid) { - if (snapshot_uuid_.empty()) { - std::string info_data; - const std::string infoPath = bgsave_info().path + "/info"; - //TODO: using file read function to replace rocksdb::ReadFileToString - rocksdb::Status s = rocksdb::ReadFileToString(rocksdb::Env::Default(), infoPath, &info_data); - if (!s.ok()) { - LOG(WARNING) << "read dump meta info failed! error:" << s.ToString(); - return Status::IOError("read dump meta info failed", infoPath); - } - pstd::MD5 md5 = pstd::MD5(info_data); - snapshot_uuid_ = md5.hexdigest(); - } - *snapshot_uuid = snapshot_uuid_; - return Status::OK(); -} - -void Slot::DoBgSave(void* arg) { - std::unique_ptr bg_task_arg(static_cast(arg)); - - // Do BgSave - bool success = bg_task_arg->slot->RunBgsaveEngine(); - - // Some output - BgSaveInfo info = bg_task_arg->slot->bgsave_info(); - std::stringstream info_content; - std::ofstream out; - out.open(info.path + "/" + kBgsaveInfoFile, std::ios::in | std::ios::trunc); - if (out.is_open()) { - info_content << (time(nullptr) - info.start_time) << "s\n" - << g_pika_server->host() << "\n" - << g_pika_server->port() << "\n" - << info.offset.b_offset.filenum << "\n" - << info.offset.b_offset.offset << "\n"; - bg_task_arg->slot->snapshot_uuid_ = md5(info_content.str()); - out << info_content.rdbuf(); - out.close(); - } - if (!success) { - std::string fail_path = info.path + "_FAILED"; - pstd::RenameFile(info.path, fail_path); - } - bg_task_arg->slot->FinishBgsave(); -} - -bool Slot::RunBgsaveEngine() { - // Prepare for Bgsaving - if (!InitBgsaveEnv() || !InitBgsaveEngine()) { - ClearBgsave(); - return false; - } - LOG(INFO) << slot_name_ << " after prepare bgsave"; - - BgSaveInfo info = bgsave_info(); - LOG(INFO) << slot_name_ << " bgsave_info: path=" << info.path << ", filenum=" << info.offset.b_offset.filenum - << ", offset=" << info.offset.b_offset.offset; - - // Backup to tmp dir - rocksdb::Status s = bgsave_engine_->CreateNewBackup(info.path); - - if (!s.ok()) { - LOG(WARNING) << slot_name_ << " create new backup failed :" << s.ToString(); - return false; - } - LOG(INFO) << slot_name_ << " create new backup finished."; - - return true; -} - -// Prepare engine, need bgsave_protector protect -bool Slot::InitBgsaveEnv() { - std::lock_guard l(bgsave_protector_); - // Prepare for bgsave dir - bgsave_info_.start_time = time(nullptr); - char s_time[32]; - int len = static_cast(strftime(s_time, sizeof(s_time), "%Y%m%d%H%M%S", localtime(&bgsave_info_.start_time))); - bgsave_info_.s_start_time.assign(s_time, len); - std::string time_sub_path = g_pika_conf->bgsave_prefix() + std::string(s_time, 8); - bgsave_info_.path = g_pika_conf->bgsave_path() + time_sub_path + "/" + bgsave_sub_path_; - if (!pstd::DeleteDirIfExist(bgsave_info_.path)) { - LOG(WARNING) << slot_name_ << " remove exist bgsave dir failed"; - return false; - } - pstd::CreatePath(bgsave_info_.path, 0755); - // Prepare for failed dir - if (!pstd::DeleteDirIfExist(bgsave_info_.path + "_FAILED")) { - LOG(WARNING) << slot_name_ << " remove exist fail bgsave dir failed :"; - return false; - } - return true; -} - -// Prepare bgsave env, need bgsave_protector protect -bool Slot::InitBgsaveEngine() { - bgsave_engine_.reset(); - rocksdb::Status s = storage::BackupEngine::Open(db().get(), bgsave_engine_); - if (!s.ok()) { - LOG(WARNING) << slot_name_ << " open backup engine failed " << s.ToString(); - return false; - } - - std::shared_ptr slot = - g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name_, slot_id_)); - if (!slot) { - LOG(WARNING) << slot_name_ << " not found"; - return false; - } - - { - std::lock_guard lock(db_rwlock_); - LogOffset bgsave_offset; - // term, index are 0 - slot->Logger()->GetProducerStatus(&(bgsave_offset.b_offset.filenum), &(bgsave_offset.b_offset.offset)); - { - std::lock_guard l(bgsave_protector_); - bgsave_info_.offset = bgsave_offset; - } - s = bgsave_engine_->SetBackupContent(); - if (!s.ok()) { - LOG(WARNING) << slot_name_ << " set backup content failed " << s.ToString(); - return false; - } - } - return true; -} - -void Slot::ClearBgsave() { - std::lock_guard l(bgsave_protector_); - bgsave_info_.Clear(); -} - -void Slot::FinishBgsave() { - std::lock_guard l(bgsave_protector_); - bgsave_info_.bgsaving = false; - g_pika_server->UpdateLastSave(time(nullptr)); -} - -bool Slot::FlushDB() { - std::lock_guard rwl(db_rwlock_); - std::lock_guard l(bgsave_protector_); - return FlushDBWithoutLock(); -} - -bool Slot::FlushDBWithoutLock() { - if (bgsave_info_.bgsaving) { - return false; - } - - LOG(INFO) << slot_name_ << " Delete old db..."; - db_.reset(); - - std::string dbpath = db_path_; - if (dbpath[dbpath.length() - 1] == '/') { - dbpath.erase(dbpath.length() - 1); - } - dbpath.append("_deleting/"); - pstd::RenameFile(db_path_, dbpath); - - db_ = std::make_shared(); - rocksdb::Status s = db_->Open(g_pika_server->storage_options(), db_path_); - assert(db_); - assert(s.ok()); - LOG(INFO) << slot_name_ << " Open new db success"; - g_pika_server->PurgeDir(dbpath); - return true; -} - -bool Slot::FlushSubDB(const std::string& db_name) { - std::lock_guard rwl(db_rwlock_); - return FlushSubDBWithoutLock(db_name); -} - -bool Slot::FlushSubDBWithoutLock(const std::string& db_name) { - std::lock_guard l(bgsave_protector_); - if (bgsave_info_.bgsaving) { - return false; - } - - LOG(INFO) << slot_name_ << " Delete old " + db_name + " db..."; - db_.reset(); - - std::string dbpath = db_path_; - if (dbpath[dbpath.length() - 1] != '/') { - dbpath.append("/"); - } - - std::string sub_dbpath = dbpath + db_name; - std::string del_dbpath = dbpath + db_name + "_deleting"; - pstd::RenameFile(sub_dbpath, del_dbpath); - - db_ = std::make_shared(); - rocksdb::Status s = db_->Open(g_pika_server->storage_options(), db_path_); - assert(db_); - assert(s.ok()); - LOG(INFO) << slot_name_ << " open new " + db_name + " db success"; - g_pika_server->PurgeDir(del_dbpath); - return true; -} - -void Slot::InitKeyScan() { - key_scan_info_.start_time = time(nullptr); - char s_time[32]; - int len = static_cast(strftime(s_time, sizeof(s_time), "%Y-%m-%d %H:%M:%S", localtime(&key_scan_info_.start_time))); - key_scan_info_.s_start_time.assign(s_time, len); - key_scan_info_.duration = -1; // duration -1 mean the task in processing -} - -KeyScanInfo Slot::GetKeyScanInfo() { - std::lock_guard l(key_info_protector_); - return key_scan_info_; -} - -DisplayCacheInfo Slot::GetCacheInfo() { - std::lock_guard l(cache_info_rwlock_); - return cache_info_; -} - -Status Slot::GetKeyNum(std::vector* key_info) { - std::lock_guard l(key_info_protector_); - if (key_scan_info_.key_scaning_) { - *key_info = key_scan_info_.key_infos; - return Status::OK(); - } - InitKeyScan(); - key_scan_info_.key_scaning_ = true; - key_scan_info_.duration = -2; // duration -2 mean the task in waiting status, - // has not been scheduled for exec - rocksdb::Status s = db_->GetKeyNum(key_info); - key_scan_info_.key_scaning_ = false; - if (!s.ok()) { - return Status::Corruption(s.ToString()); - } - key_scan_info_.key_infos = *key_info; - key_scan_info_.duration = static_cast(time(nullptr) - key_scan_info_.start_time); - return Status::OK(); -} - - -void Slot::UpdateCacheInfo(CacheInfo& cache_info) { - std::unique_lock lock(cache_info_rwlock_); - - cache_info_.status = cache_info.status; - cache_info_.cache_num = cache_info.cache_num; - cache_info_.keys_num = cache_info.keys_num; - cache_info_.used_memory = cache_info.used_memory; - cache_info_.waitting_load_keys_num = cache_info.waitting_load_keys_num; - cache_usage_ = cache_info.used_memory; - - uint64_t all_cmds = cache_info.hits + cache_info.misses; - cache_info_.hitratio_all = (0 >= all_cmds) ? 0.0 : (cache_info.hits * 100.0) / all_cmds; - - uint64_t cur_time_us = pstd::NowMicros(); - uint64_t delta_time = cur_time_us - cache_info_.last_time_us + 1; - uint64_t delta_hits = cache_info.hits - cache_info_.hits; - cache_info_.hits_per_sec = delta_hits * 1000000 / delta_time; - - uint64_t delta_all_cmds = all_cmds - (cache_info_.hits + cache_info_.misses); - cache_info_.read_cmd_per_sec = delta_all_cmds * 1000000 / delta_time; - - cache_info_.hitratio_per_sec = (0 >= delta_all_cmds) ? 0.0 : (delta_hits * 100.0) / delta_all_cmds; - - uint64_t delta_load_keys = cache_info.async_load_keys_num - cache_info_.last_load_keys_num; - cache_info_.load_keys_per_sec = delta_load_keys * 1000000 / delta_time; - - cache_info_.hits = cache_info.hits; - cache_info_.misses = cache_info.misses; - cache_info_.last_time_us = cur_time_us; - cache_info_.last_load_keys_num = cache_info.async_load_keys_num; -} - -void Slot::ResetDisplayCacheInfo(int status) { - std::unique_lock lock(cache_info_rwlock_); - cache_info_.status = status; - cache_info_.cache_num = 0; - cache_info_.keys_num = 0; - cache_info_.used_memory = 0; - cache_info_.hits = 0; - cache_info_.misses = 0; - cache_info_.hits_per_sec = 0; - cache_info_.read_cmd_per_sec = 0; - cache_info_.hitratio_per_sec = 0.0; - cache_info_.hitratio_all = 0.0; - cache_info_.load_keys_per_sec = 0; - cache_info_.waitting_load_keys_num = 0; - cache_usage_ = 0; -} diff --git a/src/pika_slot_command.cc b/src/pika_slot_command.cc index 1c7997a418..8f8e6e5830 100644 --- a/src/pika_slot_command.cc +++ b/src/pika_slot_command.cc @@ -169,8 +169,8 @@ void PikaMigrate::KillAllMigrateClient() { * -1 - error happens * >=0 - # of success migration (0 or 1) * */ -int PikaMigrate::MigrateKey(const std::string &host, const int port, int timeout, const std::string &key, - const char type, std::string &detail, const std::shared_ptr& slot) { +int PikaMigrate::MigrateKey(const std::string &host, const int port, int timeout, const std::string& key, + const char type, std::string &detail, const std::shared_ptr& db) { int send_command_num = -1; net::NetCli *migrate_cli = GetMigrateClient(host, port, timeout); @@ -180,7 +180,7 @@ int PikaMigrate::MigrateKey(const std::string &host, const int port, int timeout return -1; } - send_command_num = MigrateSend(migrate_cli, key, type, detail, slot); + send_command_num = MigrateSend(migrate_cli, key, type, detail, db); if (send_command_num <= 0) { return send_command_num; } @@ -192,8 +192,8 @@ int PikaMigrate::MigrateKey(const std::string &host, const int port, int timeout return -1; } -int PikaMigrate::MigrateSend(net::NetCli *migrate_cli, const std::string &key, const char type, std::string &detail, - const std::shared_ptr& slot) { +int PikaMigrate::MigrateSend(net::NetCli* migrate_cli, const std::string& key, const char type, std::string& detail, + const std::shared_ptr& db) { std::string wbuf_str; pstd::Status s; int command_num = -1; @@ -203,7 +203,7 @@ int PikaMigrate::MigrateSend(net::NetCli *migrate_cli, const std::string &key, c return -1; } - command_num = ParseKey(key, type, wbuf_str, slot); + command_num = ParseKey(key, type, wbuf_str, db); if (command_num < 0) { detail = "ParseKey failed"; return command_num; @@ -225,7 +225,7 @@ int PikaMigrate::MigrateSend(net::NetCli *migrate_cli, const std::string &key, c return command_num; } -bool PikaMigrate::MigrateRecv(net::NetCli *migrate_cli, int need_receive, std::string &detail) { +bool PikaMigrate::MigrateRecv(net::NetCli* migrate_cli, int need_receive, std::string& detail) { pstd::Status s; std::string reply; int64_t ret; @@ -274,26 +274,26 @@ bool PikaMigrate::MigrateRecv(net::NetCli *migrate_cli, int need_receive, std::s } // return -1 is error; 0 don't migrate; >0 the number of commond -int PikaMigrate::ParseKey(const std::string &key, const char type, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseKey(const std::string& key, const char type, std::string& wbuf_str, const std::shared_ptr& db) { int command_num = -1; int64_t ttl = 0; rocksdb::Status s; switch (type) { case 'k': - command_num = ParseKKey(key, wbuf_str, slot); + command_num = ParseKKey(key, wbuf_str, db); break; case 'h': - command_num = ParseHKey(key, wbuf_str, slot); + command_num = ParseHKey(key, wbuf_str, db); break; case 'l': - command_num = ParseLKey(key, wbuf_str, slot); + command_num = ParseLKey(key, wbuf_str, db); break; case 'z': - command_num = ParseZKey(key, wbuf_str, slot); + command_num = ParseZKey(key, wbuf_str, db); break; case 's': - command_num = ParseSKey(key, wbuf_str, slot); + command_num = ParseSKey(key, wbuf_str, db); break; default: LOG(INFO) << "ParseKey key[" << key << "], the type[" << type << "] is not support."; @@ -313,7 +313,7 @@ int PikaMigrate::ParseKey(const std::string &key, const char type, std::string & return command_num; } - ttl = TTLByType(type, key, slot); + ttl = TTLByType(type, key, db); //-1 indicates the key is valid forever if (ttl == -1) { @@ -334,7 +334,7 @@ int PikaMigrate::ParseKey(const std::string &key, const char type, std::string & return command_num; } -bool PikaMigrate::SetTTL(const std::string &key, std::string &wbuf_str, int64_t ttl) { +bool PikaMigrate::SetTTL(const std::string& key, std::string& wbuf_str, int64_t ttl) { //-1 indicates the key is valid forever if (ttl == -1) { return false; @@ -360,14 +360,14 @@ bool PikaMigrate::SetTTL(const std::string &key, std::string &wbuf_str, int64_t } // return -1 is error; 0 don't migrate; >0 the number of commond -int PikaMigrate::ParseKKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseKKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db) { net::RedisCmdArgsType argv; std::string cmd; std::string value; int64_t ttl = 0; rocksdb::Status s; - s = slot->db()->Get(key, &value); + s = db->storage()->Get(key, &value); // if key is not existed, don't migrate if (s.IsNotFound()) { @@ -382,7 +382,7 @@ int PikaMigrate::ParseKKey(const std::string &key, std::string &wbuf_str, const argv.emplace_back(key); argv.emplace_back(value); - ttl = TTLByType('k', key, slot); + ttl = TTLByType('k', key, db); // ttl = -1 indicates the key is valid forever, dont process // key is expired or not exist, dont migrate @@ -401,10 +401,10 @@ int PikaMigrate::ParseKKey(const std::string &key, std::string &wbuf_str, const return 1; } -int64_t PikaMigrate::TTLByType(const char key_type, const std::string &key, const std::shared_ptr& slot) { +int64_t PikaMigrate::TTLByType(const char key_type, const std::string& key, const std::shared_ptr& db) { std::map type_timestamp; std::map type_status; - type_timestamp = slot->db()->TTL(key, &type_status); + type_timestamp = db->storage()->TTL(key, &type_status); switch (key_type) { case 'k': { @@ -427,14 +427,14 @@ int64_t PikaMigrate::TTLByType(const char key_type, const std::string &key, cons } } -int PikaMigrate::ParseZKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseZKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db) { int command_num = 0; int64_t next_cursor = 0; std::vector score_members; do { score_members.clear(); - rocksdb::Status s = slot->db()->ZScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &score_members, &next_cursor); + rocksdb::Status s = db->storage()->ZScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &score_members, &next_cursor); if (s.ok()) { if (score_members.empty()) { break; @@ -466,13 +466,13 @@ int PikaMigrate::ParseZKey(const std::string &key, std::string &wbuf_str, const } // return -1 is error; 0 don't migrate; >0 the number of commond -int PikaMigrate::ParseHKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseHKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db) { int64_t next_cursor = 0; int command_num = 0; std::vector field_values; do { field_values.clear(); - rocksdb::Status s = slot->db()->HScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &field_values, &next_cursor); + rocksdb::Status s = db->storage()->HScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &field_values, &next_cursor); if (s.ok()) { if (field_values.empty()) { break; @@ -504,14 +504,14 @@ int PikaMigrate::ParseHKey(const std::string &key, std::string &wbuf_str, const } // return -1 is error; 0 don't migrate; >0 the number of commond -int PikaMigrate::ParseSKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseSKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db) { int command_num = 0; int64_t next_cursor = 0; std::vector members; do { members.clear(); - rocksdb::Status s = slot->db()->SScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &members, &next_cursor); + rocksdb::Status s = db->storage()->SScan(key, next_cursor, "*", MAX_MEMBERS_NUM, &members, &next_cursor); if (s.ok()) { if (members.empty()) { @@ -543,7 +543,7 @@ int PikaMigrate::ParseSKey(const std::string &key, std::string &wbuf_str, const } // return -1 is error; 0 don't migrate; >0 the number of commond -int PikaMigrate::ParseLKey(const std::string &key, std::string &wbuf_str, const std::shared_ptr& slot) { +int PikaMigrate::ParseLKey(const std::string& key, std::string& wbuf_str, const std::shared_ptr& db) { int64_t left = 0; int command_num = 0; std::vector values; @@ -560,7 +560,7 @@ int PikaMigrate::ParseLKey(const std::string &key, std::string &wbuf_str, const do { values.clear(); - rocksdb::Status s = slot->db()->LRange(key, left, left + (MAX_MEMBERS_NUM - 1), &values); + rocksdb::Status s = db->storage()->LRange(key, left, left + (MAX_MEMBERS_NUM - 1), &values); if (s.ok()) { if (values.empty()) { break; @@ -604,39 +604,39 @@ int PikaMigrate::ParseLKey(const std::string &key, std::string &wbuf_str, const * -1 - error happens * >=0 - # of success migration (0 or 1) * */ -static int SlotsMgrtOne(const std::string &host, const int port, int timeout, const std::string &key, const char type, - std::string &detail, const std::shared_ptr& slot) { +static int SlotsMgrtOne(const std::string &host, const int port, int timeout, const std::string& key, const char type, + std::string& detail, const std::shared_ptr& db) { int send_command_num = 0; rocksdb::Status s; std::map type_status; - send_command_num = g_pika_server->pika_migrate_->MigrateKey(host, port, timeout, key, type, detail, slot); + send_command_num = g_pika_server->pika_migrate_->MigrateKey(host, port, timeout, key, type, detail, db); // the key is migrated to target, delete key and slotsinfo if (send_command_num >= 1) { LOG(INFO) << "【send command success】Migrate key: " << key << " success, host: " << host << ", port: " << port; std::vector keys; keys.emplace_back(key); - int64_t count = slot->db()->Del(keys, &type_status); + int64_t count = db->storage()->Del(keys, &type_status); if (count > 0) { - WriteDelKeyToBinlog(key, slot); + WriteDelKeyToBinlog(key, db); } // del slots info - RemSlotKeyByType(std::string(1, type), key, slot); + RemSlotKeyByType(std::string(1, type), key, db); return 1; } // key is not existed, only del slotsinfo if (send_command_num == 0) { // del slots info - RemSlotKeyByType(std::string(1, type), key, slot); + RemSlotKeyByType(std::string(1, type), key, db); return 0; } return -1; } -void RemSlotKeyByType(const std::string &type, const std::string &key, const std::shared_ptr& slot) { +void RemSlotKeyByType(const std::string& type, const std::string& key, const std::shared_ptr& db) { uint32_t crc; int hastag; int slotNum = GetSlotsID(key, &crc, &hastag); @@ -646,7 +646,7 @@ void RemSlotKeyByType(const std::string &type, const std::string &key, const std std::vector members; members.emplace_back(type + key); - rocksdb::Status s = slot->db()->SRem(slot_key, members, &res); + rocksdb::Status s = db->storage()->SRem(slot_key, members, &res); if (!s.ok()) { LOG(ERROR) << "srem key[" << key << "] from slotKey[" << slot_key << "] failed, error: " << s.ToString(); return; @@ -654,7 +654,7 @@ void RemSlotKeyByType(const std::string &type, const std::string &key, const std if (hastag) { std::string tag_key = GetSlotsTagKey(crc); - s = slot->db()->SRem(tag_key, members, &res); + s = db->storage()->SRem(tag_key, members, &res); if (!s.ok()) { LOG(ERROR) << "srem key[" << key << "] from tagKey[" << tag_key << "] failed, error: " << s.ToString(); return; @@ -668,8 +668,8 @@ void RemSlotKeyByType(const std::string &type, const std::string &key, const std * -1 - error happens * >=0 - # of success migration * */ -static int SlotsMgrtTag(const std::string &host, const int port, int timeout, const std::string &key, const char type, - std::string &detail, const std::shared_ptr& slot) { +static int SlotsMgrtTag(const std::string& host, const int port, int timeout, const std::string& key, const char type, + std::string& detail, const std::shared_ptr& db) { int count = 0; uint32_t crc; int hastag; @@ -678,7 +678,7 @@ static int SlotsMgrtTag(const std::string &host, const int port, int timeout, co if (type == 0) { return 0; } - int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, slot); + int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, db); if (ret == 0) { LOG(INFO) << "slots migrate without tag failed, key: " << key << ", detail: " << detail; } @@ -689,7 +689,7 @@ static int SlotsMgrtTag(const std::string &host, const int port, int timeout, co std::vector members; // get all keys that have the same crc - rocksdb::Status s = slot->db()->SMembers(tag_key, &members); + rocksdb::Status s = db->storage()->SMembers(tag_key, &members); if (!s.ok()) { return -1; } @@ -699,7 +699,7 @@ static int SlotsMgrtTag(const std::string &host, const int port, int timeout, co std::string key = *iter; char type = key.at(0); key.erase(key.begin()); - int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, slot); + int ret = SlotsMgrtOne(host, port, timeout, key, type, detail, db); // the key is migrated to target if (ret == 1) { @@ -719,7 +719,7 @@ static int SlotsMgrtTag(const std::string &host, const int port, int timeout, co } // get slot tag -static const char *GetSlotsTag(const std::string &str, int *plen) { +static const char *GetSlotsTag(const std::string& str, int* plen) { const char *s = str.data(); int i, j, n = static_cast(str.length()); for (i = 0; i < n && s[i] != '{'; i++) { @@ -739,12 +739,12 @@ static const char *GetSlotsTag(const std::string &str, int *plen) { return s + i; } -std::string GetSlotKey(int slot) { - return SlotKeyPrefix + std::to_string(slot); +std::string GetSlotKey(int db) { + return SlotKeyPrefix + std::to_string(db); } // get slot number of the key -int GetSlotID(const std::string &str) { return GetSlotsID(str, nullptr, nullptr); } +int GetSlotID(const std::string& str) { return GetSlotsID(str, nullptr, nullptr); } // get the slot number by key int GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag) { @@ -767,10 +767,10 @@ int GetSlotsID(const std::string &str, uint32_t *pcrc, int *phastag) { return crc % g_pika_conf->default_slot_num(); } -uint32_t CRC32CheckSum(const char *buf, int len) { return CRC32Update(0, buf, len); } +uint32_t CRC32CheckSum(const char* buf, int len) { return CRC32Update(0, buf, len); } // add key to slotkey -void AddSlotKey(const std::string& type, const std::string& key, const std::shared_ptr& slot) { +void AddSlotKey(const std::string& type, const std::string& key, const std::shared_ptr& db) { if (g_pika_conf->slotmigrate() != true) { return; } @@ -783,7 +783,7 @@ void AddSlotKey(const std::string& type, const std::string& key, const std::shar std::string slot_key = GetSlotKey(slotID); std::vector members; members.emplace_back(type + key); - s = slot->db()->SAdd(slot_key, members, &res); + s = db->storage()->SAdd(slot_key, members, &res); if (!s.ok()) { LOG(ERROR) << "sadd key[" << key << "] to slotKey[" << slot_key << "] failed, error: " << s.ToString(); return; @@ -793,7 +793,7 @@ void AddSlotKey(const std::string& type, const std::string& key, const std::shar // prevent write slot_key success, but write tag_key failed, so always write tag_key if (hastag) { std::string tag_key = GetSlotsTagKey(crc); - s = slot->db()->SAdd(tag_key, members, &res); + s = db->storage()->SAdd(tag_key, members, &res); if (!s.ok()) { LOG(ERROR) << "sadd key[" << key << "] to tagKey[" << tag_key << "] failed, error: " << s.ToString(); return; @@ -802,28 +802,28 @@ void AddSlotKey(const std::string& type, const std::string& key, const std::shar } // del key from slotkey -void RemSlotKey(const std::string& key, const std::shared_ptr& slot) { +void RemSlotKey(const std::string& key, const std::shared_ptr& db) { if (g_pika_conf->slotmigrate() != true) { return; } std::string type; - if (GetKeyType(key, type, slot) < 0) { + if (GetKeyType(key, type, db) < 0) { LOG(WARNING) << "SRem key: " << key << " from slotKey error"; return; } std::string slotKey = GetSlotKey(GetSlotID(key)); int32_t count = 0; std::vector members(1, type + key); - rocksdb::Status s = slot->db()->SRem(slotKey, members, &count); + rocksdb::Status s = db->storage()->SRem(slotKey, members, &count); if (!s.ok()) { LOG(WARNING) << "SRem key: " << key << " from slotKey, error: " << s.ToString(); return; } } -int GetKeyType(const std::string& key, std::string &key_type, const std::shared_ptr& slot) { +int GetKeyType(const std::string& key, std::string& key_type, const std::shared_ptr& db) { std::vector type_str(1); - rocksdb::Status s = slot->db()->GetType(key, true, type_str); + rocksdb::Status s = db->storage()->GetType(key, true, type_str); if (!s.ok()) { LOG(WARNING) << "Get key type error: " << key << " " << s.ToString(); key_type = ""; @@ -853,7 +853,7 @@ std::string GetSlotsTagKey(uint32_t crc) { } // delete key from db -int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr& slot) { +int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr& db) { LOG(INFO) << "Del key Srem key " << key; int32_t res = 0; std::string slotKey = GetSlotKey(GetSlotID(key)); @@ -862,7 +862,7 @@ int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr // delete key from slot std::vector members; members.emplace_back(key_type + key); - rocksdb::Status s = slot->db()->SRem(slotKey, members, &res); + rocksdb::Status s = db->storage()->SRem(slotKey, members, &res); if (!s.ok()) { if (s.IsNotFound()) { LOG(INFO) << "Del key Srem key " << key << " not found"; @@ -877,12 +877,12 @@ int DeleteKey(const std::string& key, const char key_type, const std::shared_ptr members.clear(); members.emplace_back(key); std::map type_status; - int64_t del_nums = slot->db()->Del(members, &type_status); + int64_t del_nums = db->storage()->Del(members, &type_status); if (0 > del_nums) { LOG(WARNING) << "Del key: " << key << " at slot " << GetSlotID(key) << " error"; return -1; } - WriteDelKeyToBinlog(key, slot); + WriteDelKeyToBinlog(key, db); return 1; } @@ -940,7 +940,7 @@ void SlotsMgrtTagSlotCmd::DoInitial() { } } -void SlotsMgrtTagSlotCmd::Do(std::shared_ptr slot) { +void SlotsMgrtTagSlotCmd::Do() { if (g_pika_conf->slotmigrate() != true) { LOG(WARNING) << "Not in slotmigrate mode"; res_.SetRes(CmdRes::kErrOther, "not set slotmigrate"); @@ -953,7 +953,7 @@ void SlotsMgrtTagSlotCmd::Do(std::shared_ptr slot) { std::string slot_key = GetSlotKey(static_cast(slot_id_)); // first, get the count of slot_key, prevent to sscan key very slowly when the key is not found - rocksdb::Status s = slot->db()->SCard(slot_key, &len); + rocksdb::Status s = db_->storage()->SCard(slot_key, &len); LOG(INFO) << "【SlotsMgrtTagSlotCmd::Do】Get count, slot_key: " << slot_key << ", len: " << len; if (len < 0) { detail = "Get the len of slot Error"; @@ -963,13 +963,13 @@ void SlotsMgrtTagSlotCmd::Do(std::shared_ptr slot) { g_pika_server->pika_migrate_->CleanMigrateClient(); int64_t next_cursor = 0; std::vector members; - rocksdb::Status s = slot->db()->SScan(slot_key, 0, "*", 1, &members, &next_cursor); + rocksdb::Status s = db_->storage()->SScan(slot_key, 0, "*", 1, &members, &next_cursor); if (s.ok()) { for (const auto &member : members) { std::string key = member; char type = key.at(0); key.erase(key.begin()); - ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key, type, detail, slot); + ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key, type, detail, db_); } } // unlock @@ -994,9 +994,9 @@ void SlotsMgrtTagSlotCmd::Do(std::shared_ptr slot) { } // check key type -int SlotsMgrtTagOneCmd::KeyTypeCheck(const std::shared_ptr& slot) { +int SlotsMgrtTagOneCmd::KeyTypeCheck(const std::shared_ptr& db) { std::vector type_str(1); - rocksdb::Status s = slot->db()->GetType(key_, true, type_str); + rocksdb::Status s = db->storage()->GetType(key_, true, type_str); if (!s.ok()) { if (s.IsNotFound()) { LOG(INFO) << "Migrate slot key " << key_ << " not found"; @@ -1069,7 +1069,7 @@ void SlotsMgrtTagOneCmd::DoInitial() { key_ = *it++; } -void SlotsMgrtTagOneCmd::Do(std::shared_ptr slot) { +void SlotsMgrtTagOneCmd::Do() { if (!g_pika_conf->slotmigrate()) { LOG(WARNING) << "Not in slotmigrate mode"; res_.SetRes(CmdRes::kErrOther, "not set slotmigrate"); @@ -1091,7 +1091,7 @@ void SlotsMgrtTagOneCmd::Do(std::shared_ptr slot) { keys.emplace_back(key_); // check the key is not existed - ret = slot->db()->Exists(keys, &type_status); + ret = db_->storage()->Exists(keys, &type_status); // when the key is not existed, ret = 0 if (ret == -1) { @@ -1108,7 +1108,7 @@ void SlotsMgrtTagOneCmd::Do(std::shared_ptr slot) { } else { // key is tag_key, check the number of the tag_key std::string tag_key = GetSlotsTagKey(crc); - s = slot->db()->SCard(tag_key, &len); + s = db_->storage()->SCard(tag_key, &len); if (s.IsNotFound()) { res_.AppendInteger(0); return; @@ -1136,20 +1136,20 @@ void SlotsMgrtTagOneCmd::Do(std::shared_ptr slot) { keys.emplace_back(key_); // the key may be deleted by another thread std::map type_status; - ret = slot->db()->Exists(keys, &type_status); + ret = db_->storage()->Exists(keys, &type_status); // when the key is not existed, ret = 0 if (ret == -1) { detail = s.ToString(); - } else if (KeyTypeCheck(slot) != 0) { + } else if (KeyTypeCheck(db_) != 0) { detail = "cont get the key type."; ret = -1; } else { - ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key_, key_type_, detail, slot); + ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key_, key_type_, detail, db_); } } else { // key maybe doesn't exist, the key is tag key, migrate the same tag key - ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key_, 0, detail, slot); + ret = SlotsMgrtTag(dest_ip_, static_cast(dest_port_), static_cast(timeout_ms_), key_, 0, detail, db_); } // unlock the record lock @@ -1213,7 +1213,7 @@ void SlotsInfoCmd::DoInitial() { } } -void SlotsInfoCmd::Do(std::shared_ptr slot) { +void SlotsInfoCmd::Do() { int slotNum = g_pika_conf->default_slot_num(); int slots_slot[slotNum]; int slots_size[slotNum]; @@ -1226,7 +1226,7 @@ void SlotsInfoCmd::Do(std::shared_ptr slot) { for (auto i = static_cast(begin_); i < end_; i++) { slot_key = GetSlotKey(i); len = 0; - rocksdb::Status s = slot->db()->SCard(slot_key, &len); + rocksdb::Status s = db_->storage()->SCard(slot_key, &len); if (!s.ok() || len == 0) { continue; } @@ -1299,7 +1299,7 @@ void SlotsMgrtTagSlotAsyncCmd::DoInitial() { return; } -void SlotsMgrtTagSlotAsyncCmd::Do(std::shared_ptr slot) { +void SlotsMgrtTagSlotAsyncCmd::Do() { // check whether open slotmigrate if (!g_pika_conf->slotmigrate()) { res_.SetRes(CmdRes::kErrOther, "please open slotmigrate and reload slot"); @@ -1308,7 +1308,7 @@ void SlotsMgrtTagSlotAsyncCmd::Do(std::shared_ptr slot) { int32_t remained = 0; std::string slotKey = GetSlotKey(static_cast(slot_id_)); - storage::Status status = slot->db()->SCard(slotKey, &remained); + storage::Status status = db_->storage()->SCard(slotKey, &remained); if (status.IsNotFound()) { LOG(INFO) << "find no record in slot " << slot_id_; res_.AppendArrayLen(2); @@ -1322,7 +1322,7 @@ void SlotsMgrtTagSlotAsyncCmd::Do(std::shared_ptr slot) { return; } - bool ret = g_pika_server->SlotsMigrateBatch(dest_ip_, dest_port_, timeout_ms_, slot_id_, keys_num_, slot); + bool ret = g_pika_server->SlotsMigrateBatch(dest_ip_, dest_port_, timeout_ms_, slot_id_, keys_num_, db_); if (!ret) { LOG(WARNING) << "Slot batch migrate keys error"; res_.SetRes(CmdRes::kErrOther, "Slot batch migrating keys error, may be currently migrating"); @@ -1342,7 +1342,7 @@ void SlotsMgrtAsyncStatusCmd::DoInitial() { return; } -void SlotsMgrtAsyncStatusCmd::Do(std::shared_ptr slot) { +void SlotsMgrtAsyncStatusCmd::Do() { std::string status; std::string ip; int64_t port = -1, slots = -1, moved = -1, remained = -1; @@ -1376,7 +1376,7 @@ void SlotsMgrtAsyncCancelCmd::DoInitial() { return; } -void SlotsMgrtAsyncCancelCmd::Do(std::shared_ptr slot) { +void SlotsMgrtAsyncCancelCmd::Do() { bool ret = g_pika_server->SlotsMigrateAsyncCancel(); if (!ret) { res_.SetRes(CmdRes::kErrOther, "slotsmgrt-async-cancel error"); @@ -1393,14 +1393,14 @@ void SlotsDelCmd::DoInitial() { return; } -void SlotsDelCmd::Do(std::shared_ptr slot) { +void SlotsDelCmd::Do() { std::vector keys; std::vector::const_iterator iter; for (iter = slots_.begin(); iter != slots_.end(); iter++) { keys.emplace_back(SlotKeyPrefix + *iter); } std::map type_status; - int64_t count = slot->db()->Del(keys, &type_status); + int64_t count = db_->storage()->Del(keys, &type_status); if (count >= 0) { res_.AppendInteger(count); } else { @@ -1423,7 +1423,7 @@ void SlotsHashKeyCmd::DoInitial() { return; } -void SlotsHashKeyCmd::Do(std::shared_ptr slot) { +void SlotsHashKeyCmd::Do() { std::vector::const_iterator keys_it; res_.AppendArrayLenUint64(keys_.size()); @@ -1476,9 +1476,9 @@ void SlotsScanCmd::DoInitial() { return; } -void SlotsScanCmd::Do(std::shared_ptr slot) { +void SlotsScanCmd::Do() { std::vector members; - rocksdb::Status s = slot->db()->SScan(key_, cursor_, pattern_, count_, &members, &cursor_); + rocksdb::Status s = db_->storage()->SScan(key_, cursor_, pattern_, count_, &members, &cursor_); if (members.size() <= 0) { cursor_ = 0; @@ -1512,9 +1512,9 @@ void SlotsMgrtExecWrapperCmd::DoInitial() { // return 0 means key doesn't exist, or key is not migrating // return 1 means key is migrating // return -1 means something wrong -void SlotsMgrtExecWrapperCmd::Do(std::shared_ptr slot) { +void SlotsMgrtExecWrapperCmd::Do() { res_.AppendArrayLen(2); - int ret = g_pika_server->SlotsMigrateOne(key_, slot); + int ret = g_pika_server->SlotsMigrateOne(key_, db_); switch (ret) { case 0: case -2: @@ -1540,11 +1540,11 @@ void SlotsReloadCmd::DoInitial() { return; } -void SlotsReloadCmd::Do(std::shared_ptr slot) { - g_pika_server->Bgslotsreload(slot); +void SlotsReloadCmd::Do() { + g_pika_server->Bgslotsreload(db_); const PikaServer::BGSlotsReload &info = g_pika_server->bgslots_reload(); char buf[256]; - snprintf(buf, sizeof(buf), "+%s : %ld", info.s_start_time.c_str(), g_pika_server->GetSlotsreloadingCursor()); + snprintf(buf, sizeof(buf), "+%s : %lld", info.s_start_time.c_str(), g_pika_server->GetSlotsreloadingCursor()); res_.AppendContent(buf); return; } @@ -1556,7 +1556,7 @@ void SlotsReloadOffCmd::DoInitial() { return; } -void SlotsReloadOffCmd::Do(std::shared_ptr slot) { +void SlotsReloadOffCmd::Do() { g_pika_server->SetSlotsreloading(false); res_.SetRes(CmdRes::kOk); return; @@ -1583,8 +1583,8 @@ void SlotsCleanupCmd::DoInitial() { return; } -void SlotsCleanupCmd::Do(std::shared_ptr slot) { - g_pika_server->Bgslotscleanup(cleanup_slots_, slot); +void SlotsCleanupCmd::Do() { + g_pika_server->Bgslotscleanup(cleanup_slots_, db_); std::vector cleanup_slots(g_pika_server->GetCleanupSlots()); res_.AppendArrayLenUint64(cleanup_slots.size()); auto iter = cleanup_slots.begin(); @@ -1601,7 +1601,7 @@ void SlotsCleanupOffCmd::DoInitial() { return; } -void SlotsCleanupOffCmd::Do(std::shared_ptr slot) { +void SlotsCleanupOffCmd::Do() { g_pika_server->StopBgslotscleanup(); res_.SetRes(CmdRes::kOk); return; diff --git a/src/pika_stable_log.cc b/src/pika_stable_log.cc index aae650263b..b1e9fc278a 100644 --- a/src/pika_stable_log.cc +++ b/src/pika_stable_log.cc @@ -21,8 +21,8 @@ using pstd::Status; extern PikaServer* g_pika_server; extern std::unique_ptr g_pika_rm; -StableLog::StableLog(std::string db_name, uint32_t slot_id, std::string log_path) - : purging_(false), db_name_(std::move(db_name)), slot_id_(slot_id), log_path_(std::move(log_path)) { +StableLog::StableLog(std::string db_name, std::string log_path) + : purging_(false), db_name_(std::move(db_name)), log_path_(std::move(log_path)) { stable_logger_ = std::make_shared(log_path_, g_pika_conf->binlog_file_size()); std::map binlogs; if (!GetBinlogFiles(&binlogs)) { @@ -54,7 +54,7 @@ void StableLog::RemoveStableLogDir() { } g_pika_server->PurgeDir(logpath); - LOG(WARNING) << "Slot StableLog: " << db_name_ << ":" << slot_id_ << " move to trash success"; + LOG(WARNING) << "DB StableLog: " << db_name_ << " move to trash success"; } bool StableLog::PurgeStableLogs(uint32_t to, bool manual) { @@ -90,7 +90,7 @@ bool StableLog::PurgeFiles(uint32_t to, bool manual) { int delete_num = 0; struct stat file_stat; auto remain_expire_num = static_cast(binlogs.size() - g_pika_conf->expire_logs_nums()); - std::shared_ptr master_slot = nullptr; + std::shared_ptr master_db = nullptr; std::map::iterator it; for (it = binlogs.begin(); it != binlogs.end(); ++it) { if ((manual && it->first <= to) // Manual purgelogsto @@ -99,13 +99,13 @@ bool StableLog::PurgeFiles(uint32_t to, bool manual) { && stat(((log_path_ + it->second)).c_str(), &file_stat) == 0 && file_stat.st_mtime < time(nullptr) - g_pika_conf->expire_logs_days() * 24 * 3600)) { // Expire time trigger // We check this every time to avoid lock when we do file deletion - master_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(db_name_, slot_id_)); - if (!master_slot) { - LOG(WARNING) << "Slot: " << db_name_ << ":" << slot_id_ << " Not Found"; + master_db = g_pika_rm->GetSyncMasterDBByName(DBInfo(db_name_)); + if (!master_db) { + LOG(WARNING) << "DB: " << db_name_ << "Not Found"; return false; } - if (!master_slot->BinlogCloudPurge(it->first)) { + if (!master_db->BinlogCloudPurge(it->first)) { LOG(WARNING) << log_path_ << " Could not purge " << (it->first) << ", since it is already be used"; return false; } diff --git a/src/pika_stream.cc b/src/pika_stream.cc index 17278ca63d..f8841a3af5 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -7,7 +7,6 @@ #include #include "include/pika_command.h" -#include "include/pika_slot.h" #include "include/pika_stream_base.h" #include "include/pika_stream_meta_value.h" #include "include/pika_stream_types.h" @@ -179,8 +178,8 @@ void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, S } } -void AppendMessagesToRes(CmdRes &res, std::vector &field_values, const Slot *slot) { - assert(slot); +void AppendMessagesToRes(CmdRes &res, std::vector &field_values, const DB* db) { + assert(db); res.AppendArrayLenUint64(field_values.size()); for (auto &fv : field_values) { std::vector message; @@ -232,11 +231,11 @@ void XAddCmd::DoInitial() { } } -void XAddCmd::Do(std::shared_ptr slot) { +void XAddCmd::Do() { // 1 get stream meta rocksdb::Status s; StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); + s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); if (s.IsNotFound() && args_.no_mkstream) { res_.SetRes(CmdRes::kNotFound); return; @@ -277,7 +276,7 @@ void XAddCmd::Do(std::shared_ptr slot) { assert(field > serialized_last_id); #endif // DEBUG - s = StreamStorage::InsertStreamMessage(key_, args_.id, message, slot.get()); + s = StreamStorage::InsertStreamMessage(key_, args_.id, message, db_.get()); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, "error from XADD, insert stream message failed 1: " + s.ToString()); return; @@ -294,12 +293,12 @@ void XAddCmd::Do(std::shared_ptr slot) { // 4 trim the stream if needed if (args_.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { int32_t count; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, slot.get()), res_); + TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, db_.get()), res_); (void)count; } // 5 update stream meta - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()); + s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, "error from XADD, get stream meta failed 2: " + s.ToString()); return; @@ -386,13 +385,13 @@ void XRangeCmd::DoInitial() { } } -void XRangeCmd::Do(std::shared_ptr slot) { +void XRangeCmd::Do() { std::vector field_values; if (start_sid <= end_sid) { StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, false); std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); + auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); (void)next_field; if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -400,16 +399,16 @@ void XRangeCmd::Do(std::shared_ptr slot) { } } - AppendMessagesToRes(res_, field_values, slot.get()); + AppendMessagesToRes(res_, field_values, db_.get()); } -void XRevrangeCmd::Do(std::shared_ptr slot) { +void XRevrangeCmd::Do() { std::vector field_values; if (start_sid >= end_sid) { StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, true); std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); + auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); (void)next_field; if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -417,21 +416,21 @@ void XRevrangeCmd::Do(std::shared_ptr slot) { } } - AppendMessagesToRes(res_, field_values, slot.get()); + AppendMessagesToRes(res_, field_values, db_.get()); } -inline void XDelCmd::SetFirstIDOrReply(StreamMetaValue &stream_meta, const Slot *slot) { - assert(slot); - return SetFirstOrLastIDOrReply(stream_meta, slot, true); +inline void XDelCmd::SetFirstIDOrReply(StreamMetaValue &stream_meta, const DB* db) { + assert(db); + return SetFirstOrLastIDOrReply(stream_meta, db, true); } -inline void XDelCmd::SetLastIDOrReply(StreamMetaValue &stream_meta, const Slot *slot) { - assert(slot); - return SetFirstOrLastIDOrReply(stream_meta, slot, false); +inline void XDelCmd::SetLastIDOrReply(StreamMetaValue &stream_meta, const DB* db) { + assert(db); + return SetFirstOrLastIDOrReply(stream_meta, db, false); } -inline void XDelCmd::SetFirstOrLastIDOrReply(StreamMetaValue &stream_meta, const Slot *slot, bool is_set_first) { - assert(slot); +inline void XDelCmd::SetFirstOrLastIDOrReply(StreamMetaValue &stream_meta, const DB* db, bool is_set_first) { + assert(db); if (stream_meta.length() == 0) { stream_meta.set_first_id(kSTREAMID_MIN); return; @@ -443,11 +442,11 @@ inline void XDelCmd::SetFirstOrLastIDOrReply(StreamMetaValue &stream_meta, const storage::Status s; if (is_set_first) { StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MIN, kSTREAMID_MAX, 1); - s = StreamStorage::ScanStream(option, field_values, next_field, slot); + s = StreamStorage::ScanStream(option, field_values, next_field, db); } else { bool is_reverse = true; StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); - s = StreamStorage::ScanStream(option, field_values, next_field, slot); + s = StreamStorage::ScanStream(option, field_values, next_field, db); } (void)next_field; @@ -488,10 +487,10 @@ void XDelCmd::DoInitial() { } } -void XDelCmd::Do(std::shared_ptr slot) { +void XDelCmd::Do() { // 1 try to get stream meta StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); + auto s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); if (s.IsNotFound()) { res_.AppendInteger(0); return; @@ -502,7 +501,7 @@ void XDelCmd::Do(std::shared_ptr slot) { // 2 do the delete int32_t count{0}; - s = StreamStorage::DeleteStreamMessage(key_, ids_, count, slot.get()); + s = StreamStorage::DeleteStreamMessage(key_, ids_, count, db_.get()); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -515,13 +514,13 @@ void XDelCmd::Do(std::shared_ptr slot) { stream_meta.set_max_deleted_entry_id(id); } if (id == stream_meta.first_id()) { - SetFirstIDOrReply(stream_meta, slot.get()); + SetFirstIDOrReply(stream_meta, db_.get()); } else if (id == stream_meta.last_id()) { - SetLastIDOrReply(stream_meta, slot.get()); + SetLastIDOrReply(stream_meta, db_.get()); } } - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()); + s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()); if (!s.ok()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; @@ -538,10 +537,10 @@ void XLenCmd::DoInitial() { key_ = argv_[1]; } -void XLenCmd::Do(std::shared_ptr slot) { +void XLenCmd::Do() { rocksdb::Status s; StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); + s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); if (s.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); return; @@ -567,7 +566,7 @@ void XReadCmd::DoInitial() { ParseReadOrReadGroupArgsOrReply(res_, argv_, args_, false); } -void XReadCmd::Do(std::shared_ptr slot) { +void XReadCmd::Do() { rocksdb::Status s; // 1 prepare stream_metas @@ -577,7 +576,7 @@ void XReadCmd::Do(std::shared_ptr slot) { const auto &unparsed_id = args_.unparsed_ids[i]; StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key, slot.get()); + auto s = StreamStorage::GetStreamMeta(stream_meta, key, db_.get()); if (s.IsNotFound()) { continue; } else if (!s.ok()) { @@ -622,7 +621,7 @@ void XReadCmd::Do(std::shared_ptr slot) { std::vector field_values; std::string next_field; StreamStorage::ScanStreamOptions options(key, id, kSTREAMID_MAX, args_.count, true); - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); + auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); (void)next_field; if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -631,7 +630,7 @@ void XReadCmd::Do(std::shared_ptr slot) { res_.AppendArrayLen(2); res_.AppendString(key); - AppendMessagesToRes(res_, field_values, slot.get()); + AppendMessagesToRes(res_, field_values, db_.get()); } } @@ -648,10 +647,10 @@ void XTrimCmd::DoInitial() { } } -void XTrimCmd::Do(std::shared_ptr slot) { +void XTrimCmd::Do() { // 1 try to get stream meta, if not found, return error StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); + auto s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); if (s.IsNotFound()) { res_.AppendInteger(0); return; @@ -662,10 +661,10 @@ void XTrimCmd::Do(std::shared_ptr slot) { // 2 do the trim int32_t count{0}; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, slot.get()), res_); + TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, db_.get()), res_); // 3 update stream meta - TRY_CATCH_ERROR(StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()), res_); + TRY_CATCH_ERROR(StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()), res_); res_.AppendInteger(count); return; @@ -711,9 +710,9 @@ void XInfoCmd::DoInitial() { } } -void XInfoCmd::Do(std::shared_ptr slot) { +void XInfoCmd::Do() { if (!strcasecmp(subcmd_.c_str(), "STREAM")) { - this->StreamInfo(slot); + this->StreamInfo(db_); } else if (!strcasecmp(subcmd_.c_str(), "GROUPS")) { // Korpse: TODO: // this->GroupsInfo(slot); @@ -726,10 +725,10 @@ void XInfoCmd::Do(std::shared_ptr slot) { } } -void XInfoCmd::StreamInfo(std::shared_ptr &slot) { +void XInfoCmd::StreamInfo(std::shared_ptr& db) { // 1 try to get stream meta StreamMetaValue stream_meta; - TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()), res_); + TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, db.get()), res_); // 2 append the stream info res_.AppendArrayLen(10); diff --git a/src/pika_stream_base.cc b/src/pika_stream_base.cc index eebbd3512c..9437da3774 100644 --- a/src/pika_stream_base.cc +++ b/src/pika_stream_base.cc @@ -8,25 +8,27 @@ #include "include/pika_command.h" #include "include/pika_stream_meta_value.h" #include "include/pika_stream_types.h" +#include "include/pika_db.h" #include "storage/storage.h" -#define GET_NEXT_TREE_ID_AND_CHECK(slot, tid) \ + +#define GET_NEXT_TREE_ID_AND_CHECK(db, tid) \ do { \ auto &tid_gen = TreeIDGenerator::GetInstance(); \ - auto s = tid_gen.GetNextTreeID(slot, tid); \ + auto s = tid_gen.GetNextTreeID(db, tid); \ if (!s.ok()) { \ return s; \ } \ } while (0) -storage::Status TreeIDGenerator::GetNextTreeID(const Slot *slot, treeID &tid) { - assert(slot); +storage::Status TreeIDGenerator::GetNextTreeID(const DB* db, treeID &tid) { + assert(db); auto expected_id = kINVALID_TREE_ID; // if tree_id_ is not initialized (equal to kINVALID_TREE_ID), try to fetch it from storage if (tree_id_.compare_exchange_strong(expected_id, START_TREE_ID)) { std::string value; - storage::Status s = slot->db()->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); + storage::Status s = db->storage()->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); if (s.ok()) { treeID id; if (!StreamUtils::string2int32(value.c_str(), id)) { @@ -46,13 +48,13 @@ storage::Status TreeIDGenerator::GetNextTreeID(const Slot *slot, treeID &tid) { // insert tree id to storage std::string tree_id_str = std::to_string(tree_id_); int32_t res; - return slot->db()->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); + return db->storage()->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); } -storage::Status StreamStorage::GetStreamMeta(StreamMetaValue &stream_meta, const std::string &key, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::GetStreamMeta(StreamMetaValue &stream_meta, const std::string &key, const DB* db) { + assert(db); std::string value; - auto s = slot->db()->HGet(STREAM_META_HASH_KEY, key, &value); + auto s = db->storage()->HGet(STREAM_META_HASH_KEY, key, &value); if (s.ok()) { stream_meta.ParseFrom(value); return storage::Status::OK(); @@ -62,43 +64,43 @@ storage::Status StreamStorage::GetStreamMeta(StreamMetaValue &stream_meta, const // no need to be thread safe, only xadd will call this function // and xadd can be locked by the same key using current_key() -storage::Status StreamStorage::SetStreamMeta(const std::string &key, std::string &meta_value, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::SetStreamMeta(const std::string &key, std::string &meta_value, const DB* db) { + assert(db); storage::Status s; int32_t temp{0}; - s = slot->db()->HSet(STREAM_META_HASH_KEY, key, meta_value, &temp); + s = db->storage()->HSet(STREAM_META_HASH_KEY, key, meta_value, &temp); (void)temp; return s; } -storage::Status StreamStorage::DeleteStreamMeta(const std::string &key, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::DeleteStreamMeta(const std::string &key, const DB* db) { + assert(db); int32_t ret; - return slot->db()->HDel({STREAM_META_HASH_KEY}, {key}, &ret); + return db->storage()->HDel({STREAM_META_HASH_KEY}, {key}, &ret); } storage::Status StreamStorage::InsertStreamMessage(const std::string &key, const streamID &id, - const std::string &message, const Slot *slot) { - assert(slot); + const std::string &message, const DB* db) { + assert(db); std::string true_key = STREAM_DATA_HASH_PREFIX + key; int32_t temp{0}; std::string serialized_id; id.SerializeTo(serialized_id); - storage::Status s = slot->db()->HSet(true_key, serialized_id, message, &temp); + storage::Status s = db->storage()->HSet(true_key, serialized_id, message, &temp); (void)temp; return s; } storage::Status StreamStorage::GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const Slot *slot) { - assert(slot); + const DB* db) { + assert(db); std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return slot->db()->HGet(true_key, sid, &message); + return db->storage()->HGet(true_key, sid, &message); } storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, const std::vector &id, - int32_t &ret, const Slot *slot) { - assert(slot); + int32_t &ret, const DB* db) { + assert(db); std::string true_key = STREAM_DATA_HASH_PREFIX + key; std::vector serialized_ids; for (auto &sid : id) { @@ -106,20 +108,20 @@ storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, const sid.SerializeTo(serialized_id); serialized_ids.emplace_back(std::move(serialized_id)); } - return slot->db()->HDel(true_key, {serialized_ids}, &ret); + return db->storage()->HDel(true_key, {serialized_ids}, &ret); } storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, const std::vector &serialized_ids, int32_t &ret, - const Slot *slot) { - assert(slot); + const DB* db) { + assert(db); std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return slot->db()->HDel(true_key, {serialized_ids}, &ret); + return db->storage()->HDel(true_key, {serialized_ids}, &ret); } storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vector &field_values, - std::string &next_field, const Slot *slot) { - assert(slot); + std::string &next_field, const DB* db) { + assert(db); std::string start_field; std::string end_field; storage::Slice pattern = "*"; // match all the fields from start_field to end_field @@ -134,7 +136,7 @@ storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vect } else { op.start_sid.SerializeTo(start_field); } - s = slot->db()->PKHRScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); + s = db->storage()->PKHRScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); } else { op.start_sid.SerializeTo(start_field); if (op.end_sid == kSTREAMID_MAX) { @@ -142,7 +144,7 @@ storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vect } else { op.end_sid.SerializeTo(end_field); } - s = slot->db()->PKHScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); + s = db->storage()->PKHScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); } // 2 exclude the start_sid and end_sid if needed @@ -165,76 +167,76 @@ storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vect return s; } -storage::Status StreamStorage::DeleteStreamData(const std::string &key, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::DeleteStreamData(const std::string &key, const DB* db) { + assert(db); std::string true_key = STREAM_DATA_HASH_PREFIX + key; std::map type_status; - int64_t count = slot->db()->Del({true_key}, &type_status); + int64_t count = db->storage()->Del({true_key}, &type_status); storage::Status s = type_status[storage::DataType::kHashes]; return s; } storage::Status StreamStorage::GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, - const Slot *slot) { - assert(slot); + const DB* db) { + assert(db); auto key = std::move(StreamUtils::TreeID2Key(tid)); storage::Status s; - s = slot->db()->HGet(key, field, &value); + s = db->storage()->HGet(key, field, &value); return s; } storage::Status StreamStorage::InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const Slot *slot) { - assert(slot); + const DB* db) { + assert(db); auto key = std::move(StreamUtils::TreeID2Key(tid)); storage::Status s; int res; - s = slot->db()->HSet(key, filed, value, &res); + s = db->storage()->HSet(key, filed, value, &res); (void)res; return s; } -storage::Status StreamStorage::DeleteTreeNode(const treeID tid, const std::string &field, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::DeleteTreeNode(const treeID tid, const std::string &field, const DB* db) { + assert(db); auto key = std::move(StreamUtils::TreeID2Key(tid)); storage::Status s; int res; - s = slot->db()->HDel(key, std::vector{field}, &res); + s = db->storage()->HDel(key, std::vector{field}, &res); (void)res; return s; } storage::Status StreamStorage::GetAllTreeNode(const treeID tid, std::vector &field_values, - const Slot *slot) { - assert(slot); + const DB* db) { + assert(db); auto key = std::move(StreamUtils::TreeID2Key(tid)); - return slot->db()->PKHScanRange(key, "", "", "*", INT_MAX, &field_values, nullptr); + return db->storage()->PKHScanRange(key, "", "", "*", INT_MAX, &field_values, nullptr); } -storage::Status StreamStorage::DeleteTree(const treeID tid, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::DeleteTree(const treeID tid, const DB* db) { + assert(db); assert(tid != kINVALID_TREE_ID); storage::Status s; auto key = std::move(StreamUtils::TreeID2Key(tid)); std::map type_status; - int64_t count = slot->db()->Del({key}, &type_status); + int64_t count = db->storage()->Del({key}, &type_status); s = type_status[storage::DataType::kStrings]; return s; } -storage::Status StreamStorage::CreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::CreateConsumer(treeID consumer_tid, std::string &consumername, const DB* db) { + assert(db); std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, slot); + auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, db); if (s.IsNotFound()) { treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(slot, pel_tid); + GET_NEXT_TREE_ID_AND_CHECK(db, pel_tid); StreamConsumerMetaValue consumer_meta; consumer_meta.Init(pel_tid); - s = StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), slot); + s = StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), db); if (!s.ok()) { LOG(ERROR) << "Insert consumer meta failed"; return s; @@ -244,19 +246,19 @@ storage::Status StreamStorage::CreateConsumer(treeID consumer_tid, std::string & return s; } -storage::Status StreamStorage::GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot, +storage::Status StreamStorage::GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const DB* db, StreamConsumerMetaValue &consumer_meta) { - assert(slot); + assert(db); std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, slot); + auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, db); if (s.ok()) { consumer_meta.ParseFrom(consumer_meta_value); } else if (s.IsNotFound()) { treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(slot, pel_tid); + GET_NEXT_TREE_ID_AND_CHECK(db, pel_tid); consumer_meta.Init(pel_tid); - return StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), slot); + return StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), db); } return s; @@ -336,8 +338,8 @@ bool StreamUtils::StreamParseIntervalId(const std::string &var, streamID &id, bo } storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args) { - assert(slot); + const DB* db, const StreamAddTrimArgs &args) { + assert(db); storage::Status s; // we delete the message in batchs, prevent from using too much memory while (stream_meta.length() - trim_ret.count > args.maxlen) { @@ -347,7 +349,7 @@ storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue & StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), kSTREAMID_MAX, cur_batch, false, false, false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, slot); + s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, db); if (!s.ok() && !s.IsNotFound()) { return s; } @@ -363,7 +365,7 @@ storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue & fields_to_del.emplace_back(std::move(fv.field)); } int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, slot); + s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, db); if (!s.ok()) { return s; } @@ -375,8 +377,8 @@ storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue & } storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args) { - assert(slot); + const DB* db, const StreamAddTrimArgs &args) { + assert(db); storage::Status s; std::string serialized_min_id; stream_meta.first_id().SerializeTo(trim_ret.next_field); @@ -388,7 +390,7 @@ storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &s std::vector filed_values; StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), args.minid, cur_batch, false, false, false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, slot); + s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, db); if (!s.ok() && !s.IsNotFound()) { return s; } @@ -415,7 +417,7 @@ storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &s fields_to_del.emplace_back(std::move(fv.field)); } int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, slot); + s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, db); if (!s.ok()) { return s; } @@ -427,17 +429,17 @@ storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &s } storage::Status StreamStorage::TrimStream(int32_t &count, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const Slot *slot) { - assert(slot); + StreamAddTrimArgs &args, const DB* db) { + assert(db); count = 0; // 1 do the trim TrimRet trim_ret; storage::Status s; if (args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MAXLEN) { - s = TrimByMaxlen(trim_ret, stream_meta, key, slot, args); + s = TrimByMaxlen(trim_ret, stream_meta, key, db, args); } else { assert(args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MINID); - s = TrimByMinid(trim_ret, stream_meta, key, slot, args); + s = TrimByMinid(trim_ret, stream_meta, key, db, args); } if (!s.ok()) { @@ -472,13 +474,13 @@ storage::Status StreamStorage::TrimStream(int32_t &count, StreamMetaValue &strea return s; } -storage::Status StreamStorage::DestoryStreams(std::vector &keys, const Slot *slot) { - assert(slot); +storage::Status StreamStorage::DestoryStreams(std::vector &keys, const DB* db) { + assert(db); storage::Status s; for (const auto &key : keys) { // 1.try to get the stream meta StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key, slot); + s = StreamStorage::GetStreamMeta(stream_meta, key, db); if (s.IsNotFound()) { continue; } else if (!s.ok()) { @@ -494,13 +496,13 @@ storage::Status StreamStorage::DestoryStreams(std::vector &keys, co } // 3 delete stream meta - s = StreamStorage::DeleteStreamMeta(key, slot); + s = StreamStorage::DeleteStreamMeta(key, db); if (!s.ok()) { return s; } // 4 delete stream data - s = StreamStorage::DeleteStreamData(key, slot); + s = StreamStorage::DeleteStreamData(key, db); if (!s.ok()) { return s; } diff --git a/src/pika_transaction.cc b/src/pika_transaction.cc index 0ef4974269..7e4f0194ed 100644 --- a/src/pika_transaction.cc +++ b/src/pika_transaction.cc @@ -13,11 +13,12 @@ #include "include/pika_rm.h" #include "include/pika_server.h" #include "include/pika_transaction.h" +#include "src/pstd/include/scope_record_lock.h" extern std::unique_ptr g_pika_server; extern std::unique_ptr g_pika_rm; -void MultiCmd::Do(std::shared_ptr partition) { +void MultiCmd::Do() { auto conn = GetConn(); auto client_conn = std::dynamic_pointer_cast(conn); if (conn == nullptr || client_conn == nullptr) { @@ -39,7 +40,7 @@ void MultiCmd::DoInitial() { } } -void ExecCmd::Do(std::shared_ptr slot) { +void ExecCmd::Do() { auto conn = GetConn(); auto client_conn = std::dynamic_pointer_cast(conn); std::vector res_vec = {}; @@ -51,8 +52,8 @@ void ExecCmd::Do(std::shared_ptr slot) { std::for_each(cmds_.begin(), cmds_.end(), [&client_conn, &res_vec, &resp_strs_iter](CmdInfo& each_cmd_info) { each_cmd_info.cmd_->SetResp(*resp_strs_iter++); auto& cmd = each_cmd_info.cmd_; - auto& slot = each_cmd_info.slot_; - auto sync_slot = each_cmd_info.sync_slot_; + auto& db = each_cmd_info.db_; + auto sync_db = each_cmd_info.sync_db_; cmd->res() = {}; if (cmd->name() == kCmdNameFlushall) { auto flushall = std::dynamic_pointer_cast(cmd); @@ -60,15 +61,15 @@ void ExecCmd::Do(std::shared_ptr slot) { client_conn->SetAllTxnFailed(); } else if (cmd->name() == kCmdNameFlushdb) { auto flushdb = std::dynamic_pointer_cast(cmd); - flushdb->FlushAllSlotsWithoutLock(each_cmd_info.db_); + flushdb->FlushAllDBsWithoutLock(); if (cmd->res().ok()) { cmd->res().SetRes(CmdRes::kOk); } client_conn->SetTxnFailedFromDBs(each_cmd_info.db_->GetDBName()); } else { - cmd->Do(slot); + cmd->Do(); if (cmd->res().ok() && cmd->is_write()) { - cmd->DoBinlog(sync_slot); + cmd->DoBinlog(); auto db_keys = cmd->current_key(); for (auto& item : db_keys) { item = cmd->db_name().append(item); @@ -139,34 +140,34 @@ bool ExecCmd::IsTxnFailedAndSetState() { void ExecCmd::Lock() { g_pika_server->DBLockShared(); std::for_each(lock_db_.begin(), lock_db_.end(), [](auto& need_lock_db) { - need_lock_db->SlotLock(); + need_lock_db->DBLock(); }); - if (is_lock_rm_slots_) { - g_pika_rm->SlotLock(); + if (is_lock_rm_dbs_) { + g_pika_rm->DBLock(); } - std::for_each(r_lock_slots_.begin(), r_lock_slots_.end(), [this](auto& need_lock_slot) { - if (lock_slot_keys_.count(need_lock_slot) != 0) { + std::for_each(r_lock_dbs_.begin(), r_lock_dbs_.end(), [this](auto& need_lock_slot) { + if (lock_db_keys_.count(need_lock_slot) != 0) { pstd::lock::MultiRecordLock record_lock(need_lock_slot->LockMgr()); - record_lock.Lock(lock_slot_keys_[need_lock_slot]); + record_lock.Lock(lock_db_keys_[need_lock_slot]); } need_lock_slot->DbRWLockReader(); }); } void ExecCmd::Unlock() { - std::for_each(r_lock_slots_.begin(), r_lock_slots_.end(), [this](auto& need_lock_slot) { - if (lock_slot_keys_.count(need_lock_slot) != 0) { + std::for_each(r_lock_dbs_.begin(), r_lock_dbs_.end(), [this](auto& need_lock_slot) { + if (lock_db_keys_.count(need_lock_slot) != 0) { pstd::lock::MultiRecordLock record_lock(need_lock_slot->LockMgr()); - record_lock.Unlock(lock_slot_keys_[need_lock_slot]); + record_lock.Unlock(lock_db_keys_[need_lock_slot]); } need_lock_slot->DbRWUnLock(); }); - if (is_lock_rm_slots_) { - g_pika_rm->SlotUnlock(); + if (is_lock_rm_dbs_) { + g_pika_rm->DBUnlock(); } std::for_each(lock_db_.begin(), lock_db_.end(), [](auto& need_lock_db) { - need_lock_db->SlotUnlock(); + need_lock_db->DBUnlock(); }); g_pika_server->DBUnlockShared(); } @@ -178,26 +179,26 @@ void ExecCmd::SetCmdsVec() { while (!cmd_que.empty()) { auto cmd = cmd_que.front(); auto cmd_db = client_conn->GetCurrentTable(); - auto cmd_slot = g_pika_server->GetSlotByDBName(cmd_db); - auto sync_slot = g_pika_rm->GetSyncMasterSlotByName(SlotInfo(cmd->db_name(), cmd_slot->GetSlotID())); - cmds_.emplace_back(cmd, g_pika_server->GetDB(cmd_db), cmd_slot, sync_slot); + auto db = g_pika_server->GetDB(cmd_db); + auto sync_db = g_pika_rm->GetSyncMasterDBByName(DBInfo(cmd->db_name())); + cmds_.emplace_back(cmd, db, sync_db); if (cmd->name() == kCmdNameSelect) { - cmd->Do(cmd_slot); + cmd->Do(); } else if (cmd->name() == kCmdNameFlushdb) { - is_lock_rm_slots_ = true; + is_lock_rm_dbs_ = true; lock_db_.emplace(g_pika_server->GetDB(cmd_db)); } else if (cmd->name() == kCmdNameFlushall) { - is_lock_rm_slots_ = true; + is_lock_rm_dbs_ = true; for (const auto& db_item : g_pika_server->GetDB()) { lock_db_.emplace(db_item.second); } } else { - r_lock_slots_.emplace(cmd_slot); - if (lock_slot_keys_.count(cmd_slot) == 0) { - lock_slot_keys_.emplace(cmd_slot, std::vector{}); + r_lock_dbs_.emplace(db); + if (lock_db_keys_.count(db) == 0) { + lock_db_keys_.emplace(db, std::vector{}); } auto cmd_keys = cmd->current_key(); - lock_slot_keys_[cmd_slot].insert(lock_slot_keys_[cmd_slot].end(), cmd_keys.begin(), cmd_keys.end()); + lock_db_keys_[db].insert(lock_db_keys_[db].end(), cmd_keys.begin(), cmd_keys.end()); if (cmd->name() == kCmdNameLPush || cmd->name() == kCmdNameRPush) { list_cmd_.insert(list_cmd_.end(), cmds_.back()); } @@ -214,15 +215,15 @@ void ExecCmd::ServeToBLrPopWithKeys() { auto push_key = push_keys[0]; if (auto push_list_cmd = std::dynamic_pointer_cast(each_list_cmd.cmd_); push_list_cmd != nullptr) { - push_list_cmd->TryToServeBLrPopWithThisKey(push_key, each_list_cmd.slot_); + push_list_cmd->TryToServeBLrPopWithThisKey(push_key, each_list_cmd.db_); } } } -void WatchCmd::Do(std::shared_ptr slot) { +void WatchCmd::Do() { auto mp = std::map{}; for (const auto& key : keys_) { - auto type_count = slot->db()->IsExist(key, &mp); + auto type_count = db_->storage()->IsExist(key, &mp); if (type_count > 1) { res_.SetRes(CmdRes::CmdRet::kErrOther, "EXEC WATCH watch key must be unique"); return; @@ -246,8 +247,7 @@ void WatchCmd::Do(std::shared_ptr slot) { } void WatchCmd::Execute() { - std::shared_ptr slot = g_pika_server->GetSlotByDBName(db_name_); - Do(slot); + Do(); } void WatchCmd::DoInitial() { @@ -262,7 +262,7 @@ void WatchCmd::DoInitial() { } } -void UnwatchCmd::Do(std::shared_ptr slot) { +void UnwatchCmd::Do() { auto conn = GetConn(); auto client_conn = std::dynamic_pointer_cast(conn); if (client_conn == nullptr) { @@ -294,7 +294,7 @@ void DiscardCmd::DoInitial() { } } -void DiscardCmd::Do(std::shared_ptr partition) { +void DiscardCmd::Do() { auto conn = GetConn(); auto client_conn = std::dynamic_pointer_cast(conn); if (client_conn == nullptr) { diff --git a/src/pika_zset.cc b/src/pika_zset.cc index 37a3fcba7f..d5dced37bd 100644 --- a/src/pika_zset.cc +++ b/src/pika_zset.cc @@ -34,25 +34,25 @@ void ZAddCmd::DoInitial() { } } -void ZAddCmd::Do(std::shared_ptr slot) { +void ZAddCmd::Do() { int32_t count = 0; - s_ = slot->db()->ZAdd(key_, score_members, &count); + s_ = db_->storage()->ZAdd(key_, score_members, &count); if (s_.ok()) { res_.AppendInteger(count); - AddSlotKey("z", key_, slot); + AddSlotKey("z", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void ZAddCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZAddCmd::DoThroughDB() { + Do(); } -void ZAddCmd::DoUpdateCache(std::shared_ptr slot) { +void ZAddCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZAddIfKeyExist(CachePrefixKeyZ, score_members); + db_->cache()->ZAddIfKeyExist(CachePrefixKeyZ, score_members); } } @@ -64,9 +64,9 @@ void ZCardCmd::DoInitial() { key_ = argv_[1]; } -void ZCardCmd::Do(std::shared_ptr slot) { +void ZCardCmd::Do() { int32_t card = 0; - s_ = slot->db()->ZCard(key_, &card); + s_ = db_->storage()->ZCard(key_, &card); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(card); } else { @@ -74,16 +74,16 @@ void ZCardCmd::Do(std::shared_ptr slot) { } } -void ZCardCmd::ReadCache(std::shared_ptr slot){ +void ZCardCmd::ReadCache(){ res_.SetRes(CmdRes::kCacheMiss); } -void ZCardCmd::DoThroughDB(std::shared_ptr slot) { +void ZCardCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZCardCmd::DoUpdateCache(std::shared_ptr slot) { +void ZCardCmd::DoUpdateCache() { return; } @@ -125,10 +125,10 @@ void ZScanCmd::DoInitial() { } } -void ZScanCmd::Do(std::shared_ptr slot) { +void ZScanCmd::Do() { int64_t next_cursor = 0; std::vector score_members; - rocksdb::Status s = slot->db()->ZScan(key_, cursor_, pattern_, count_, &score_members, &next_cursor); + rocksdb::Status s = db_->storage()->ZScan(key_, cursor_, pattern_, count_, &score_members, &next_cursor); if (s.ok() || s.IsNotFound()) { res_.AppendContent("*2"); char buf[32]; @@ -162,29 +162,29 @@ void ZIncrbyCmd::DoInitial() { member_ = argv_[3]; } -void ZIncrbyCmd::Do(std::shared_ptr slot) { +void ZIncrbyCmd::Do() { double score = 0.0; - rocksdb::Status s = slot->db()->ZIncrby(key_, member_, by_, &score); + rocksdb::Status s = db_->storage()->ZIncrby(key_, member_, by_, &score); if (s.ok()) { score_ = score; char buf[32]; int64_t len = pstd::d2string(buf, sizeof(buf), score); res_.AppendStringLen(len); res_.AppendContent(buf); - AddSlotKey("z", key_, slot); + AddSlotKey("z", key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s.ToString()); } } -void ZIncrbyCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZIncrbyCmd::DoThroughDB() { + Do(); } -void ZIncrbyCmd::DoUpdateCache(std::shared_ptr slot) { +void ZIncrbyCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZIncrbyIfKeyExist(CachePrefixKeyZ, member_, by_, this, slot); + db_->cache()->ZIncrbyIfKeyExist(CachePrefixKeyZ, member_, by_, this, db_); } } @@ -214,9 +214,9 @@ void ZRangeCmd::DoInitial() { ZsetRangeParentCmd::DoInitial(); } -void ZRangeCmd::Do(std::shared_ptr slot) { +void ZRangeCmd::Do() { std::vector score_members; - s_ = slot->db()->ZRange(key_, static_cast(start_), static_cast(stop_), &score_members); + s_ = db_->storage()->ZRange(key_, static_cast(start_), static_cast(stop_), &score_members); if (s_.ok() || s_.IsNotFound()) { if (is_ws_) { char buf[32]; @@ -241,9 +241,9 @@ void ZRangeCmd::Do(std::shared_ptr slot) { } } -void ZRangeCmd::ReadCache(std::shared_ptr slot) { +void ZRangeCmd::ReadCache() { std::vector score_members; - auto s = slot ->cache()->ZRange(key_, start_, stop_, &score_members, slot); + auto s = db_->cache()->ZRange(key_, start_, stop_, &score_members, db_); if (s.ok()) { if (is_ws_) { char buf[32]; @@ -271,14 +271,14 @@ void ZRangeCmd::ReadCache(std::shared_ptr slot) { return; } -void ZRangeCmd::DoThroughDB(std::shared_ptr slot) { +void ZRangeCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRangeCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRangeCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -290,9 +290,9 @@ void ZRevrangeCmd::DoInitial() { ZsetRangeParentCmd::DoInitial(); } -void ZRevrangeCmd::Do(std::shared_ptr slot) { +void ZRevrangeCmd::Do() { std::vector score_members; - s_ = slot->db()->ZRevrange(key_, static_cast(start_), static_cast(stop_), &score_members); + s_ = db_->storage()->ZRevrange(key_, static_cast(start_), static_cast(stop_), &score_members); if (s_.ok() || s_.IsNotFound()) { if (is_ws_) { char buf[32]; @@ -317,9 +317,9 @@ void ZRevrangeCmd::Do(std::shared_ptr slot) { } } -void ZRevrangeCmd::ReadCache(std::shared_ptr slot) { +void ZRevrangeCmd::ReadCache() { std::vector score_members; - auto s = slot->cache()->ZRevrange(key_, start_, stop_, &score_members, slot); + auto s = db_->cache()->ZRevrange(key_, start_, stop_, &score_members, db_); if (s.ok()) { if (is_ws_) { @@ -348,14 +348,14 @@ void ZRevrangeCmd::ReadCache(std::shared_ptr slot) { return; } -void ZRevrangeCmd::DoThroughDB(std::shared_ptr slot) { +void ZRevrangeCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRevrangeCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRevrangeCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -441,13 +441,13 @@ void ZRangebyscoreCmd::DoInitial() { ZsetRangebyscoreParentCmd::DoInitial(); } -void ZRangebyscoreCmd::Do(std::shared_ptr slot) { +void ZRangebyscoreCmd::Do() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent("*0"); return; } std::vector score_members; - s_ = slot->db()->ZRangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); + s_ = db_->storage()->ZRangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -475,7 +475,7 @@ void ZRangebyscoreCmd::Do(std::shared_ptr slot) { } } -void ZRangebyscoreCmd::ReadCache(std::shared_ptr slot) { +void ZRangebyscoreCmd::ReadCache() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent("*0"); return; @@ -484,7 +484,7 @@ void ZRangebyscoreCmd::ReadCache(std::shared_ptr slot) { std::vector score_members; min_ = std::to_string(min_score_); max_ = std::to_string(max_score_); - auto s = slot->cache()->ZRangebyscore(key_, min_, max_, &score_members, this); + auto s = db_->cache()->ZRangebyscore(key_, min_, max_, &score_members, this); if (s.ok()) { auto sm_count = score_members.size(); if (with_scores_) { @@ -512,14 +512,14 @@ void ZRangebyscoreCmd::ReadCache(std::shared_ptr slot) { } } -void ZRangebyscoreCmd::DoThroughDB(std::shared_ptr slot) { +void ZRangebyscoreCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRangebyscoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -540,13 +540,13 @@ void ZRevrangebyscoreCmd::DoInitial() { right_close_ = tmp_close; } -void ZRevrangebyscoreCmd::Do(std::shared_ptr slot) { +void ZRevrangebyscoreCmd::Do() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent("*0"); return; } std::vector score_members; - s_ = slot->db()->ZRevrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); + s_ = db_->storage()->ZRevrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -574,14 +574,14 @@ void ZRevrangebyscoreCmd::Do(std::shared_ptr slot) { } } -void ZRevrangebyscoreCmd::ReadCache(std::shared_ptr slot){ +void ZRevrangebyscoreCmd::ReadCache(){ if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN || max_score_ < min_score_) { res_.AppendContent("*0"); return; } std::vector score_members; - auto s = slot->cache()->ZRevrangebyscore(key_, min_, max_, &score_members, this, slot); + auto s = db_->cache()->ZRevrangebyscore(key_, min_, max_, &score_members, this, db_); if (s.ok()) { auto sm_count = score_members.size(); if (with_scores_) { @@ -609,14 +609,14 @@ void ZRevrangebyscoreCmd::ReadCache(std::shared_ptr slot){ } } -void ZRevrangebyscoreCmd::DoThroughDB(std::shared_ptr slot) { +void ZRevrangebyscoreCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRevrangebyscoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRevrangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -626,7 +626,7 @@ void ZCountCmd::DoInitial() { return; } key_ = argv_[1]; - min_= argv_[2]; + min_ = argv_[2]; max_ = argv_[3]; int32_t ret = DoScoreStrRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_score_, &max_score_); if (ret == -1) { @@ -635,14 +635,14 @@ void ZCountCmd::DoInitial() { } } -void ZCountCmd::Do(std::shared_ptr slot) { +void ZCountCmd::Do() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent("*0"); return; } int32_t count = 0; - s_ = slot->db()->ZCount(key_, min_score_, max_score_, left_close_, right_close_, &count); + s_ = db_->storage()->ZCount(key_, min_score_, max_score_, left_close_, right_close_, &count); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(count); } else { @@ -650,13 +650,13 @@ void ZCountCmd::Do(std::shared_ptr slot) { } } -void ZCountCmd::ReadCache(std::shared_ptr slot) { +void ZCountCmd::ReadCache() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent("*0"); return; } uint64_t count = 0; - auto s = slot->cache()->ZCount(key_, min_, max_, &count, this); + auto s = db_->cache()->ZCount(key_, min_, max_, &count, this); if (s.ok()) { res_.AppendInteger(count); } else if (s.IsNotFound()) { @@ -666,14 +666,14 @@ void ZCountCmd::ReadCache(std::shared_ptr slot) { } } -void ZCountCmd::DoThroughDB(std::shared_ptr slot) { +void ZCountCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZCountCmd::DoUpdateCache(std::shared_ptr slot) { +void ZCountCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -687,8 +687,8 @@ void ZRemCmd::DoInitial() { members_.assign(iter, argv_.end()); } -void ZRemCmd::Do(std::shared_ptr slot) { - s_ = slot->db()->ZRem(key_, members_, &deleted_); +void ZRemCmd::Do() { + s_ = db_->storage()->ZRem(key_, members_, &deleted_); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(deleted_); } else { @@ -696,14 +696,14 @@ void ZRemCmd::Do(std::shared_ptr slot) { } } -void ZRemCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZRemCmd::DoThroughDB() { + Do(); } -void ZRemCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRemCmd::DoUpdateCache() { if (s_.ok() && deleted_ > 0) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZRem(CachePrefixKeyZ, members_, slot); + db_->cache()->ZRem(CachePrefixKeyZ, members_, db_); } } @@ -773,37 +773,38 @@ void ZUnionstoreCmd::DoInitial() { ZsetUIstoreParentCmd::DoInitial(); } -void ZUnionstoreCmd::Do(std::shared_ptr slot) { +void ZUnionstoreCmd::Do() { int32_t count = 0; - s_ = slot->db()->ZUnionstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); + s_ = db_->storage()->ZUnionstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); - AddSlotKey("z", dest_key_, slot); + AddSlotKey("z", dest_key_, db_); } else { res_.SetRes(CmdRes::kErrOther, s_.ToString()); } } -void ZUnionstoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZUnionstoreCmd::DoThroughDB() { + Do(); } -void ZUnionstoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZUnionstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; - v.emplace_back(PCacheKeyPrefixZ+dest_key_); - slot->cache()->Del(v); + v.emplace_back(PCacheKeyPrefixZ + dest_key_); + db_->cache()->Del(v); } } -void ZUnionstoreCmd::DoBinlog(const std::shared_ptr& slot) { +void ZUnionstoreCmd::DoBinlog() { PikaCmdArgsType del_args; del_args.emplace_back("del"); del_args.emplace_back(dest_key_); - del_cmd_->Initial(del_args, db_name_); - del_cmd_->SetConn(GetConn()); - del_cmd_->SetResp(resp_.lock()); - del_cmd_->DoBinlog(slot); + std::shared_ptr del_cmd = std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB); + del_cmd->Initial(del_args, db_name_); + del_cmd->SetConn(GetConn()); + del_cmd->SetResp(resp_.lock()); + del_cmd->DoBinlog(); if(value_to_dest_.empty()){ // The union operation got an empty set, only use del to simulate overwrite the dest_key with empty set @@ -829,7 +830,7 @@ void ZUnionstoreCmd::DoBinlog(const std::shared_ptr& slot) { for (const auto& it : value_to_dest_) { if (data_size >= kDataSize) { // If the binlog has reached the size of 128KB. (131,072 bytes = 128KB) - zadd_cmd_->DoBinlog(slot); + zadd_cmd_->DoBinlog(); zadd_argv.clear(); zadd_argv.emplace_back("zadd"); zadd_argv.emplace_back(dest_key_); @@ -840,7 +841,7 @@ void ZUnionstoreCmd::DoBinlog(const std::shared_ptr& slot) { zadd_argv.emplace_back(it.first); data_size += (d_len + it.first.size()); } - zadd_cmd_->DoBinlog(slot); + zadd_cmd_->DoBinlog(); } void ZInterstoreCmd::DoInitial() { @@ -851,9 +852,9 @@ void ZInterstoreCmd::DoInitial() { ZsetUIstoreParentCmd::DoInitial(); } -void ZInterstoreCmd::Do(std::shared_ptr slot) { +void ZInterstoreCmd::Do() { int32_t count = 0; - s_ = slot->db()->ZInterstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); + s_ = db_->storage()->ZInterstore(dest_key_, keys_, weights_, aggregate_, value_to_dest_, &count); if (s_.ok()) { res_.AppendInteger(count); } else { @@ -861,26 +862,27 @@ void ZInterstoreCmd::Do(std::shared_ptr slot) { } } -void ZInterstoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZInterstoreCmd::DoThroughDB() { + Do(); } -void ZInterstoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZInterstoreCmd::DoUpdateCache() { if (s_.ok()) { std::vector v; v.emplace_back(PCacheKeyPrefixZ + dest_key_); - slot->cache()->Del(v); + db_->cache()->Del(v); } } -void ZInterstoreCmd::DoBinlog(const std::shared_ptr& slot) { +void ZInterstoreCmd::DoBinlog() { PikaCmdArgsType del_args; del_args.emplace_back("del"); del_args.emplace_back(dest_key_); - del_cmd_->Initial(del_args, db_name_); - del_cmd_->SetConn(GetConn()); - del_cmd_->SetResp(resp_.lock()); - del_cmd_->DoBinlog(slot); + std::shared_ptr del_cmd = std::make_unique(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsKv | kCmdFlagsDoThroughDB); + del_cmd->Initial(del_args, db_name_); + del_cmd->SetConn(GetConn()); + del_cmd->SetResp(resp_.lock()); + del_cmd->DoBinlog(); if (value_to_dest_.size() == 0) { //The inter operation got an empty set, just exec del to simulate overwrite an empty set to dest_key @@ -904,7 +906,7 @@ void ZInterstoreCmd::DoBinlog(const std::shared_ptr& slot) { for (size_t i = 1; i < value_to_dest_.size(); i++) { if (data_size >= kDataSize) { // If the binlog has reached the size of 128KB. (131,072 bytes = 128KB) - zadd_cmd_->DoBinlog(slot); + zadd_cmd_->DoBinlog(); zadd_argv.clear(); zadd_argv.emplace_back("zadd"); zadd_argv.emplace_back(dest_key_); @@ -915,7 +917,7 @@ void ZInterstoreCmd::DoBinlog(const std::shared_ptr& slot) { zadd_argv.emplace_back(value_to_dest_[i].member); data_size += (value_to_dest_[i].member.size() + d_len); } - zadd_cmd_->DoBinlog(slot); + zadd_cmd_->DoBinlog(); } void ZsetRankParentCmd::DoInitial() { @@ -931,9 +933,9 @@ void ZRankCmd::DoInitial() { ZsetRankParentCmd::DoInitial(); } -void ZRankCmd::Do(std::shared_ptr slot) { +void ZRankCmd::Do() { int32_t rank = 0; - s_ = slot->db()->ZRank(key_, member_, &rank); + s_ = db_->storage()->ZRank(key_, member_, &rank); if (s_.ok()) { res_.AppendInteger(rank); } else if (s_.IsNotFound()) { @@ -943,9 +945,9 @@ void ZRankCmd::Do(std::shared_ptr slot) { } } -void ZRankCmd::ReadCache(std::shared_ptr slot) { +void ZRankCmd::ReadCache() { int64_t rank = 0; - auto s = slot->cache()->ZRank(key_, member_, &rank, slot); + auto s = db_->cache()->ZRank(key_, member_, &rank, db_); if (s.ok()) { res_.AppendInteger(rank); } else if (s.IsNotFound()){ @@ -955,14 +957,14 @@ void ZRankCmd::ReadCache(std::shared_ptr slot) { } } -void ZRankCmd::DoThroughDB(std::shared_ptr slot) { +void ZRankCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRankCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRankCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -974,9 +976,9 @@ void ZRevrankCmd::DoInitial() { ZsetRankParentCmd::DoInitial(); } -void ZRevrankCmd::Do(std::shared_ptr slot) { +void ZRevrankCmd::Do() { int32_t revrank = 0; - s_ = slot->db()->ZRevrank(key_, member_, &revrank); + s_ = db_->storage()->ZRevrank(key_, member_, &revrank); if (s_.ok()) { res_.AppendInteger(revrank); } else if (s_.IsNotFound()) { @@ -986,9 +988,9 @@ void ZRevrankCmd::Do(std::shared_ptr slot) { } } -void ZRevrankCmd::ReadCache(std::shared_ptr slot) { +void ZRevrankCmd::ReadCache() { int64_t revrank = 0; - auto s = slot->cache()->ZRevrank(key_, member_, &revrank, slot); + auto s = db_->cache()->ZRevrank(key_, member_, &revrank, db_); if (s.ok()) { res_.AppendInteger(revrank); } else if (s.IsNotFound()){ @@ -998,14 +1000,14 @@ void ZRevrankCmd::ReadCache(std::shared_ptr slot) { } } -void ZRevrankCmd::DoThroughDB(std::shared_ptr slot) { +void ZRevrankCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRevrankCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRevrankCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1018,9 +1020,9 @@ void ZScoreCmd::DoInitial() { member_ = argv_[2]; } -void ZScoreCmd::Do(std::shared_ptr slot) { +void ZScoreCmd::Do() { double score = 0.0; - s_ = slot->db()->ZScore(key_, member_, &score); + s_ = db_->storage()->ZScore(key_, member_, &score); if (s_.ok()) { char buf[32]; int64_t len = pstd::d2string(buf, sizeof(buf), score); @@ -1033,10 +1035,10 @@ void ZScoreCmd::Do(std::shared_ptr slot) { } } -void ZScoreCmd::ReadCache(std::shared_ptr slot) { +void ZScoreCmd::ReadCache() { double score = 0.0; std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - auto s = slot->cache()->ZScore(CachePrefixKeyZ, member_, &score, slot); + auto s = db_->cache()->ZScore(CachePrefixKeyZ, member_, &score, db_); if (s.ok()) { char buf[32]; int64_t len = pstd::d2string(buf, sizeof(buf), score); @@ -1049,12 +1051,12 @@ void ZScoreCmd::ReadCache(std::shared_ptr slot) { } } -void ZScoreCmd::DoThroughDB(std::shared_ptr slot) { +void ZScoreCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZScoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZScoreCmd::DoUpdateCache() { return; } @@ -1126,13 +1128,13 @@ void ZRangebylexCmd::DoInitial() { ZsetRangebylexParentCmd::DoInitial(); } -void ZRangebylexCmd::Do(std::shared_ptr slot) { +void ZRangebylexCmd::Do() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent("*0"); return; } std::vector members; - s_ = slot->db()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); + s_ = db_->storage()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -1148,13 +1150,13 @@ void ZRangebylexCmd::Do(std::shared_ptr slot) { } } -void ZRangebylexCmd::ReadCache(std::shared_ptr slot) { +void ZRangebylexCmd::ReadCache() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent("*0"); return; } std::vector members; - auto s = slot->cache()->ZRangebylex(key_, min_, max_, &members, slot); + auto s = db_->cache()->ZRangebylex(key_, min_, max_, &members, db_); if (s.ok()) { FitLimit(count_, offset_, members.size()); @@ -1172,14 +1174,14 @@ void ZRangebylexCmd::ReadCache(std::shared_ptr slot) { } } -void ZRangebylexCmd::DoThroughDB(std::shared_ptr slot) { +void ZRangebylexCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRangebylexCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRangebylexCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1201,13 +1203,13 @@ void ZRevrangebylexCmd::DoInitial() { right_close_ = tmp_b; } -void ZRevrangebylexCmd::Do(std::shared_ptr slot) { +void ZRevrangebylexCmd::Do() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent("*0"); return; } std::vector members; - s_ = slot->db()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); + s_ = db_->storage()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -1223,13 +1225,13 @@ void ZRevrangebylexCmd::Do(std::shared_ptr slot) { } } -void ZRevrangebylexCmd::ReadCache(std::shared_ptr slot) { +void ZRevrangebylexCmd::ReadCache() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent("*0"); return; } std::vector members; - auto s = slot->cache()->ZRevrangebylex(key_, min_, max_, &members, slot); + auto s = db_->cache()->ZRevrangebylex(key_, min_, max_, &members, db_); if (s.ok()) { auto size = count_ < members.size() ? count_ : members.size(); res_.AppendArrayLen(static_cast(size)); @@ -1243,14 +1245,14 @@ void ZRevrangebylexCmd::ReadCache(std::shared_ptr slot) { } } -void ZRevrangebylexCmd::DoThroughDB(std::shared_ptr slot) { +void ZRevrangebylexCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZRevrangebylexCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRevrangebylexCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1269,13 +1271,13 @@ void ZLexcountCmd::DoInitial() { } } -void ZLexcountCmd::Do(std::shared_ptr slot) { +void ZLexcountCmd::Do() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent(":0"); return; } int32_t count = 0; - s_ = slot->db()->ZLexcount(key_, min_member_, max_member_, left_close_, right_close_, &count); + s_ = db_->storage()->ZLexcount(key_, min_member_, max_member_, left_close_, right_close_, &count); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -1283,13 +1285,13 @@ void ZLexcountCmd::Do(std::shared_ptr slot) { res_.AppendInteger(count); } -void ZLexcountCmd::ReadCache(std::shared_ptr slot) { +void ZLexcountCmd::ReadCache() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent(":0"); return; } uint64_t count = 0; - auto s = slot->cache()->ZLexcount(key_, min_, max_, &count, slot); + auto s = db_->cache()->ZLexcount(key_, min_, max_, &count, db_); if (s.ok()) { res_.AppendInteger(count); } else if (s.IsNotFound()) { @@ -1299,14 +1301,14 @@ void ZLexcountCmd::ReadCache(std::shared_ptr slot) { } } -void ZLexcountCmd::DoThroughDB(std::shared_ptr slot) { +void ZLexcountCmd::DoThroughDB() { res_.clear(); - Do(slot); + Do(); } -void ZLexcountCmd::DoUpdateCache(std::shared_ptr slot) { +void ZLexcountCmd::DoUpdateCache() { if (s_.ok()) { - slot->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, slot); + db_->cache()->PushKeyToAsyncLoadQueue(PIKA_KEY_TYPE_ZSET, key_, db_); } } @@ -1326,9 +1328,9 @@ void ZRemrangebyrankCmd::DoInitial() { } } -void ZRemrangebyrankCmd::Do(std::shared_ptr slot) { +void ZRemrangebyrankCmd::Do() { int32_t count = 0; - s_ = slot->db()->ZRemrangebyrank(key_, static_cast(start_rank_), static_cast(stop_rank_), &count); + s_ = db_->storage()->ZRemrangebyrank(key_, static_cast(start_rank_), static_cast(stop_rank_), &count); if (s_.ok() || s_.IsNotFound()) { res_.AppendInteger(count); } else { @@ -1336,14 +1338,14 @@ void ZRemrangebyrankCmd::Do(std::shared_ptr slot) { } } -void ZRemrangebyrankCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZRemrangebyrankCmd::DoThroughDB() { + Do(); } -void ZRemrangebyrankCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRemrangebyrankCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZRemrangebyrank(CachePrefixKeyZ, min_, max_, ele_deleted_); + db_->cache()->ZRemrangebyrank(CachePrefixKeyZ, min_, max_, ele_deleted_); } } @@ -1360,13 +1362,13 @@ void ZRemrangebyscoreCmd::DoInitial() { } } -void ZRemrangebyscoreCmd::Do(std::shared_ptr slot) { +void ZRemrangebyscoreCmd::Do() { if (min_score_ == storage::ZSET_SCORE_MAX || max_score_ == storage::ZSET_SCORE_MIN) { res_.AppendContent(":0"); return; } int32_t count = 0; - s_ = slot->db()->ZRemrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &count); + s_ = db_->storage()->ZRemrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &count); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -1374,14 +1376,14 @@ void ZRemrangebyscoreCmd::Do(std::shared_ptr slot) { res_.AppendInteger(count); } -void ZRemrangebyscoreCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZRemrangebyscoreCmd::DoThroughDB() { + Do(); } -void ZRemrangebyscoreCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRemrangebyscoreCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZRemrangebyscore(CachePrefixKeyZ, min_, max_, slot); + db_->cache()->ZRemrangebyscore(CachePrefixKeyZ, min_, max_, db_); } } @@ -1398,14 +1400,14 @@ void ZRemrangebylexCmd::DoInitial() { } } -void ZRemrangebylexCmd::Do(std::shared_ptr slot) { +void ZRemrangebylexCmd::Do() { if (min_member_ == "+" || max_member_ == "-") { res_.AppendContent("*0"); return; } int32_t count = 0; - s_ = slot->db()->ZRemrangebylex(key_, min_member_, max_member_, left_close_, right_close_, &count); + s_ = db_->storage()->ZRemrangebylex(key_, min_member_, max_member_, left_close_, right_close_, &count); if (!s_.ok() && !s_.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s_.ToString()); return; @@ -1413,14 +1415,14 @@ void ZRemrangebylexCmd::Do(std::shared_ptr slot) { res_.AppendInteger(count); } -void ZRemrangebylexCmd::DoThroughDB(std::shared_ptr slot) { - Do(slot); +void ZRemrangebylexCmd::DoThroughDB() { + Do(); } -void ZRemrangebylexCmd::DoUpdateCache(std::shared_ptr slot) { +void ZRemrangebylexCmd::DoUpdateCache() { if (s_.ok()) { std::string CachePrefixKeyZ = PCacheKeyPrefixZ + key_; - slot->cache()->ZRemrangebylex(CachePrefixKeyZ, min_, max_, slot); + db_->cache()->ZRemrangebylex(CachePrefixKeyZ, min_, max_, db_); } } @@ -1440,9 +1442,9 @@ void ZPopmaxCmd::DoInitial() { } } -void ZPopmaxCmd::Do(std::shared_ptr slot) { +void ZPopmaxCmd::Do() { std::vector score_members; - rocksdb::Status s = slot->db()->ZPopMax(key_, count_, &score_members); + rocksdb::Status s = db_->storage()->ZPopMax(key_, count_, &score_members); if (s.ok() || s.IsNotFound()) { char buf[32]; int64_t len = 0; @@ -1474,9 +1476,9 @@ void ZPopminCmd::DoInitial() { } } -void ZPopminCmd::Do(std::shared_ptr slot) { +void ZPopminCmd::Do() { std::vector score_members; - rocksdb::Status s = slot->db()->ZPopMin(key_, count_, &score_members); + rocksdb::Status s = db_->storage()->ZPopMin(key_, count_, &score_members); if (s.ok() || s.IsNotFound()) { char buf[32]; int64_t len = 0; diff --git a/src/rsync_client.cc b/src/rsync_client.cc index 8a8c049a4f..be1a75cbb9 100644 --- a/src/rsync_client.cc +++ b/src/rsync_client.cc @@ -22,8 +22,8 @@ const int kBytesPerRequest = 4 << 20; const int kThrottleCheckCycle = 10; namespace rsync { -RsyncClient::RsyncClient(const std::string& dir, const std::string& db_name, const uint32_t slot_id) - : snapshot_uuid_(""), dir_(dir), db_name_(db_name), slot_id_(slot_id), +RsyncClient::RsyncClient(const std::string& dir, const std::string& db_name) + : snapshot_uuid_(""), dir_(dir), db_name_(db_name), state_(IDLE), max_retries_(10), master_ip_(""), master_port_(0), parallel_num_(g_pika_conf->max_rsync_parallel_num()) { wo_mgr_.reset(new WaitObjectManager()); @@ -177,7 +177,12 @@ Status RsyncClient::CopyRemoteFile(const std::string& filename, int index) { request.set_reader_index(index); request.set_type(kRsyncFile); request.set_db_name(db_name_); - request.set_slot_id(slot_id_); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + request.set_slot_id(0); FileRequest* file_req = request.mutable_file_req(); file_req->set_filename(filename); file_req->set_offset(offset); @@ -309,7 +314,7 @@ bool RsyncClient::Recover() { } state_ = RUNNING; - LOG(INFO) << "copy meta data done, slot_id: " << slot_id_ + LOG(INFO) << "copy meta data done, db name: " << db_name_ << " snapshot_uuid: " << snapshot_uuid_ << " file count: " << file_set_.size() << " expired file count: " << expired_files.size() @@ -329,7 +334,12 @@ Status RsyncClient::CopyRemoteMeta(std::string* snapshot_uuid, std::set req = task_arg->req; std::shared_ptr conn = task_arg->conn; std::string db_name = req->db_name(); - uint32_t slot_id = req->slot_id(); - std::shared_ptr slot = g_pika_server->GetDBSlotById(db_name, slot_id); - if (!slot || slot->IsBgSaving()) { + std::shared_ptr db = g_pika_server->GetDB(db_name); + if (!db || db->IsBgSaving()) { LOG(WARNING) << "waiting bgsave done..."; return; } @@ -127,11 +126,16 @@ void RsyncServerConn::HandleMetaRsyncRequest(void* arg) { response.set_code(RsyncService::kOk); response.set_type(RsyncService::kRsyncMeta); response.set_db_name(db_name); - response.set_slot_id(slot_id); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + response.set_slot_id(0); std::vector filenames; std::string snapshot_uuid; - g_pika_server->GetDumpMeta(db_name, slot_id, &filenames, &snapshot_uuid); + g_pika_server->GetDumpMeta(db_name, &filenames, &snapshot_uuid); response.set_snapshot_uuid(snapshot_uuid); LOG(INFO) << "Rsync Meta request, snapshot_uuid: " << snapshot_uuid @@ -152,7 +156,6 @@ void RsyncServerConn::HandleFileRsyncRequest(void* arg) { const std::shared_ptr req = task_arg->req; std::shared_ptr conn = task_arg->conn; - uint32_t slot_id = req->slot_id(); std::string db_name = req->db_name(); std::string filename = req->file_req().filename(); size_t offset = req->file_req().offset(); @@ -163,10 +166,15 @@ void RsyncServerConn::HandleFileRsyncRequest(void* arg) { response.set_code(RsyncService::kOk); response.set_type(RsyncService::kRsyncFile); response.set_db_name(db_name); - response.set_slot_id(slot_id); + /* + * Since the slot field is written in protobuffer, + * slot_id is set to the default value 0 for compatibility + * with older versions, but slot_id is not used + */ + response.set_slot_id(0); std::string snapshot_uuid; - Status s = g_pika_server->GetDumpUUID(db_name, slot_id, &snapshot_uuid); + Status s = g_pika_server->GetDumpUUID(db_name, &snapshot_uuid); response.set_snapshot_uuid(snapshot_uuid); if (!s.ok()) { LOG(WARNING) << "rsyncserver get snapshotUUID failed"; @@ -175,15 +183,14 @@ void RsyncServerConn::HandleFileRsyncRequest(void* arg) { return; } - std::shared_ptr slot = g_pika_server->GetDBSlotById(db_name, slot_id); - if (!slot) { - LOG(WARNING) << "cannot find slot for db_name: " << db_name - << " slot_id: " << slot_id; + std::shared_ptr db = g_pika_server->GetDB(db_name); + if (!db) { + LOG(WARNING) << "cannot find db for db_name: " << db_name; response.set_code(RsyncService::kErr); RsyncWriteResp(response, conn); } - const std::string filepath = slot->bgsave_info().path + "/" + filename; + const std::string filepath = db->bgsave_info().path + "/" + filename; char* buffer = new char[req->file_req().count() + 1]; size_t bytes_read{0}; std::string checksum = ""; @@ -214,20 +221,20 @@ RsyncServerThread::RsyncServerThread(const std::set& ips, int port, : HolyThread(ips, port, &conn_factory_, cron_interval, &handle_, true), conn_factory_(arg) {} RsyncServerThread::~RsyncServerThread() { - LOG(WARNING) << "RsyncServerThread destroyed"; + LOG(WARNING) << "RsyncServerThread destroyed"; } void RsyncServerThread::RsyncServerHandle::FdClosedHandle(int fd, const std::string& ip_port) const { - LOG(WARNING) << "ip_port: " << ip_port << " connection closed"; + LOG(WARNING) << "ip_port: " << ip_port << " connection closed"; } void RsyncServerThread::RsyncServerHandle::FdTimeoutHandle(int fd, const std::string& ip_port) const { - LOG(WARNING) << "ip_port: " << ip_port << " connection timeout"; + LOG(WARNING) << "ip_port: " << ip_port << " connection timeout"; } bool RsyncServerThread::RsyncServerHandle::AccessHandle(int fd, std::string& ip_port) const { - LOG(WARNING) << "fd: "<< fd << " ip_port: " << ip_port << " connection accepted"; - return true; + LOG(WARNING) << "fd: "<< fd << " ip_port: " << ip_port << " connection accepted"; + return true; } void RsyncServerThread::RsyncServerHandle::CronHandle() const { diff --git a/src/storage/src/redis_strings.cc b/src/storage/src/redis_strings.cc index ba238afb35..dd4c63dbbb 100644 --- a/src/storage/src/redis_strings.cc +++ b/src/storage/src/redis_strings.cc @@ -692,7 +692,7 @@ Status RedisStrings::MGetWithTTL(const std::vector& keys, std::vect Status RedisStrings::MSet(const std::vector& kvs) { std::vector keys; keys.reserve(kvs.size()); -for (const auto& kv : kvs) { + for (const auto& kv : kvs) { keys.push_back(kv.key); } diff --git a/tests/integration/integrate_test.sh b/tests/integration/integrate_test.sh index 8c3319acec..87fc34e21a 100755 --- a/tests/integration/integrate_test.sh +++ b/tests/integration/integrate_test.sh @@ -4,4 +4,4 @@ # of patent rights can be found in the PATENTS file in the same directory. go mod tidy -go test \ No newline at end of file +go test -timeout 30m \ No newline at end of file diff --git a/tests/integration/replication_test.go b/tests/integration/replication_test.go index 662b3e731d..49b23ab7d3 100644 --- a/tests/integration/replication_test.go +++ b/tests/integration/replication_test.go @@ -3,6 +3,7 @@ package pika_integration import ( "context" "fmt" + "log" "math/rand" "strings" "sync" @@ -218,6 +219,8 @@ func test_del_replication(ctx *context.Context, clientMaster, clientSlave *redis for i := int64(0); i < clientMaster.LLen(*ctx, "blist3").Val(); i++ { Expect(clientMaster.LIndex(*ctx, "blist3", i)).To(Equal(clientSlave.LIndex(*ctx, "blist3", i))) } + clientMaster.Del(*ctx, "blist100", "blist101", "blist102", "blist103") + clientMaster.Del(*ctx, "blist0", "blist1", "blist2", "blist3") } @@ -346,19 +349,19 @@ func issuePushPopFrequency(ctx *context.Context, clientMaster *redis.Client, wg letters9 := randomString(5) clientMaster.LPush(*ctx, "blist0", letters1, letters2, letters3, letters4, letters5) - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") clientMaster.RPush(*ctx, "blist0", letters9, letters8, letters7, letters6, letters5) - clientMaster.BRPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BRPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BRPop(*ctx, 1 * time.Second, "blist0") + clientMaster.BRPop(*ctx, 1*time.Second, "blist0") + clientMaster.BRPop(*ctx, 1*time.Second, "blist0") + clientMaster.BRPop(*ctx, 1*time.Second, "blist0") clientMaster.RPush(*ctx, "blist0", letters7, letters8, letters9, letters1, letters2) - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") - clientMaster.BLPop(*ctx, 1 * time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") + clientMaster.BLPop(*ctx, 1*time.Second, "blist0") } var _ = Describe("should replication ", func() { @@ -373,15 +376,16 @@ var _ = Describe("should replication ", func() { cleanEnv(ctx, clientMaster, clientSlave) Expect(clientSlave.FlushDB(ctx).Err()).NotTo(HaveOccurred()) Expect(clientMaster.FlushDB(ctx).Err()).NotTo(HaveOccurred()) - time.Sleep(1 * time.Second) + time.Sleep(3 * time.Second) }) AfterEach(func() { cleanEnv(ctx, clientMaster, clientSlave) Expect(clientSlave.FlushDB(ctx).Err()).NotTo(HaveOccurred()) Expect(clientMaster.FlushDB(ctx).Err()).NotTo(HaveOccurred()) - time.Sleep(1 * time.Second) + time.Sleep(3 * time.Second) Expect(clientSlave.Close()).NotTo(HaveOccurred()) Expect(clientMaster.Close()).NotTo(HaveOccurred()) + log.Println("Replication test case done") }) It("Let The slave become a replica of The master ", func() { @@ -417,38 +421,45 @@ var _ = Describe("should replication ", func() { slaveWrite := clientSlave.Set(ctx, "foo", "bar", 0) Expect(slaveWrite.Err()).To(MatchError("ERR Server in read-only")) - clientMaster.Del(ctx, "blist0", "blist1", "blist") + log.Println("rpoplpush test start") + Expect(clientMaster.Del(ctx, "blist0", "blist1", "blist").Err()).NotTo(HaveOccurred()) execute(&ctx, clientMaster, 4, rpoplpushThread) for i := int64(0); i < clientMaster.LLen(ctx, "blist").Val(); i++ { Expect(clientMaster.LIndex(ctx, "blist", i)).To(Equal(clientSlave.LIndex(ctx, "blist", i))) } + Expect(clientMaster.Del(ctx, "blist0", "blist1", "blist").Err()).NotTo(HaveOccurred()) + log.Println("rpoplpush test success") + log.Println("randomBitop test start") Expect(clientMaster.Del(ctx, "bitkey1", "bitkey2", "bitkey_out1", "bitkey_out2").Err()).NotTo(HaveOccurred()) execute(&ctx, clientMaster, 4, randomBitopThread) master_key_out_count1 := clientMaster.Do(ctx, "bitcount", "bitkey_out1", 0, -1) slave_key_out_count1 := clientSlave.Do(ctx, "bitcount", "bitkey_out1", 0, -1) Expect(master_key_out_count1.Val()).To(Equal(slave_key_out_count1.Val())) - master_key_out_count2 := clientMaster.Do(ctx, "bitcount", "bitkey_out2", 0, -1) slave_key_out_count2 := clientSlave.Do(ctx, "bitcount", "bitkey_out2", 0, -1) Expect(master_key_out_count2.Val()).To(Equal(slave_key_out_count2.Val())) + Expect(clientMaster.Del(ctx, "bitkey1", "bitkey2", "bitkey_out1", "bitkey_out2").Err()).NotTo(HaveOccurred()) + log.Println("randomBitop test success") - clientMaster.Del(ctx, "source_set", "dest_set") + log.Println("randomSmove test start") + Expect(clientMaster.Del(ctx, "sourceSet", "destSet").Err()).NotTo(HaveOccurred()) execute(&ctx, clientMaster, 4, randomSmoveThread) master_source_set := clientMaster.SMembers(ctx, "sourceSet") Expect(master_source_set.Err()).NotTo(HaveOccurred()) slave_source_set := clientSlave.SMembers(ctx, "sourceSet") Expect(slave_source_set.Err()).NotTo(HaveOccurred()) Expect(master_source_set.Val()).To(Equal(slave_source_set.Val())) - master_dest_set := clientMaster.SMembers(ctx, "destSet") Expect(master_dest_set.Err()).NotTo(HaveOccurred()) slave_dest_set := clientSlave.SMembers(ctx, "destSet") Expect(slave_dest_set.Err()).NotTo(HaveOccurred()) Expect(master_dest_set.Val()).To(Equal(slave_dest_set.Val())) - + Expect(clientMaster.Del(ctx, "sourceSet", "destSet").Err()).NotTo(HaveOccurred()) test_del_replication(&ctx, clientMaster, clientSlave) + log.Println("randomSmove test success") + log.Println("randomSdiffstore test start") clientMaster.Del(ctx, "set1", "set2", "dest_set") execute(&ctx, clientMaster, 4, randomSdiffstoreThread) master_set1 := clientMaster.SMembers(ctx, "set1") @@ -456,19 +467,20 @@ var _ = Describe("should replication ", func() { slave_set1 := clientSlave.SMembers(ctx, "set1") Expect(slave_set1.Err()).NotTo(HaveOccurred()) Expect(master_set1.Val()).To(Equal(slave_set1.Val())) - master_set2 := clientMaster.SMembers(ctx, "set2") Expect(master_set2.Err()).NotTo(HaveOccurred()) slave_set2 := clientSlave.SMembers(ctx, "set2") Expect(slave_set2.Err()).NotTo(HaveOccurred()) Expect(master_set2.Val()).To(Equal(slave_set2.Val())) - master_dest_store_set := clientMaster.SMembers(ctx, "dest_set") Expect(master_dest_store_set.Err()).NotTo(HaveOccurred()) slave_dest_store_set := clientSlave.SMembers(ctx, "dest_set") Expect(slave_dest_store_set.Err()).NotTo(HaveOccurred()) Expect(master_dest_store_set.Val()).To(Equal(slave_dest_store_set.Val())) + clientMaster.Del(ctx, "set1", "set2", "dest_set") + log.Println("randomSdiffstore test success") + log.Println("randomSinterstore test start") clientMaster.Del(ctx, "set1", "set2", "dest_set") execute(&ctx, clientMaster, 4, randomSinterstoreThread) master_dest_interstore_set := clientMaster.SMembers(ctx, "dest_set") @@ -476,7 +488,8 @@ var _ = Describe("should replication ", func() { slave_dest_interstore_set := clientSlave.SMembers(ctx, "dest_set") Expect(slave_dest_interstore_set.Err()).NotTo(HaveOccurred()) Expect(master_dest_interstore_set.Val()).To(Equal(slave_dest_interstore_set.Val())) - + clientMaster.Del(ctx, "set1", "set2", "dest_set") + log.Println("randomSinterstore test success") //clientMaster.FlushAll(ctx) //time.Sleep(3 * time.Second) //go randomPfmergeThread(&ctx, clientMaster) @@ -489,7 +502,7 @@ var _ = Describe("should replication ", func() { //slave_hll_out := clientSlave.PFCount(ctx, "hll_out") //Expect(slave_hll_out.Err()).NotTo(HaveOccurred()) //Expect(master_hll_out.Val()).To(Equal(slave_hll_out.Val())) - + log.Println("randomZunionstore test start") clientMaster.Del(ctx, "zset1", "zset2", "zset_out") execute(&ctx, clientMaster, 4, randomZunionstoreThread) master_zset_out := clientMaster.ZRange(ctx, "zset_out", 0, -1) @@ -497,7 +510,10 @@ var _ = Describe("should replication ", func() { slave_zset_out := clientSlave.ZRange(ctx, "zset_out", 0, -1) Expect(slave_zset_out.Err()).NotTo(HaveOccurred()) Expect(master_zset_out.Val()).To(Equal(slave_zset_out.Val())) + clientMaster.Del(ctx, "zset1", "zset2", "zset_out") + log.Println("randomZunionstore test success") + log.Println("randomZinterstore test start") clientMaster.Del(ctx, "zset1", "zset2", "zset_out") execute(&ctx, clientMaster, 4, randomZinterstoreThread) master_dest_interstore_set = clientMaster.SMembers(ctx, "dest_set") @@ -505,16 +521,21 @@ var _ = Describe("should replication ", func() { slave_dest_interstore_set = clientSlave.SMembers(ctx, "dest_set") Expect(slave_dest_interstore_set.Err()).NotTo(HaveOccurred()) Expect(master_dest_interstore_set.Val()).To(Equal(slave_dest_interstore_set.Val())) - clientMaster.Del(ctx, "set1", "set2", "set_out") + log.Println("randomZinterstore test success") + + log.Println("randomSunionstore test start") execute(&ctx, clientMaster, 4, randomSunionstroeThread) master_unionstore_set := clientMaster.SMembers(ctx, "set_out") Expect(master_unionstore_set.Err()).NotTo(HaveOccurred()) slave_unionstore_set := clientSlave.SMembers(ctx, "set_out") Expect(slave_unionstore_set.Err()).NotTo(HaveOccurred()) Expect(master_unionstore_set.Val()).To(Equal(slave_unionstore_set.Val())) + clientMaster.Del(ctx, "set1", "set2", "set_out") + log.Println("randomSunionstore test success") // Stream replication test + log.Println("randomXadd test start") clientMaster.Del(ctx, "mystream") execute(&ctx, clientMaster, 4, randomXaddThread) masterStreamMessages := clientMaster.XRange(ctx, "mystream", "-", "+") @@ -522,8 +543,11 @@ var _ = Describe("should replication ", func() { Expect(masterStreamMessages.Err()).NotTo(HaveOccurred()) Expect(slaveStreamMessages.Err()).NotTo(HaveOccurred()) Expect(masterStreamMessages.Val()).To(Equal(slaveStreamMessages.Val())) + clientMaster.Del(ctx, "mystream") + log.Println("randomXadd test success") // Blocked master-slave replication test + log.Println("master-slave replication test start") lists := []string{"list0", "list1"} err := clientMaster.Del(ctx, lists...) Expect(err.Err()).NotTo(HaveOccurred()) @@ -563,6 +587,7 @@ var _ = Describe("should replication ", func() { Expect(clientMaster.LIndex(ctx, "list1", i)).To(Equal(clientSlave.LIndex(ctx, "list1", i))) } } + err = clientMaster.Del(ctx, lists...) // High frequency pop/push during unblocking process lists = []string{"blist0", "blist1"} @@ -582,6 +607,8 @@ var _ = Describe("should replication ", func() { for i := int64(0); i < clientMaster.LLen(ctx, "blist0").Val(); i++ { Expect(clientMaster.LIndex(ctx, "blist0", i)).To(Equal(clientSlave.LIndex(ctx, "blist0", i))) } + err = clientMaster.Del(ctx, lists...) + log.Println("master-slave replication test success") }) })