From 522c96b171e135e485aa1455df5e9775b74fd97e Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Sat, 23 Dec 2023 16:43:22 +0800 Subject: [PATCH 01/10] feat: move stream implememtation to storage with some bugs. --- include/pika_stream.h | 32 +- include/pika_stream_base.h | 255 ----- src/pika_hash.cc | 8 - src/pika_kv.cc | 8 - src/pika_stream.cc | 430 ++------ src/pika_stream_base.cc | 617 ------------ src/storage/include/storage/storage.h | 44 +- src/storage/src/base_data_key_format.h | 8 + src/storage/src/pika_stream_base.cc | 233 +++++ src/storage/src/pika_stream_base.h | 155 +++ .../storage/src}/pika_stream_meta_value.h | 189 +++- .../storage/src}/pika_stream_types.h | 28 +- src/storage/src/redis_streams.cc | 944 ++++++++++++++++++ src/storage/src/redis_streams.h | 176 ++++ src/storage/src/storage.cc | 57 ++ 15 files changed, 1896 insertions(+), 1288 deletions(-) delete mode 100644 include/pika_stream_base.h delete mode 100644 src/pika_stream_base.cc create mode 100644 src/storage/src/pika_stream_base.cc create mode 100644 src/storage/src/pika_stream_base.h rename {include => src/storage/src}/pika_stream_meta_value.h (70%) rename {include => src/storage/src}/pika_stream_types.h (84%) create mode 100644 src/storage/src/redis_streams.cc create mode 100644 src/storage/src/redis_streams.h diff --git a/include/pika_stream.h b/include/pika_stream.h index f7834243e6..ca7075141e 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -8,20 +8,20 @@ #include "include/pika_command.h" #include "include/pika_slot.h" -#include "include/pika_stream_base.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" +#include "storage/src/pika_stream_base.h" +// #include "include/pika_stream_meta_value.h" +// #include "include/pika_stream_types.h" #include "storage/storage.h" /* * stream */ -inline void ParseAddOrTrimArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, StreamAddTrimArgs& args, int* idpos, - bool is_xadd); +inline void ParseAddOrTrimArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, storage::StreamAddTrimArgs& args, + int* idpos, bool is_xadd); -inline void ParseReadOrReadGroupArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, StreamReadGroupReadArgs& args, - bool is_xreadgroup); +inline void ParseReadOrReadGroupArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, + storage::StreamReadGroupReadArgs& args, bool is_xreadgroup); // @field_values is the result of ScanStream. // field is the serialized message id, @@ -39,11 +39,10 @@ class XAddCmd : public Cmd { private: std::string key_; - StreamAddTrimArgs args_; + storage::StreamAddTrimArgs args_; int field_pos_{0}; void DoInitial() override; - inline void GenerateStreamIDOrReply(const StreamMetaValue& stream_meta); }; class XDelCmd : public Cmd { @@ -57,13 +56,10 @@ class XDelCmd : public Cmd { private: std::string key_; - std::vector ids_; + std::vector ids_; void DoInitial() override; void Clear() override { ids_.clear(); } - inline void SetFirstOrLastIDOrReply(StreamMetaValue& stream_meta, const Slot* slot, bool is_set_first); - inline void SetFirstIDOrReply(StreamMetaValue& stream_meta, const Slot* slot); - inline void SetLastIDOrReply(StreamMetaValue& stream_meta, const Slot* slot); }; class XReadCmd : public Cmd { @@ -75,7 +71,7 @@ class XReadCmd : public Cmd { Cmd* Clone() override { return new XReadCmd(*this); } private: - StreamReadGroupReadArgs args_; + storage::StreamReadGroupReadArgs args_; void DoInitial() override; void Clear() override { @@ -94,11 +90,7 @@ class XRangeCmd : public Cmd { protected: std::string key_; - streamID start_sid; - streamID end_sid; - int32_t count_{INT32_MAX}; - bool start_ex_{false}; - bool end_ex_{false}; + storage::StreamScanArgs args_; void DoInitial() override; }; @@ -137,7 +129,7 @@ class XTrimCmd : public Cmd { private: std::string key_; - StreamAddTrimArgs args_; + storage::StreamAddTrimArgs args_; void DoInitial() override; }; diff --git a/include/pika_stream_base.h b/include/pika_stream_base.h deleted file mode 100644 index 1091e19ebc..0000000000 --- a/include/pika_stream_base.h +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef SRC_STREAM_UTIL_H_ -#define SRC_STREAM_UTIL_H_ - -#include "include/pika_slot_command.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" -#include "storage/storage.h" - -// each abstracted tree has a prefix, -// prefix + treeID will be the key of the hash to store the tree, -// notice: we need to ban the use of this prefix when using "HSET". -static const std::string STERAM_TREE_PREFIX = "STR_TREE"; - -// key of the hash to store stream meta, -// notice: we need to ban the use of this key when using "HSET". -static const std::string STREAM_META_HASH_KEY = "STREAM"; - -// each stream's data (messages) are stored in a hash, -// to avoid stream key confilict with hash key, we add a prefix to each key for stream data, -// it's ok to use the same string as STREAM_META_HASH_KEY to be the prefix. -// notice: we need to ban the use of this prefix when using "HSET". -static const std::string STREAM_DATA_HASH_PREFIX = STREAM_META_HASH_KEY; - -// I need to find a place to store the last generated tree id, -// so I stored it as a field-value pair in the hash storing stream meta. -// the field is a fixed string, and the value is the last generated tree id. -// notice: we need to ban the use of this key when using stream, to avoid conflict with stream's key. -static const std::string STREAM_LAST_GENERATED_TREE_ID_FIELD = "STREAM"; - -// the max number of each delete operation in XTRIM command，to avoid too much memory usage. -// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / -// kDEFAULT_TRIM_BATCH_SIZE) times -const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; - -struct StreamAddTrimArgs { - // XADD options - streamID id; - bool id_given{false}; - bool seq_given{false}; - bool no_mkstream{false}; - - // XADD + XTRIM common options - StreamTrimStrategy trim_strategy{0}; - int trim_strategy_arg_idx{0}; - - // TRIM_STRATEGY_MAXLEN options - uint64_t maxlen{0}; - streamID minid; -}; - -struct StreamReadGroupReadArgs { - // XREAD + XREADGROUP common options - std::vector keys; - std::vector unparsed_ids; - int32_t count{INT32_MAX}; // in redis this is uint64_t, but PKHScanRange only support int32_t - uint64_t block{0}; // 0 means no block - - // XREADGROUP options - std::string group_name; - std::string consumer_name; - bool noack_{false}; -}; - -// get next tree id thread safe -class TreeIDGenerator { - private: - TreeIDGenerator() = default; - void operator=(const TreeIDGenerator &) = delete; - - public: - ~TreeIDGenerator() = default; - - // work in singeletone mode - static TreeIDGenerator &GetInstance() { - static TreeIDGenerator instance; - return instance; - } - - storage::Status GetNextTreeID(const Slot *slot, treeID &tid); - - private: - static const treeID START_TREE_ID = 0; - std::atomic tree_id_ = kINVALID_TREE_ID; -}; - -// Implement all the functions that related to blackwidow derctly. -// if we want to change the storage engine, we need to rewrite this class. -class StreamStorage { - public: - struct ScanStreamOptions { - std::string key; // the key of the stream - streamID start_sid; - streamID end_sid; - int32_t count; - bool start_ex; // exclude first message - bool end_ex; // exclude last message - bool is_reverse; // scan in reverse order - ScanStreamOptions(std::string skey, streamID start_sid, streamID end_sid, int32_t count, - bool start_ex = false, bool end_ex = false, bool is_reverse = false) - : key(skey), - start_sid(start_sid), - end_sid(end_sid), - count(count), - start_ex(start_ex), - end_ex(end_ex), - is_reverse(is_reverse) {} - }; - - static storage::Status ScanStream(const ScanStreamOptions &option, std::vector &field_values, - std::string &next_field, const Slot *slot); - - // get and parse the stream meta if found - // @return ok only when the stream meta exists - static storage::Status GetStreamMeta(StreamMetaValue &tream_meta, const std::string &key, const Slot *slot); - - // will create stream meta hash if it dosent't exist. - // return !s.ok() only when insert failed - static storage::Status SetStreamMeta(const std::string &key, std::string &meta_value, const Slot *slot); - - static storage::Status InsertStreamMessage(const std::string &key, const streamID &id, const std::string &message, - const Slot *slot); - - static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &ids, int32_t &ret, - const Slot *slot); - - static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &serialized_ids, - int32_t &ret, const Slot *slot); - - static storage::Status GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const Slot *slot); - - static storage::Status DeleteStreamData(const std::string &key, const Slot *slot); - - static storage::Status TrimStream(int32_t &res, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const Slot *slot); - - // get the abstracted tree node, e.g. get a message in pel, get a consumer meta or get a cgroup meta. - // the behavior of abstracted tree is similar to radix-tree in redis; - // in cgroup tree, field is groupname - // in consumer tree, field is consumername - // in pel tree, field is messageID - static storage::Status GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, const Slot *slot); - static storage::Status InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const Slot *slot); - static storage::Status DeleteTreeNode(const treeID tid, const std::string &field, const Slot *slot); - static storage::Status GetAllTreeNode(const treeID tid, std::vector &field_values, - const Slot *slot); - - // delete the stream meta - // @return true if the stream meta exists and deleted - static storage::Status DeleteStreamMeta(const std::string &key, const Slot *slot); - - // note: the tree must exist - // @return true if the tree exists and is deleted - static storage::Status DeleteTree(const treeID tid, const Slot *slot); - - // get consumer meta value. - // if the consumer meta value does not exist, create a new one and return it. - static storage::Status GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot, - StreamConsumerMetaValue &consumer_meta); - - static storage::Status CreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot); - - // delete the pels, consumers, cgroups and stream meta of a stream - // note: this function do not delete the stream data value - static storage::Status DestoryStreams(std::vector &keys, const Slot *slot); - - private: - StreamStorage(); - ~StreamStorage(); - struct TrimRet { - // the count of deleted messages - int32_t count{0}; - // the next field after trim - std::string next_field; - // the max deleted field, will be empty if no message is deleted - std::string max_deleted_field; - }; - - static storage::Status TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args); - - static storage::Status TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args); -}; - -// Helper function of stream command. -// Should be reconstructed when transfer to another command framework. -// any function that has Reply in its name will reply to the client if error occurs. -class StreamCmdBase { - public: - private: - StreamCmdBase(); - ~StreamCmdBase(); -}; - -class StreamUtils { - public: - StreamUtils() = default; - ~StreamUtils() = default; - - static bool string2uint64(const char *s, uint64_t &value); - static bool string2int64(const char *s, int64_t &value); - static bool string2int32(const char *s, int32_t &value); - static std::string TreeID2Key(const treeID &tid); - - static uint64_t GetCurrentTimeMs(); - - // serialize the message to a string. - // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} - static bool SerializeMessage(const std::vector &field_values, std::string &serialized_message, - int field_pos); - - // deserialize the message from a string with the format of SerializeMessage. - static bool DeserializeMessage(const std::string &message, std::vector &parsed_message); - - // Parse a stream ID in the format given by clients to Pika, that is - // -, and converts it into a streamID structure. The ID may be in incomplete - // form, just stating the milliseconds time part of the stream. In such a case - // the missing part is set according to the value of 'missing_seq' parameter. - // - // The IDs "-" and "+" specify respectively the minimum and maximum IDs - // that can be represented. If 'strict' is set to 1, "-" and "+" will be - // treated as an invalid ID. - // - // The ID form -* specifies a millisconds-only ID, leaving the sequence part - // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this - // form is accepted and the argument is set to 0 unless the sequence part is - // specified. - static bool StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 0, to be used when - and + are acceptable IDs. - static bool StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 1, to be used when we want to return an error if the special IDs + or - - // are provided. - static bool StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given); - - // Helper for parsing a stream ID that is a range query interval. When the - // exclude argument is NULL, streamParseID() is called and the interval - // is treated as close (inclusive). Otherwise, the exclude argument is set if - // the interval is open (the "(" prefix) and streamParseStrictID() is - // called in that case. - static bool StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq); -}; - -#endif diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 4e5cb1e362..798ac1c4e4 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -9,7 +9,6 @@ #include "include/pika_conf.h" #include "include/pika_slot_command.h" -#include "include/pika_stream_base.h" #include "include/pika_cache.h" extern std::unique_ptr g_pika_conf; @@ -54,13 +53,6 @@ void HSetCmd::DoInitial() { key_ = argv_[1]; field_ = argv_[2]; value_ = argv_[3]; - - // check the conflict of stream used prefix, see details in defination of STREAM_TREE_PREFIX - if (key_.compare(0, STERAM_TREE_PREFIX.size(), STERAM_TREE_PREFIX) == 0 || - key_.compare(0, STREAM_DATA_HASH_PREFIX.size(), STREAM_DATA_HASH_PREFIX) == 0) { - res_.SetRes(CmdRes::kErrOther, "hash key can't start with " + STERAM_TREE_PREFIX + " or " + STREAM_META_HASH_KEY); - return; - } } void HSetCmd::Do(std::shared_ptr slot) { diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 5c3d788be1..659ddf9bb7 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -8,7 +8,6 @@ #include "include/pika_client_conn.h" #include "include/pika_command.h" -#include "include/pika_stream_base.h" #include "pstd/include/pstd_string.h" #include "include/pika_binlog_transverter.h" @@ -207,13 +206,6 @@ void DelCmd::Do(std::shared_ptr slot) { std::map type_status; int64_t count = slot->db()->Del(keys_, &type_status); - - // stream's destory need to be treated specially - auto s = StreamStorage::DestoryStreams(keys_, slot.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "stream delete error: " + s.ToString()); - return; - } if (count >= 0) { res_.AppendInteger(count); diff --git a/src/pika_stream.cc b/src/pika_stream.cc index 17278ca63d..62e1f6e4f6 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -5,12 +5,15 @@ #include "include/pika_stream.h" #include +#include +#include #include "include/pika_command.h" +#include "include/pika_define.h" #include "include/pika_slot.h" -#include "include/pika_stream_base.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" +#include "storage/src/pika_stream_base.h" +#include "storage/src/pika_stream_meta_value.h" +#include "storage/src/pika_stream_types.h" #include "storage/storage.h" // s : rocksdb::Status @@ -24,7 +27,7 @@ } \ } while (0) -void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamAddTrimArgs &args, int *idpos, +void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, storage::StreamAddTrimArgs &args, int *idpos, bool is_xadd) { int i = 2; bool limit_given = false; @@ -38,7 +41,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA } else if (strcasecmp(opt.c_str(), "maxlen") == 0 && moreargs) { // case: XADD mystream ... MAXLEN [= | ~] threshold ... - if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + if (args.trim_strategy != storage::StreamTrimStrategy::TRIM_STRATEGY_NONE) { res.SetRes(CmdRes::kSyntaxErr, "syntax error, MAXLEN and MINID options at the same time are not compatible"); return; } @@ -48,16 +51,16 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA i++; } // parse threshold as uint64 - if (!StreamUtils::string2uint64(argv[i + 1].c_str(), args.maxlen)) { + if (!storage::StreamUtils::string2uint64(argv[i + 1].c_str(), args.maxlen)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid MAXLEN argument"); } i++; - args.trim_strategy = StreamTrimStrategy::TRIM_STRATEGY_MAXLEN; + args.trim_strategy = storage::StreamTrimStrategy::TRIM_STRATEGY_MAXLEN; args.trim_strategy_arg_idx = i; } else if (strcasecmp(opt.c_str(), "minid") == 0 && moreargs) { // case: XADD mystream ... MINID [= | ~] threshold ... - if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + if (args.trim_strategy != storage::StreamTrimStrategy::TRIM_STRATEGY_NONE) { res.SetRes(CmdRes::kSyntaxErr, "syntax error, MAXLEN and MINID options at the same time are not compatible"); return; } @@ -67,12 +70,12 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA i++; } // parse threshold as stremID - if (!StreamUtils::StreamParseID(argv[i + 1], args.minid, 0)) { + if (!storage::StreamUtils::StreamParseID(argv[i + 1], args.minid, 0)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } i++; - args.trim_strategy = StreamTrimStrategy::TRIM_STRATEGY_MINID; + args.trim_strategy = storage::StreamTrimStrategy::TRIM_STRATEGY_MINID; args.trim_strategy_arg_idx = i; } else if (strcasecmp(opt.c_str(), "limit") == 0 && moreargs) { @@ -87,7 +90,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA } else if (is_xadd) { // case: XADD mystream ... ID ... - if (!StreamUtils::StreamParseStrictID(argv[i], args.id, 0, &args.seq_given)) { + if (!storage::StreamUtils::StreamParseStrictID(argv[i], args.id, 0, &args.seq_given)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } @@ -110,7 +113,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA * [NOACK] STREAMS key [key ...] id [id ...] * XREAD [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id * [id ...] */ -void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamReadGroupReadArgs &args, +void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, storage::StreamReadGroupReadArgs &args, bool is_xreadgroup) { int streams_arg_idx{0}; // the index of stream keys arg size_t streams_cnt{0}; // the count of stream keys @@ -120,13 +123,13 @@ void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, S const std::string &o = argv[i]; if (strcasecmp(o.c_str(), "BLOCK") == 0 && moreargs) { i++; - if (!StreamUtils::string2uint64(argv[i].c_str(), args.block)) { + if (!storage::StreamUtils::string2uint64(argv[i].c_str(), args.block)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid BLOCK argument"); return; } } else if (strcasecmp(o.c_str(), "COUNT") == 0 && moreargs) { i++; - if (!StreamUtils::string2int32(argv[i].c_str(), args.count)) { + if (!storage::StreamUtils::string2int32(argv[i].c_str(), args.count)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid COUNT argument"); return; } @@ -179,12 +182,12 @@ void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, S } } -void AppendMessagesToRes(CmdRes &res, std::vector &field_values, const Slot *slot) { +void AppendMessagesToRes(CmdRes &res, std::vector &id_messages, const Slot *slot) { assert(slot); - res.AppendArrayLenUint64(field_values.size()); - for (auto &fv : field_values) { + res.AppendArrayLenUint64(id_messages.size()); + for (auto &fv : id_messages) { std::vector message; - if (!StreamUtils::DeserializeMessage(fv.value, message)) { + if (!storage::StreamUtils::DeserializeMessage(fv.value, message)) { LOG(ERROR) << "Deserialize message failed"; res.SetRes(CmdRes::kErrOther, "Deserialize message failed"); return; @@ -192,7 +195,7 @@ void AppendMessagesToRes(CmdRes &res, std::vector &field_va assert(message.size() % 2 == 0); res.AppendArrayLen(2); - streamID sid; + storage::streamID sid; sid.DeserializeFrom(fv.field); res.AppendString(sid.ToString()); // field here is the stream id res.AppendArrayLenUint64(message.size()); @@ -209,11 +212,7 @@ void XAddCmd::DoInitial() { } key_ = argv_[1]; - // check the conflict of stream used prefix, see detail in defination of STREAM_TREE_PREFIX - if (key_ == STREAM_LAST_GENERATED_TREE_ID_FIELD) { - res_.SetRes(CmdRes::kErrOther, "Can not use " + STREAM_LAST_GENERATED_TREE_ID_FIELD + "as stream key"); - return; - } + // FIXME: check the conflict of stream used prefix, see detail in defination of STERAM_TREE_PREFIX int idpos{-1}; ParseAddOrTrimArgsOrReply(res_, argv_, args_, &idpos, true); @@ -233,28 +232,15 @@ void XAddCmd::DoInitial() { } void XAddCmd::Do(std::shared_ptr slot) { - // 1 get stream meta - rocksdb::Status s; - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); - if (s.IsNotFound() && args_.no_mkstream) { - res_.SetRes(CmdRes::kNotFound); - return; - } else if (s.IsNotFound()) { - stream_meta.Init(); - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, get stream meta failed: " + s.ToString()); - return; - } - - if (stream_meta.last_id().ms == UINT64_MAX && stream_meta.last_id().seq == UINT64_MAX) { - res_.SetRes(CmdRes::kErrOther, "Fatal! Sequence number overflow !"); + std::string message; + if (!storage::StreamUtils::SerializeMessage(argv_, message, field_pos_)) { + res_.SetRes(CmdRes::kErrOther, "Serialize message failed"); return; } - // 2 append the message to storage - GenerateStreamIDOrReply(stream_meta); - if (res_.ret() != CmdRes::kNone) { + auto s = slot->db()->XAdd(key_, message, args_); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } @@ -264,122 +250,32 @@ void XAddCmd::Do(std::shared_ptr slot) { argv_[field_pos_ - 1] = args_.id.ToString(); } - std::string message; - if (!StreamUtils::SerializeMessage(argv_, message, field_pos_)) { - res_.SetRes(CmdRes::kErrOther, "Serialize message failed"); - return; - } - - // check the serialized current id is larger than last_id -#ifdef DEBUG - std::string serialized_last_id; - stream_meta.last_id().SerializeTo(serialized_last_id); - assert(field > serialized_last_id); -#endif // DEBUG - - s = StreamStorage::InsertStreamMessage(key_, args_.id, message, slot.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, insert stream message failed 1: " + s.ToString()); - return; - } - - // 3 update stream meta - if (stream_meta.length() == 0) { - stream_meta.set_first_id(args_.id); - } - stream_meta.set_entries_added(stream_meta.entries_added() + 1); - stream_meta.set_last_id(args_.id); - stream_meta.set_length(stream_meta.length() + 1); - - // 4 trim the stream if needed - if (args_.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { - int32_t count; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, slot.get()), res_); - (void)count; - } - - // 5 update stream meta - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, get stream meta failed 2: " + s.ToString()); - return; - } - res_.AppendString(args_.id.ToString()); } -void XAddCmd::GenerateStreamIDOrReply(const StreamMetaValue &stream_meta) { - auto &id = args_.id; - if (args_.id_given && args_.seq_given && id.ms == 0 && id.seq == 0) { - res_.SetRes(CmdRes::kInvalidParameter, "The ID specified in XADD must be greater than 0-0"); - return; - } - - if (!args_.id_given || !args_.seq_given) { - // if id not given, generate one - if (!args_.id_given) { - id.ms = StreamUtils::GetCurrentTimeMs(); - - if (id.ms < stream_meta.last_id().ms) { - id.ms = stream_meta.last_id().ms; - if (stream_meta.last_id().seq == UINT64_MAX) { - id.ms++; - id.seq = 0; - } else { - id.seq++; - } - return; - } - } - - // generate seq - auto last_id = stream_meta.last_id(); - if (id.ms < last_id.ms) { - res_.SetRes(CmdRes::kErrOther, "The ID specified in XADD is equal or smaller"); - return; - } else if (id.ms == last_id.ms) { - if (last_id.seq == UINT64_MAX) { - res_.SetRes(CmdRes::kErrOther, "The ID specified in XADD is equal or smaller"); - return; - } - id.seq = last_id.seq + 1; - } else { - id.seq = 0; - } - - } else { - // Full ID given, check id - auto last_id = stream_meta.last_id(); - if (id.ms < last_id.ms || (id.ms == last_id.ms && id.seq <= last_id.seq)) { - res_.SetRes(CmdRes::kErrOther, "INVALID ID given"); - return; - } - } -} - void XRangeCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameXRange); return; } key_ = argv_[1]; - if (!StreamUtils::StreamParseIntervalId(argv_[2], start_sid, &start_ex_, 0) || - !StreamUtils::StreamParseIntervalId(argv_[3], end_sid, &end_ex_, UINT64_MAX)) { + if (!storage::StreamUtils::StreamParseIntervalId(argv_[2], args_.start_sid, &args_.start_ex, 0) || + !storage::StreamUtils::StreamParseIntervalId(argv_[3], args_.end_sid, &args_.end_ex, UINT64_MAX)) { res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } - if (start_ex_ && start_sid.ms == UINT64_MAX && start_sid.seq == UINT64_MAX) { + if (args_.start_ex && args_.start_sid.ms == UINT64_MAX && args_.start_sid.seq == UINT64_MAX) { res_.SetRes(CmdRes::kInvalidParameter, "invalid start id"); return; } - if (end_ex_ && end_sid.ms == 0 && end_sid.seq == 0) { + if (args_.end_ex && args_.end_sid.ms == 0 && args_.end_sid.seq == 0) { res_.SetRes(CmdRes::kInvalidParameter, "invalid end id"); return; } if (argv_.size() == 6) { // pika's PKHScanRange() only sopport max count of INT32_MAX // but redis supports max count of UINT64_MAX - if (!StreamUtils::string2int32(argv_[5].c_str(), count_)) { + if (!storage::StreamUtils::string2uint64(argv_[5].c_str(), args_.limit)) { res_.SetRes(CmdRes::kInvalidParameter, "COUNT should be a integer greater than 0 and not bigger than INT32_MAX"); return; } @@ -387,97 +283,43 @@ void XRangeCmd::DoInitial() { } void XRangeCmd::Do(std::shared_ptr slot) { - std::vector field_values; + std::vector id_messages; - if (start_sid <= end_sid) { - StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, false); - std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); - (void)next_field; + if (args_.start_sid <= args_.end_sid) { + auto s = slot->db()->XRange(key_, args_, id_messages); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } } - AppendMessagesToRes(res_, field_values, slot.get()); + AppendMessagesToRes(res_, id_messages, slot.get()); } void XRevrangeCmd::Do(std::shared_ptr slot) { - std::vector field_values; + std::vector id_messages; - if (start_sid >= end_sid) { - StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, true); - std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); - (void)next_field; + if (args_.start_sid >= args_.end_sid) { + auto s = slot->db()->XRevrange(key_, args_, id_messages); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } } - AppendMessagesToRes(res_, field_values, slot.get()); -} - -inline void XDelCmd::SetFirstIDOrReply(StreamMetaValue &stream_meta, const Slot *slot) { - assert(slot); - return SetFirstOrLastIDOrReply(stream_meta, slot, true); -} - -inline void XDelCmd::SetLastIDOrReply(StreamMetaValue &stream_meta, const Slot *slot) { - assert(slot); - return SetFirstOrLastIDOrReply(stream_meta, slot, false); -} - -inline void XDelCmd::SetFirstOrLastIDOrReply(StreamMetaValue &stream_meta, const Slot *slot, bool is_set_first) { - assert(slot); - if (stream_meta.length() == 0) { - stream_meta.set_first_id(kSTREAMID_MIN); - return; - } - - std::vector field_values; - std::string next_field; - - storage::Status s; - if (is_set_first) { - StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MIN, kSTREAMID_MAX, 1); - s = StreamStorage::ScanStream(option, field_values, next_field, slot); - } else { - bool is_reverse = true; - StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); - s = StreamStorage::ScanStream(option, field_values, next_field, slot); - } - (void)next_field; - - if (!s.ok() && !s.IsNotFound()) { - LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - if (field_values.empty()) { - LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; - res_.SetRes(CmdRes::kErrOther, "Internal error: no messages found but stream length is not 0"); - return; - } - - streamID id; - id.DeserializeFrom(field_values[0].field); - stream_meta.set_first_id(id); + AppendMessagesToRes(res_, id_messages, slot.get()); } void XDelCmd::DoInitial() { if (!CheckArg(argv_.size())) { - res_.SetRes(CmdRes::kWrongNum, kCmdNameXDel); + res_.SetRes(CmdRes::kWrongNum, kCmdNameXAdd); return; } key_ = argv_[1]; for (int i = 2; i < argv_.size(); i++) { - streamID id; - if (!StreamUtils::StreamParseStrictID(argv_[i], id, 0, nullptr)) { + storage::streamID id; + if (!storage::StreamUtils::StreamParseStrictID(argv_[i], id, 0, nullptr)) { res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } @@ -489,45 +331,17 @@ void XDelCmd::DoInitial() { } void XDelCmd::Do(std::shared_ptr slot) { - // 1 try to get stream meta - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); - if (s.IsNotFound()) { - res_.AppendInteger(0); - return; - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - // 2 do the delete - int32_t count{0}; - s = StreamStorage::DeleteStreamMessage(key_, ids_, count, slot.get()); - if (!s.ok()) { + size_t count{0}; + auto s = slot->db()->XDel(key_, ids_, count); + if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - // 3 update stream meta - stream_meta.set_length(stream_meta.length() - count); - for (const auto &id : ids_) { - if (id > stream_meta.max_deleted_entry_id()) { - stream_meta.set_max_deleted_entry_id(id); - } - if (id == stream_meta.first_id()) { - SetFirstIDOrReply(stream_meta, slot.get()); - } else if (id == stream_meta.last_id()) { - SetLastIDOrReply(stream_meta, slot.get()); - } } - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; + if (count > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); } - res_.AppendInteger(count); + res_.AppendInteger(static_cast(count)); } void XLenCmd::DoInitial() { @@ -539,9 +353,8 @@ void XLenCmd::DoInitial() { } void XLenCmd::Do(std::shared_ptr slot) { - rocksdb::Status s; - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); + size_t len{0}; + auto s = slot->db()->XLen(key_, len); if (s.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); return; @@ -550,11 +363,11 @@ void XLenCmd::Do(std::shared_ptr slot) { return; } - if (stream_meta.length() > INT_MAX) { - res_.SetRes(CmdRes::kErrOther, "stream's length is larger than INT_MAX"); - return; + if (len > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "stream's length is larger than INT_MAX"); } - res_.AppendInteger(static_cast(stream_meta.length())); + + res_.AppendInteger(static_cast(len)); return; } @@ -568,70 +381,34 @@ void XReadCmd::DoInitial() { } void XReadCmd::Do(std::shared_ptr slot) { - rocksdb::Status s; - - // 1 prepare stream_metas - std::vector> streammeta_idx; - for (int i = 0; i < args_.unparsed_ids.size(); i++) { - const auto &key = args_.keys[i]; - const auto &unparsed_id = args_.unparsed_ids[i]; - - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key, slot.get()); - if (s.IsNotFound()) { - continue; - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - streammeta_idx.emplace_back(std::move(stream_meta), i); + std::vector> results; + // The wrong key will not trigger error, just be ignored, + // we need to save the right key，and return it to client. + std::vector reserved_keys; + auto s = slot->db()->XRead(args_, results, reserved_keys); + + if (!s.ok() && s.ToString() == + "The > ID can be specified only when calling " + "XREADGROUP using the GROUP " + " option.") { + res_.SetRes(CmdRes::kSyntaxErr, s.ToString()); + } else if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); } - if (streammeta_idx.empty()) { + if (results.empty()) { res_.AppendArrayLen(-1); return; } - // 2 do the scan - res_.AppendArrayLenUint64(streammeta_idx.size()); - for (const auto &stream_meta_id : streammeta_idx) { - const auto &stream_meta = stream_meta_id.first; - const auto &idx = stream_meta_id.second; - const auto &unparsed_id = args_.unparsed_ids[idx]; - const auto &key = args_.keys[idx]; - - // 2.1 try to parse id - streamID id; - if (unparsed_id == "<") { - res_.SetRes(CmdRes::kSyntaxErr, - "The > ID can be specified only when calling " - "XREADGROUP using the GROUP " - " option."); - return; - } else if (unparsed_id == "$") { - id = stream_meta.last_id(); - } else { - if (!StreamUtils::StreamParseStrictID(unparsed_id, id, 0, nullptr)) { - res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); - return; - } - } - - // 2.2 scan - std::vector field_values; - std::string next_field; - StreamStorage::ScanStreamOptions options(key, id, kSTREAMID_MAX, args_.count, true); - auto s = StreamStorage::ScanStream(options, field_values, next_field, slot.get()); - (void)next_field; - if (!s.ok() && !s.IsNotFound()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } + assert(results.size() == reserved_keys.size()); + // 2 do the scan + res_.AppendArrayLenUint64(results.size()); + for (size_t i = 0; i < results.size(); ++i) { res_.AppendArrayLen(2); - res_.AppendString(key); - AppendMessagesToRes(res_, field_values, slot.get()); + res_.AppendString(reserved_keys[i]); + AppendMessagesToRes(res_, results[i], slot.get()); } } @@ -649,25 +426,18 @@ void XTrimCmd::DoInitial() { } void XTrimCmd::Do(std::shared_ptr slot) { - // 1 try to get stream meta, if not found, return error - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()); - if (s.IsNotFound()) { - res_.AppendInteger(0); - return; - } else if (!s.ok()) { + size_t count{0}; + auto s = slot->db()->XTrim(key_, args_, count); + if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } - // 2 do the trim - int32_t count{0}; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, slot.get()), res_); - - // 3 update stream meta - TRY_CATCH_ERROR(StreamStorage::SetStreamMeta(key_, stream_meta.value(), slot.get()), res_); + if (count > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); + } - res_.AppendInteger(count); + res_.AppendInteger(static_cast(count)); return; } @@ -682,7 +452,7 @@ void XInfoCmd::DoInitial() { if (!strcasecmp(subcmd_.c_str(), "STREAM")) { if (argv_.size() > 3 && strcasecmp(subcmd_.c_str(), "FULL") != 0) { is_full_ = true; - if (argv_.size() > 4 && !StreamUtils::string2uint64(argv_[4].c_str(), count_)) { + if (argv_.size() > 4 && !storage::StreamUtils::string2uint64(argv_[4].c_str(), count_)) { res_.SetRes(CmdRes::kInvalidParameter, "invalid count"); return; } @@ -728,21 +498,21 @@ void XInfoCmd::Do(std::shared_ptr slot) { void XInfoCmd::StreamInfo(std::shared_ptr &slot) { // 1 try to get stream meta - StreamMetaValue stream_meta; - TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()), res_); - - // 2 append the stream info - res_.AppendArrayLen(10); - res_.AppendString("length"); - res_.AppendInteger(static_cast(stream_meta.length())); - res_.AppendString("last-generated-id"); - res_.AppendString(stream_meta.last_id().ToString()); - res_.AppendString("max-deleted-entry-id"); - res_.AppendString(stream_meta.max_deleted_entry_id().ToString()); - res_.AppendString("entries-added"); - res_.AppendInteger(static_cast(stream_meta.entries_added())); - res_.AppendString("recorded-first-entry-id"); - res_.AppendString(stream_meta.first_id().ToString()); + // StreamMetaValue stream_meta; + // TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()), res_); + + // // 2 append the stream info + // res_.AppendArrayLen(10); + // res_.AppendString("length"); + // res_.AppendInteger(static_cast(stream_meta.length())); + // res_.AppendString("last-generated-id"); + // res_.AppendString(stream_meta.last_id().ToString()); + // res_.AppendString("max-deleted-entry-id"); + // res_.AppendString(stream_meta.max_deleted_entry_id().ToString()); + // res_.AppendString("entries-added"); + // res_.AppendInteger(static_cast(stream_meta.entries_added())); + // res_.AppendString("recorded-first-entry-id"); + // res_.AppendString(stream_meta.first_id().ToString()); // Korpse TODO: add group info } diff --git a/src/pika_stream_base.cc b/src/pika_stream_base.cc deleted file mode 100644 index eebbd3512c..0000000000 --- a/src/pika_stream_base.cc +++ /dev/null @@ -1,617 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "include/pika_stream_base.h" - -#include "include/pika_command.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" -#include "storage/storage.h" - -#define GET_NEXT_TREE_ID_AND_CHECK(slot, tid) \ - do { \ - auto &tid_gen = TreeIDGenerator::GetInstance(); \ - auto s = tid_gen.GetNextTreeID(slot, tid); \ - if (!s.ok()) { \ - return s; \ - } \ - } while (0) - -storage::Status TreeIDGenerator::GetNextTreeID(const Slot *slot, treeID &tid) { - assert(slot); - auto expected_id = kINVALID_TREE_ID; - - // if tree_id_ is not initialized (equal to kINVALID_TREE_ID), try to fetch it from storage - if (tree_id_.compare_exchange_strong(expected_id, START_TREE_ID)) { - std::string value; - storage::Status s = slot->db()->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); - if (s.ok()) { - treeID id; - if (!StreamUtils::string2int32(value.c_str(), id)) { - LOG(ERROR) << "Invalid tree id: " << value; - return storage::Status::Corruption("Invalid tree id"); - } - tree_id_.store(id); - - } else if (!s.IsNotFound()) { - return s; - } - } - - tree_id_.fetch_add(1); - tid = tree_id_; - - // insert tree id to storage - std::string tree_id_str = std::to_string(tree_id_); - int32_t res; - return slot->db()->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); -} - -storage::Status StreamStorage::GetStreamMeta(StreamMetaValue &stream_meta, const std::string &key, const Slot *slot) { - assert(slot); - std::string value; - auto s = slot->db()->HGet(STREAM_META_HASH_KEY, key, &value); - if (s.ok()) { - stream_meta.ParseFrom(value); - return storage::Status::OK(); - } - return s; -} - -// no need to be thread safe, only xadd will call this function -// and xadd can be locked by the same key using current_key() -storage::Status StreamStorage::SetStreamMeta(const std::string &key, std::string &meta_value, const Slot *slot) { - assert(slot); - storage::Status s; - int32_t temp{0}; - s = slot->db()->HSet(STREAM_META_HASH_KEY, key, meta_value, &temp); - (void)temp; - return s; -} - -storage::Status StreamStorage::DeleteStreamMeta(const std::string &key, const Slot *slot) { - assert(slot); - int32_t ret; - return slot->db()->HDel({STREAM_META_HASH_KEY}, {key}, &ret); -} - -storage::Status StreamStorage::InsertStreamMessage(const std::string &key, const streamID &id, - const std::string &message, const Slot *slot) { - assert(slot); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - int32_t temp{0}; - std::string serialized_id; - id.SerializeTo(serialized_id); - storage::Status s = slot->db()->HSet(true_key, serialized_id, message, &temp); - (void)temp; - return s; -} - -storage::Status StreamStorage::GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const Slot *slot) { - assert(slot); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return slot->db()->HGet(true_key, sid, &message); -} - -storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, const std::vector &id, - int32_t &ret, const Slot *slot) { - assert(slot); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - std::vector serialized_ids; - for (auto &sid : id) { - std::string serialized_id; - sid.SerializeTo(serialized_id); - serialized_ids.emplace_back(std::move(serialized_id)); - } - return slot->db()->HDel(true_key, {serialized_ids}, &ret); -} - -storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, - const std::vector &serialized_ids, int32_t &ret, - const Slot *slot) { - assert(slot); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return slot->db()->HDel(true_key, {serialized_ids}, &ret); -} - -storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vector &field_values, - std::string &next_field, const Slot *slot) { - assert(slot); - std::string start_field; - std::string end_field; - storage::Slice pattern = "*"; // match all the fields from start_field to end_field - storage::Status s; - std::string true_key = STREAM_DATA_HASH_PREFIX + op.key; - - // 1 do the scan - if (op.is_reverse) { - op.end_sid.SerializeTo(start_field); - if (op.start_sid == kSTREAMID_MAX) { - start_field = ""; - } else { - op.start_sid.SerializeTo(start_field); - } - s = slot->db()->PKHRScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); - } else { - op.start_sid.SerializeTo(start_field); - if (op.end_sid == kSTREAMID_MAX) { - end_field = ""; - } else { - op.end_sid.SerializeTo(end_field); - } - s = slot->db()->PKHScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); - } - - // 2 exclude the start_sid and end_sid if needed - if (op.start_ex && !field_values.empty()) { - streamID sid; - sid.DeserializeFrom(field_values.front().field); - if (sid == op.start_sid) { - field_values.erase(field_values.begin()); - } - } - - if (op.end_ex && !field_values.empty()) { - streamID sid; - sid.DeserializeFrom(field_values.back().field); - if (sid == op.end_sid) { - field_values.pop_back(); - } - } - - return s; -} - -storage::Status StreamStorage::DeleteStreamData(const std::string &key, const Slot *slot) { - assert(slot); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - std::map type_status; - int64_t count = slot->db()->Del({true_key}, &type_status); - storage::Status s = type_status[storage::DataType::kHashes]; - return s; -} - -storage::Status StreamStorage::GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, - const Slot *slot) { - assert(slot); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - storage::Status s; - s = slot->db()->HGet(key, field, &value); - return s; -} - -storage::Status StreamStorage::InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const Slot *slot) { - assert(slot); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - - storage::Status s; - int res; - s = slot->db()->HSet(key, filed, value, &res); - (void)res; - return s; -} - -storage::Status StreamStorage::DeleteTreeNode(const treeID tid, const std::string &field, const Slot *slot) { - assert(slot); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - - storage::Status s; - int res; - s = slot->db()->HDel(key, std::vector{field}, &res); - (void)res; - return s; -} - -storage::Status StreamStorage::GetAllTreeNode(const treeID tid, std::vector &field_values, - const Slot *slot) { - assert(slot); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - return slot->db()->PKHScanRange(key, "", "", "*", INT_MAX, &field_values, nullptr); -} - -storage::Status StreamStorage::DeleteTree(const treeID tid, const Slot *slot) { - assert(slot); - assert(tid != kINVALID_TREE_ID); - storage::Status s; - auto key = std::move(StreamUtils::TreeID2Key(tid)); - std::map type_status; - int64_t count = slot->db()->Del({key}, &type_status); - s = type_status[storage::DataType::kStrings]; - return s; -} - -storage::Status StreamStorage::CreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot) { - assert(slot); - std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, slot); - if (s.IsNotFound()) { - treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(slot, pel_tid); - - StreamConsumerMetaValue consumer_meta; - consumer_meta.Init(pel_tid); - s = StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), slot); - if (!s.ok()) { - LOG(ERROR) << "Insert consumer meta failed"; - return s; - } - } - // consumer meta already exists or other error - return s; -} - -storage::Status StreamStorage::GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const Slot *slot, - StreamConsumerMetaValue &consumer_meta) { - assert(slot); - std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, slot); - if (s.ok()) { - consumer_meta.ParseFrom(consumer_meta_value); - - } else if (s.IsNotFound()) { - treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(slot, pel_tid); - consumer_meta.Init(pel_tid); - return StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), slot); - } - - return s; -} - -bool StreamUtils::StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given) { - char buf[128]; - if (var.size() > sizeof(buf) - 1) { - return false; - } - - memcpy(buf, var.data(), var.size()); - buf[var.size()] = '\0'; - - if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { - // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); - return false; - } - - if (seq_given != nullptr) { - *seq_given = true; - } - - if (buf[0] == '-' && buf[1] == '\0') { - id.ms = 0; - id.seq = 0; - return true; - } else if (buf[0] == '+' && buf[1] == '\0') { - id.ms = UINT64_MAX; - id.seq = UINT64_MAX; - return true; - } - - uint64_t ms; - uint64_t seq; - char *dot = strchr(buf, '-'); - if (dot) { - *dot = '\0'; - } - if (!StreamUtils::string2uint64(buf, ms)) { - return false; - }; - if (dot) { - size_t seqlen = strlen(dot + 1); - if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { - seq = 0; - *seq_given = false; - } else if (!StreamUtils::string2uint64(dot + 1, seq)) { - return false; - } - } else { - seq = missing_seq; - } - id.ms = ms; - id.seq = seq; - return true; -} - -bool StreamUtils::StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq) { - return StreamGenericParseID(var, id, missing_seq, false, nullptr); -} - -bool StreamUtils::StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given) { - return StreamGenericParseID(var, id, missing_seq, true, seq_given); -} - -bool StreamUtils::StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq) { - if (exclude != nullptr) { - *exclude = (var.size() > 1 && var[0] == '('); - } - if (exclude != nullptr && *exclude) { - return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); - } else { - return StreamParseID(var, id, missing_seq); - } -} - -storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args) { - assert(slot); - storage::Status s; - // we delete the message in batchs, prevent from using too much memory - while (stream_meta.length() - trim_ret.count > args.maxlen) { - auto cur_batch = - (std::min(static_cast(stream_meta.length() - trim_ret.count - args.maxlen), kDEFAULT_TRIM_BATCH_SIZE)); - std::vector filed_values; - - StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), kSTREAMID_MAX, cur_batch, false, false, - false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, slot); - if (!s.ok() && !s.IsNotFound()) { - return s; - } - - assert(filed_values.size() == cur_batch); - trim_ret.count += cur_batch; - trim_ret.max_deleted_field = filed_values.back().field; - - // delete the message in batchs - std::vector fields_to_del; - fields_to_del.reserve(filed_values.size()); - for (auto &fv : filed_values) { - fields_to_del.emplace_back(std::move(fv.field)); - } - int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, slot); - if (!s.ok()) { - return s; - } - assert(ret == fields_to_del.size()); - } - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const Slot *slot, const StreamAddTrimArgs &args) { - assert(slot); - storage::Status s; - std::string serialized_min_id; - stream_meta.first_id().SerializeTo(trim_ret.next_field); - args.minid.SerializeTo(serialized_min_id); - - // we delete the message in batchs, prevent from using too much memory - while (trim_ret.next_field < serialized_min_id && stream_meta.length() - trim_ret.count > 0) { - auto cur_batch = static_cast(std::min(static_cast(stream_meta.length() - trim_ret.count), kDEFAULT_TRIM_BATCH_SIZE)); - std::vector filed_values; - - StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), args.minid, cur_batch, false, false, false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, slot); - if (!s.ok() && !s.IsNotFound()) { - return s; - } - - if (!filed_values.empty()) { - if (filed_values.back().field == serialized_min_id) { - // we do not need to delete the message that it's id matches the minid - filed_values.pop_back(); - trim_ret.next_field = serialized_min_id; - } - // duble check - if (!filed_values.empty()) { - trim_ret.max_deleted_field = filed_values.back().field; - } - } - - assert(filed_values.size() <= cur_batch); - trim_ret.count += static_cast(filed_values.size()); - - // do the delete in batch - std::vector fields_to_del; - fields_to_del.reserve(filed_values.size()); - for (auto &fv : filed_values) { - fields_to_del.emplace_back(std::move(fv.field)); - } - int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, slot); - if (!s.ok()) { - return s; - } - assert(ret == fields_to_del.size()); - } - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::TrimStream(int32_t &count, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const Slot *slot) { - assert(slot); - count = 0; - // 1 do the trim - TrimRet trim_ret; - storage::Status s; - if (args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MAXLEN) { - s = TrimByMaxlen(trim_ret, stream_meta, key, slot, args); - } else { - assert(args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MINID); - s = TrimByMinid(trim_ret, stream_meta, key, slot, args); - } - - if (!s.ok()) { - return s; - } - - if (trim_ret.count == 0) { - return s; - } - - // 2 update stream meta - streamID first_id; - streamID max_deleted_id; - if (stream_meta.length() == trim_ret.count) { - // all the message in the stream were deleted - first_id = kSTREAMID_MIN; - } else { - first_id.DeserializeFrom(trim_ret.next_field); - } - assert(!trim_ret.max_deleted_field.empty()); - max_deleted_id.DeserializeFrom(trim_ret.max_deleted_field); - - stream_meta.set_first_id(first_id); - if (max_deleted_id > stream_meta.max_deleted_entry_id()) { - stream_meta.set_max_deleted_entry_id(max_deleted_id); - } - stream_meta.set_length(stream_meta.length() - trim_ret.count); - - count = trim_ret.count; - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::DestoryStreams(std::vector &keys, const Slot *slot) { - assert(slot); - storage::Status s; - for (const auto &key : keys) { - // 1.try to get the stream meta - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key, slot); - if (s.IsNotFound()) { - continue; - } else if (!s.ok()) { - LOG(ERROR) << "Get stream meta failed"; - return s; - } - - // 2 destroy all the cgroup - // 2.1 find all the cgroups' meta - auto cgroup_tid = stream_meta.groups_id(); - if (cgroup_tid != kINVALID_TREE_ID) { - // Korpse TODO: delete the cgroup, now we don't have groups. - } - - // 3 delete stream meta - s = StreamStorage::DeleteStreamMeta(key, slot); - if (!s.ok()) { - return s; - } - - // 4 delete stream data - s = StreamStorage::DeleteStreamData(key, slot); - if (!s.ok()) { - return s; - } - } - - s = storage::Status::OK(); - return s; -} - -bool StreamUtils::string2uint64(const char *s, uint64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - uint64_t tmp = strtoull(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int64(const char *s, int64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - int64_t tmp = std::strtoll(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int32(const char *s, int32_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - long tmp = strtol(s, &end, 10); - if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { - // Conversion either didn't consume the entire string, - // or overflow or underflow occurred - return false; - } - - value = static_cast(tmp); - return true; -} - -std::string StreamUtils::TreeID2Key(const treeID &tid) { - std::string key; - key.reserve(STERAM_TREE_PREFIX.size() + sizeof(tid)); - key.append(STERAM_TREE_PREFIX); - key.append(reinterpret_cast(&tid), sizeof(tid)); - return key; -} - -bool StreamUtils::SerializeMessage(const std::vector &field_values, std::string &message, int field_pos) { - assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); - assert(message.empty()); - // count the size of serizlized message - size_t size = 0; - for (int i = field_pos; i < field_values.size(); i++) { - size += field_values[i].size() + sizeof(size_t); - } - message.reserve(size); - - // serialize message - for (int i = field_pos; i < field_values.size(); i++) { - size_t len = field_values[i].size(); - message.append(reinterpret_cast(&len), sizeof(len)); - message.append(field_values[i]); - } - - return true; -} - -bool StreamUtils::DeserializeMessage(const std::string &message, std::vector &parsed_message) { - size_t pos = 0; - while (pos < message.size()) { - // Read the length of the next field value from the message - size_t len = *reinterpret_cast(&message[pos]); - pos += sizeof(size_t); - - // Check if the calculated end of the string is still within the message bounds - if (pos + len > message.size()) { - LOG(ERROR) << "Invalid message format, failed to parse message"; - return false; // Error: not enough data in the message string - } - - // Extract the field value and add it to the vector - parsed_message.push_back(message.substr(pos, len)); - pos += len; - } - - return true; -} - -uint64_t StreamUtils::GetCurrentTimeMs() { - return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - .count(); -} diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index fce6d546c5..ba77c17621 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -7,6 +7,7 @@ #define INCLUDE_STORAGE_STORAGE_H_ #include +#include #include #include #include @@ -39,6 +40,7 @@ inline const std::string HASHES_DB = "hashes"; inline const std::string LISTS_DB = "lists"; inline const std::string ZSETS_DB = "zsets"; inline const std::string SETS_DB = "sets"; +inline const std::string STREAMS_DB = "streams"; inline constexpr size_t BATCH_DELETE_LIMIT = 100; inline constexpr size_t COMPACT_THRESHOLD_COUNT = 2000; @@ -53,9 +55,15 @@ class RedisHashes; class RedisSets; class RedisLists; class RedisZSets; +class RedisStreams; class HyperLogLog; enum class OptionType; +struct StreamAddTrimArgs; +struct StreamReadGroupReadArgs; +struct StreamScanArgs; +struct streamID; + template class LRUCache; @@ -96,6 +104,12 @@ struct FieldValue { bool operator==(const FieldValue& fv) const { return (fv.field == field && fv.value == value); } }; +struct IdMessage { + std::string field; + std::string value; + bool operator==(const IdMessage& fv) const { return (fv.field == field && fv.value == value); } +}; + struct KeyVersion { std::string key; int32_t version; @@ -110,9 +124,9 @@ struct ScoreMember { enum BeforeOrAfter { Before, After }; -enum DataType { kAll, kStrings, kHashes, kLists, kZSets, kSets }; +enum DataType { kAll, kStrings, kHashes, kLists, kZSets, kSets, kStreams }; -const char DataTypeTag[] = {'a', 'k', 'h', 'l', 'z', 's'}; +const char DataTypeTag[] = {'a', 'k', 'h', 'l', 'z', 's', 'x'}; enum class OptionType { kDB, @@ -125,16 +139,25 @@ enum AGGREGATE { SUM, MIN, MAX }; enum BitOpType { kBitOpAnd = 1, kBitOpOr, kBitOpXor, kBitOpNot, kBitOpDefault }; -enum Operation { kNone = 0, kCleanAll, kCleanStrings, kCleanHashes, kCleanZSets, kCleanSets, kCleanLists, kCompactRange }; +enum Operation { + kNone = 0, + kCleanAll, + kCleanStrings, + kCleanHashes, + kCleanZSets, + kCleanSets, + kCleanLists, + kCompactRange +}; struct BGTask { DataType type; Operation operation; std::vector argv; - BGTask(const DataType& _type = DataType::kAll, - const Operation& _opeation = Operation::kNone, - const std::vector& _argv = {}) : type(_type), operation(_opeation), argv(_argv) {} + BGTask(const DataType& _type = DataType::kAll, const Operation& _opeation = Operation::kNone, + const std::vector& _argv = {}) + : type(_type), operation(_opeation), argv(_argv) {} }; class Storage { @@ -902,6 +925,14 @@ class Storage { Status ZScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, std::vector* score_members, int64_t* next_cursor); + Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); + Status XDel(const Slice& key, const std::vector& ids, size_t& ret); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count); + Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XLen(const Slice& key, uint64_t& len); + Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys); // Keys Commands // Note: @@ -1057,6 +1088,7 @@ class Storage { std::unique_ptr sets_db_; std::unique_ptr zsets_db_; std::unique_ptr lists_db_; + std::unique_ptr streams_db_; std::atomic is_opened_ = false; std::unique_ptr> cursors_store_; diff --git a/src/storage/src/base_data_key_format.h b/src/storage/src/base_data_key_format.h index 316f473070..ffac531046 100644 --- a/src/storage/src/base_data_key_format.h +++ b/src/storage/src/base_data_key_format.h @@ -113,9 +113,17 @@ class ParsedZSetsMemberKey : public ParsedBaseDataKey { Slice member() { return data_; } }; +class ParsedStreamDataKey : public ParsedBaseDataKey { + public: + explicit ParsedStreamDataKey(const std::string* key) : ParsedBaseDataKey(key) {} + explicit ParsedStreamDataKey(const Slice& key) : ParsedBaseDataKey(key) {} + Slice id() { return data_; } +}; + using HashesDataKey = BaseDataKey; using SetsMemberKey = BaseDataKey; using ZSetsMemberKey = BaseDataKey; +using StreamDataKey = BaseDataKey; } // namespace storage #endif // SRC_BASE_DATA_KEY_FORMAT_H_ diff --git a/src/storage/src/pika_stream_base.cc b/src/storage/src/pika_stream_base.cc new file mode 100644 index 0000000000..a06c08d6ca --- /dev/null +++ b/src/storage/src/pika_stream_base.cc @@ -0,0 +1,233 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "pika_stream_base.h" + +// #include "pika_command.h" +#include "pika_stream_meta_value.h" +#include "pika_stream_types.h" +#include "storage/storage.h" + +namespace storage { + +#define GET_NEXT_TREE_ID_AND_CHECK(slot, tid) \ + do { \ + auto &tid_gen = TreeIDGenerator::GetInstance(); \ + auto s = tid_gen.GetNextTreeID(slot, tid); \ + if (!s.ok()) { \ + return s; \ + } \ + } while (0) + +storage::Status TreeIDGenerator::GetNextTreeID(const rocksdb::DB *db, treeID &tid) { + // assert(slot); + // auto expected_id = kINVALID_TREE_ID; + + // // if tree_id_ is not initialized (equal to kINVALID_TREE_ID), try to fetch it from storage + // if (tree_id_.compare_exchange_strong(expected_id, START_TREE_ID)) { + // std::string value; + // storage::Status s = db->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); + // if (s.ok()) { + // treeID id; + // if (!StreamUtils::string2int32(value.c_str(), id)) { + // LOG(ERROR) << "Invalid tree id: " << value; + // return storage::Status::Corruption("Invalid tree id"); + // } + // tree_id_.store(id); + + // } else if (!s.IsNotFound()) { + // return s; + // } + // } + + // tree_id_.fetch_add(1); + // tid = tree_id_; + + // // insert tree id to storage + // std::string tree_id_str = std::to_string(tree_id_); + // int32_t res; + // return db->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); +} + +bool StreamUtils::StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, + bool *seq_given) { + char buf[128]; + if (var.size() > sizeof(buf) - 1) { + return false; + } + + memcpy(buf, var.data(), var.size()); + buf[var.size()] = '\0'; + + if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { + // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); + return false; + } + + if (seq_given != nullptr) { + *seq_given = true; + } + + if (buf[0] == '-' && buf[1] == '\0') { + id.ms = 0; + id.seq = 0; + return true; + } else if (buf[0] == '+' && buf[1] == '\0') { + id.ms = UINT64_MAX; + id.seq = UINT64_MAX; + return true; + } + + uint64_t ms; + uint64_t seq; + char *dot = strchr(buf, '-'); + if (dot) { + *dot = '\0'; + } + if (!StreamUtils::string2uint64(buf, ms)) { + return false; + }; + if (dot) { + size_t seqlen = strlen(dot + 1); + if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { + seq = 0; + *seq_given = false; + } else if (!StreamUtils::string2uint64(dot + 1, seq)) { + return false; + } + } else { + seq = missing_seq; + } + id.ms = ms; + id.seq = seq; + return true; +} + +bool StreamUtils::StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq) { + return StreamGenericParseID(var, id, missing_seq, false, nullptr); +} + +bool StreamUtils::StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given) { + return StreamGenericParseID(var, id, missing_seq, true, seq_given); +} + +bool StreamUtils::StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq) { + if (exclude != nullptr) { + *exclude = (var.size() > 1 && var[0] == '('); + } + if (exclude != nullptr && *exclude) { + return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); + } else { + return StreamParseID(var, id, missing_seq); + } +} + +bool StreamUtils::string2uint64(const char *s, uint64_t &value) { + if (!s || !*s) { + return false; + } + + char *end; + errno = 0; + uint64_t tmp = strtoull(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int64(const char *s, int64_t &value) { + if (!s || !*s) { + return false; + } + + char *end; + errno = 0; + int64_t tmp = std::strtoll(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int32(const char *s, int32_t &value) { + if (!s || !*s) { + return false; + } + + char *end; + errno = 0; + long tmp = strtol(s, &end, 10); + if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { + // Conversion either didn't consume the entire string, + // or overflow or underflow occurred + return false; + } + + value = static_cast(tmp); + return true; +} + +std::string StreamUtils::TreeID2Key(const treeID &tid) { + std::string key; + key.reserve(STERAM_TREE_PREFIX.size() + sizeof(tid)); + key.append(STERAM_TREE_PREFIX); + key.append(reinterpret_cast(&tid), sizeof(tid)); + return key; +} + +bool StreamUtils::SerializeMessage(const std::vector &field_values, std::string &message, int field_pos) { + assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); + assert(message.empty()); + // count the size of serizlized message + size_t size = 0; + for (int i = field_pos; i < field_values.size(); i++) { + size += field_values[i].size() + sizeof(size_t); + } + message.reserve(size); + + // serialize message + for (int i = field_pos; i < field_values.size(); i++) { + size_t len = field_values[i].size(); + message.append(reinterpret_cast(&len), sizeof(len)); + message.append(field_values[i]); + } + + return true; +} + +bool StreamUtils::DeserializeMessage(const std::string &message, std::vector &parsed_message) { + size_t pos = 0; + while (pos < message.size()) { + // Read the length of the next field value from the message + size_t len = *reinterpret_cast(&message[pos]); + pos += sizeof(size_t); + + // Check if the calculated end of the string is still within the message bounds + if (pos + len > message.size()) { + LOG(ERROR) << "Invalid message format, failed to parse message"; + return false; // Error: not enough data in the message string + } + + // Extract the field value and add it to the vector + parsed_message.push_back(message.substr(pos, len)); + pos += len; + } + + return true; +} + +uint64_t StreamUtils::GetCurrentTimeMs() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); +} + +} // namespace storage \ No newline at end of file diff --git a/src/storage/src/pika_stream_base.h b/src/storage/src/pika_stream_base.h new file mode 100644 index 0000000000..e3d0775515 --- /dev/null +++ b/src/storage/src/pika_stream_base.h @@ -0,0 +1,155 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +#include +#include "include/storage/storage.h" +#include "pika_stream_meta_value.h" +#include "pika_stream_types.h" +#include "rocksdb/db.h" +#include "rocksdb/slice.h" + +namespace storage { + +// each abstracted tree has a prefix, +// prefix + treeID will be the key of the hash to store the tree, +// notice: we need to ban the use of this prefix when using "HSET". +static const std::string STERAM_TREE_PREFIX = "STR_TREE"; + +// I need to find a place to store the last generated tree id, +// so I stored it as a field-value pair in the hash storing stream meta. +// the field is a fixed string, and the value is the last generated tree id. +// notice: we need to ban the use of this key when using stream, to avoid conflict with stream's key. +static const std::string STREAM_LAST_GENERATED_TREE_ID_FIELD = "STREAM"; + +// the max number of each delete operation in XTRIM command，to avoid too much memory usage. +// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / +// kDEFAULT_TRIM_BATCH_SIZE) times +const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; +struct StreamAddTrimArgs { + // XADD options + streamID id; + bool id_given{false}; + bool seq_given{false}; + bool no_mkstream{false}; + + // XADD + XTRIM common options + StreamTrimStrategy trim_strategy{TRIM_STRATEGY_NONE}; + int trim_strategy_arg_idx{0}; + + // TRIM_STRATEGY_MAXLEN options + uint64_t maxlen{0}; + streamID minid; +}; + +struct StreamReadGroupReadArgs { + // XREAD + XREADGROUP common options + std::vector keys; + std::vector unparsed_ids; + int32_t count{INT32_MAX}; // The limit of read, in redis this is uint64_t, but PKHScanRange only support int32_t + uint64_t block{0}; // 0 means no block + + // XREADGROUP options + std::string group_name; + std::string consumer_name; + bool noack_{false}; +}; + +struct StreamScanArgs { + streamID start_sid; + streamID end_sid; + size_t limit{INT32_MAX}; + bool start_ex{false}; // exclude first message + bool end_ex{false}; // exclude last message + bool is_reverse{false}; // scan in reverse order +}; + +// get next tree id thread safe +class TreeIDGenerator { + private: + TreeIDGenerator() = default; + void operator=(const TreeIDGenerator &) = delete; + + public: + ~TreeIDGenerator() = default; + + // work in singeletone mode + static TreeIDGenerator &GetInstance() { + static TreeIDGenerator instance; + return instance; + } + + storage::Status GetNextTreeID(const rocksdb::DB *db, treeID &tid); + + private: + static const treeID START_TREE_ID = 0; + std::atomic tree_id_{kINVALID_TREE_ID}; +}; + +// Helper function of stream command. +// Should be reconstructed when transfer to another command framework. +// any function that has Reply in its name will reply to the client if error occurs. +class StreamCmdBase { + public: + private: + StreamCmdBase(); + ~StreamCmdBase(); +}; + +class StreamUtils { + public: + StreamUtils() = default; + ~StreamUtils() = default; + + static bool string2uint64(const char *s, uint64_t &value); + static bool string2int64(const char *s, int64_t &value); + static bool string2int32(const char *s, int32_t &value); + static std::string TreeID2Key(const treeID &tid); + + static uint64_t GetCurrentTimeMs(); + + // serialize the message to a string. + // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} + static bool SerializeMessage(const std::vector &field_values, std::string &serialized_message, + int field_pos); + + // deserialize the message from a string with the format of SerializeMessage. + static bool DeserializeMessage(const std::string &message, std::vector &parsed_message); + + // Parse a stream ID in the format given by clients to Pika, that is + // -, and converts it into a streamID structure. The ID may be in incomplete + // form, just stating the milliseconds time part of the stream. In such a case + // the missing part is set according to the value of 'missing_seq' parameter. + // + // The IDs "-" and "+" specify respectively the minimum and maximum IDs + // that can be represented. If 'strict' is set to 1, "-" and "+" will be + // treated as an invalid ID. + // + // The ID form -* specifies a millisconds-only ID, leaving the sequence part + // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this + // form is accepted and the argument is set to 0 unless the sequence part is + // specified. + static bool StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, + bool *seq_given); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 0, to be used when - and + are acceptable IDs. + static bool StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 1, to be used when we want to return an error if the special IDs + or - + // are provided. + static bool StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given); + + // Helper for parsing a stream ID that is a range query interval. When the + // exclude argument is NULL, streamParseID() is called and the interval + // is treated as close (inclusive). Otherwise, the exclude argument is set if + // the interval is open (the "(" prefix) and streamParseStrictID() is + // called in that case. + static bool StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq); +}; + +} // namespace storage diff --git a/include/pika_stream_meta_value.h b/src/storage/src/pika_stream_meta_value.h similarity index 70% rename from include/pika_stream_meta_value.h rename to src/storage/src/pika_stream_meta_value.h index e0bd763950..ecbc6a0958 100644 --- a/include/pika_stream_meta_value.h +++ b/src/storage/src/pika_stream_meta_value.h @@ -3,41 +3,66 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#ifndef SRC_STREAM_META_VALUE_FORMAT_H_ -#define SRC_STREAM_META_VALUE_FORMAT_H_ +#pragma once +#include #include "glog/logging.h" -#include "include/pika_stream_types.h" +#include "pika_stream_types.h" +#include "src/coding.h" +#include "storage/storage.h" + +namespace storage { static const size_t kDefaultStreamValueLength = - sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t); + sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t) + sizeof(int32_t); class StreamMetaValue { public: explicit StreamMetaValue() = default; // used only when create a new stream - void Init() { + void InitMetaValue() { + groups_id_ = kINVALID_TREE_ID; + entries_added_ = 0; + first_id_ = streamID(); + last_id_ = streamID(); + max_deleted_entry_id_ = streamID(); + length_ = 0; + + // We do not reset version_ here, because we want to keep the version of the old stream meta. + // Each time we delete a stream, we will increase the version of the stream meta, so that the old stream date will + // not be seen by the new stream with the same key. + ++version_; + size_t needed = kDefaultStreamValueLength; - assert(value_.size() == 0); - if (value_.size() != 0) { - LOG(ERROR) << "Init on a existed stream meta value!"; - return; - } value_.resize(needed); - char* dst = value_.data(); + char* dst = &value_[0]; // Encode each member into the string - memcpy(dst, &groups_id_, sizeof(treeID)); + EncodeFixed64(dst, groups_id_); dst += sizeof(treeID); - memcpy(dst, &entries_added_, sizeof(size_t)); + + EncodeFixed64(dst, entries_added_); dst += sizeof(size_t); - memcpy(dst, &first_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &last_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &max_deleted_entry_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &length_, sizeof(size_t)); + + EncodeFixed64(dst, first_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, first_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, last_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, last_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, max_deleted_entry_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, max_deleted_entry_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, length_); + dst += sizeof(int64_t); + + EncodeFixed32(dst, version_); } // used only when parse a existed stream meta @@ -49,20 +74,36 @@ class StreamMetaValue { LOG(ERROR) << "Invalid stream meta value length: "; return; } - char* pos = value_.data(); - memcpy(&groups_id_, pos, sizeof(treeID)); + char* pos = &value_[0]; + groups_id_ = DecodeFixed32(pos); pos += sizeof(treeID); - memcpy(&entries_added_, pos, sizeof(size_t)); + + entries_added_ = DecodeFixed64(pos); pos += sizeof(size_t); - memcpy(&first_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&last_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&max_deleted_entry_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&length_, pos, sizeof(size_t)); + + first_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + first_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + last_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + last_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + max_deleted_entry_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + max_deleted_entry_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + length_ = DecodeFixed64(pos); + pos += sizeof(size_t); + + version_ = static_cast(DecodeFixed32(pos)); } + int32_t version() const { return version_; } + treeID groups_id() const { return groups_id_; } size_t entries_added() const { return entries_added_; } @@ -84,7 +125,7 @@ class StreamMetaValue { std::string(", entries_added: ") + std::to_string(entries_added_) + std::string(", first_id: ") + first_id_.ToString() + std::string(", last_id: ") + last_id_.ToString() + std::string(", max_deleted_entry_id: ") + max_deleted_entry_id_.ToString() + std::string(", length: ") + - std::to_string(length_); + std::to_string(length_) + std::string(", version: ") + std::to_string(version_); } void set_groups_id(treeID groups_id) { @@ -129,6 +170,14 @@ class StreamMetaValue { memcpy(dst, &length_, sizeof(size_t)); } + void set_version(int32_t version) { + assert(value_.size() == kDefaultStreamValueLength); + version_ = version; + char* dst = + const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(size_t); + EncodeFixed32(dst, version_); + } + private: treeID groups_id_ = kINVALID_TREE_ID; size_t entries_added_{0}; @@ -136,10 +185,80 @@ class StreamMetaValue { streamID last_id_; streamID max_deleted_entry_id_; size_t length_{0}; // number of the messages in the stream + int32_t version_{0}; std::string value_{}; }; +// Used only for reading ! +class ParsedStreamMetaValue { + public: + ParsedStreamMetaValue(const Slice& value) { + assert(value.size() == kDefaultStreamValueLength); + if (value.size() != kDefaultStreamValueLength) { + LOG(ERROR) << "Invalid stream meta value length: "; + return; + } + char* pos = const_cast(value.data()); + groups_id_ = DecodeFixed32(pos); + pos += sizeof(treeID); + + entries_added_ = DecodeFixed64(pos); + pos += sizeof(size_t); + + first_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + first_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + last_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + last_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + max_deleted_entry_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + max_deleted_entry_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + length_ = DecodeFixed64(pos); + pos += sizeof(size_t); + + version_ = static_cast(DecodeFixed32(pos)); + } + + int32_t version() const { return version_; } + + treeID groups_id() const { return groups_id_; } + + size_t entries_added() const { return entries_added_; } + + streamID first_id() const { return first_id_; } + + streamID last_id() const { return last_id_; } + + streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } + + size_t length() const { return length_; } + + std::string ToString() { + return "stream_meta: " + std::string("groups_id: ") + std::to_string(groups_id_) + + std::string(", entries_added: ") + std::to_string(entries_added_) + std::string(", first_id: ") + + first_id_.ToString() + std::string(", last_id: ") + last_id_.ToString() + + std::string(", max_deleted_entry_id: ") + max_deleted_entry_id_.ToString() + std::string(", length: ") + + std::to_string(length_) + std::string(", version: ") + std::to_string(version_); + } + + private: + treeID groups_id_ = kINVALID_TREE_ID; + size_t entries_added_{0}; + streamID first_id_; + streamID last_id_; + streamID max_deleted_entry_id_; + size_t length_{0}; // number of the messages in the stream + int32_t version_{0}; +}; + static const size_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(treeID); class StreamCGroupMetaValue { public: @@ -157,7 +276,7 @@ class StreamCGroupMetaValue { } value_.resize(needed); - auto dst = value_.data(); + char* dst = &value_[0]; memcpy(dst, &last_id_, sizeof(streamID)); dst += sizeof(uint64_t); @@ -254,7 +373,7 @@ class StreamConsumerMetaValue { } size_t needed = kDefaultStreamConsumerValueLength; value_.resize(needed); - auto dst = value_.data(); + char* dst = &value_[0]; memcpy(dst, &seen_time_, sizeof(stream_ms_t)); dst += sizeof(stream_ms_t); @@ -310,7 +429,7 @@ class StreamPelMeta { return; } value_.resize(needed); - char* dst = value_.data(); + char* dst = &value_[0]; memcpy(dst, &delivery_time_, sizeof(stream_ms_t)); dst += sizeof(stream_ms_t); @@ -369,4 +488,4 @@ class StreamPelMeta { std::string consumer_; }; -#endif // SRC_STREAM_META_VALUE_FORMAT_H_ +} // namespace storage \ No newline at end of file diff --git a/include/pika_stream_types.h b/src/storage/src/pika_stream_types.h similarity index 84% rename from include/pika_stream_types.h rename to src/storage/src/pika_stream_types.h index 0a76ba925d..86a08d03bf 100644 --- a/include/pika_stream_types.h +++ b/src/storage/src/pika_stream_types.h @@ -3,15 +3,18 @@ // LICENSE file in the root directory of this source tree. An additional grant // of patent rights can be found in the PATENTS file in the same directory. -#ifndef SRC_STREAM_TYPE_H_ -#define SRC_STREAM_TYPE_H_ +#pragma once #include #include #include #include #include -#include "src/storage/src/coding.h" +#include "src/coding.h" + +namespace storage { + +#define kINVALID_TREE_ID 0 using streamID = struct streamID { streamID(uint64_t _ms, uint64_t _seq) : ms(_ms), seq(_seq) {} @@ -25,7 +28,7 @@ using streamID = struct streamID { // We must store the streamID in memory in big-endian format. This way, our comparison of the serialized streamID byte // code will be equivalent to the comparison of the uint64_t numbers. inline void EncodeUint64InBigEndian(char* buf, uint64_t value) const { - if (storage::kLittleEndian) { + if (kLittleEndian) { // little endian, reverse the bytes for (int i = 7; i >= 0; --i) { buf[i] = static_cast(value & 0xff); @@ -39,7 +42,7 @@ using streamID = struct streamID { inline uint64_t DecodeUint64OfBigEndian(const char* ptr) { uint64_t value; - if (storage::kLittleEndian) { + if (kLittleEndian) { // little endian, reverse the bytes value = 0; for (int i = 0; i < 8; ++i) { @@ -59,6 +62,14 @@ using streamID = struct streamID { EncodeUint64InBigEndian(&dst[0] + sizeof(ms), seq); } + std::string Serialize() const { + std::string dst; + dst.resize(sizeof(ms) + sizeof(seq)); + EncodeUint64InBigEndian(&dst[0], ms); + EncodeUint64InBigEndian(&dst[0] + sizeof(ms), seq); + return dst; + } + void DeserializeFrom(std::string& src) { assert(src.size() == sizeof(ms) + sizeof(seq)); ms = DecodeUint64OfBigEndian(&src[0]); @@ -73,11 +84,10 @@ using streamID = struct streamID { static const streamID kSTREAMID_MAX = streamID(UINT64_MAX, UINT64_MAX); static const streamID kSTREAMID_MIN = streamID(0, 0); -enum class StreamTrimStrategy { TRIM_STRATEGY_NONE, TRIM_STRATEGY_MAXLEN, TRIM_STRATEGY_MINID }; +enum StreamTrimStrategy { TRIM_STRATEGY_NONE, TRIM_STRATEGY_MAXLEN, TRIM_STRATEGY_MINID }; -using treeID = int32_t; -static const treeID kINVALID_TREE_ID = -1; +using treeID = uint32_t; using stream_ms_t = uint64_t; -#endif // SRC_STREAM_TYPE_H_ +} // namespace storage \ No newline at end of file diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc new file mode 100644 index 0000000000..b32b80ee86 --- /dev/null +++ b/src/storage/src/redis_streams.cc @@ -0,0 +1,944 @@ +#include "src/redis_streams.h" +#include +#include +#include +#include +#include +#include "pika_stream_base.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "src/base_data_key_format.h" +#include "src/base_filter.h" +#include "src/debug.h" +#include "src/pika_stream_meta_value.h" +#include "src/scope_record_lock.h" +#include "src/scope_snapshot.h" +#include "storage/storage.h" +#include "storage/util.h" + +namespace storage { + +Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args) { + // With the lock, we do not need snapshot for read. + // And it's bugy to use snapshot for read when we try to add message with trim. + // such as: XADD key 1-0 field value MINID 1-0 + ScopeRecordLock l(lock_mgr_, key); + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, default_read_options_); + if (s.IsNotFound() && args.no_mkstream) { + return Status::NotFound("no_mkstream"); + } else if (s.IsNotFound()) { + stream_meta.InitMetaValue(); + } else if (!s.ok()) { + return Status::Corruption("error from XADD, get stream meta failed: " + s.ToString()); + } + + if (stream_meta.length() == 0) { + if (args.no_mkstream) { + return Status::NotFound("no_mkstream"); + } + stream_meta.InitMetaValue(); + } + + if (stream_meta.last_id().ms == UINT64_MAX && stream_meta.last_id().seq == UINT64_MAX) { + return Status::Corruption("Fatal! Sequence number overflow !"); + } + + // 2 append the message to storage + s = GenerateStreamID(stream_meta, args); + if (!s.ok()) { + return s; + } + +#ifdef DEBUG + // check the serialized current id is larger than last_id + std::string serialized_last_id; + std::string current_id; + stream_meta.last_id().SerializeTo(serialized_last_id); + args.id.SerializeTo(current_id); + assert(current_id > serialized_last_id); +#endif + + StreamDataKey stream_data_key(key, stream_meta.version(), args.id.Serialize()); + s = db_->Put(default_write_options_, handles_[1], stream_data_key.Encode(), serialized_message); + if (!s.ok()) { + return Status::Corruption("error from XADD, insert stream message failed 1: " + s.ToString()); + } + + // 3 update stream meta + if (stream_meta.length() == 0) { + stream_meta.set_first_id(args.id); + } + stream_meta.set_entries_added(stream_meta.entries_added() + 1); + stream_meta.set_last_id(args.id); + stream_meta.set_length(stream_meta.length() + 1); + + // 4 trim the stream if needed + if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + size_t count{0}; + s = TrimStream(count, stream_meta, key, args, default_read_options_); + if (!s.ok()) { + return Status::Corruption("error from XADD, trim stream failed: " + s.ToString()); + } + (void)count; + } + + // 5 update stream meta + s = db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); + if (!s.ok()) { + return s; + } + + return Status::OK(); +} + +Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count) { + ScopeRecordLock l(lock_mgr_, key); + + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the trim + count = 0; + s = TrimStream(count, stream_meta, key, args, read_options); + if (!s.ok()) { + return s; + } + + // 3 update stream meta + s = db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); + if (!s.ok()) { + return s; + } + + return Status::OK(); +} + +Status RedisStreams::XDel(const Slice& key, const std::vector& ids, size_t& count) { + ScopeRecordLock l(lock_mgr_, key); + + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 try to get stream meta + StreamMetaValue stream_meta; + auto s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the delete + count = ids.size(); + std::string unused; + for (auto id : ids) { + StreamDataKey stream_data_key(key, stream_meta.version(), id.Serialize()); + s = db_->Get(read_options, handles_[1], stream_data_key.Encode(), &unused); + if (s.IsNotFound()) { + --count; + continue; + } else if (!s.ok()) { + return s; + } + } + s = DeleteStreamMessages(key, stream_meta, ids, read_options); + if (!s.ok()) { + return s; + } + + // 3 update stream meta + stream_meta.set_length(stream_meta.length() - count); + for (const auto& id : ids) { + if (id > stream_meta.max_deleted_entry_id()) { + stream_meta.set_max_deleted_entry_id(id); + } + if (id == stream_meta.first_id()) { + s = SetFirstID(key, stream_meta, read_options); + } else if (id == stream_meta.last_id()) { + s = SetLastID(key, stream_meta, read_options); + } + if (!s.ok()) { + return s; + } + } + + return db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); +} + +Status RedisStreams::XRange(const Slice& key, const StreamScanArgs& args, std::vector& field_values) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the scan + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), args.start_sid, args.end_sid, args.limit, args.start_ex, + args.end_ex, false); + s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + + return s; +} + +Status RedisStreams::XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& field_values) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the scan + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), args.start_sid, args.end_sid, args.limit, args.start_ex, + args.end_ex, true); + s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + + return s; +} + +Status RedisStreams::XLen(const Slice& key, uint64_t& len) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + len = stream_meta.length(); + return Status::OK(); +} + +Status RedisStreams::XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 prepare stream_metas + rocksdb::Status s; + std::vector> streammeta_idx; + for (int i = 0; i < args.unparsed_ids.size(); i++) { + const auto& key = args.keys[i]; + + StreamMetaValue stream_meta; + auto s = GetStreamMeta(stream_meta, key, read_options); + if (s.IsNotFound()) { + continue; + } else if (!s.ok()) { + return s; + } + + streammeta_idx.emplace_back(std::move(stream_meta), i); + } + + if (streammeta_idx.empty()) { + return Status::OK(); + } + + // 2 do the scan + for (const auto& stream_meta_id : streammeta_idx) { + const auto& stream_meta = stream_meta_id.first; + const auto& idx = stream_meta_id.second; + const auto& unparsed_id = args.unparsed_ids[idx]; + const auto& key = args.keys[idx]; + + // 2.1 try to parse id + storage::streamID id; + if (unparsed_id == "<") { + return Status::Corruption( + "The > ID can be specified only when calling " + "XREADGROUP using the GROUP " + " option."); + } else if (unparsed_id == "$") { + id = stream_meta.last_id(); + } else { + if (!storage::StreamUtils::StreamParseStrictID(unparsed_id, id, 0, nullptr)) { + return Status::Corruption("Invalid stream ID specified as stream "); + } + } + + // 2.2 scan + std::vector field_values; + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), id, storage::kSTREAMID_MAX, args.count, true); + auto s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + if (!s.ok() && !s.IsNotFound()) { + return s; + } + results.emplace_back(std::move(field_values)); + reserved_keys.emplace_back(args.keys[idx]); + } + + return Status::OK(); +} + +Status RedisStreams::Open(const StorageOptions& storage_options, const std::string& db_path) { + statistics_store_->SetCapacity(storage_options.statistics_max_size); + small_compaction_threshold_ = storage_options.small_compaction_threshold; + + rocksdb::Options ops(storage_options.options); + Status s = rocksdb::DB::Open(ops, db_path, &db_); + if (s.ok()) { + // create column family + rocksdb::ColumnFamilyHandle* cf; + // FIXME: Dose stream data need a comparater ? + s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "data_cf", &cf); + if (!s.ok()) { + return s; + } + // close DB + delete cf; + delete db_; + } + + // Open + rocksdb::DBOptions db_ops(storage_options.options); + rocksdb::ColumnFamilyOptions meta_cf_ops(storage_options.options); + rocksdb::ColumnFamilyOptions data_cf_ops(storage_options.options); + // Notice: Stream's Meta dose not have timestamp and version, so it does not need to be filtered. + + // use the bloom filter policy to reduce disk reads + rocksdb::BlockBasedTableOptions table_ops(storage_options.table_options); + table_ops.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true)); + rocksdb::BlockBasedTableOptions meta_cf_table_ops(table_ops); + rocksdb::BlockBasedTableOptions data_cf_table_ops(table_ops); + if (!storage_options.share_block_cache && storage_options.block_cache_size > 0) { + meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); + data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); + } + meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(meta_cf_table_ops)); + data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(data_cf_table_ops)); + + std::vector column_families; + // Meta CF + column_families.emplace_back(rocksdb::kDefaultColumnFamilyName, meta_cf_ops); + // Data CF + column_families.emplace_back("data_cf", data_cf_ops); + return rocksdb::DB::Open(db_ops, db_path, column_families, &handles_, &db_); +} + +Status RedisStreams::CompactRange(const rocksdb::Slice* begin, const rocksdb::Slice* end, + const ColumnFamilyType& type) { + if (type == kMeta || type == kMetaAndData) { + db_->CompactRange(default_compact_range_options_, handles_[0], begin, end); + } + if (type == kData || type == kMetaAndData) { + db_->CompactRange(default_compact_range_options_, handles_[1], begin, end); + } + return Status::OK(); +} + +Status RedisStreams::GetProperty(const std::string& property, uint64_t* out) { + std::string value; + db_->GetProperty(handles_[0], property, &value); + *out = std::strtoull(value.c_str(), nullptr, 10); + db_->GetProperty(handles_[1], property, &value); + *out += std::strtoull(value.c_str(), nullptr, 10); + return Status::OK(); +} + +// Check if the key has prefix of STERAM_TREE_PREFIX. +// That means the key-value is a virtual tree node, not a stream meta. +bool IsVirtualTree(rocksdb::Slice key) { + if (key.size() < STERAM_TREE_PREFIX.size()) { + return false; + } + + if (memcmp(key.data(), STERAM_TREE_PREFIX.data(), STERAM_TREE_PREFIX.size()) == 0) { + return true; + } + + return false; +} + +Status RedisStreams::ScanKeyNum(KeyInfo* key_info) { + uint64_t keys = 0; + uint64_t expires = 0; + uint64_t ttl_sum = 0; + uint64_t invaild_keys = 0; + + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + int64_t curtime; + rocksdb::Env::Default()->GetCurrentTime(&curtime); + + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (IsVirtualTree(iter->key())) { + continue; + } + ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); + if (parsed_stream_meta_value.length() == 0) { + invaild_keys++; + } else { + keys++; + } + } + delete iter; + + key_info->keys = keys; + key_info->invaild_keys = invaild_keys; + return Status::OK(); +} + +Status RedisStreams::ScanKeys(const std::string& pattern, std::vector* keys) { + std::string key; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + if (IsVirtualTree(iter->key())) { + continue; + } + ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); + if (parsed_stream_meta_value.length() != 0) { + key = iter->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + } + } + delete iter; + return Status::OK(); +} + +Status RedisStreams::PKPatternMatchDel(const std::string& pattern, int32_t* ret) { + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + std::string key; + std::string meta_value; + int32_t total_delete = 0; + Status s; + rocksdb::WriteBatch batch; + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + iter->SeekToFirst(); + while (iter->Valid()) { + if (IsVirtualTree(iter->key())) { + iter->Next(); + continue; + } + key = iter->key().ToString(); + meta_value = iter->value().ToString(); + StreamMetaValue stream_meta_value; + stream_meta_value.ParseFrom(meta_value); + if ((stream_meta_value.length() != 0) && + (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0)) { + stream_meta_value.InitMetaValue(); + batch.Put(handles_[0], key, stream_meta_value.value()); + } + if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { + s = db_->Write(default_write_options_, &batch); + if (s.ok()) { + total_delete += static_cast(batch.Count()); + batch.Clear(); + } else { + *ret = total_delete; + return s; + } + } + iter->Next(); + } + if (batch.Count() != 0U) { + s = db_->Write(default_write_options_, &batch); + if (s.ok()) { + total_delete += static_cast(batch.Count()); + batch.Clear(); + } + } + + *ret = total_delete; + return s; +} + +Status RedisStreams::Del(const Slice& key) { + // FIXME: Check the prefix of key + // stream TODO: Delete all the cgroup and consumers + std::string meta_value; + ScopeRecordLock l(lock_mgr_, key); + Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value); + if (s.ok()) { + StreamMetaValue stream_meta_value; + stream_meta_value.ParseFrom(meta_value); + if (stream_meta_value.length() == 0) { + return Status::NotFound(); + } else { + uint32_t statistic = stream_meta_value.length(); + stream_meta_value.InitMetaValue(); + s = db_->Put(default_write_options_, handles_[0], key, stream_meta_value.value()); + UpdateSpecificKeyStatistics(key.ToString(), statistic); + } + } + return s; +} + +bool RedisStreams::Scan(const std::string& start_key, const std::string& pattern, std::vector* keys, + int64_t* count, std::string* next_key) { + std::string meta_key; + bool is_finish = true; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + + it->Seek(start_key); + while (it->Valid() && (*count) > 0) { + if (IsVirtualTree(it->key())) { + it->Next(); + continue; + } + ParsedStreamMetaValue parsed_stream_meta_value(it->value()); + if (parsed_stream_meta_value.length() == 0) { + it->Next(); + continue; + } else { + meta_key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), meta_key.data(), meta_key.size(), 0) != 0) { + keys->push_back(meta_key); + } + (*count)--; + it->Next(); + } + } + + std::string prefix = isTailWildcard(pattern) ? pattern.substr(0, pattern.size() - 1) : ""; + if (it->Valid() && (it->key().compare(prefix) <= 0 || it->key().starts_with(prefix))) { + *next_key = it->key().ToString(); + is_finish = false; + } else { + *next_key = ""; + } + delete it; + return is_finish; +} + +Status RedisStreams::Expire(const Slice& key, int32_t ttl) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Expire not supported by stream")); + return Status::Corruption(s.ToString()); +} + +bool RedisStreams::PKExpireScan(const std::string& start_key, int32_t min_timestamp, int32_t max_timestamp, + std::vector* keys, int64_t* leftover_visits, std::string* next_key) { + TRACE("RedisStreams::PKExpireScan not supported by stream"); + return false; +} + +Status RedisStreams::Expireat(const Slice& key, int32_t timestamp) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Expireat not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::Persist(const Slice& key) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Persist not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::TTL(const Slice& key, int64_t* timestamp) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::TTL not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::GetStreamMeta(StreamMetaValue& stream_meta, const rocksdb::Slice& key, + rocksdb::ReadOptions& read_options) { + std::string value; + auto s = db_->Get(read_options, handles_[0], key, &value); + if (s.ok()) { + stream_meta.ParseFrom(value); + return Status::OK(); + } + return s; +} + +Status RedisStreams::TrimStream(size_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + count = 0; + // 1 do the trim + TrimRet trim_ret; + Status s; + if (args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MAXLEN) { + s = TrimByMaxlen(trim_ret, stream_meta, key, args, read_options); + } else { + assert(args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MINID); + s = TrimByMinid(trim_ret, stream_meta, key, args, read_options); + } + + if (!s.ok()) { + return s; + } + + if (trim_ret.count == 0) { + return s; + } + + // 2 update stream meta + streamID first_id; + streamID max_deleted_id; + if (stream_meta.length() == trim_ret.count) { + // all the message in the stream were deleted + first_id = kSTREAMID_MIN; + } else { + first_id.DeserializeFrom(trim_ret.next_field); + } + assert(!trim_ret.max_deleted_field.empty()); + max_deleted_id.DeserializeFrom(trim_ret.max_deleted_field); + + stream_meta.set_first_id(first_id); + if (max_deleted_id > stream_meta.max_deleted_entry_id()) { + stream_meta.set_max_deleted_entry_id(max_deleted_id); + } + stream_meta.set_length(stream_meta.length() - trim_ret.count); + + count = trim_ret.count; + return Status::OK(); +} + +Status RedisStreams::ScanStream(const ScanStreamOptions& op, std::vector& field_values, + std::string& next_field, rocksdb::ReadOptions& read_options) { + std::string start_field; + std::string end_field; + Slice pattern = "*"; // match all the fields from start_field to end_field + Status s; + + // 1 do the scan + if (op.is_reverse) { + start_field = op.end_sid.Serialize(); + if (op.start_sid == kSTREAMID_MAX) { + start_field = ""; + } else { + start_field = op.start_sid.Serialize(); + } + s = ReScanRange(op.key, op.version, start_field, end_field, pattern, op.limit, field_values, next_field, + read_options); + } else { + start_field = op.start_sid.Serialize(); + if (op.end_sid == kSTREAMID_MAX) { + end_field = ""; + } else { + end_field = op.end_sid.Serialize(); + } + s = ScanRange(op.key, op.version, start_field, end_field, pattern, op.limit, field_values, next_field, + read_options); + } + + // 2 exclude the start_sid and end_sid if needed + if (op.start_ex && !field_values.empty()) { + streamID sid; + sid.DeserializeFrom(field_values.front().field); + if (sid == op.start_sid) { + field_values.erase(field_values.begin()); + } + } + + if (op.end_ex && !field_values.empty()) { + streamID sid; + sid.DeserializeFrom(field_values.back().field); + if (sid == op.end_sid) { + field_values.pop_back(); + } + } + + return s; +} + +Status RedisStreams::GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args) { + auto& id = args.id; + if (args.id_given && args.seq_given && id.ms == 0 && id.seq == 0) { + return Status::InvalidArgument("The ID specified in XADD must be greater than 0-0"); + } + + if (!args.id_given || !args.seq_given) { + // if id not given, generate one + if (!args.id_given) { + id.ms = StreamUtils::GetCurrentTimeMs(); + + if (id.ms < stream_meta.last_id().ms) { + id.ms = stream_meta.last_id().ms; + if (stream_meta.last_id().seq == UINT64_MAX) { + id.ms++; + id.seq = 0; + } else { + id.seq++; + } + return Status::OK(); + } + } + + // generate seq + auto last_id = stream_meta.last_id(); + if (id.ms < last_id.ms) { + return Status::InvalidArgument("The ID specified in XADD is equal or smaller"); + } else if (id.ms == last_id.ms) { + if (last_id.seq == UINT64_MAX) { + return Status::InvalidArgument("The ID specified in XADD is equal or smaller"); + } + id.seq = last_id.seq + 1; + } else { + id.seq = 0; + } + + } else { + // Full ID given, check id + auto last_id = stream_meta.last_id(); + if (id.ms < last_id.ms || (id.ms == last_id.ms && id.seq <= last_id.seq)) { + return Status::InvalidArgument("INVALID ID given"); + } + } + return Status::OK(); +} + +Status RedisStreams::TrimByMaxlen(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + Status s; + // we delete the message in batchs, prevent from using too much memory + while (stream_meta.length() - trim_ret.count > args.maxlen) { + auto cur_batch = + (std::min(static_cast(stream_meta.length() - trim_ret.count - args.maxlen), kDEFAULT_TRIM_BATCH_SIZE)); + std::vector id_messages; + + RedisStreams::ScanStreamOptions options(key, stream_meta.version(), stream_meta.first_id(), kSTREAMID_MAX, + cur_batch, false, false, false); + s = RedisStreams::ScanStream(options, id_messages, trim_ret.next_field, read_options); + if (!s.ok()) { + assert(!s.IsNotFound()); + return s; + } + + assert(id_messages.size() == cur_batch); + trim_ret.count += cur_batch; + trim_ret.max_deleted_field = id_messages.back().field; + + // delete the message in batchs + std::vector ids_to_del; + ids_to_del.reserve(id_messages.size()); + for (auto& fv : id_messages) { + ids_to_del.emplace_back(std::move(fv.field)); + } + s = DeleteStreamMessages(key, stream_meta, ids_to_del, read_options); + if (!s.ok()) { + return s; + } + } + + s = Status::OK(); + return s; +} + +Status RedisStreams::TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + Status s; + std::string serialized_min_id; + trim_ret.next_field = stream_meta.first_id().Serialize(); + serialized_min_id = args.minid.Serialize(); + + // we delete the message in batchs, prevent from using too much memory + while (trim_ret.next_field < serialized_min_id && stream_meta.length() - trim_ret.count > 0) { + auto cur_batch = static_cast( + std::min(static_cast(stream_meta.length() - trim_ret.count), kDEFAULT_TRIM_BATCH_SIZE)); + std::vector id_messages; + + RedisStreams::ScanStreamOptions options(key, stream_meta.version(), stream_meta.first_id(), args.minid, cur_batch, + false, false, false); + s = RedisStreams::ScanStream(options, id_messages, trim_ret.next_field, read_options); + if (!s.ok()) { + assert(!s.IsNotFound()); + return s; + } + + if (!id_messages.empty()) { + if (id_messages.back().field == serialized_min_id) { + // we do not need to delete the message that it's id matches the minid + id_messages.pop_back(); + trim_ret.next_field = serialized_min_id; + } + // duble check + if (!id_messages.empty()) { + trim_ret.max_deleted_field = id_messages.back().field; + } + } + + assert(id_messages.size() <= cur_batch); + trim_ret.count += static_cast(id_messages.size()); + + // do the delete in batch + std::vector fields_to_del; + fields_to_del.reserve(id_messages.size()); + for (auto& fv : id_messages) { + fields_to_del.emplace_back(std::move(fv.field)); + } + + s = DeleteStreamMessages(key, stream_meta, fields_to_del, read_options); + if (!s.ok()) { + return s; + } + } + + s = Status::OK(); + return s; +} + +Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Slice& id_start, + const std::string& id_end, const Slice& pattern, int32_t limit, + std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options) { + next_id.clear(); + id_messages.clear(); + + int64_t remain = limit; + std::string meta_value; + + bool start_no_limit = id_start.compare("") == 0; + bool end_no_limit = id_end.empty(); + + if (!start_no_limit && !end_no_limit && (id_start.compare(id_end) > 0)) { + return Status::InvalidArgument("error in given range"); + } + + StreamDataKey streams_data_prefix(key, version, Slice()); + StreamDataKey streams_start_data_key(key, version, id_start); + std::string prefix = streams_data_prefix.Encode().ToString(); + rocksdb::Iterator* iter = db_->NewIterator(read_options, handles_[1]); + for (iter->Seek(start_no_limit ? prefix : streams_start_data_key.Encode()); + iter->Valid() && remain > 0 && iter->key().starts_with(prefix); iter->Next()) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + std::string id = parsed_streams_data_key.id().ToString(); + if (!end_no_limit && id.compare(id_end) > 0) { + break; + } + if (StringMatch(pattern.data(), pattern.size(), id.data(), id.size(), 0) != 0) { + id_messages.push_back({id, iter->value().ToString()}); + } + remain--; + } + + if (iter->Valid() && iter->key().starts_with(prefix)) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + if (end_no_limit || parsed_streams_data_key.id().compare(id_end) <= 0) { + next_id = parsed_streams_data_key.id().ToString(); + } + } + delete iter; + + return Status::OK(); +} + +Status RedisStreams::ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, + const std::string& id_end, const Slice& pattern, int32_t limit, + std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options) { + next_id.clear(); + id_messages.clear(); + + int64_t remain = limit; + std::string meta_value; + + bool start_no_limit = id_start.compare("") == 0; + bool end_no_limit = id_end.empty(); + + if (!start_no_limit && !end_no_limit && (id_start.compare(id_end) < 0)) { + return Status::InvalidArgument("error in given range"); + } + + int32_t start_key_version = start_no_limit ? version + 1 : version; + std::string start_key_id = start_no_limit ? "" : id_start.ToString(); + StreamDataKey streams_data_prefix(key, version, Slice()); + StreamDataKey streams_start_data_key(key, start_key_version, start_key_id); + std::string prefix = streams_data_prefix.Encode().ToString(); + rocksdb::Iterator* iter = db_->NewIterator(read_options, handles_[1]); + for (iter->SeekForPrev(streams_start_data_key.Encode().ToString()); + iter->Valid() && remain > 0 && iter->key().starts_with(prefix); iter->Prev()) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + std::string id = parsed_streams_data_key.id().ToString(); + if (!end_no_limit && id.compare(id_end) < 0) { + break; + } + if (StringMatch(pattern.data(), pattern.size(), id.data(), id.size(), 0) != 0) { + id_messages.push_back({id, iter->value().ToString()}); + } + remain--; + } + + if (iter->Valid() && iter->key().starts_with(prefix)) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + if (end_no_limit || parsed_streams_data_key.id().compare(id_end) >= 0) { + next_id = parsed_streams_data_key.id().ToString(); + } + } + delete iter; + + return Status::OK(); +} + +Status RedisStreams::DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& ids, rocksdb::ReadOptions& read_options) { + std::vector serialized_ids; + serialized_ids.reserve(ids.size()); + for (const auto& id : ids) { + serialized_ids.emplace_back(id.Serialize()); + } + return DeleteStreamMessages(key, stream_meta, serialized_ids, read_options); +} + +Status RedisStreams::DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& serialized_ids, + rocksdb::ReadOptions& read_options) { + rocksdb::WriteBatch batch; + for (auto& sid : serialized_ids) { + StreamDataKey stream_data_key(key, stream_meta.version(), sid); + batch.Delete(handles_[1], stream_data_key.Encode()); + } + return db_->Write(default_write_options_, &batch); +} +}; // namespace storage \ No newline at end of file diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h new file mode 100644 index 0000000000..799fa1a2be --- /dev/null +++ b/src/storage/src/redis_streams.h @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include +#include "include/storage/storage.h" +#include "pika_stream_base.h" +#include "pika_stream_meta_value.h" +#include "pika_stream_types.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "src/redis.h" +#include "storage/storage.h" + +namespace storage { + +class RedisStreams : public Redis { + public: + RedisStreams(Storage* const s, const DataType& type) : Redis(s, type) {} + ~RedisStreams() override = default; + + //===--------------------------------------------------------------------===// + // Commands + //===--------------------------------------------------------------------===// + Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); + Status XDel(const Slice& key, const std::vector& ids, size_t& count); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count); + Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XLen(const Slice& key, size_t& len); + Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys); + + //===--------------------------------------------------------------------===// + // Common Commands + //===--------------------------------------------------------------------===// + Status Open(const StorageOptions& storage_options, const std::string& db_path) override; + Status CompactRange(const rocksdb::Slice* begin, const rocksdb::Slice* end, + const ColumnFamilyType& type = kMetaAndData) override; + Status GetProperty(const std::string& property, uint64_t* out) override; + Status ScanKeyNum(KeyInfo* keyinfo) override; + Status ScanKeys(const std::string& pattern, std::vector* keys) override; + Status PKPatternMatchDel(const std::string& pattern, int32_t* ret) override; + + //===--------------------------------------------------------------------===// + // Keys Commands + //===--------------------------------------------------------------------===// + Status Del(const Slice& key) override; + bool Scan(const std::string& start_key, const std::string& pattern, std::vector* keys, int64_t* count, + std::string* next_key) override; + + //===--------------------------------------------------------------------===// + // Not needed for streams + //===--------------------------------------------------------------------===// + Status Expire(const Slice& key, int32_t ttl) override; + bool PKExpireScan(const std::string& start_key, int32_t min_timestamp, int32_t max_timestamp, + std::vector* keys, int64_t* leftover_visits, std::string* next_key) override; + Status Expireat(const Slice& key, int32_t timestamp) override; + Status Persist(const Slice& key) override; + Status TTL(const Slice& key, int64_t* timestamp) override; + + //===--------------------------------------------------------------------===// + // Storage API + //===--------------------------------------------------------------------===// + struct ScanStreamOptions { + const rocksdb::Slice key; // the key of the stream + int32_t version; // the version of the stream + streamID start_sid; + streamID end_sid; + size_t limit; + bool start_ex; // exclude first message + bool end_ex; // exclude last message + bool is_reverse; // scan in reverse order + ScanStreamOptions(const rocksdb::Slice skey, int32_t version, streamID start_sid, streamID end_sid, size_t count, + bool start_ex = false, bool end_ex = false, bool is_reverse = false) + : key(skey), + version(version), + start_sid(start_sid), + end_sid(end_sid), + limit(count), + start_ex(start_ex), + end_ex(end_ex), + is_reverse(is_reverse) {} + }; + + Status ScanStream(const ScanStreamOptions& option, std::vector& id_messages, std::string& next_field, + rocksdb::ReadOptions& read_options); + // get and parse the stream meta if found + // @return ok only when the stream meta exists + Status GetStreamMeta(StreamMetaValue& tream_meta, const rocksdb::Slice& key, rocksdb::ReadOptions& read_options); + + // Before calling this function, the caller should ensure that the ids are valid + Status DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& ids, rocksdb::ReadOptions& read_options); + + // Before calling this function, the caller should ensure that the ids are valid + Status DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& serialized_ids, rocksdb::ReadOptions& read_options); + + Status TrimStream(size_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, StreamAddTrimArgs& args, + rocksdb::ReadOptions& read_options); + + private: + Status GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args); + + Status ScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, + const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options); + Status ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, + const Slice& pattern, int32_t limit, std::vector& id_values, std::string& next_id, + rocksdb::ReadOptions& read_options); + + struct TrimRet { + // the count of deleted messages + size_t count{0}; + // the next field after trim + std::string next_field; + // the max deleted field, will be empty if no message is deleted + std::string max_deleted_field; + }; + + Status TrimByMaxlen(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); + + Status TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); + + inline Status SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, + rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, true, read_options); + } + + inline Status SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, false, read_options); + } + + inline Status SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, + rocksdb::ReadOptions& read_options) { + if (stream_meta.length() == 0) { + stream_meta.set_first_id(kSTREAMID_MIN); + return Status::OK(); + } + + std::vector id_messages; + std::string next_field; + + storage::Status s; + if (is_set_first) { + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MIN, kSTREAMID_MAX, 1); + s = ScanStream(option, id_messages, next_field, read_options); + } else { + bool is_reverse = true; + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); + s = ScanStream(option, id_messages, next_field, read_options); + } + (void)next_field; + + if (!s.ok() && !s.IsNotFound()) { + LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); + return Status::Corruption("Internal error: scan stream failed: " + s.ToString()); + } + + if (id_messages.empty()) { + LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; + return Status::Corruption("Internal error: no messages found but stream length is not 0"); + } + + streamID id; + id.DeserializeFrom(id_messages[0].field); + stream_meta.set_first_id(id); + return Status::OK(); + } +}; +} // namespace storage \ No newline at end of file diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index d16548b9c2..04c6c8d055 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -18,6 +18,7 @@ #include "src/redis_hyperloglog.h" #include "src/redis_lists.h" #include "src/redis_sets.h" +#include "src/redis_streams.h" #include "src/redis_strings.h" #include "src/redis_zsets.h" @@ -69,6 +70,7 @@ Storage::~Storage() { rocksdb::CancelAllBackgroundWork(sets_db_->GetDB(), true); rocksdb::CancelAllBackgroundWork(lists_db_->GetDB(), true); rocksdb::CancelAllBackgroundWork(zsets_db_->GetDB(), true); + rocksdb::CancelAllBackgroundWork(streams_db_->GetDB(), true); } int ret = 0; @@ -117,6 +119,13 @@ Status Storage::Open(const StorageOptions& storage_options, const std::string& d if (!s.ok()) { LOG(FATAL) << "open zset db failed, " << s.ToString(); } + + streams_db_ = std::make_unique(this, kStreams); + s = streams_db_->Open(storage_options, AppendSubDirectory(db_path, "streams")); + if (!s.ok()) { + LOG(FATAL) << "open stream db failed, " << s.ToString(); + } + is_opened_.store(true); return Status::OK(); } @@ -526,6 +535,35 @@ Status Storage::ZScan(const Slice& key, int64_t cursor, const std::string& patte return zsets_db_->ZScan(key, cursor, pattern, count, score_members, next_cursor); } +Status Storage::XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args) { + return streams_db_->XAdd(key, serialized_message, args); +} + +Status Storage::XDel(const Slice& key, const std::vector& ids, size_t& ret) { + return streams_db_->XDel(key, ids, ret); +} + +Status Storage::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count) { + return streams_db_->XTrim(key, args, count); +} + +Status Storage::XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages) { + return streams_db_->XRange(key, args, id_messages); +} + +Status Storage::XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages) { + return streams_db_->XRevrange(key, args, id_messages); +} + +Status Storage::XLen(const Slice& key, uint64_t& len) { + return streams_db_->XLen(key, len); +} + +Status Storage::XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys) { + return streams_db_->XRead(args, results, reserved_keys); +} + // Keys Commands int32_t Storage::Expire(const Slice& key, int32_t ttl, std::map* type_status) { int32_t ret = 0; @@ -633,6 +671,15 @@ int64_t Storage::Del(const std::vector& keys, std::mapDel(key); + if (s.ok()) { + count++; + } else if (!s.IsNotFound()) { + is_corruption = true; + (*type_status)[DataType::kStreams] = s; + } } if (is_corruption) { @@ -699,6 +746,16 @@ int64_t Storage::DelByType(const std::vector& keys, const DataType& } break; } + // Stream + case DataType::kStreams: { + s = streams_db_->Del(key); + if (s.ok()) { + count++; + } else if (!s.IsNotFound()) { + is_corruption = true; + } + break; + } case DataType::kAll: { return -1; } From a3335d269edc91a092ee13d1c488523e3f6e7b87 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Sat, 23 Dec 2023 20:19:48 +0800 Subject: [PATCH 02/10] fix: fix all problem of stream commands --- src/pika_stream.cc | 37 +++++++++++++---------- src/storage/include/storage/storage.h | 2 ++ src/storage/src/pika_stream_base.h | 8 +++++ src/storage/src/redis_streams.cc | 43 ++++++++++++++++----------- src/storage/src/redis_streams.h | 1 + src/storage/src/storage.cc | 4 +++ 6 files changed, 62 insertions(+), 33 deletions(-) diff --git a/src/pika_stream.cc b/src/pika_stream.cc index 62e1f6e4f6..e007890cb5 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -497,22 +497,27 @@ void XInfoCmd::Do(std::shared_ptr slot) { } void XInfoCmd::StreamInfo(std::shared_ptr &slot) { - // 1 try to get stream meta - // StreamMetaValue stream_meta; - // TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, slot.get()), res_); + storage::StreamInfoResult info; + auto s = slot->db()->XInfo(key_, info); + + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kNotFound); + return; + } // // 2 append the stream info - // res_.AppendArrayLen(10); - // res_.AppendString("length"); - // res_.AppendInteger(static_cast(stream_meta.length())); - // res_.AppendString("last-generated-id"); - // res_.AppendString(stream_meta.last_id().ToString()); - // res_.AppendString("max-deleted-entry-id"); - // res_.AppendString(stream_meta.max_deleted_entry_id().ToString()); - // res_.AppendString("entries-added"); - // res_.AppendInteger(static_cast(stream_meta.entries_added())); - // res_.AppendString("recorded-first-entry-id"); - // res_.AppendString(stream_meta.first_id().ToString()); - - // Korpse TODO: add group info + res_.AppendArrayLen(10); + res_.AppendString("length"); + res_.AppendInteger(static_cast(info.length)); + res_.AppendString("last-generated-id"); + res_.AppendString(info.last_id_str); + res_.AppendString("max-deleted-entry-id"); + res_.AppendString(info.max_deleted_entry_id_str); + res_.AppendString("entries-added"); + res_.AppendInteger(static_cast(info.entries_added)); + res_.AppendString("recorded-first-entry-id"); + res_.AppendString(info.first_id_str); } diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index ba77c17621..1500dc804c 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -63,6 +63,7 @@ struct StreamAddTrimArgs; struct StreamReadGroupReadArgs; struct StreamScanArgs; struct streamID; +struct StreamInfoResult; template class LRUCache; @@ -933,6 +934,7 @@ class Storage { Status XLen(const Slice& key, uint64_t& len); Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, std::vector& reserved_keys); + Status XInfo(const Slice& key, StreamInfoResult &result); // Keys Commands // Note: diff --git a/src/storage/src/pika_stream_base.h b/src/storage/src/pika_stream_base.h index e3d0775515..1d69dc9dee 100644 --- a/src/storage/src/pika_stream_base.h +++ b/src/storage/src/pika_stream_base.h @@ -67,6 +67,14 @@ struct StreamScanArgs { bool is_reverse{false}; // scan in reverse order }; +struct StreamInfoResult { + uint64_t length{0}; + std::string last_id_str; + std::string max_deleted_entry_id_str; + uint64_t entries_added{0}; + std::string first_id_str; +}; + // get next tree id thread safe class TreeIDGenerator { private: diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index b32b80ee86..2baaa1c775 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -98,22 +98,17 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count) { ScopeRecordLock l(lock_mgr_, key); - rocksdb::ReadOptions read_options; - const rocksdb::Snapshot* snapshot; - ScopeSnapshot ss(db_, &snapshot); - read_options.snapshot = snapshot; - // 1 get stream meta rocksdb::Status s; StreamMetaValue stream_meta; - s = GetStreamMeta(stream_meta, key, read_options); + s = GetStreamMeta(stream_meta, key, default_read_options_); if (!s.ok()) { return s; } // 2 do the trim count = 0; - s = TrimStream(count, stream_meta, key, args, read_options); + s = TrimStream(count, stream_meta, key, args, default_read_options_); if (!s.ok()) { return s; } @@ -130,14 +125,9 @@ Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& co Status RedisStreams::XDel(const Slice& key, const std::vector& ids, size_t& count) { ScopeRecordLock l(lock_mgr_, key); - rocksdb::ReadOptions read_options; - const rocksdb::Snapshot* snapshot; - ScopeSnapshot ss(db_, &snapshot); - read_options.snapshot = snapshot; - // 1 try to get stream meta StreamMetaValue stream_meta; - auto s = GetStreamMeta(stream_meta, key, read_options); + auto s = GetStreamMeta(stream_meta, key, default_read_options_); if (!s.ok()) { return s; } @@ -147,7 +137,7 @@ Status RedisStreams::XDel(const Slice& key, const std::vector& ids, si std::string unused; for (auto id : ids) { StreamDataKey stream_data_key(key, stream_meta.version(), id.Serialize()); - s = db_->Get(read_options, handles_[1], stream_data_key.Encode(), &unused); + s = db_->Get(default_read_options_, handles_[1], stream_data_key.Encode(), &unused); if (s.IsNotFound()) { --count; continue; @@ -155,7 +145,7 @@ Status RedisStreams::XDel(const Slice& key, const std::vector& ids, si return s; } } - s = DeleteStreamMessages(key, stream_meta, ids, read_options); + s = DeleteStreamMessages(key, stream_meta, ids, default_read_options_); if (!s.ok()) { return s; } @@ -167,9 +157,9 @@ Status RedisStreams::XDel(const Slice& key, const std::vector& ids, si stream_meta.set_max_deleted_entry_id(id); } if (id == stream_meta.first_id()) { - s = SetFirstID(key, stream_meta, read_options); + s = SetFirstID(key, stream_meta, default_read_options_); } else if (id == stream_meta.last_id()) { - s = SetLastID(key, stream_meta, read_options); + s = SetLastID(key, stream_meta, default_read_options_); } if (!s.ok()) { return s; @@ -311,6 +301,25 @@ Status RedisStreams::XRead(const StreamReadGroupReadArgs& args, std::vectorSetCapacity(storage_options.statistics_max_size); small_compaction_threshold_ = storage_options.small_compaction_threshold; diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index 799fa1a2be..6ea229e2d1 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -32,6 +32,7 @@ class RedisStreams : public Redis { Status XLen(const Slice& key, size_t& len); Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, std::vector& reserved_keys); + Status XInfo(const Slice& key, StreamInfoResult& result); //===--------------------------------------------------------------------===// // Common Commands diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 04c6c8d055..2cae7bb167 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -564,6 +564,10 @@ Status Storage::XRead(const StreamReadGroupReadArgs& args, std::vectorXRead(args, results, reserved_keys); } +Status Storage::XInfo(const Slice& key, StreamInfoResult &result) { + return streams_db_->XInfo(key, result); +} + // Keys Commands int32_t Storage::Expire(const Slice& key, int32_t ttl, std::map* type_status) { int32_t ret = 0; From f98aa3f00bc943559678125547dc2ce7f6b9e4df Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Mon, 25 Dec 2023 17:10:08 +0800 Subject: [PATCH 03/10] refactor: adjust stream code structure. 1. remove logic of tree_id generate, the tree will be implemented when supporting XGROUP commands. 2. move all the helper functions to redis_stream.h. --- include/pika_stream.h | 4 +- src/pika_stream.cc | 3 - src/storage/src/pika_stream_base.cc | 233 -------------------- src/storage/src/pika_stream_base.h | 163 -------------- src/storage/src/pika_stream_meta_value.h | 70 +++--- src/storage/src/pika_stream_types.h | 8 +- src/storage/src/redis_streams.cc | 261 +++++++++++++++++++---- src/storage/src/redis_streams.h | 152 +++++++++---- 8 files changed, 368 insertions(+), 526 deletions(-) delete mode 100644 src/storage/src/pika_stream_base.cc delete mode 100644 src/storage/src/pika_stream_base.h diff --git a/include/pika_stream.h b/include/pika_stream.h index ca7075141e..be5f386550 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -8,10 +8,8 @@ #include "include/pika_command.h" #include "include/pika_slot.h" -#include "storage/src/pika_stream_base.h" -// #include "include/pika_stream_meta_value.h" -// #include "include/pika_stream_types.h" #include "storage/storage.h" +#include "storage/src/redis_streams.h" /* * stream diff --git a/src/pika_stream.cc b/src/pika_stream.cc index e007890cb5..5e531b89d7 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -11,9 +11,6 @@ #include "include/pika_command.h" #include "include/pika_define.h" #include "include/pika_slot.h" -#include "storage/src/pika_stream_base.h" -#include "storage/src/pika_stream_meta_value.h" -#include "storage/src/pika_stream_types.h" #include "storage/storage.h" // s : rocksdb::Status diff --git a/src/storage/src/pika_stream_base.cc b/src/storage/src/pika_stream_base.cc deleted file mode 100644 index a06c08d6ca..0000000000 --- a/src/storage/src/pika_stream_base.cc +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "pika_stream_base.h" - -// #include "pika_command.h" -#include "pika_stream_meta_value.h" -#include "pika_stream_types.h" -#include "storage/storage.h" - -namespace storage { - -#define GET_NEXT_TREE_ID_AND_CHECK(slot, tid) \ - do { \ - auto &tid_gen = TreeIDGenerator::GetInstance(); \ - auto s = tid_gen.GetNextTreeID(slot, tid); \ - if (!s.ok()) { \ - return s; \ - } \ - } while (0) - -storage::Status TreeIDGenerator::GetNextTreeID(const rocksdb::DB *db, treeID &tid) { - // assert(slot); - // auto expected_id = kINVALID_TREE_ID; - - // // if tree_id_ is not initialized (equal to kINVALID_TREE_ID), try to fetch it from storage - // if (tree_id_.compare_exchange_strong(expected_id, START_TREE_ID)) { - // std::string value; - // storage::Status s = db->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); - // if (s.ok()) { - // treeID id; - // if (!StreamUtils::string2int32(value.c_str(), id)) { - // LOG(ERROR) << "Invalid tree id: " << value; - // return storage::Status::Corruption("Invalid tree id"); - // } - // tree_id_.store(id); - - // } else if (!s.IsNotFound()) { - // return s; - // } - // } - - // tree_id_.fetch_add(1); - // tid = tree_id_; - - // // insert tree id to storage - // std::string tree_id_str = std::to_string(tree_id_); - // int32_t res; - // return db->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); -} - -bool StreamUtils::StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given) { - char buf[128]; - if (var.size() > sizeof(buf) - 1) { - return false; - } - - memcpy(buf, var.data(), var.size()); - buf[var.size()] = '\0'; - - if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { - // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); - return false; - } - - if (seq_given != nullptr) { - *seq_given = true; - } - - if (buf[0] == '-' && buf[1] == '\0') { - id.ms = 0; - id.seq = 0; - return true; - } else if (buf[0] == '+' && buf[1] == '\0') { - id.ms = UINT64_MAX; - id.seq = UINT64_MAX; - return true; - } - - uint64_t ms; - uint64_t seq; - char *dot = strchr(buf, '-'); - if (dot) { - *dot = '\0'; - } - if (!StreamUtils::string2uint64(buf, ms)) { - return false; - }; - if (dot) { - size_t seqlen = strlen(dot + 1); - if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { - seq = 0; - *seq_given = false; - } else if (!StreamUtils::string2uint64(dot + 1, seq)) { - return false; - } - } else { - seq = missing_seq; - } - id.ms = ms; - id.seq = seq; - return true; -} - -bool StreamUtils::StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq) { - return StreamGenericParseID(var, id, missing_seq, false, nullptr); -} - -bool StreamUtils::StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given) { - return StreamGenericParseID(var, id, missing_seq, true, seq_given); -} - -bool StreamUtils::StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq) { - if (exclude != nullptr) { - *exclude = (var.size() > 1 && var[0] == '('); - } - if (exclude != nullptr && *exclude) { - return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); - } else { - return StreamParseID(var, id, missing_seq); - } -} - -bool StreamUtils::string2uint64(const char *s, uint64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - uint64_t tmp = strtoull(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int64(const char *s, int64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - int64_t tmp = std::strtoll(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int32(const char *s, int32_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - long tmp = strtol(s, &end, 10); - if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { - // Conversion either didn't consume the entire string, - // or overflow or underflow occurred - return false; - } - - value = static_cast(tmp); - return true; -} - -std::string StreamUtils::TreeID2Key(const treeID &tid) { - std::string key; - key.reserve(STERAM_TREE_PREFIX.size() + sizeof(tid)); - key.append(STERAM_TREE_PREFIX); - key.append(reinterpret_cast(&tid), sizeof(tid)); - return key; -} - -bool StreamUtils::SerializeMessage(const std::vector &field_values, std::string &message, int field_pos) { - assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); - assert(message.empty()); - // count the size of serizlized message - size_t size = 0; - for (int i = field_pos; i < field_values.size(); i++) { - size += field_values[i].size() + sizeof(size_t); - } - message.reserve(size); - - // serialize message - for (int i = field_pos; i < field_values.size(); i++) { - size_t len = field_values[i].size(); - message.append(reinterpret_cast(&len), sizeof(len)); - message.append(field_values[i]); - } - - return true; -} - -bool StreamUtils::DeserializeMessage(const std::string &message, std::vector &parsed_message) { - size_t pos = 0; - while (pos < message.size()) { - // Read the length of the next field value from the message - size_t len = *reinterpret_cast(&message[pos]); - pos += sizeof(size_t); - - // Check if the calculated end of the string is still within the message bounds - if (pos + len > message.size()) { - LOG(ERROR) << "Invalid message format, failed to parse message"; - return false; // Error: not enough data in the message string - } - - // Extract the field value and add it to the vector - parsed_message.push_back(message.substr(pos, len)); - pos += len; - } - - return true; -} - -uint64_t StreamUtils::GetCurrentTimeMs() { - return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - .count(); -} - -} // namespace storage \ No newline at end of file diff --git a/src/storage/src/pika_stream_base.h b/src/storage/src/pika_stream_base.h deleted file mode 100644 index 1d69dc9dee..0000000000 --- a/src/storage/src/pika_stream_base.h +++ /dev/null @@ -1,163 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#pragma once - -#include -#include "include/storage/storage.h" -#include "pika_stream_meta_value.h" -#include "pika_stream_types.h" -#include "rocksdb/db.h" -#include "rocksdb/slice.h" - -namespace storage { - -// each abstracted tree has a prefix, -// prefix + treeID will be the key of the hash to store the tree, -// notice: we need to ban the use of this prefix when using "HSET". -static const std::string STERAM_TREE_PREFIX = "STR_TREE"; - -// I need to find a place to store the last generated tree id, -// so I stored it as a field-value pair in the hash storing stream meta. -// the field is a fixed string, and the value is the last generated tree id. -// notice: we need to ban the use of this key when using stream, to avoid conflict with stream's key. -static const std::string STREAM_LAST_GENERATED_TREE_ID_FIELD = "STREAM"; - -// the max number of each delete operation in XTRIM command，to avoid too much memory usage. -// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / -// kDEFAULT_TRIM_BATCH_SIZE) times -const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; -struct StreamAddTrimArgs { - // XADD options - streamID id; - bool id_given{false}; - bool seq_given{false}; - bool no_mkstream{false}; - - // XADD + XTRIM common options - StreamTrimStrategy trim_strategy{TRIM_STRATEGY_NONE}; - int trim_strategy_arg_idx{0}; - - // TRIM_STRATEGY_MAXLEN options - uint64_t maxlen{0}; - streamID minid; -}; - -struct StreamReadGroupReadArgs { - // XREAD + XREADGROUP common options - std::vector keys; - std::vector unparsed_ids; - int32_t count{INT32_MAX}; // The limit of read, in redis this is uint64_t, but PKHScanRange only support int32_t - uint64_t block{0}; // 0 means no block - - // XREADGROUP options - std::string group_name; - std::string consumer_name; - bool noack_{false}; -}; - -struct StreamScanArgs { - streamID start_sid; - streamID end_sid; - size_t limit{INT32_MAX}; - bool start_ex{false}; // exclude first message - bool end_ex{false}; // exclude last message - bool is_reverse{false}; // scan in reverse order -}; - -struct StreamInfoResult { - uint64_t length{0}; - std::string last_id_str; - std::string max_deleted_entry_id_str; - uint64_t entries_added{0}; - std::string first_id_str; -}; - -// get next tree id thread safe -class TreeIDGenerator { - private: - TreeIDGenerator() = default; - void operator=(const TreeIDGenerator &) = delete; - - public: - ~TreeIDGenerator() = default; - - // work in singeletone mode - static TreeIDGenerator &GetInstance() { - static TreeIDGenerator instance; - return instance; - } - - storage::Status GetNextTreeID(const rocksdb::DB *db, treeID &tid); - - private: - static const treeID START_TREE_ID = 0; - std::atomic tree_id_{kINVALID_TREE_ID}; -}; - -// Helper function of stream command. -// Should be reconstructed when transfer to another command framework. -// any function that has Reply in its name will reply to the client if error occurs. -class StreamCmdBase { - public: - private: - StreamCmdBase(); - ~StreamCmdBase(); -}; - -class StreamUtils { - public: - StreamUtils() = default; - ~StreamUtils() = default; - - static bool string2uint64(const char *s, uint64_t &value); - static bool string2int64(const char *s, int64_t &value); - static bool string2int32(const char *s, int32_t &value); - static std::string TreeID2Key(const treeID &tid); - - static uint64_t GetCurrentTimeMs(); - - // serialize the message to a string. - // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} - static bool SerializeMessage(const std::vector &field_values, std::string &serialized_message, - int field_pos); - - // deserialize the message from a string with the format of SerializeMessage. - static bool DeserializeMessage(const std::string &message, std::vector &parsed_message); - - // Parse a stream ID in the format given by clients to Pika, that is - // -, and converts it into a streamID structure. The ID may be in incomplete - // form, just stating the milliseconds time part of the stream. In such a case - // the missing part is set according to the value of 'missing_seq' parameter. - // - // The IDs "-" and "+" specify respectively the minimum and maximum IDs - // that can be represented. If 'strict' is set to 1, "-" and "+" will be - // treated as an invalid ID. - // - // The ID form -* specifies a millisconds-only ID, leaving the sequence part - // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this - // form is accepted and the argument is set to 0 unless the sequence part is - // specified. - static bool StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 0, to be used when - and + are acceptable IDs. - static bool StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 1, to be used when we want to return an error if the special IDs + or - - // are provided. - static bool StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given); - - // Helper for parsing a stream ID that is a range query interval. When the - // exclude argument is NULL, streamParseID() is called and the interval - // is treated as close (inclusive). Otherwise, the exclude argument is set if - // the interval is open (the "(" prefix) and streamParseStrictID() is - // called in that case. - static bool StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq); -}; - -} // namespace storage diff --git a/src/storage/src/pika_stream_meta_value.h b/src/storage/src/pika_stream_meta_value.h index ecbc6a0958..448d4b6710 100644 --- a/src/storage/src/pika_stream_meta_value.h +++ b/src/storage/src/pika_stream_meta_value.h @@ -14,7 +14,7 @@ namespace storage { static const size_t kDefaultStreamValueLength = - sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t) + sizeof(int32_t); + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t) + sizeof(int32_t); class StreamMetaValue { public: explicit StreamMetaValue() = default; @@ -39,7 +39,7 @@ class StreamMetaValue { // Encode each member into the string EncodeFixed64(dst, groups_id_); - dst += sizeof(treeID); + dst += sizeof(tree_id_t); EncodeFixed64(dst, entries_added_); dst += sizeof(size_t); @@ -76,7 +76,7 @@ class StreamMetaValue { } char* pos = &value_[0]; groups_id_ = DecodeFixed32(pos); - pos += sizeof(treeID); + pos += sizeof(tree_id_t); entries_added_ = DecodeFixed64(pos); pos += sizeof(size_t); @@ -104,7 +104,7 @@ class StreamMetaValue { int32_t version() const { return version_; } - treeID groups_id() const { return groups_id_; } + tree_id_t groups_id() const { return groups_id_; } size_t entries_added() const { return entries_added_; } @@ -128,45 +128,45 @@ class StreamMetaValue { std::to_string(length_) + std::string(", version: ") + std::to_string(version_); } - void set_groups_id(treeID groups_id) { + void set_groups_id(tree_id_t groups_id) { assert(value_.size() == kDefaultStreamValueLength); groups_id_ = groups_id; char* dst = const_cast(value_.data()); - memcpy(dst, &groups_id_, sizeof(treeID)); + memcpy(dst, &groups_id_, sizeof(tree_id_t)); } void set_entries_added(uint64_t entries_added) { assert(value_.size() == kDefaultStreamValueLength); entries_added_ = entries_added; - char* dst = const_cast(value_.data()) + sizeof(treeID); + char* dst = const_cast(value_.data()) + sizeof(tree_id_t); memcpy(dst, &entries_added_, sizeof(uint64_t)); } void set_first_id(streamID first_id) { assert(value_.size() == kDefaultStreamValueLength); first_id_ = first_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t); + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t); memcpy(dst, &first_id_, sizeof(uint64_t)); } void set_last_id(streamID last_id) { assert(value_.size() == kDefaultStreamValueLength); last_id_ = last_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + sizeof(streamID); + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + sizeof(streamID); memcpy(dst, &last_id_, sizeof(streamID)); } void set_max_deleted_entry_id(streamID max_deleted_entry_id) { assert(value_.size() == kDefaultStreamValueLength); max_deleted_entry_id_ = max_deleted_entry_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 2 * sizeof(streamID); + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 2 * sizeof(streamID); memcpy(dst, &max_deleted_entry_id_, sizeof(streamID)); } void set_length(size_t length) { assert(value_.size() == kDefaultStreamValueLength); length_ = length; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID); + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID); memcpy(dst, &length_, sizeof(size_t)); } @@ -174,12 +174,12 @@ class StreamMetaValue { assert(value_.size() == kDefaultStreamValueLength); version_ = version; char* dst = - const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(size_t); + const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(size_t); EncodeFixed32(dst, version_); } private: - treeID groups_id_ = kINVALID_TREE_ID; + tree_id_t groups_id_ = kINVALID_TREE_ID; size_t entries_added_{0}; streamID first_id_; streamID last_id_; @@ -201,7 +201,7 @@ class ParsedStreamMetaValue { } char* pos = const_cast(value.data()); groups_id_ = DecodeFixed32(pos); - pos += sizeof(treeID); + pos += sizeof(tree_id_t); entries_added_ = DecodeFixed64(pos); pos += sizeof(size_t); @@ -229,7 +229,7 @@ class ParsedStreamMetaValue { int32_t version() const { return version_; } - treeID groups_id() const { return groups_id_; } + tree_id_t groups_id() const { return groups_id_; } size_t entries_added() const { return entries_added_; } @@ -250,7 +250,7 @@ class ParsedStreamMetaValue { } private: - treeID groups_id_ = kINVALID_TREE_ID; + tree_id_t groups_id_ = kINVALID_TREE_ID; size_t entries_added_{0}; streamID first_id_; streamID last_id_; @@ -259,13 +259,13 @@ class ParsedStreamMetaValue { int32_t version_{0}; }; -static const size_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(treeID); +static const size_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(tree_id_t); class StreamCGroupMetaValue { public: explicit StreamCGroupMetaValue() = default; // tid and consumers should be set at beginning - void Init(treeID tid, treeID consumers) { + void Init(tree_id_t tid, tree_id_t consumers) { pel_ = tid; consumers_ = consumers; size_t needed = kDefaultStreamCGroupValueLength; @@ -282,9 +282,9 @@ class StreamCGroupMetaValue { dst += sizeof(uint64_t); memcpy(dst, &entries_read_, sizeof(uint64_t)); dst += sizeof(uint64_t); - memcpy(dst, &pel_, sizeof(treeID)); - dst += sizeof(treeID); - memcpy(dst, &consumers_, sizeof(treeID)); + memcpy(dst, &pel_, sizeof(tree_id_t)); + dst += sizeof(tree_id_t); + memcpy(dst, &consumers_, sizeof(tree_id_t)); } void ParseFrom(std::string& value) { @@ -300,9 +300,9 @@ class StreamCGroupMetaValue { pos += sizeof(streamID); memcpy(&entries_read_, pos, sizeof(uint64_t)); pos += sizeof(uint64_t); - memcpy(&pel_, pos, sizeof(treeID)); - pos += sizeof(treeID); - memcpy(&consumers_, pos, sizeof(treeID)); + memcpy(&pel_, pos, sizeof(tree_id_t)); + pos += sizeof(tree_id_t); + memcpy(&consumers_, pos, sizeof(tree_id_t)); } } @@ -325,9 +325,9 @@ class StreamCGroupMetaValue { } // pel and consumers were set in constructor, can't be modified - treeID pel() { return pel_; } + tree_id_t pel() { return pel_; } - treeID consumers() { return consumers_; } + tree_id_t consumers() { return consumers_; } std::string& value() { return value_; } @@ -336,11 +336,11 @@ class StreamCGroupMetaValue { streamID last_id_; uint64_t entries_read_ = 0; - treeID pel_ = 0; - treeID consumers_ = 0; + tree_id_t pel_ = 0; + tree_id_t consumers_ = 0; }; -static const size_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(treeID); +static const size_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(tree_id_t); class StreamConsumerMetaValue { public: // pel must been set at beginning @@ -360,11 +360,11 @@ class StreamConsumerMetaValue { pos += sizeof(stream_ms_t); memcpy(&active_time_, pos, sizeof(stream_ms_t)); pos += sizeof(stream_ms_t); - memcpy(&pel_, pos, sizeof(treeID)); + memcpy(&pel_, pos, sizeof(tree_id_t)); } } - void Init(treeID pel) { + void Init(tree_id_t pel) { pel_ = pel; assert(value_.size() == 0); if (value_.size() != 0) { @@ -379,7 +379,7 @@ class StreamConsumerMetaValue { dst += sizeof(stream_ms_t); memcpy(dst, &active_time_, sizeof(stream_ms_t)); dst += sizeof(stream_ms_t); - memcpy(dst, &pel_, sizeof(treeID)); + memcpy(dst, &pel_, sizeof(tree_id_t)); } stream_ms_t seen_time() { return seen_time_; } @@ -401,7 +401,7 @@ class StreamConsumerMetaValue { } // pel was set in constructor, can't be modified - treeID pel_tid() { return pel_; } + tree_id_t pel_tid() { return pel_; } std::string& value() { return value_; } @@ -410,10 +410,10 @@ class StreamConsumerMetaValue { stream_ms_t seen_time_ = 0; stream_ms_t active_time_ = 0; - treeID pel_ = 0; + tree_id_t pel_ = 0; }; -static const size_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(treeID); +static const size_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(tree_id_t); class StreamPelMeta { public: // consumer must been set at beginning diff --git a/src/storage/src/pika_stream_types.h b/src/storage/src/pika_stream_types.h index 86a08d03bf..db893e8237 100644 --- a/src/storage/src/pika_stream_types.h +++ b/src/storage/src/pika_stream_types.h @@ -56,12 +56,6 @@ using streamID = struct streamID { return value; } - void SerializeTo(std::string& dst) const { - dst.resize(sizeof(ms) + sizeof(seq)); - EncodeUint64InBigEndian(&dst[0], ms); - EncodeUint64InBigEndian(&dst[0] + sizeof(ms), seq); - } - std::string Serialize() const { std::string dst; dst.resize(sizeof(ms) + sizeof(seq)); @@ -86,7 +80,7 @@ static const streamID kSTREAMID_MIN = streamID(0, 0); enum StreamTrimStrategy { TRIM_STRATEGY_NONE, TRIM_STRATEGY_MAXLEN, TRIM_STRATEGY_MINID }; -using treeID = uint32_t; +using tree_id_t = uint32_t; using stream_ms_t = uint64_t; diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index 2baaa1c775..d88b49e5ae 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -4,7 +4,6 @@ #include #include #include -#include "pika_stream_base.h" #include "rocksdb/slice.h" #include "rocksdb/status.h" #include "src/base_data_key_format.h" @@ -55,10 +54,8 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag #ifdef DEBUG // check the serialized current id is larger than last_id - std::string serialized_last_id; - std::string current_id; - stream_meta.last_id().SerializeTo(serialized_last_id); - args.id.SerializeTo(current_id); + std::string serialized_last_id = stream_meta.last_id().Serialize(); + std::string current_id = args.id.Serialize(); assert(current_id > serialized_last_id); #endif @@ -385,20 +382,6 @@ Status RedisStreams::GetProperty(const std::string& property, uint64_t* out) { return Status::OK(); } -// Check if the key has prefix of STERAM_TREE_PREFIX. -// That means the key-value is a virtual tree node, not a stream meta. -bool IsVirtualTree(rocksdb::Slice key) { - if (key.size() < STERAM_TREE_PREFIX.size()) { - return false; - } - - if (memcmp(key.data(), STERAM_TREE_PREFIX.data(), STERAM_TREE_PREFIX.size()) == 0) { - return true; - } - - return false; -} - Status RedisStreams::ScanKeyNum(KeyInfo* key_info) { uint64_t keys = 0; uint64_t expires = 0; @@ -416,9 +399,6 @@ Status RedisStreams::ScanKeyNum(KeyInfo* key_info) { rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - if (IsVirtualTree(iter->key())) { - continue; - } ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); if (parsed_stream_meta_value.length() == 0) { invaild_keys++; @@ -443,9 +423,6 @@ Status RedisStreams::ScanKeys(const std::string& pattern, std::vectorNewIterator(iterator_options, handles_[0]); for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { - if (IsVirtualTree(iter->key())) { - continue; - } ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); if (parsed_stream_meta_value.length() != 0) { key = iter->key().ToString(); @@ -473,10 +450,6 @@ Status RedisStreams::PKPatternMatchDel(const std::string& pattern, int32_t* ret) rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); iter->SeekToFirst(); while (iter->Valid()) { - if (IsVirtualTree(iter->key())) { - iter->Next(); - continue; - } key = iter->key().ToString(); meta_value = iter->value().ToString(); StreamMetaValue stream_meta_value; @@ -545,10 +518,6 @@ bool RedisStreams::Scan(const std::string& start_key, const std::string& pattern it->Seek(start_key); while (it->Valid() && (*count) > 0) { - if (IsVirtualTree(it->key())) { - it->Next(); - continue; - } ParsedStreamMetaValue parsed_stream_meta_value(it->value()); if (parsed_stream_meta_value.length() == 0) { it->Next(); @@ -839,13 +808,13 @@ Status RedisStreams::TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta } Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Slice& id_start, - const std::string& id_end, const Slice& pattern, int32_t limit, + const std::string& id_end, const Slice& pattern, size_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options) { next_id.clear(); id_messages.clear(); - int64_t remain = limit; + auto remain = limit; std::string meta_value; bool start_no_limit = id_start.compare("") == 0; @@ -884,13 +853,13 @@ Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Sl } Status RedisStreams::ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, - const std::string& id_end, const Slice& pattern, int32_t limit, + const std::string& id_end, const Slice& pattern, size_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options) { next_id.clear(); id_messages.clear(); - int64_t remain = limit; + auto remain = limit; std::string meta_value; bool start_no_limit = id_start.compare("") == 0; @@ -950,4 +919,222 @@ Status RedisStreams::DeleteStreamMessages(const rocksdb::Slice& key, const Strea } return db_->Write(default_write_options_, &batch); } + +inline Status RedisStreams::SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, + rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, true, read_options); +} + +inline Status RedisStreams::SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, + rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, false, read_options); +} + +inline Status RedisStreams::SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, + rocksdb::ReadOptions& read_options) { + if (stream_meta.length() == 0) { + stream_meta.set_first_id(kSTREAMID_MIN); + return Status::OK(); + } + + std::vector id_messages; + std::string next_field; + + storage::Status s; + if (is_set_first) { + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MIN, kSTREAMID_MAX, 1); + s = ScanStream(option, id_messages, next_field, read_options); + } else { + bool is_reverse = true; + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); + s = ScanStream(option, id_messages, next_field, read_options); + } + (void)next_field; + + if (!s.ok() && !s.IsNotFound()) { + LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); + return Status::Corruption("Internal error: scan stream failed: " + s.ToString()); + } + + if (id_messages.empty()) { + LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; + return Status::Corruption("Internal error: no messages found but stream length is not 0"); + } + + streamID id; + id.DeserializeFrom(id_messages[0].field); + stream_meta.set_first_id(id); + return Status::OK(); +} + +bool StreamUtils::StreamGenericParseID(const std::string& var, streamID& id, uint64_t missing_seq, bool strict, + bool* seq_given) { + char buf[128]; + if (var.size() > sizeof(buf) - 1) { + return false; + } + + memcpy(buf, var.data(), var.size()); + buf[var.size()] = '\0'; + + if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { + // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); + return false; + } + + if (seq_given != nullptr) { + *seq_given = true; + } + + if (buf[0] == '-' && buf[1] == '\0') { + id.ms = 0; + id.seq = 0; + return true; + } else if (buf[0] == '+' && buf[1] == '\0') { + id.ms = UINT64_MAX; + id.seq = UINT64_MAX; + return true; + } + + uint64_t ms; + uint64_t seq; + char* dot = strchr(buf, '-'); + if (dot) { + *dot = '\0'; + } + if (!StreamUtils::string2uint64(buf, ms)) { + return false; + }; + if (dot) { + size_t seqlen = strlen(dot + 1); + if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { + seq = 0; + *seq_given = false; + } else if (!StreamUtils::string2uint64(dot + 1, seq)) { + return false; + } + } else { + seq = missing_seq; + } + id.ms = ms; + id.seq = seq; + return true; +} + +bool StreamUtils::StreamParseID(const std::string& var, streamID& id, uint64_t missing_seq) { + return StreamGenericParseID(var, id, missing_seq, false, nullptr); +} + +bool StreamUtils::StreamParseStrictID(const std::string& var, streamID& id, uint64_t missing_seq, bool* seq_given) { + return StreamGenericParseID(var, id, missing_seq, true, seq_given); +} + +bool StreamUtils::StreamParseIntervalId(const std::string& var, streamID& id, bool* exclude, uint64_t missing_seq) { + if (exclude != nullptr) { + *exclude = (var.size() > 1 && var[0] == '('); + } + if (exclude != nullptr && *exclude) { + return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); + } else { + return StreamParseID(var, id, missing_seq); + } +} + +bool StreamUtils::string2uint64(const char* s, uint64_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + uint64_t tmp = strtoull(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int64(const char* s, int64_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + int64_t tmp = std::strtoll(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int32(const char* s, int32_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + long tmp = strtol(s, &end, 10); + if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { + // Conversion either didn't consume the entire string, + // or overflow or underflow occurred + return false; + } + + value = static_cast(tmp); + return true; +} + +bool StreamUtils::SerializeMessage(const std::vector& field_values, std::string& message, int field_pos) { + assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); + assert(message.empty()); + // count the size of serizlized message + size_t size = 0; + for (int i = field_pos; i < field_values.size(); i++) { + size += field_values[i].size() + sizeof(size_t); + } + message.reserve(size); + + // serialize message + for (int i = field_pos; i < field_values.size(); i++) { + size_t len = field_values[i].size(); + message.append(reinterpret_cast(&len), sizeof(len)); + message.append(field_values[i]); + } + + return true; +} + +bool StreamUtils::DeserializeMessage(const std::string& message, std::vector& parsed_message) { + size_t pos = 0; + while (pos < message.size()) { + // Read the length of the next field value from the message + size_t len = *reinterpret_cast(&message[pos]); + pos += sizeof(size_t); + + // Check if the calculated end of the string is still within the message bounds + if (pos + len > message.size()) { + LOG(ERROR) << "Invalid message format, failed to parse message"; + return false; // Error: not enough data in the message string + } + + // Extract the field value and add it to the vector + parsed_message.push_back(message.substr(pos, len)); + pos += len; + } + + return true; +} + +uint64_t StreamUtils::GetCurrentTimeMs() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); +} }; // namespace storage \ No newline at end of file diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index 6ea229e2d1..4d8806b9f5 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -5,7 +5,6 @@ #include #include #include "include/storage/storage.h" -#include "pika_stream_base.h" #include "pika_stream_meta_value.h" #include "pika_stream_types.h" #include "rocksdb/options.h" @@ -16,6 +15,108 @@ namespace storage { +// the max number of each delete operation in XTRIM command，to avoid too much memory usage. +// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / +// kDEFAULT_TRIM_BATCH_SIZE) times +const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; +struct StreamAddTrimArgs { + // XADD options + streamID id; + bool id_given{false}; + bool seq_given{false}; + bool no_mkstream{false}; + + // XADD + XTRIM common options + StreamTrimStrategy trim_strategy{TRIM_STRATEGY_NONE}; + int trim_strategy_arg_idx{0}; + + // TRIM_STRATEGY_MAXLEN options + uint64_t maxlen{0}; + streamID minid; +}; + +struct StreamReadGroupReadArgs { + // XREAD + XREADGROUP common options + std::vector keys; + std::vector unparsed_ids; + int32_t count{INT32_MAX}; // The limit of read, in redis this is uint64_t, but PKHScanRange only support int32_t + uint64_t block{0}; // 0 means no block + + // XREADGROUP options + std::string group_name; + std::string consumer_name; + bool noack_{false}; +}; + +struct StreamScanArgs { + streamID start_sid; + streamID end_sid; + size_t limit{INT32_MAX}; + bool start_ex{false}; // exclude first message + bool end_ex{false}; // exclude last message + bool is_reverse{false}; // scan in reverse order +}; + +struct StreamInfoResult { + uint64_t length{0}; + std::string last_id_str; + std::string max_deleted_entry_id_str; + uint64_t entries_added{0}; + std::string first_id_str; +}; + +class StreamUtils { + public: + StreamUtils() = default; + ~StreamUtils() = default; + + static bool string2uint64(const char* s, uint64_t& value); + static bool string2int64(const char* s, int64_t& value); + static bool string2int32(const char* s, int32_t& value); + + static uint64_t GetCurrentTimeMs(); + + // serialize the message to a string. + // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} + static bool SerializeMessage(const std::vector& field_values, std::string& serialized_message, + int field_pos); + + // deserialize the message from a string with the format of SerializeMessage. + static bool DeserializeMessage(const std::string& message, std::vector& parsed_message); + + // Parse a stream ID in the format given by clients to Pika, that is + // -, and converts it into a streamID structure. The ID may be in incomplete + // form, just stating the milliseconds time part of the stream. In such a case + // the missing part is set according to the value of 'missing_seq' parameter. + // + // The IDs "-" and "+" specify respectively the minimum and maximum IDs + // that can be represented. If 'strict' is set to 1, "-" and "+" will be + // treated as an invalid ID. + // + // The ID form -* specifies a millisconds-only ID, leaving the sequence part + // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this + // form is accepted and the argument is set to 0 unless the sequence part is + // specified. + static bool StreamGenericParseID(const std::string& var, streamID& id, uint64_t missing_seq, bool strict, + bool* seq_given); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 0, to be used when - and + are acceptable IDs. + static bool StreamParseID(const std::string& var, streamID& id, uint64_t missing_seq); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 1, to be used when we want to return an error if the special IDs + or - + // are provided. + static bool StreamParseStrictID(const std::string& var, streamID& id, uint64_t missing_seq, bool* seq_given); + + // Helper for parsing a stream ID that is a range query interval. When the + // exclude argument is NULL, streamParseID() is called and the interval + // is treated as close (inclusive). Otherwise, the exclude argument is set if + // the interval is open (the "(" prefix) and streamParseStrictID() is + // called in that case. + static bool StreamParseIntervalId(const std::string& var, streamID& id, bool* exclude, uint64_t missing_seq); +}; + class RedisStreams : public Redis { public: RedisStreams(Storage* const s, const DataType& type) : Redis(s, type) {} @@ -107,10 +208,10 @@ class RedisStreams : public Redis { Status GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args); Status ScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, - const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, + const Slice& pattern, size_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options); Status ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, - const Slice& pattern, int32_t limit, std::vector& id_values, std::string& next_id, + const Slice& pattern, size_t limit, std::vector& id_values, std::string& next_id, rocksdb::ReadOptions& read_options); struct TrimRet { @@ -128,50 +229,11 @@ class RedisStreams : public Redis { Status TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); - inline Status SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, - rocksdb::ReadOptions& read_options) { - return SetFirstOrLastID(key, stream_meta, true, read_options); - } + inline Status SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options); - inline Status SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options) { - return SetFirstOrLastID(key, stream_meta, false, read_options); - } + inline Status SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options); inline Status SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, - rocksdb::ReadOptions& read_options) { - if (stream_meta.length() == 0) { - stream_meta.set_first_id(kSTREAMID_MIN); - return Status::OK(); - } - - std::vector id_messages; - std::string next_field; - - storage::Status s; - if (is_set_first) { - ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MIN, kSTREAMID_MAX, 1); - s = ScanStream(option, id_messages, next_field, read_options); - } else { - bool is_reverse = true; - ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); - s = ScanStream(option, id_messages, next_field, read_options); - } - (void)next_field; - - if (!s.ok() && !s.IsNotFound()) { - LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); - return Status::Corruption("Internal error: scan stream failed: " + s.ToString()); - } - - if (id_messages.empty()) { - LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; - return Status::Corruption("Internal error: no messages found but stream length is not 0"); - } - - streamID id; - id.DeserializeFrom(id_messages[0].field); - stream_meta.set_first_id(id); - return Status::OK(); - } + rocksdb::ReadOptions& read_options); }; } // namespace storage \ No newline at end of file From 373b42bf1e9b1b61efe0f92b34c9a77686fc71f8 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Mon, 25 Dec 2023 19:42:18 +0800 Subject: [PATCH 04/10] feat: support some basic function of storage layer. --- src/storage/include/storage/storage.h | 1 - src/storage/src/backupable.cc | 3 ++- src/storage/src/redis_streams.cc | 3 --- src/storage/src/storage.cc | 13 +++++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index 8a72ab48db..d1249d3200 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -7,7 +7,6 @@ #define INCLUDE_STORAGE_STORAGE_H_ #include -#include #include #include #include diff --git a/src/storage/src/backupable.cc b/src/storage/src/backupable.cc index 3de6ae2ac6..9ff3ef6e07 100644 --- a/src/storage/src/backupable.cc +++ b/src/storage/src/backupable.cc @@ -7,6 +7,7 @@ #include #include "storage/backupable.h" +#include "storage/storage.h" namespace storage { @@ -36,7 +37,7 @@ Status BackupEngine::Open(storage::Storage* storage, std::shared_ptrGetDBByType(type))) { s = Status::Corruption("Error db type"); diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index d88b49e5ae..b71633ff0e 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -326,7 +326,6 @@ Status RedisStreams::Open(const StorageOptions& storage_options, const std::stri if (s.ok()) { // create column family rocksdb::ColumnFamilyHandle* cf; - // FIXME: Dose stream data need a comparater ? s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "data_cf", &cf); if (!s.ok()) { return s; @@ -484,8 +483,6 @@ Status RedisStreams::PKPatternMatchDel(const std::string& pattern, int32_t* ret) } Status RedisStreams::Del(const Slice& key) { - // FIXME: Check the prefix of key - // stream TODO: Delete all the cgroup and consumers std::string meta_value; ScopeRecordLock l(lock_mgr_, key); Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value); diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 3bf5093e68..2879b096d8 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -1763,6 +1763,7 @@ Status Storage::GetUsage(const std::string& property, std::mapGetProperty(property, &out); result += out; } + if (db_type == ALL_DB || db_type == STREAMS_DB) { + streams_db_->GetProperty(property, &out); + result += out; + } return result; } @@ -1827,6 +1832,8 @@ rocksdb::DB* Storage::GetDBByType(const std::string& type) { return sets_db_->GetDB(); } else if (type == ZSETS_DB) { return zsets_db_->GetDB(); + } else if (type == STREAMS_DB) { + return streams_db_->GetDB(); } else { return nullptr; } @@ -1865,6 +1872,12 @@ Status Storage::SetOptions(const OptionType& option_type, const std::string& db_ return s; } } + if (db_type == ALL_DB || db_type == STREAMS_DB) { + s = streams_db_->SetOptions(option_type, options); + if (!s.ok()) { + return s; + } + } return s; } From bda2b008378c0f6e1c46d35a2c7f282d70f03443 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Mon, 25 Dec 2023 20:16:51 +0800 Subject: [PATCH 05/10] fix: Add licence. --- include/pika_stream.h | 8 ++++---- src/pika_stream.cc | 8 ++++---- src/storage/src/pika_stream_meta_value.h | 10 +++++----- src/storage/src/pika_stream_types.h | 10 +++++----- src/storage/src/redis_streams.cc | 7 ++++++- src/storage/src/redis_streams.h | 7 ++++++- 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/include/pika_stream.h b/include/pika_stream.h index be5f386550..b9b29131b8 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -1,7 +1,7 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #ifndef PIKA_STREAM_H_ #define PIKA_STREAM_H_ diff --git a/src/pika_stream.cc b/src/pika_stream.cc index 5e531b89d7..f1de29c5a7 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -1,7 +1,7 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #include "include/pika_stream.h" #include diff --git a/src/storage/src/pika_stream_meta_value.h b/src/storage/src/pika_stream_meta_value.h index 448d4b6710..e779dfdcd1 100644 --- a/src/storage/src/pika_stream_meta_value.h +++ b/src/storage/src/pika_stream_meta_value.h @@ -1,7 +1,7 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #pragma once @@ -488,4 +488,4 @@ class StreamPelMeta { std::string consumer_; }; -} // namespace storage \ No newline at end of file +} // namespace storage diff --git a/src/storage/src/pika_stream_types.h b/src/storage/src/pika_stream_types.h index db893e8237..69c4733334 100644 --- a/src/storage/src/pika_stream_types.h +++ b/src/storage/src/pika_stream_types.h @@ -1,7 +1,7 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #pragma once @@ -84,4 +84,4 @@ using tree_id_t = uint32_t; using stream_ms_t = uint64_t; -} // namespace storage \ No newline at end of file +} // namespace storage diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index b71633ff0e..66e0a3588a 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -1,3 +1,8 @@ +// Copyright (c) 2017-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + #include "src/redis_streams.h" #include #include @@ -1134,4 +1139,4 @@ uint64_t StreamUtils::GetCurrentTimeMs() { return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) .count(); } -}; // namespace storage \ No newline at end of file +}; // namespace storage diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index 4d8806b9f5..ff0ee7b064 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -1,3 +1,8 @@ +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + #pragma once #include @@ -236,4 +241,4 @@ class RedisStreams : public Redis { inline Status SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, rocksdb::ReadOptions& read_options); }; -} // namespace storage \ No newline at end of file +} // namespace storage From 9a3dabf2345db011d691e996ed154b5324067595 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Fri, 29 Dec 2023 10:47:38 +0800 Subject: [PATCH 06/10] fix: compile problem in macos. --- src/storage/src/redis_streams.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index 66e0a3588a..f42ba8c490 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -219,7 +219,7 @@ Status RedisStreams::XRevrange(const Slice& key, const StreamScanArgs& args, std return s; } -Status RedisStreams::XLen(const Slice& key, uint64_t& len) { +Status RedisStreams::XLen(const Slice& key, size_t& len) { rocksdb::ReadOptions read_options; const rocksdb::Snapshot* snapshot; ScopeSnapshot ss(db_, &snapshot); From ce0f1979980e032920b891d727b1164420476c30 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Fri, 29 Dec 2023 16:40:42 +0800 Subject: [PATCH 07/10] fix: length of stream now using int32_t. --- src/pika_stream.cc | 18 ++--- src/storage/include/storage/storage.h | 6 +- src/storage/src/pika_stream_meta_value.h | 86 +++++++++++++----------- src/storage/src/redis_streams.cc | 34 +++++----- src/storage/src/redis_streams.h | 22 +++--- src/storage/src/storage.cc | 6 +- 6 files changed, 91 insertions(+), 81 deletions(-) diff --git a/src/pika_stream.cc b/src/pika_stream.cc index f1de29c5a7..681c870530 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -6,8 +6,10 @@ #include "include/pika_stream.h" #include #include +#include #include +#include "glog/logging.h" #include "include/pika_command.h" #include "include/pika_define.h" #include "include/pika_slot.h" @@ -270,9 +272,7 @@ void XRangeCmd::DoInitial() { return; } if (argv_.size() == 6) { - // pika's PKHScanRange() only sopport max count of INT32_MAX - // but redis supports max count of UINT64_MAX - if (!storage::StreamUtils::string2uint64(argv_[5].c_str(), args_.limit)) { + if (!storage::StreamUtils::string2int32(argv_[5].c_str(), args_.limit)) { res_.SetRes(CmdRes::kInvalidParameter, "COUNT should be a integer greater than 0 and not bigger than INT32_MAX"); return; } @@ -328,7 +328,7 @@ void XDelCmd::DoInitial() { } void XDelCmd::Do(std::shared_ptr slot) { - size_t count{0}; + int32_t count{0}; auto s = slot->db()->XDel(key_, ids_, count); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -338,7 +338,7 @@ void XDelCmd::Do(std::shared_ptr slot) { return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); } - res_.AppendInteger(static_cast(count)); + res_.AppendInteger(count); } void XLenCmd::DoInitial() { @@ -350,7 +350,7 @@ void XLenCmd::DoInitial() { } void XLenCmd::Do(std::shared_ptr slot) { - size_t len{0}; + int32_t len{0}; auto s = slot->db()->XLen(key_, len); if (s.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); @@ -364,7 +364,7 @@ void XLenCmd::Do(std::shared_ptr slot) { return res_.SetRes(CmdRes::kErrOther, "stream's length is larger than INT_MAX"); } - res_.AppendInteger(static_cast(len)); + res_.AppendInteger(len); return; } @@ -423,7 +423,7 @@ void XTrimCmd::DoInitial() { } void XTrimCmd::Do(std::shared_ptr slot) { - size_t count{0}; + int32_t count{0}; auto s = slot->db()->XTrim(key_, args_, count); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); @@ -434,7 +434,7 @@ void XTrimCmd::Do(std::shared_ptr slot) { return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); } - res_.AppendInteger(static_cast(count)); + res_.AppendInteger(count); return; } diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index d1249d3200..a4751a8812 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -932,11 +932,11 @@ class Storage { std::vector* score_members, int64_t* next_cursor); Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); - Status XDel(const Slice& key, const std::vector& ids, size_t& ret); - Status XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count); + Status XDel(const Slice& key, const std::vector& ids, int32_t& ret); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count); Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); - Status XLen(const Slice& key, uint64_t& len); + Status XLen(const Slice& key, int32_t& len); Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, std::vector& reserved_keys); Status XInfo(const Slice& key, StreamInfoResult &result); diff --git a/src/storage/src/pika_stream_meta_value.h b/src/storage/src/pika_stream_meta_value.h index e779dfdcd1..1a43f9dcbe 100644 --- a/src/storage/src/pika_stream_meta_value.h +++ b/src/storage/src/pika_stream_meta_value.h @@ -13,8 +13,8 @@ namespace storage { -static const size_t kDefaultStreamValueLength = - sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t) + sizeof(int32_t); +static const uint64_t kDefaultStreamValueLength = + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(int32_t) + sizeof(int32_t); class StreamMetaValue { public: explicit StreamMetaValue() = default; @@ -32,7 +32,7 @@ class StreamMetaValue { // not be seen by the new stream with the same key. ++version_; - size_t needed = kDefaultStreamValueLength; + uint64_t needed = kDefaultStreamValueLength; value_.resize(needed); char* dst = &value_[0]; @@ -42,7 +42,7 @@ class StreamMetaValue { dst += sizeof(tree_id_t); EncodeFixed64(dst, entries_added_); - dst += sizeof(size_t); + dst += sizeof(uint64_t); EncodeFixed64(dst, first_id_.ms); dst += sizeof(uint64_t); @@ -59,8 +59,8 @@ class StreamMetaValue { EncodeFixed64(dst, max_deleted_entry_id_.seq); dst += sizeof(uint64_t); - EncodeFixed64(dst, length_); - dst += sizeof(int64_t); + EncodeFixed32(dst, length_); + dst += sizeof(length_); EncodeFixed32(dst, version_); } @@ -79,7 +79,7 @@ class StreamMetaValue { pos += sizeof(tree_id_t); entries_added_ = DecodeFixed64(pos); - pos += sizeof(size_t); + pos += sizeof(uint64_t); first_id_.ms = DecodeFixed64(pos); pos += sizeof(uint64_t); @@ -96,8 +96,8 @@ class StreamMetaValue { max_deleted_entry_id_.seq = DecodeFixed64(pos); pos += sizeof(uint64_t); - length_ = DecodeFixed64(pos); - pos += sizeof(size_t); + length_ = static_cast(DecodeFixed32(pos)); + pos += sizeof(length_); version_ = static_cast(DecodeFixed32(pos)); } @@ -106,9 +106,9 @@ class StreamMetaValue { tree_id_t groups_id() const { return groups_id_; } - size_t entries_added() const { return entries_added_; } + uint64_t entries_added() const { return entries_added_; } - void ModifyEntriesAdded(size_t delta) { set_entries_added(entries_added_ + delta); } + void ModifyEntriesAdded(uint64_t delta) { set_entries_added(entries_added_ + delta); } streamID first_id() const { return first_id_; } @@ -116,7 +116,7 @@ class StreamMetaValue { streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } - size_t length() const { return length_; } + int32_t length() const { return length_; } std::string& value() { return value_; } @@ -132,59 +132,65 @@ class StreamMetaValue { assert(value_.size() == kDefaultStreamValueLength); groups_id_ = groups_id; char* dst = const_cast(value_.data()); - memcpy(dst, &groups_id_, sizeof(tree_id_t)); + EncodeFixed32(dst, groups_id_); } void set_entries_added(uint64_t entries_added) { assert(value_.size() == kDefaultStreamValueLength); entries_added_ = entries_added; char* dst = const_cast(value_.data()) + sizeof(tree_id_t); - memcpy(dst, &entries_added_, sizeof(uint64_t)); + EncodeFixed64(dst, entries_added_); } void set_first_id(streamID first_id) { assert(value_.size() == kDefaultStreamValueLength); first_id_ = first_id; char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t); - memcpy(dst, &first_id_, sizeof(uint64_t)); + EncodeFixed64(dst, first_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, first_id_.seq); } void set_last_id(streamID last_id) { assert(value_.size() == kDefaultStreamValueLength); last_id_ = last_id; char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + sizeof(streamID); - memcpy(dst, &last_id_, sizeof(streamID)); + EncodeFixed64(dst, last_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, last_id_.seq); } void set_max_deleted_entry_id(streamID max_deleted_entry_id) { assert(value_.size() == kDefaultStreamValueLength); max_deleted_entry_id_ = max_deleted_entry_id; char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 2 * sizeof(streamID); - memcpy(dst, &max_deleted_entry_id_, sizeof(streamID)); + EncodeFixed64(dst, max_deleted_entry_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, max_deleted_entry_id_.seq); } - void set_length(size_t length) { + void set_length(int32_t length) { assert(value_.size() == kDefaultStreamValueLength); length_ = length; char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID); - memcpy(dst, &length_, sizeof(size_t)); + EncodeFixed32(dst, length_); } void set_version(int32_t version) { assert(value_.size() == kDefaultStreamValueLength); version_ = version; char* dst = - const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(size_t); + const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(length_); EncodeFixed32(dst, version_); } private: tree_id_t groups_id_ = kINVALID_TREE_ID; - size_t entries_added_{0}; + uint64_t entries_added_{0}; streamID first_id_; streamID last_id_; streamID max_deleted_entry_id_; - size_t length_{0}; // number of the messages in the stream + int32_t length_{0}; // number of the messages in the stream int32_t version_{0}; std::string value_{}; @@ -204,7 +210,7 @@ class ParsedStreamMetaValue { pos += sizeof(tree_id_t); entries_added_ = DecodeFixed64(pos); - pos += sizeof(size_t); + pos += sizeof(uint64_t); first_id_.ms = DecodeFixed64(pos); pos += sizeof(uint64_t); @@ -221,8 +227,8 @@ class ParsedStreamMetaValue { max_deleted_entry_id_.seq = DecodeFixed64(pos); pos += sizeof(uint64_t); - length_ = DecodeFixed64(pos); - pos += sizeof(size_t); + length_ = static_cast(DecodeFixed32(pos)); + pos += sizeof(length_); version_ = static_cast(DecodeFixed32(pos)); } @@ -231,7 +237,7 @@ class ParsedStreamMetaValue { tree_id_t groups_id() const { return groups_id_; } - size_t entries_added() const { return entries_added_; } + uint64_t entries_added() const { return entries_added_; } streamID first_id() const { return first_id_; } @@ -239,7 +245,7 @@ class ParsedStreamMetaValue { streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } - size_t length() const { return length_; } + int32_t length() const { return length_; } std::string ToString() { return "stream_meta: " + std::string("groups_id: ") + std::to_string(groups_id_) + @@ -251,15 +257,15 @@ class ParsedStreamMetaValue { private: tree_id_t groups_id_ = kINVALID_TREE_ID; - size_t entries_added_{0}; + uint64_t entries_added_{0}; streamID first_id_; streamID last_id_; streamID max_deleted_entry_id_; - size_t length_{0}; // number of the messages in the stream + int32_t length_{0}; // number of the messages in the stream int32_t version_{0}; }; -static const size_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(tree_id_t); +static const uint64_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(tree_id_t); class StreamCGroupMetaValue { public: explicit StreamCGroupMetaValue() = default; @@ -268,7 +274,7 @@ class StreamCGroupMetaValue { void Init(tree_id_t tid, tree_id_t consumers) { pel_ = tid; consumers_ = consumers; - size_t needed = kDefaultStreamCGroupValueLength; + uint64_t needed = kDefaultStreamCGroupValueLength; assert(value_.size() == 0); if (value_.size() != 0) { LOG(FATAL) << "Init on a existed stream cgroup meta value!"; @@ -340,7 +346,7 @@ class StreamCGroupMetaValue { tree_id_t consumers_ = 0; }; -static const size_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(tree_id_t); +static const uint64_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(tree_id_t); class StreamConsumerMetaValue { public: // pel must been set at beginning @@ -371,7 +377,7 @@ class StreamConsumerMetaValue { LOG(FATAL) << "Invalid stream consumer meta value length: " << value_.size() << " expected: 0"; return; } - size_t needed = kDefaultStreamConsumerValueLength; + uint64_t needed = kDefaultStreamConsumerValueLength; value_.resize(needed); char* dst = &value_[0]; @@ -413,7 +419,7 @@ class StreamConsumerMetaValue { tree_id_t pel_ = 0; }; -static const size_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(tree_id_t); +static const uint64_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(tree_id_t); class StreamPelMeta { public: // consumer must been set at beginning @@ -422,7 +428,7 @@ class StreamPelMeta { void Init(std::string consumer, stream_ms_t delivery_time) { consumer_ = std::move(consumer); delivery_time_ = delivery_time; - size_t needed = kDefaultStreamPelMetaValueLength; + uint64_t needed = kDefaultStreamPelMetaValueLength; assert(value_.size() == 0); if (value_.size() != 0) { LOG(ERROR) << "Init on a existed stream pel meta value!"; @@ -435,8 +441,8 @@ class StreamPelMeta { dst += sizeof(stream_ms_t); memcpy(dst, &delivery_count_, sizeof(uint64_t)); dst += sizeof(uint64_t); - memcpy(dst, &cname_len_, sizeof(size_t)); - dst += sizeof(size_t); + memcpy(dst, &cname_len_, sizeof(uint64_t)); + dst += sizeof(uint64_t); memcpy(dst, consumer_.data(), cname_len_); } @@ -452,8 +458,8 @@ class StreamPelMeta { pos += sizeof(stream_ms_t); memcpy(&delivery_count_, pos, sizeof(uint64_t)); pos += sizeof(uint64_t); - memcpy(&cname_len_, pos, sizeof(size_t)); - pos += sizeof(size_t); + memcpy(&cname_len_, pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); consumer_.assign(pos, cname_len_); } @@ -484,7 +490,7 @@ class StreamPelMeta { stream_ms_t delivery_time_ = 0; uint64_t delivery_count_ = 1; - size_t cname_len_ = 0; + uint64_t cname_len_ = 0; std::string consumer_; }; diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index f42ba8c490..f0e82dc337 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -4,6 +4,7 @@ // of patent rights can be found in the PATENTS file in the same directory. #include "src/redis_streams.h" +#include #include #include #include @@ -80,7 +81,7 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag // 4 trim the stream if needed if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { - size_t count{0}; + int32_t count{0}; s = TrimStream(count, stream_meta, key, args, default_read_options_); if (!s.ok()) { return Status::Corruption("error from XADD, trim stream failed: " + s.ToString()); @@ -97,7 +98,7 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag return Status::OK(); } -Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count) { +Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count) { ScopeRecordLock l(lock_mgr_, key); // 1 get stream meta @@ -124,7 +125,7 @@ Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& co return Status::OK(); } -Status RedisStreams::XDel(const Slice& key, const std::vector& ids, size_t& count) { +Status RedisStreams::XDel(const Slice& key, const std::vector& ids, int32_t& count) { ScopeRecordLock l(lock_mgr_, key); // 1 try to get stream meta @@ -135,7 +136,10 @@ Status RedisStreams::XDel(const Slice& key, const std::vector& ids, si } // 2 do the delete - count = ids.size(); + if (ids.size() > INT32_MAX) { + return Status::InvalidArgument("Too many IDs specified"); + } + count = static_cast(ids.size()); std::string unused; for (auto id : ids) { StreamDataKey stream_data_key(key, stream_meta.version(), id.Serialize()); @@ -219,7 +223,7 @@ Status RedisStreams::XRevrange(const Slice& key, const StreamScanArgs& args, std return s; } -Status RedisStreams::XLen(const Slice& key, size_t& len) { +Status RedisStreams::XLen(const Slice& key, int32_t& len) { rocksdb::ReadOptions read_options; const rocksdb::Snapshot* snapshot; ScopeSnapshot ss(db_, &snapshot); @@ -582,7 +586,7 @@ Status RedisStreams::GetStreamMeta(StreamMetaValue& stream_meta, const rocksdb:: return s; } -Status RedisStreams::TrimStream(size_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, +Status RedisStreams::TrimStream(int32_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { count = 0; // 1 do the trim @@ -810,7 +814,7 @@ Status RedisStreams::TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta } Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Slice& id_start, - const std::string& id_end, const Slice& pattern, size_t limit, + const std::string& id_end, const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options) { next_id.clear(); @@ -855,7 +859,7 @@ Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Sl } Status RedisStreams::ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, - const std::string& id_end, const Slice& pattern, size_t limit, + const std::string& id_end, const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options) { next_id.clear(); @@ -1008,7 +1012,7 @@ bool StreamUtils::StreamGenericParseID(const std::string& var, streamID& id, uin return false; }; if (dot) { - size_t seqlen = strlen(dot + 1); + auto seqlen = strlen(dot + 1); if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { seq = 0; *seq_given = false; @@ -1098,15 +1102,15 @@ bool StreamUtils::SerializeMessage(const std::vector& field_values, assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); assert(message.empty()); // count the size of serizlized message - size_t size = 0; + uint32_t size = 0; for (int i = field_pos; i < field_values.size(); i++) { - size += field_values[i].size() + sizeof(size_t); + size += field_values[i].size() + sizeof(uint32_t); } message.reserve(size); // serialize message for (int i = field_pos; i < field_values.size(); i++) { - size_t len = field_values[i].size(); + uint32_t len = field_values[i].size(); message.append(reinterpret_cast(&len), sizeof(len)); message.append(field_values[i]); } @@ -1115,11 +1119,11 @@ bool StreamUtils::SerializeMessage(const std::vector& field_values, } bool StreamUtils::DeserializeMessage(const std::string& message, std::vector& parsed_message) { - size_t pos = 0; + uint32_t pos = 0; while (pos < message.size()) { // Read the length of the next field value from the message - size_t len = *reinterpret_cast(&message[pos]); - pos += sizeof(size_t); + uint32_t len = *reinterpret_cast(&message[pos]); + pos += sizeof(uint32_t); // Check if the calculated end of the string is still within the message bounds if (pos + len > message.size()) { diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index ff0ee7b064..c36aa69d77 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -56,14 +56,14 @@ struct StreamReadGroupReadArgs { struct StreamScanArgs { streamID start_sid; streamID end_sid; - size_t limit{INT32_MAX}; + int32_t limit{INT32_MAX}; bool start_ex{false}; // exclude first message bool end_ex{false}; // exclude last message bool is_reverse{false}; // scan in reverse order }; struct StreamInfoResult { - uint64_t length{0}; + int32_t length{0}; std::string last_id_str; std::string max_deleted_entry_id_str; uint64_t entries_added{0}; @@ -131,11 +131,11 @@ class RedisStreams : public Redis { // Commands //===--------------------------------------------------------------------===// Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); - Status XDel(const Slice& key, const std::vector& ids, size_t& count); - Status XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count); + Status XDel(const Slice& key, const std::vector& ids, int32_t& count); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count); Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); - Status XLen(const Slice& key, size_t& len); + Status XLen(const Slice& key, int32_t& len); Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, std::vector& reserved_keys); Status XInfo(const Slice& key, StreamInfoResult& result); @@ -176,11 +176,11 @@ class RedisStreams : public Redis { int32_t version; // the version of the stream streamID start_sid; streamID end_sid; - size_t limit; + int32_t limit; bool start_ex; // exclude first message bool end_ex; // exclude last message bool is_reverse; // scan in reverse order - ScanStreamOptions(const rocksdb::Slice skey, int32_t version, streamID start_sid, streamID end_sid, size_t count, + ScanStreamOptions(const rocksdb::Slice skey, int32_t version, streamID start_sid, streamID end_sid, int32_t count, bool start_ex = false, bool end_ex = false, bool is_reverse = false) : key(skey), version(version), @@ -206,22 +206,22 @@ class RedisStreams : public Redis { Status DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, const std::vector& serialized_ids, rocksdb::ReadOptions& read_options); - Status TrimStream(size_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, StreamAddTrimArgs& args, + Status TrimStream(int32_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); private: Status GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args); Status ScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, - const Slice& pattern, size_t limit, std::vector& id_messages, std::string& next_id, + const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, rocksdb::ReadOptions& read_options); Status ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, - const Slice& pattern, size_t limit, std::vector& id_values, std::string& next_id, + const Slice& pattern, int32_t limit, std::vector& id_values, std::string& next_id, rocksdb::ReadOptions& read_options); struct TrimRet { // the count of deleted messages - size_t count{0}; + int32_t count{0}; // the next field after trim std::string next_field; // the max deleted field, will be empty if no message is deleted diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index 2879b096d8..1ee70acd39 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -543,11 +543,11 @@ Status Storage::XAdd(const Slice& key, const std::string& serialized_message, St return streams_db_->XAdd(key, serialized_message, args); } -Status Storage::XDel(const Slice& key, const std::vector& ids, size_t& ret) { +Status Storage::XDel(const Slice& key, const std::vector& ids, int32_t& ret) { return streams_db_->XDel(key, ids, ret); } -Status Storage::XTrim(const Slice& key, StreamAddTrimArgs& args, size_t& count) { +Status Storage::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count) { return streams_db_->XTrim(key, args, count); } @@ -559,7 +559,7 @@ Status Storage::XRevrange(const Slice& key, const StreamScanArgs& args, std::vec return streams_db_->XRevrange(key, args, id_messages); } -Status Storage::XLen(const Slice& key, uint64_t& len) { +Status Storage::XLen(const Slice& key, int32_t& len) { return streams_db_->XLen(key, len); } From e560b899c7875b42d9103b7785188be824ceae90 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Thu, 11 Jan 2024 22:26:38 +0800 Subject: [PATCH 08/10] fix: Support some commands and API of blackwidow. * KEYS * Storage::DoCompactRange() * Storage::PKHScanRange() * Storage::PKHRSranRange() --- include/pika_stream.h | 3 +- src/pika_kv.cc | 2 + src/storage/include/storage/storage.h | 1 + src/storage/src/redis_streams.cc | 106 ++++++++++++++++++++++++++ src/storage/src/redis_streams.h | 4 + src/storage/src/storage.cc | 43 +++++++++++ tests/integration/stream_test.go | 26 +++++++ 7 files changed, 184 insertions(+), 1 deletion(-) diff --git a/include/pika_stream.h b/include/pika_stream.h index 356cca5a46..a86f0d21cf 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -6,10 +6,11 @@ #ifndef PIKA_STREAM_H_ #define PIKA_STREAM_H_ +#include "include/acl.h" #include "include/pika_command.h" #include "include/pika_slot.h" -#include "storage/storage.h" #include "storage/src/redis_streams.h" +#include "storage/storage.h" /* * stream diff --git a/src/pika_kv.cc b/src/pika_kv.cc index f021195898..8837653c05 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -611,6 +611,8 @@ void KeysCmd::DoInitial() { type_ = storage::DataType::kLists; } else if (strcasecmp(opt.data(), "hash") == 0) { type_ = storage::DataType::kHashes; + } else if (strcasecmp(opt.data(), "stream") == 0) { + type_ = storage::DataType::kStreams; } else { res_.SetRes(CmdRes::kSyntaxErr); } diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index d872f41dc2..cbd55ae3a3 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -148,6 +148,7 @@ enum Operation { kCleanZSets, kCleanSets, kCleanLists, + kCleanStreams, kCompactRange }; diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index f0e82dc337..2ea34efb3c 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -491,6 +491,112 @@ Status RedisStreams::PKPatternMatchDel(const std::string& pattern, int32_t* ret) return s; } +rocksdb::Status RedisStreams::PKScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, + int32_t limit, std::vector* keys, std::string* next_key) { + next_key->clear(); + + std::string key; + int32_t remain = limit; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + bool start_no_limit = key_start.compare("") == 0; + bool end_no_limit = key_end.compare("") == 0; + + if (!start_no_limit && !end_no_limit && (key_start.compare(key_end) > 0)) { + return rocksdb::Status::InvalidArgument("error in given range"); + } + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + if (start_no_limit) { + it->SeekToFirst(); + } else { + it->Seek(key_start); + } + + while (it->Valid() && remain > 0 && (end_no_limit || it->key().compare(key_end) <= 0)) { + ParsedStreamMetaValue parsed_meta_value(it->value()); + if (parsed_meta_value.length() == 0) { + it->Next(); + } else { + key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + remain--; + it->Next(); + } + } + + while (it->Valid() && (end_no_limit || it->key().compare(key_end) <= 0)) { + ParsedStreamMetaValue parsed_meta_value(it->value()); + if (parsed_meta_value.length() == 0) { + it->Next(); + } else { + *next_key = it->key().ToString(); + break; + } + } + delete it; + return rocksdb::Status::OK(); +} + +Status RedisStreams::PKRScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key) { + next_key->clear(); + + std::string key; + int32_t remain = limit; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + bool start_no_limit = key_start.compare("") == 0; + bool end_no_limit = key_end.compare("") == 0; + + if (!start_no_limit && !end_no_limit && (key_start.compare(key_end) < 0)) { + return Status::InvalidArgument("error in given range"); + } + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + if (start_no_limit) { + it->SeekToLast(); + } else { + it->SeekForPrev(key_start); + } + + while (it->Valid() && remain > 0 && (end_no_limit || it->key().compare(key_end) >= 0)) { + ParsedStreamMetaValue parsed_streams_meta_value(it->value()); + if (parsed_streams_meta_value.length() == 0) { + it->Prev(); + } else { + key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + remain--; + it->Prev(); + } + } + + while (it->Valid() && (end_no_limit || it->key().compare(key_end) >= 0)) { + ParsedStreamMetaValue parsed_streams_meta_value(it->value()); + if (parsed_streams_meta_value.length() == 0) { + it->Prev(); + } else { + *next_key = it->key().ToString(); + break; + } + } + delete it; + return Status::OK(); +} + Status RedisStreams::Del(const Slice& key) { std::string meta_value; ScopeRecordLock l(lock_mgr_, key); diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index c36aa69d77..df4da61411 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -150,6 +150,10 @@ class RedisStreams : public Redis { Status ScanKeyNum(KeyInfo* keyinfo) override; Status ScanKeys(const std::string& pattern, std::vector* keys) override; Status PKPatternMatchDel(const std::string& pattern, int32_t* ret) override; + Status PKScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key); + Status PKRScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key); //===--------------------------------------------------------------------===// // Keys Commands diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index b326102b26..3dbb6c29b4 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -929,6 +929,22 @@ int64_t Storage::Scan(const DataType& dtype, int64_t cursor, const std::string& } } start_key = prefix; + case 'x': + is_finish = streams_db_->Scan(start_key, pattern, keys, &leftover_visits, &next_key); + if ((leftover_visits == 0) && !is_finish) { + cursor_ret = cursor + step_length; + StoreCursorStartKey(DataType::kStreams, cursor_ret, std::string("x") + next_key); + break; + } else if (is_finish) { + if (DataType::kStreams == dtype) { + cursor_ret = 0; + break; + } else if (leftover_visits == 0) { + cursor_ret = cursor + step_length; + StoreCursorStartKey(DataType::kStreams, cursor_ret, std::string("k") + prefix); + break; + } + } case 'z': is_finish = zsets_db_->Scan(start_key, pattern, keys, &leftover_visits, &next_key); if ((leftover_visits == 0) && !is_finish) { @@ -1079,6 +1095,9 @@ Status Storage::PKScanRange(const DataType& data_type, const Slice& key_start, c case DataType::kSets: s = sets_db_->PKScanRange(key_start, key_end, pattern, limit, keys, next_key); break; + case DataType::kStreams: + s = streams_db_->PKScanRange(key_start, key_end, pattern, limit, keys, next_key); + break; default: s = Status::Corruption("Unsupported data types"); break; @@ -1108,6 +1127,9 @@ Status Storage::PKRScanRange(const DataType& data_type, const Slice& key_start, case DataType::kSets: s = sets_db_->PKRScanRange(key_start, key_end, pattern, limit, keys, next_key); break; + case DataType::kStreams: + s = streams_db_->PKRScanRange(key_start, key_end, pattern, limit, keys, next_key); + break; default: s = Status::Corruption("Unsupported data types"); break; @@ -1161,6 +1183,9 @@ Status Storage::Scanx(const DataType& data_type, const std::string& start_key, c case DataType::kSets: sets_db_->Scan(start_key, pattern, keys, &count, next_key); break; + case DataType::kStreams: + streams_db_->Scan(start_key, pattern, keys, &count, next_key); + break; default: Status::Corruption("Unsupported data types"); break; @@ -1407,6 +1432,11 @@ Status Storage::Keys(const DataType& data_type, const std::string& pattern, std: if (!s.ok()) { return s; } + } else if (data_type == DataType::kStreams) { + s = streams_db_->ScanKeys(pattern, keys); + if (!s.ok()) { + return s; + } } else { s = strings_db_->ScanKeys(pattern, keys); if (!s.ok()) { @@ -1428,6 +1458,10 @@ Status Storage::Keys(const DataType& data_type, const std::string& pattern, std: if (!s.ok()) { return s; } + s = streams_db_->ScanKeys(pattern, keys); + if (!s.ok()) { + return s; + } } return s; } @@ -1648,6 +1682,9 @@ Status Storage::DoCompact(const DataType& type) { } else if (type == kLists) { current_task_type_ = Operation::kCleanLists; s = lists_db_->CompactRange(nullptr, nullptr); + } else if (type == kStreams) { + current_task_type_ = Operation::kCleanStreams; + s = streams_db_->CompactRange(nullptr, nullptr); } else { current_task_type_ = Operation::kCleanAll; s = strings_db_->CompactRange(nullptr, nullptr); @@ -1655,6 +1692,7 @@ Status Storage::DoCompact(const DataType& type) { s = sets_db_->CompactRange(nullptr, nullptr); s = zsets_db_->CompactRange(nullptr, nullptr); s = lists_db_->CompactRange(nullptr, nullptr); + s = streams_db_->CompactRange(nullptr, nullptr); } current_task_type_ = Operation::kNone; return s; @@ -1702,6 +1740,9 @@ Status Storage::DoCompactRange(const DataType& type, const std::string& start, c } else if (type == kLists) { s = lists_db_->CompactRange(&slice_meta_begin, &slice_meta_end, kMeta); s = lists_db_->CompactRange(&slice_data_begin, &slice_data_end, kData); + } else if (type == kStreams) { + s = streams_db_->CompactRange(&slice_meta_begin, &slice_meta_end, kMeta); + s = streams_db_->CompactRange(&slice_data_begin, &slice_data_end, kData); } return s; } @@ -1745,6 +1786,8 @@ std::string Storage::GetCurrentTaskType() { return "Set"; case kCleanLists: return "List"; + case kCleanStreams: + return "Stream"; case kNone: default: return "No"; diff --git a/tests/integration/stream_test.go b/tests/integration/stream_test.go index 82d47a8e1e..3c49ad9973 100644 --- a/tests/integration/stream_test.go +++ b/tests/integration/stream_test.go @@ -204,6 +204,32 @@ var _ = Describe("Stream Commands", func() { return atomic.LoadInt32(&messageCount) }).Should(Equal(int32(totalMessages))) }) + + It("should create a stream and find it using keys * command", func() { + Expect(client.Del(ctx, "mystream").Err()).NotTo(HaveOccurred()) + // Creating a stream and adding entries + _, err := client.XAdd(ctx, &redis.XAddArgs{ + Stream: "mystream", + ID: "*", + Values: map[string]interface{}{"key1": "value1", "key2": "value2"}, + }).Result() + Expect(err).NotTo(HaveOccurred()) + + // Using keys * to find all keys including the stream + keys, err := client.Keys(ctx, "*").Result() + Expect(err).NotTo(HaveOccurred()) + + // Checking if the stream 'mystream' exists in the returned keys + found := false + for _, key := range keys { + if key == "mystream" { + found = true + break + } + } + Expect(found).To(BeTrue(), "Stream 'mystream' should exist in keys") + }) + It("XADD wrong number of args", func() { From 7ea89dbb8e57280598d175f649e485e67852f446 Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Mon, 29 Jan 2024 20:12:46 +0800 Subject: [PATCH 09/10] fix: build problem on ubuntu. --- src/storage/src/redis_streams.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h index df4da61411..c964efef7f 100644 --- a/src/storage/src/redis_streams.h +++ b/src/storage/src/redis_streams.h @@ -9,7 +9,6 @@ #include #include #include -#include "include/storage/storage.h" #include "pika_stream_meta_value.h" #include "pika_stream_types.h" #include "rocksdb/options.h" @@ -136,7 +135,7 @@ class RedisStreams : public Redis { Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); Status XLen(const Slice& key, int32_t& len); - Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, std::vector& reserved_keys); Status XInfo(const Slice& key, StreamInfoResult& result); From 25452589ccd4bceb9370c72a8b2812335f9d974a Mon Sep 17 00:00:00 2001 From: Korpse <543514071@qq.com> Date: Thu, 1 Feb 2024 15:11:11 +0800 Subject: [PATCH 10/10] fix: remote key lock. --- src/storage/src/redis_streams.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc index 2ea34efb3c..84e11832be 100644 --- a/src/storage/src/redis_streams.cc +++ b/src/storage/src/redis_streams.cc @@ -27,7 +27,6 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag // With the lock, we do not need snapshot for read. // And it's bugy to use snapshot for read when we try to add message with trim. // such as: XADD key 1-0 field value MINID 1-0 - ScopeRecordLock l(lock_mgr_, key); // 1 get stream meta rocksdb::Status s; @@ -99,7 +98,6 @@ Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_messag } Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count) { - ScopeRecordLock l(lock_mgr_, key); // 1 get stream meta rocksdb::Status s; @@ -126,7 +124,6 @@ Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& c } Status RedisStreams::XDel(const Slice& key, const std::vector& ids, int32_t& count) { - ScopeRecordLock l(lock_mgr_, key); // 1 try to get stream meta StreamMetaValue stream_meta; @@ -599,7 +596,6 @@ Status RedisStreams::PKRScanRange(const Slice& key_start, const Slice& key_end, Status RedisStreams::Del(const Slice& key) { std::string meta_value; - ScopeRecordLock l(lock_mgr_, key); Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value); if (s.ok()) { StreamMetaValue stream_meta_value;