diff --git a/include/pika_stream.h b/include/pika_stream.h index f9c6366acb..bf61a96c6b 100644 --- a/include/pika_stream.h +++ b/include/pika_stream.h @@ -1,26 +1,25 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #ifndef PIKA_STREAM_H_ #define PIKA_STREAM_H_ +#include "include/acl.h" #include "include/pika_command.h" -#include "include/pika_stream_base.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" +#include "storage/src/redis_streams.h" #include "storage/storage.h" /* * stream */ -inline void ParseAddOrTrimArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, StreamAddTrimArgs& args, int* idpos, - bool is_xadd); +inline void ParseAddOrTrimArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, storage::StreamAddTrimArgs& args, + int* idpos, bool is_xadd); -inline void ParseReadOrReadGroupArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, StreamReadGroupReadArgs& args, - bool is_xreadgroup); +inline void ParseReadOrReadGroupArgsOrReply(CmdRes& res, const PikaCmdArgsType& argv, + storage::StreamReadGroupReadArgs& args, bool is_xreadgroup); // @field_values is the result of ScanStream. // field is the serialized message id, @@ -39,11 +38,10 @@ class XAddCmd : public Cmd { private: std::string key_; - StreamAddTrimArgs args_; + storage::StreamAddTrimArgs args_; int field_pos_{0}; void DoInitial() override; - inline void GenerateStreamIDOrReply(const StreamMetaValue& stream_meta); }; class XDelCmd : public Cmd { @@ -58,13 +56,10 @@ class XDelCmd : public Cmd { private: std::string key_; - std::vector ids_; + std::vector ids_; void DoInitial() override; void Clear() override { ids_.clear(); } - inline void SetFirstOrLastIDOrReply(StreamMetaValue& stream_meta, const DB* db, bool is_set_first); - inline void SetFirstIDOrReply(StreamMetaValue& stream_meta, const DB* db); - inline void SetLastIDOrReply(StreamMetaValue& stream_meta, const DB* db); }; class XReadCmd : public Cmd { @@ -77,7 +72,7 @@ class XReadCmd : public Cmd { Cmd* Clone() override { return new XReadCmd(*this); } private: - StreamReadGroupReadArgs args_; + storage::StreamReadGroupReadArgs args_; void DoInitial() override; void Clear() override { @@ -97,11 +92,7 @@ class XRangeCmd : public Cmd { protected: std::string key_; - streamID start_sid; - streamID end_sid; - int32_t count_{INT32_MAX}; - bool start_ex_{false}; - bool end_ex_{false}; + storage::StreamScanArgs args_; void DoInitial() override; }; @@ -141,7 +132,7 @@ class XTrimCmd : public Cmd { private: std::string key_; - StreamAddTrimArgs args_; + storage::StreamAddTrimArgs args_; void DoInitial() override; }; diff --git a/include/pika_stream_base.h b/include/pika_stream_base.h deleted file mode 100644 index eb9dc6b34c..0000000000 --- a/include/pika_stream_base.h +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef SRC_STREAM_UTIL_H_ -#define SRC_STREAM_UTIL_H_ - -#include "include/pika_slot_command.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" -#include "storage/storage.h" - -// each abstracted tree has a prefix, -// prefix + treeID will be the key of the hash to store the tree, -// notice: we need to ban the use of this prefix when using "HSET". -static const std::string STERAM_TREE_PREFIX = "STR_TREE"; - -// key of the hash to store stream meta, -// notice: we need to ban the use of this key when using "HSET". -static const std::string STREAM_META_HASH_KEY = "STREAM"; - -// each stream's data (messages) are stored in a hash, -// to avoid stream key confilict with hash key, we add a prefix to each key for stream data, -// it's ok to use the same string as STREAM_META_HASH_KEY to be the prefix. -// notice: we need to ban the use of this prefix when using "HSET". -static const std::string STREAM_DATA_HASH_PREFIX = STREAM_META_HASH_KEY; - -// I need to find a place to store the last generated tree id, -// so I stored it as a field-value pair in the hash storing stream meta. -// the field is a fixed string, and the value is the last generated tree id. -// notice: we need to ban the use of this key when using stream, to avoid conflict with stream's key. -static const std::string STREAM_LAST_GENERATED_TREE_ID_FIELD = "STREAM"; - -// the max number of each delete operation in XTRIM command，to avoid too much memory usage. -// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / -// kDEFAULT_TRIM_BATCH_SIZE) times -const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; - -struct StreamAddTrimArgs { - // XADD options - streamID id; - bool id_given{false}; - bool seq_given{false}; - bool no_mkstream{false}; - - // XADD + XTRIM common options - StreamTrimStrategy trim_strategy{0}; - int trim_strategy_arg_idx{0}; - - // TRIM_STRATEGY_MAXLEN options - uint64_t maxlen{0}; - streamID minid; -}; - -struct StreamReadGroupReadArgs { - // XREAD + XREADGROUP common options - std::vector keys; - std::vector unparsed_ids; - int32_t count{INT32_MAX}; // in redis this is uint64_t, but PKHScanRange only support int32_t - uint64_t block{0}; // 0 means no block - - // XREADGROUP options - std::string group_name; - std::string consumer_name; - bool noack_{false}; -}; - -// get next tree id thread safe -class TreeIDGenerator { - private: - TreeIDGenerator() = default; - void operator=(const TreeIDGenerator &) = delete; - - public: - ~TreeIDGenerator() = default; - - // work in singeletone mode - static TreeIDGenerator &GetInstance() { - static TreeIDGenerator instance; - return instance; - } - - storage::Status GetNextTreeID(const DB *db, treeID &tid); - - private: - static const treeID START_TREE_ID = 0; - std::atomic tree_id_ = kINVALID_TREE_ID; -}; - -// Implement all the functions that related to blackwidow derctly. -// if we want to change the storage engine, we need to rewrite this class. -class StreamStorage { - public: - struct ScanStreamOptions { - std::string key; // the key of the stream - streamID start_sid; - streamID end_sid; - int32_t count; - bool start_ex; // exclude first message - bool end_ex; // exclude last message - bool is_reverse; // scan in reverse order - ScanStreamOptions(std::string skey, streamID start_sid, streamID end_sid, int32_t count, - bool start_ex = false, bool end_ex = false, bool is_reverse = false) - : key(skey), - start_sid(start_sid), - end_sid(end_sid), - count(count), - start_ex(start_ex), - end_ex(end_ex), - is_reverse(is_reverse) {} - }; - - static storage::Status ScanStream(const ScanStreamOptions &option, std::vector &field_values, - std::string &next_field, const DB *db); - - // get and parse the stream meta if found - // @return ok only when the stream meta exists - static storage::Status GetStreamMeta(StreamMetaValue &tream_meta, const std::string &key, const DB *db); - - // will create stream meta hash if it dosent't exist. - // return !s.ok() only when insert failed - static storage::Status SetStreamMeta(const std::string &key, std::string &meta_value, const DB *db); - - static storage::Status InsertStreamMessage(const std::string &key, const streamID &id, const std::string &message, - const DB *db); - - static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &ids, int32_t &ret, - const DB *db); - - static storage::Status DeleteStreamMessage(const std::string &key, const std::vector &serialized_ids, - int32_t &ret, const DB *db); - - static storage::Status GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const DB *db); - - static storage::Status DeleteStreamData(const std::string &key, const DB *db); - - static storage::Status TrimStream(int32_t &res, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const DB *db); - - // get the abstracted tree node, e.g. get a message in pel, get a consumer meta or get a cgroup meta. - // the behavior of abstracted tree is similar to radix-tree in redis; - // in cgroup tree, field is groupname - // in consumer tree, field is consumername - // in pel tree, field is messageID - static storage::Status GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, const DB *db); - static storage::Status InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const DB *db); - static storage::Status DeleteTreeNode(const treeID tid, const std::string &field, const DB *db); - static storage::Status GetAllTreeNode(const treeID tid, std::vector &field_values, - const DB *db); - - // delete the stream meta - // @return true if the stream meta exists and deleted - static storage::Status DeleteStreamMeta(const std::string &key, const DB *db); - - // note: the tree must exist - // @return true if the tree exists and is deleted - static storage::Status DeleteTree(const treeID tid, const DB *db); - - // get consumer meta value. - // if the consumer meta value does not exist, create a new one and return it. - static storage::Status GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const DB *db, - StreamConsumerMetaValue &consumer_meta); - - static storage::Status CreateConsumer(treeID consumer_tid, std::string &consumername, const DB *db); - - // delete the pels, consumers, cgroups and stream meta of a stream - // note: this function do not delete the stream data value - static storage::Status DestoryStreams(std::vector &keys, const DB *db); - - private: - StreamStorage(); - ~StreamStorage(); - struct TrimRet { - // the count of deleted messages - int32_t count{0}; - // the next field after trim - std::string next_field; - // the max deleted field, will be empty if no message is deleted - std::string max_deleted_field; - }; - - static storage::Status TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const DB *db, const StreamAddTrimArgs &args); - - static storage::Status TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const DB *db, const StreamAddTrimArgs &args); -}; - -// Helper function of stream command. -// Should be reconstructed when transfer to another command framework. -// any function that has Reply in its name will reply to the client if error occurs. -class StreamCmdBase { - public: - private: - StreamCmdBase(); - ~StreamCmdBase(); -}; - -class StreamUtils { - public: - StreamUtils() = default; - ~StreamUtils() = default; - - static bool string2uint64(const char *s, uint64_t &value); - static bool string2int64(const char *s, int64_t &value); - static bool string2int32(const char *s, int32_t &value); - static std::string TreeID2Key(const treeID &tid); - - static uint64_t GetCurrentTimeMs(); - - // serialize the message to a string. - // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} - static bool SerializeMessage(const std::vector &field_values, std::string &serialized_message, - int field_pos); - - // deserialize the message from a string with the format of SerializeMessage. - static bool DeserializeMessage(const std::string &message, std::vector &parsed_message); - - // Parse a stream ID in the format given by clients to Pika, that is - // -, and converts it into a streamID structure. The ID may be in incomplete - // form, just stating the milliseconds time part of the stream. In such a case - // the missing part is set according to the value of 'missing_seq' parameter. - // - // The IDs "-" and "+" specify respectively the minimum and maximum IDs - // that can be represented. If 'strict' is set to 1, "-" and "+" will be - // treated as an invalid ID. - // - // The ID form -* specifies a millisconds-only ID, leaving the sequence part - // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this - // form is accepted and the argument is set to 0 unless the sequence part is - // specified. - static bool StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 0, to be used when - and + are acceptable IDs. - static bool StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq); - - // Wrapper for streamGenericParseID() with 'strict' argument set to - // 1, to be used when we want to return an error if the special IDs + or - - // are provided. - static bool StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given); - - // Helper for parsing a stream ID that is a range query interval. When the - // exclude argument is NULL, streamParseID() is called and the interval - // is treated as close (inclusive). Otherwise, the exclude argument is set if - // the interval is open (the "(" prefix) and streamParseStrictID() is - // called in that case. - static bool StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq); -}; - -#endif diff --git a/include/pika_stream_meta_value.h b/include/pika_stream_meta_value.h deleted file mode 100644 index e0bd763950..0000000000 --- a/include/pika_stream_meta_value.h +++ /dev/null @@ -1,372 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef SRC_STREAM_META_VALUE_FORMAT_H_ -#define SRC_STREAM_META_VALUE_FORMAT_H_ - -#include "glog/logging.h" -#include "include/pika_stream_types.h" - -static const size_t kDefaultStreamValueLength = - sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(uint64_t); -class StreamMetaValue { - public: - explicit StreamMetaValue() = default; - // used only when create a new stream - void Init() { - size_t needed = kDefaultStreamValueLength; - assert(value_.size() == 0); - if (value_.size() != 0) { - LOG(ERROR) << "Init on a existed stream meta value!"; - return; - } - value_.resize(needed); - - char* dst = value_.data(); - - // Encode each member into the string - memcpy(dst, &groups_id_, sizeof(treeID)); - dst += sizeof(treeID); - memcpy(dst, &entries_added_, sizeof(size_t)); - dst += sizeof(size_t); - memcpy(dst, &first_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &last_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &max_deleted_entry_id_, sizeof(streamID)); - dst += sizeof(streamID); - memcpy(dst, &length_, sizeof(size_t)); - } - - // used only when parse a existed stream meta - // value_ = std::move(value); - void ParseFrom(std::string& value) { - value_ = std::move(value); - assert(value_.size() == kDefaultStreamValueLength); - if (value_.size() != kDefaultStreamValueLength) { - LOG(ERROR) << "Invalid stream meta value length: "; - return; - } - char* pos = value_.data(); - memcpy(&groups_id_, pos, sizeof(treeID)); - pos += sizeof(treeID); - memcpy(&entries_added_, pos, sizeof(size_t)); - pos += sizeof(size_t); - memcpy(&first_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&last_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&max_deleted_entry_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&length_, pos, sizeof(size_t)); - } - - treeID groups_id() const { return groups_id_; } - - size_t entries_added() const { return entries_added_; } - - void ModifyEntriesAdded(size_t delta) { set_entries_added(entries_added_ + delta); } - - streamID first_id() const { return first_id_; } - - streamID last_id() const { return last_id_; } - - streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } - - size_t length() const { return length_; } - - std::string& value() { return value_; } - - std::string ToString() { - return "stream_meta: " + std::string("groups_id: ") + std::to_string(groups_id_) + - std::string(", entries_added: ") + std::to_string(entries_added_) + std::string(", first_id: ") + - first_id_.ToString() + std::string(", last_id: ") + last_id_.ToString() + - std::string(", max_deleted_entry_id: ") + max_deleted_entry_id_.ToString() + std::string(", length: ") + - std::to_string(length_); - } - - void set_groups_id(treeID groups_id) { - assert(value_.size() == kDefaultStreamValueLength); - groups_id_ = groups_id; - char* dst = const_cast(value_.data()); - memcpy(dst, &groups_id_, sizeof(treeID)); - } - - void set_entries_added(uint64_t entries_added) { - assert(value_.size() == kDefaultStreamValueLength); - entries_added_ = entries_added; - char* dst = const_cast(value_.data()) + sizeof(treeID); - memcpy(dst, &entries_added_, sizeof(uint64_t)); - } - - void set_first_id(streamID first_id) { - assert(value_.size() == kDefaultStreamValueLength); - first_id_ = first_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t); - memcpy(dst, &first_id_, sizeof(uint64_t)); - } - - void set_last_id(streamID last_id) { - assert(value_.size() == kDefaultStreamValueLength); - last_id_ = last_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + sizeof(streamID); - memcpy(dst, &last_id_, sizeof(streamID)); - } - - void set_max_deleted_entry_id(streamID max_deleted_entry_id) { - assert(value_.size() == kDefaultStreamValueLength); - max_deleted_entry_id_ = max_deleted_entry_id; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 2 * sizeof(streamID); - memcpy(dst, &max_deleted_entry_id_, sizeof(streamID)); - } - - void set_length(size_t length) { - assert(value_.size() == kDefaultStreamValueLength); - length_ = length; - char* dst = const_cast(value_.data()) + sizeof(treeID) + sizeof(uint64_t) + 3 * sizeof(streamID); - memcpy(dst, &length_, sizeof(size_t)); - } - - private: - treeID groups_id_ = kINVALID_TREE_ID; - size_t entries_added_{0}; - streamID first_id_; - streamID last_id_; - streamID max_deleted_entry_id_; - size_t length_{0}; // number of the messages in the stream - - std::string value_{}; -}; - -static const size_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(treeID); -class StreamCGroupMetaValue { - public: - explicit StreamCGroupMetaValue() = default; - - // tid and consumers should be set at beginning - void Init(treeID tid, treeID consumers) { - pel_ = tid; - consumers_ = consumers; - size_t needed = kDefaultStreamCGroupValueLength; - assert(value_.size() == 0); - if (value_.size() != 0) { - LOG(FATAL) << "Init on a existed stream cgroup meta value!"; - return; - } - value_.resize(needed); - - auto dst = value_.data(); - - memcpy(dst, &last_id_, sizeof(streamID)); - dst += sizeof(uint64_t); - memcpy(dst, &entries_read_, sizeof(uint64_t)); - dst += sizeof(uint64_t); - memcpy(dst, &pel_, sizeof(treeID)); - dst += sizeof(treeID); - memcpy(dst, &consumers_, sizeof(treeID)); - } - - void ParseFrom(std::string& value) { - value_ = std::move(value); - assert(value_.size() == kDefaultStreamCGroupValueLength); - if (value_.size() != kDefaultStreamCGroupValueLength) { - LOG(FATAL) << "Invalid stream cgroup meta value length: "; - return; - } - if (value_.size() == kDefaultStreamCGroupValueLength) { - auto pos = value_.data(); - memcpy(&last_id_, pos, sizeof(streamID)); - pos += sizeof(streamID); - memcpy(&entries_read_, pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&pel_, pos, sizeof(treeID)); - pos += sizeof(treeID); - memcpy(&consumers_, pos, sizeof(treeID)); - } - } - - streamID last_id() { return last_id_; } - - void set_last_id(streamID last_id) { - assert(value_.size() == kDefaultStreamCGroupValueLength); - last_id_ = last_id; - char* dst = const_cast(value_.data()); - memcpy(dst, &last_id_, sizeof(streamID)); - } - - uint64_t entries_read() { return entries_read_; } - - void set_entries_read(uint64_t entries_read) { - assert(value_.size() == kDefaultStreamCGroupValueLength); - entries_read_ = entries_read; - char* dst = const_cast(value_.data()) + sizeof(streamID); - memcpy(dst, &entries_read_, sizeof(uint64_t)); - } - - // pel and consumers were set in constructor, can't be modified - treeID pel() { return pel_; } - - treeID consumers() { return consumers_; } - - std::string& value() { return value_; } - - private: - std::string value_; - - streamID last_id_; - uint64_t entries_read_ = 0; - treeID pel_ = 0; - treeID consumers_ = 0; -}; - -static const size_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(treeID); -class StreamConsumerMetaValue { - public: - // pel must been set at beginning - StreamConsumerMetaValue() = default; - - void ParseFrom(std::string& value) { - value_ = std::move(value); - assert(value_.size() == kDefaultStreamConsumerValueLength); - if (value_.size() != kDefaultStreamConsumerValueLength) { - LOG(FATAL) << "Invalid stream consumer meta value length: " << value_.size() - << " expected: " << kDefaultStreamConsumerValueLength; - return; - } - if (value_.size() == kDefaultStreamConsumerValueLength) { - auto pos = value_.data(); - memcpy(&seen_time_, pos, sizeof(stream_ms_t)); - pos += sizeof(stream_ms_t); - memcpy(&active_time_, pos, sizeof(stream_ms_t)); - pos += sizeof(stream_ms_t); - memcpy(&pel_, pos, sizeof(treeID)); - } - } - - void Init(treeID pel) { - pel_ = pel; - assert(value_.size() == 0); - if (value_.size() != 0) { - LOG(FATAL) << "Invalid stream consumer meta value length: " << value_.size() << " expected: 0"; - return; - } - size_t needed = kDefaultStreamConsumerValueLength; - value_.resize(needed); - auto dst = value_.data(); - - memcpy(dst, &seen_time_, sizeof(stream_ms_t)); - dst += sizeof(stream_ms_t); - memcpy(dst, &active_time_, sizeof(stream_ms_t)); - dst += sizeof(stream_ms_t); - memcpy(dst, &pel_, sizeof(treeID)); - } - - stream_ms_t seen_time() { return seen_time_; } - - void set_seen_time(stream_ms_t seen_time) { - seen_time_ = seen_time; - assert(value_.size() == kDefaultStreamConsumerValueLength); - char* dst = const_cast(value_.data()); - memcpy(dst, &seen_time_, sizeof(stream_ms_t)); - } - - stream_ms_t active_time() { return active_time_; } - - void set_active_time(stream_ms_t active_time) { - active_time_ = active_time; - assert(value_.size() == kDefaultStreamConsumerValueLength); - char* dst = const_cast(value_.data()) + sizeof(stream_ms_t); - memcpy(dst, &active_time_, sizeof(stream_ms_t)); - } - - // pel was set in constructor, can't be modified - treeID pel_tid() { return pel_; } - - std::string& value() { return value_; } - - private: - std::string value_; - - stream_ms_t seen_time_ = 0; - stream_ms_t active_time_ = 0; - treeID pel_ = 0; -}; - -static const size_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(treeID); -class StreamPelMeta { - public: - // consumer must been set at beginning - StreamPelMeta() = default; - - void Init(std::string consumer, stream_ms_t delivery_time) { - consumer_ = std::move(consumer); - delivery_time_ = delivery_time; - size_t needed = kDefaultStreamPelMetaValueLength; - assert(value_.size() == 0); - if (value_.size() != 0) { - LOG(ERROR) << "Init on a existed stream pel meta value!"; - return; - } - value_.resize(needed); - char* dst = value_.data(); - - memcpy(dst, &delivery_time_, sizeof(stream_ms_t)); - dst += sizeof(stream_ms_t); - memcpy(dst, &delivery_count_, sizeof(uint64_t)); - dst += sizeof(uint64_t); - memcpy(dst, &cname_len_, sizeof(size_t)); - dst += sizeof(size_t); - memcpy(dst, consumer_.data(), cname_len_); - } - - void ParseFrom(std::string& value) { - value_ = std::move(value); - assert(value_.size() == kDefaultStreamPelMetaValueLength); - if (value_.size() != kDefaultStreamPelMetaValueLength) { - LOG(ERROR) << "Invalid stream pel meta value length: "; - return; - } - auto pos = value_.data(); - memcpy(&delivery_time_, pos, sizeof(stream_ms_t)); - pos += sizeof(stream_ms_t); - memcpy(&delivery_count_, pos, sizeof(uint64_t)); - pos += sizeof(uint64_t); - memcpy(&cname_len_, pos, sizeof(size_t)); - pos += sizeof(size_t); - consumer_.assign(pos, cname_len_); - } - - stream_ms_t delivery_time() { return delivery_time_; } - - void set_delivery_time(stream_ms_t delivery_time) { - assert(value_.size() == kDefaultStreamPelMetaValueLength); - delivery_time_ = delivery_time; - char* dst = const_cast(value_.data()); - memcpy(dst, &delivery_time_, sizeof(stream_ms_t)); - } - - uint64_t delivery_count() { return delivery_count_; } - - void set_delivery_count(uint64_t delivery_count) { - assert(value_.size() == kDefaultStreamPelMetaValueLength); - delivery_count_ = delivery_count; - char* dst = const_cast(value_.data()); - memcpy(dst + sizeof(stream_ms_t), &delivery_count_, sizeof(uint64_t)); - } - - std::string& consumer() { return consumer_; } - - std::string& value() { return value_; } - - private: - std::string value_; - - stream_ms_t delivery_time_ = 0; - uint64_t delivery_count_ = 1; - size_t cname_len_ = 0; - std::string consumer_; -}; - -#endif // SRC_STREAM_META_VALUE_FORMAT_H_ diff --git a/src/pika_hash.cc b/src/pika_hash.cc index 976ed5ae5b..367ff70aa9 100644 --- a/src/pika_hash.cc +++ b/src/pika_hash.cc @@ -9,7 +9,6 @@ #include "include/pika_conf.h" #include "include/pika_slot_command.h" -#include "include/pika_stream_base.h" #include "include/pika_cache.h" extern std::unique_ptr g_pika_conf; @@ -54,13 +53,6 @@ void HSetCmd::DoInitial() { key_ = argv_[1]; field_ = argv_[2]; value_ = argv_[3]; - - // check the conflict of stream used prefix, see details in defination of STREAM_TREE_PREFIX - if (key_.compare(0, STERAM_TREE_PREFIX.size(), STERAM_TREE_PREFIX) == 0 || - key_.compare(0, STREAM_DATA_HASH_PREFIX.size(), STREAM_DATA_HASH_PREFIX) == 0) { - res_.SetRes(CmdRes::kErrOther, "hash key can't start with " + STERAM_TREE_PREFIX + " or " + STREAM_META_HASH_KEY); - return; - } } void HSetCmd::Do() { diff --git a/src/pika_kv.cc b/src/pika_kv.cc index 0342105548..6a04dd726b 100644 --- a/src/pika_kv.cc +++ b/src/pika_kv.cc @@ -7,7 +7,6 @@ #include #include "include/pika_command.h" -#include "include/pika_stream_base.h" #include "pstd/include/pstd_string.h" #include "include/pika_cache.h" @@ -202,14 +201,9 @@ void DelCmd::DoInitial() { void DelCmd::Do() { std::map type_status; + int64_t count = db_->storage()->Del(keys_, &type_status); - - // stream's destory need to be treated specially - auto s = StreamStorage::DestoryStreams(keys_, db_.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "stream delete error: " + s.ToString()); - return; - } + if (count >= 0) { res_.AppendInteger(count); s_ = rocksdb::Status::OK(); @@ -617,6 +611,8 @@ void KeysCmd::DoInitial() { type_ = storage::DataType::kLists; } else if (strcasecmp(opt.data(), "hash") == 0) { type_ = storage::DataType::kHashes; + } else if (strcasecmp(opt.data(), "stream") == 0) { + type_ = storage::DataType::kStreams; } else { res_.SetRes(CmdRes::kSyntaxErr); } diff --git a/src/pika_stream.cc b/src/pika_stream.cc index f8841a3af5..60d799f59f 100644 --- a/src/pika_stream.cc +++ b/src/pika_stream.cc @@ -1,15 +1,18 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. #include "include/pika_stream.h" #include +#include +#include +#include +#include "glog/logging.h" #include "include/pika_command.h" -#include "include/pika_stream_base.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" +#include "include/pika_db.h" +#include "include/pika_define.h" #include "storage/storage.h" // s : rocksdb::Status @@ -23,7 +26,7 @@ } \ } while (0) -void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamAddTrimArgs &args, int *idpos, +void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, storage::StreamAddTrimArgs &args, int *idpos, bool is_xadd) { int i = 2; bool limit_given = false; @@ -37,7 +40,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA } else if (strcasecmp(opt.c_str(), "maxlen") == 0 && moreargs) { // case: XADD mystream ... MAXLEN [= | ~] threshold ... - if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + if (args.trim_strategy != storage::StreamTrimStrategy::TRIM_STRATEGY_NONE) { res.SetRes(CmdRes::kSyntaxErr, "syntax error, MAXLEN and MINID options at the same time are not compatible"); return; } @@ -47,16 +50,16 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA i++; } // parse threshold as uint64 - if (!StreamUtils::string2uint64(argv[i + 1].c_str(), args.maxlen)) { + if (!storage::StreamUtils::string2uint64(argv[i + 1].c_str(), args.maxlen)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid MAXLEN argument"); } i++; - args.trim_strategy = StreamTrimStrategy::TRIM_STRATEGY_MAXLEN; + args.trim_strategy = storage::StreamTrimStrategy::TRIM_STRATEGY_MAXLEN; args.trim_strategy_arg_idx = i; } else if (strcasecmp(opt.c_str(), "minid") == 0 && moreargs) { // case: XADD mystream ... MINID [= | ~] threshold ... - if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + if (args.trim_strategy != storage::StreamTrimStrategy::TRIM_STRATEGY_NONE) { res.SetRes(CmdRes::kSyntaxErr, "syntax error, MAXLEN and MINID options at the same time are not compatible"); return; } @@ -66,12 +69,12 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA i++; } // parse threshold as stremID - if (!StreamUtils::StreamParseID(argv[i + 1], args.minid, 0)) { + if (!storage::StreamUtils::StreamParseID(argv[i + 1], args.minid, 0)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } i++; - args.trim_strategy = StreamTrimStrategy::TRIM_STRATEGY_MINID; + args.trim_strategy = storage::StreamTrimStrategy::TRIM_STRATEGY_MINID; args.trim_strategy_arg_idx = i; } else if (strcasecmp(opt.c_str(), "limit") == 0 && moreargs) { @@ -86,7 +89,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA } else if (is_xadd) { // case: XADD mystream ... ID ... - if (!StreamUtils::StreamParseStrictID(argv[i], args.id, 0, &args.seq_given)) { + if (!storage::StreamUtils::StreamParseStrictID(argv[i], args.id, 0, &args.seq_given)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } @@ -109,7 +112,7 @@ void ParseAddOrTrimArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamA * [NOACK] STREAMS key [key ...] id [id ...] * XREAD [COUNT count] [BLOCK milliseconds] STREAMS key [key ...] id * [id ...] */ -void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, StreamReadGroupReadArgs &args, +void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, storage::StreamReadGroupReadArgs &args, bool is_xreadgroup) { int streams_arg_idx{0}; // the index of stream keys arg size_t streams_cnt{0}; // the count of stream keys @@ -119,13 +122,13 @@ void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, S const std::string &o = argv[i]; if (strcasecmp(o.c_str(), "BLOCK") == 0 && moreargs) { i++; - if (!StreamUtils::string2uint64(argv[i].c_str(), args.block)) { + if (!storage::StreamUtils::string2uint64(argv[i].c_str(), args.block)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid BLOCK argument"); return; } } else if (strcasecmp(o.c_str(), "COUNT") == 0 && moreargs) { i++; - if (!StreamUtils::string2int32(argv[i].c_str(), args.count)) { + if (!storage::StreamUtils::string2int32(argv[i].c_str(), args.count)) { res.SetRes(CmdRes::kInvalidParameter, "Invalid COUNT argument"); return; } @@ -178,12 +181,12 @@ void ParseReadOrReadGroupArgsOrReply(CmdRes &res, const PikaCmdArgsType &argv, S } } -void AppendMessagesToRes(CmdRes &res, std::vector &field_values, const DB* db) { +void AppendMessagesToRes(CmdRes &res, std::vector &id_messages, const DB* db) { assert(db); - res.AppendArrayLenUint64(field_values.size()); - for (auto &fv : field_values) { + res.AppendArrayLenUint64(id_messages.size()); + for (auto &fv : id_messages) { std::vector message; - if (!StreamUtils::DeserializeMessage(fv.value, message)) { + if (!storage::StreamUtils::DeserializeMessage(fv.value, message)) { LOG(ERROR) << "Deserialize message failed"; res.SetRes(CmdRes::kErrOther, "Deserialize message failed"); return; @@ -191,7 +194,7 @@ void AppendMessagesToRes(CmdRes &res, std::vector &field_va assert(message.size() % 2 == 0); res.AppendArrayLen(2); - streamID sid; + storage::streamID sid; sid.DeserializeFrom(fv.field); res.AppendString(sid.ToString()); // field here is the stream id res.AppendArrayLenUint64(message.size()); @@ -208,12 +211,6 @@ void XAddCmd::DoInitial() { } key_ = argv_[1]; - // check the conflict of stream used prefix, see detail in defination of STREAM_TREE_PREFIX - if (key_ == STREAM_LAST_GENERATED_TREE_ID_FIELD) { - res_.SetRes(CmdRes::kErrOther, "Can not use " + STREAM_LAST_GENERATED_TREE_ID_FIELD + "as stream key"); - return; - } - int idpos{-1}; ParseAddOrTrimArgsOrReply(res_, argv_, args_, &idpos, true); if (res_.ret() != CmdRes::kNone) { @@ -232,28 +229,15 @@ void XAddCmd::DoInitial() { } void XAddCmd::Do() { - // 1 get stream meta - rocksdb::Status s; - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); - if (s.IsNotFound() && args_.no_mkstream) { - res_.SetRes(CmdRes::kNotFound); - return; - } else if (s.IsNotFound()) { - stream_meta.Init(); - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, get stream meta failed: " + s.ToString()); - return; - } - - if (stream_meta.last_id().ms == UINT64_MAX && stream_meta.last_id().seq == UINT64_MAX) { - res_.SetRes(CmdRes::kErrOther, "Fatal! Sequence number overflow !"); + std::string message; + if (!storage::StreamUtils::SerializeMessage(argv_, message, field_pos_)) { + res_.SetRes(CmdRes::kErrOther, "Serialize message failed"); return; } - // 2 append the message to storage - GenerateStreamIDOrReply(stream_meta); - if (res_.ret() != CmdRes::kNone) { + auto s = db_->storage()->XAdd(key_, message, args_); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } @@ -263,122 +247,30 @@ void XAddCmd::Do() { argv_[field_pos_ - 1] = args_.id.ToString(); } - std::string message; - if (!StreamUtils::SerializeMessage(argv_, message, field_pos_)) { - res_.SetRes(CmdRes::kErrOther, "Serialize message failed"); - return; - } - - // check the serialized current id is larger than last_id -#ifdef DEBUG - std::string serialized_last_id; - stream_meta.last_id().SerializeTo(serialized_last_id); - assert(field > serialized_last_id); -#endif // DEBUG - - s = StreamStorage::InsertStreamMessage(key_, args_.id, message, db_.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, insert stream message failed 1: " + s.ToString()); - return; - } - - // 3 update stream meta - if (stream_meta.length() == 0) { - stream_meta.set_first_id(args_.id); - } - stream_meta.set_entries_added(stream_meta.entries_added() + 1); - stream_meta.set_last_id(args_.id); - stream_meta.set_length(stream_meta.length() + 1); - - // 4 trim the stream if needed - if (args_.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { - int32_t count; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, db_.get()), res_); - (void)count; - } - - // 5 update stream meta - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, "error from XADD, get stream meta failed 2: " + s.ToString()); - return; - } - res_.AppendString(args_.id.ToString()); } -void XAddCmd::GenerateStreamIDOrReply(const StreamMetaValue &stream_meta) { - auto &id = args_.id; - if (args_.id_given && args_.seq_given && id.ms == 0 && id.seq == 0) { - res_.SetRes(CmdRes::kInvalidParameter, "The ID specified in XADD must be greater than 0-0"); - return; - } - - if (!args_.id_given || !args_.seq_given) { - // if id not given, generate one - if (!args_.id_given) { - id.ms = StreamUtils::GetCurrentTimeMs(); - - if (id.ms < stream_meta.last_id().ms) { - id.ms = stream_meta.last_id().ms; - if (stream_meta.last_id().seq == UINT64_MAX) { - id.ms++; - id.seq = 0; - } else { - id.seq++; - } - return; - } - } - - // generate seq - auto last_id = stream_meta.last_id(); - if (id.ms < last_id.ms) { - res_.SetRes(CmdRes::kErrOther, "The ID specified in XADD is equal or smaller"); - return; - } else if (id.ms == last_id.ms) { - if (last_id.seq == UINT64_MAX) { - res_.SetRes(CmdRes::kErrOther, "The ID specified in XADD is equal or smaller"); - return; - } - id.seq = last_id.seq + 1; - } else { - id.seq = 0; - } - - } else { - // Full ID given, check id - auto last_id = stream_meta.last_id(); - if (id.ms < last_id.ms || (id.ms == last_id.ms && id.seq <= last_id.seq)) { - res_.SetRes(CmdRes::kErrOther, "INVALID ID given"); - return; - } - } -} - void XRangeCmd::DoInitial() { if (!CheckArg(argv_.size())) { res_.SetRes(CmdRes::kWrongNum, kCmdNameXRange); return; } key_ = argv_[1]; - if (!StreamUtils::StreamParseIntervalId(argv_[2], start_sid, &start_ex_, 0) || - !StreamUtils::StreamParseIntervalId(argv_[3], end_sid, &end_ex_, UINT64_MAX)) { + if (!storage::StreamUtils::StreamParseIntervalId(argv_[2], args_.start_sid, &args_.start_ex, 0) || + !storage::StreamUtils::StreamParseIntervalId(argv_[3], args_.end_sid, &args_.end_ex, UINT64_MAX)) { res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } - if (start_ex_ && start_sid.ms == UINT64_MAX && start_sid.seq == UINT64_MAX) { + if (args_.start_ex && args_.start_sid.ms == UINT64_MAX && args_.start_sid.seq == UINT64_MAX) { res_.SetRes(CmdRes::kInvalidParameter, "invalid start id"); return; } - if (end_ex_ && end_sid.ms == 0 && end_sid.seq == 0) { + if (args_.end_ex && args_.end_sid.ms == 0 && args_.end_sid.seq == 0) { res_.SetRes(CmdRes::kInvalidParameter, "invalid end id"); return; } if (argv_.size() == 6) { - // pika's PKHScanRange() only sopport max count of INT32_MAX - // but redis supports max count of UINT64_MAX - if (!StreamUtils::string2int32(argv_[5].c_str(), count_)) { + if (!storage::StreamUtils::string2int32(argv_[5].c_str(), args_.limit)) { res_.SetRes(CmdRes::kInvalidParameter, "COUNT should be a integer greater than 0 and not bigger than INT32_MAX"); return; } @@ -386,97 +278,43 @@ void XRangeCmd::DoInitial() { } void XRangeCmd::Do() { - std::vector field_values; + std::vector id_messages; - if (start_sid <= end_sid) { - StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, false); - std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); - (void)next_field; + if (args_.start_sid <= args_.end_sid) { + auto s = db_->storage()->XRange(key_, args_, id_messages); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } } - AppendMessagesToRes(res_, field_values, db_.get()); + AppendMessagesToRes(res_, id_messages, db_.get()); } void XRevrangeCmd::Do() { - std::vector field_values; + std::vector id_messages; - if (start_sid >= end_sid) { - StreamStorage::ScanStreamOptions options(key_, start_sid, end_sid, count_, start_ex_, end_ex_, true); - std::string next_field; - auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); - (void)next_field; + if (args_.start_sid >= args_.end_sid) { + auto s = db_->storage()->XRevrange(key_, args_, id_messages); if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } } - AppendMessagesToRes(res_, field_values, db_.get()); -} - -inline void XDelCmd::SetFirstIDOrReply(StreamMetaValue &stream_meta, const DB* db) { - assert(db); - return SetFirstOrLastIDOrReply(stream_meta, db, true); -} - -inline void XDelCmd::SetLastIDOrReply(StreamMetaValue &stream_meta, const DB* db) { - assert(db); - return SetFirstOrLastIDOrReply(stream_meta, db, false); -} - -inline void XDelCmd::SetFirstOrLastIDOrReply(StreamMetaValue &stream_meta, const DB* db, bool is_set_first) { - assert(db); - if (stream_meta.length() == 0) { - stream_meta.set_first_id(kSTREAMID_MIN); - return; - } - - std::vector field_values; - std::string next_field; - - storage::Status s; - if (is_set_first) { - StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MIN, kSTREAMID_MAX, 1); - s = StreamStorage::ScanStream(option, field_values, next_field, db); - } else { - bool is_reverse = true; - StreamStorage::ScanStreamOptions option(key_, kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); - s = StreamStorage::ScanStream(option, field_values, next_field, db); - } - (void)next_field; - - if (!s.ok() && !s.IsNotFound()) { - LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - if (field_values.empty()) { - LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; - res_.SetRes(CmdRes::kErrOther, "Internal error: no messages found but stream length is not 0"); - return; - } - - streamID id; - id.DeserializeFrom(field_values[0].field); - stream_meta.set_first_id(id); + AppendMessagesToRes(res_, id_messages, db_.get()); } void XDelCmd::DoInitial() { if (!CheckArg(argv_.size())) { - res_.SetRes(CmdRes::kWrongNum, kCmdNameXDel); + res_.SetRes(CmdRes::kWrongNum, kCmdNameXAdd); return; } key_ = argv_[1]; for (int i = 2; i < argv_.size(); i++) { - streamID id; - if (!StreamUtils::StreamParseStrictID(argv_[i], id, 0, nullptr)) { + storage::streamID id; + if (!storage::StreamUtils::StreamParseStrictID(argv_[i], id, 0, nullptr)) { res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); return; } @@ -488,42 +326,14 @@ void XDelCmd::DoInitial() { } void XDelCmd::Do() { - // 1 try to get stream meta - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); - if (s.IsNotFound()) { - res_.AppendInteger(0); - return; - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - // 2 do the delete int32_t count{0}; - s = StreamStorage::DeleteStreamMessage(key_, ids_, count, db_.get()); - if (!s.ok()) { + auto s = db_->storage()->XDel(key_, ids_, count); + if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - // 3 update stream meta - stream_meta.set_length(stream_meta.length() - count); - for (const auto &id : ids_) { - if (id > stream_meta.max_deleted_entry_id()) { - stream_meta.set_max_deleted_entry_id(id); - } - if (id == stream_meta.first_id()) { - SetFirstIDOrReply(stream_meta, db_.get()); - } else if (id == stream_meta.last_id()) { - SetLastIDOrReply(stream_meta, db_.get()); - } } - s = StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()); - if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; + if (count > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); } res_.AppendInteger(count); @@ -538,9 +348,8 @@ void XLenCmd::DoInitial() { } void XLenCmd::Do() { - rocksdb::Status s; - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); + int32_t len{0}; + auto s = db_->storage()->XLen(key_, len); if (s.IsNotFound()) { res_.SetRes(CmdRes::kNotFound); return; @@ -549,11 +358,11 @@ void XLenCmd::Do() { return; } - if (stream_meta.length() > INT_MAX) { - res_.SetRes(CmdRes::kErrOther, "stream's length is larger than INT_MAX"); - return; + if (len > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "stream's length is larger than INT_MAX"); } - res_.AppendInteger(static_cast(stream_meta.length())); + + res_.AppendInteger(len); return; } @@ -567,70 +376,34 @@ void XReadCmd::DoInitial() { } void XReadCmd::Do() { - rocksdb::Status s; - - // 1 prepare stream_metas - std::vector> streammeta_idx; - for (int i = 0; i < args_.unparsed_ids.size(); i++) { - const auto &key = args_.keys[i]; - const auto &unparsed_id = args_.unparsed_ids[i]; - - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key, db_.get()); - if (s.IsNotFound()) { - continue; - } else if (!s.ok()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } - - streammeta_idx.emplace_back(std::move(stream_meta), i); + std::vector> results; + // The wrong key will not trigger error, just be ignored, + // we need to save the right key，and return it to client. + std::vector reserved_keys; + auto s = db_->storage()->XRead(args_, results, reserved_keys); + + if (!s.ok() && s.ToString() == + "The > ID can be specified only when calling " + "XREADGROUP using the GROUP " + " option.") { + res_.SetRes(CmdRes::kSyntaxErr, s.ToString()); + } else if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); } - if (streammeta_idx.empty()) { + if (results.empty()) { res_.AppendArrayLen(-1); return; } - // 2 do the scan - res_.AppendArrayLenUint64(streammeta_idx.size()); - for (const auto &stream_meta_id : streammeta_idx) { - const auto &stream_meta = stream_meta_id.first; - const auto &idx = stream_meta_id.second; - const auto &unparsed_id = args_.unparsed_ids[idx]; - const auto &key = args_.keys[idx]; - - // 2.1 try to parse id - streamID id; - if (unparsed_id == "<") { - res_.SetRes(CmdRes::kSyntaxErr, - "The > ID can be specified only when calling " - "XREADGROUP using the GROUP " - " option."); - return; - } else if (unparsed_id == "$") { - id = stream_meta.last_id(); - } else { - if (!StreamUtils::StreamParseStrictID(unparsed_id, id, 0, nullptr)) { - res_.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); - return; - } - } - - // 2.2 scan - std::vector field_values; - std::string next_field; - StreamStorage::ScanStreamOptions options(key, id, kSTREAMID_MAX, args_.count, true); - auto s = StreamStorage::ScanStream(options, field_values, next_field, db_.get()); - (void)next_field; - if (!s.ok() && !s.IsNotFound()) { - res_.SetRes(CmdRes::kErrOther, s.ToString()); - return; - } + assert(results.size() == reserved_keys.size()); + // 2 do the scan + res_.AppendArrayLenUint64(results.size()); + for (size_t i = 0; i < results.size(); ++i) { res_.AppendArrayLen(2); - res_.AppendString(key); - AppendMessagesToRes(res_, field_values, db_.get()); + res_.AppendString(reserved_keys[i]); + AppendMessagesToRes(res_, results[i], db_.get()); } } @@ -648,23 +421,16 @@ void XTrimCmd::DoInitial() { } void XTrimCmd::Do() { - // 1 try to get stream meta, if not found, return error - StreamMetaValue stream_meta; - auto s = StreamStorage::GetStreamMeta(stream_meta, key_, db_.get()); - if (s.IsNotFound()) { - res_.AppendInteger(0); - return; - } else if (!s.ok()) { + int32_t count{0}; + auto s = db_->storage()->XTrim(key_, args_, count); + if (!s.ok() && !s.IsNotFound()) { res_.SetRes(CmdRes::kErrOther, s.ToString()); return; } - // 2 do the trim - int32_t count{0}; - TRY_CATCH_ERROR(StreamStorage::TrimStream(count, stream_meta, key_, args_, db_.get()), res_); - - // 3 update stream meta - TRY_CATCH_ERROR(StreamStorage::SetStreamMeta(key_, stream_meta.value(), db_.get()), res_); + if (count > INT_MAX) { + return res_.SetRes(CmdRes::kErrOther, "count is larger than INT_MAX"); + } res_.AppendInteger(count); return; @@ -681,7 +447,7 @@ void XInfoCmd::DoInitial() { if (!strcasecmp(subcmd_.c_str(), "STREAM")) { if (argv_.size() > 3 && strcasecmp(subcmd_.c_str(), "FULL") != 0) { is_full_ = true; - if (argv_.size() > 4 && !StreamUtils::string2uint64(argv_[4].c_str(), count_)) { + if (argv_.size() > 4 && !storage::StreamUtils::string2uint64(argv_[4].c_str(), count_)) { res_.SetRes(CmdRes::kInvalidParameter, "invalid count"); return; } @@ -726,22 +492,27 @@ void XInfoCmd::Do() { } void XInfoCmd::StreamInfo(std::shared_ptr& db) { - // 1 try to get stream meta - StreamMetaValue stream_meta; - TRY_CATCH_ERROR(StreamStorage::GetStreamMeta(stream_meta, key_, db.get()), res_); + storage::StreamInfoResult info; + auto s = db_->storage()->XInfo(key_, info); + + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kNotFound); + return; + } - // 2 append the stream info + // // 2 append the stream info res_.AppendArrayLen(10); res_.AppendString("length"); - res_.AppendInteger(static_cast(stream_meta.length())); + res_.AppendInteger(static_cast(info.length)); res_.AppendString("last-generated-id"); - res_.AppendString(stream_meta.last_id().ToString()); + res_.AppendString(info.last_id_str); res_.AppendString("max-deleted-entry-id"); - res_.AppendString(stream_meta.max_deleted_entry_id().ToString()); + res_.AppendString(info.max_deleted_entry_id_str); res_.AppendString("entries-added"); - res_.AppendInteger(static_cast(stream_meta.entries_added())); + res_.AppendInteger(static_cast(info.entries_added)); res_.AppendString("recorded-first-entry-id"); - res_.AppendString(stream_meta.first_id().ToString()); - - // Korpse TODO: add group info + res_.AppendString(info.first_id_str); } diff --git a/src/pika_stream_base.cc b/src/pika_stream_base.cc deleted file mode 100644 index 9437da3774..0000000000 --- a/src/pika_stream_base.cc +++ /dev/null @@ -1,619 +0,0 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "include/pika_stream_base.h" - -#include "include/pika_command.h" -#include "include/pika_stream_meta_value.h" -#include "include/pika_stream_types.h" -#include "include/pika_db.h" -#include "storage/storage.h" - - -#define GET_NEXT_TREE_ID_AND_CHECK(db, tid) \ - do { \ - auto &tid_gen = TreeIDGenerator::GetInstance(); \ - auto s = tid_gen.GetNextTreeID(db, tid); \ - if (!s.ok()) { \ - return s; \ - } \ - } while (0) - -storage::Status TreeIDGenerator::GetNextTreeID(const DB* db, treeID &tid) { - assert(db); - auto expected_id = kINVALID_TREE_ID; - - // if tree_id_ is not initialized (equal to kINVALID_TREE_ID), try to fetch it from storage - if (tree_id_.compare_exchange_strong(expected_id, START_TREE_ID)) { - std::string value; - storage::Status s = db->storage()->HGet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, &value); - if (s.ok()) { - treeID id; - if (!StreamUtils::string2int32(value.c_str(), id)) { - LOG(ERROR) << "Invalid tree id: " << value; - return storage::Status::Corruption("Invalid tree id"); - } - tree_id_.store(id); - - } else if (!s.IsNotFound()) { - return s; - } - } - - tree_id_.fetch_add(1); - tid = tree_id_; - - // insert tree id to storage - std::string tree_id_str = std::to_string(tree_id_); - int32_t res; - return db->storage()->HSet(STREAM_META_HASH_KEY, STREAM_LAST_GENERATED_TREE_ID_FIELD, tree_id_str, &res); -} - -storage::Status StreamStorage::GetStreamMeta(StreamMetaValue &stream_meta, const std::string &key, const DB* db) { - assert(db); - std::string value; - auto s = db->storage()->HGet(STREAM_META_HASH_KEY, key, &value); - if (s.ok()) { - stream_meta.ParseFrom(value); - return storage::Status::OK(); - } - return s; -} - -// no need to be thread safe, only xadd will call this function -// and xadd can be locked by the same key using current_key() -storage::Status StreamStorage::SetStreamMeta(const std::string &key, std::string &meta_value, const DB* db) { - assert(db); - storage::Status s; - int32_t temp{0}; - s = db->storage()->HSet(STREAM_META_HASH_KEY, key, meta_value, &temp); - (void)temp; - return s; -} - -storage::Status StreamStorage::DeleteStreamMeta(const std::string &key, const DB* db) { - assert(db); - int32_t ret; - return db->storage()->HDel({STREAM_META_HASH_KEY}, {key}, &ret); -} - -storage::Status StreamStorage::InsertStreamMessage(const std::string &key, const streamID &id, - const std::string &message, const DB* db) { - assert(db); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - int32_t temp{0}; - std::string serialized_id; - id.SerializeTo(serialized_id); - storage::Status s = db->storage()->HSet(true_key, serialized_id, message, &temp); - (void)temp; - return s; -} - -storage::Status StreamStorage::GetStreamMessage(const std::string &key, const std::string &sid, std::string &message, - const DB* db) { - assert(db); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return db->storage()->HGet(true_key, sid, &message); -} - -storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, const std::vector &id, - int32_t &ret, const DB* db) { - assert(db); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - std::vector serialized_ids; - for (auto &sid : id) { - std::string serialized_id; - sid.SerializeTo(serialized_id); - serialized_ids.emplace_back(std::move(serialized_id)); - } - return db->storage()->HDel(true_key, {serialized_ids}, &ret); -} - -storage::Status StreamStorage::DeleteStreamMessage(const std::string &key, - const std::vector &serialized_ids, int32_t &ret, - const DB* db) { - assert(db); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - return db->storage()->HDel(true_key, {serialized_ids}, &ret); -} - -storage::Status StreamStorage::ScanStream(const ScanStreamOptions &op, std::vector &field_values, - std::string &next_field, const DB* db) { - assert(db); - std::string start_field; - std::string end_field; - storage::Slice pattern = "*"; // match all the fields from start_field to end_field - storage::Status s; - std::string true_key = STREAM_DATA_HASH_PREFIX + op.key; - - // 1 do the scan - if (op.is_reverse) { - op.end_sid.SerializeTo(start_field); - if (op.start_sid == kSTREAMID_MAX) { - start_field = ""; - } else { - op.start_sid.SerializeTo(start_field); - } - s = db->storage()->PKHRScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); - } else { - op.start_sid.SerializeTo(start_field); - if (op.end_sid == kSTREAMID_MAX) { - end_field = ""; - } else { - op.end_sid.SerializeTo(end_field); - } - s = db->storage()->PKHScanRange(true_key, start_field, end_field, pattern, op.count, &field_values, &next_field); - } - - // 2 exclude the start_sid and end_sid if needed - if (op.start_ex && !field_values.empty()) { - streamID sid; - sid.DeserializeFrom(field_values.front().field); - if (sid == op.start_sid) { - field_values.erase(field_values.begin()); - } - } - - if (op.end_ex && !field_values.empty()) { - streamID sid; - sid.DeserializeFrom(field_values.back().field); - if (sid == op.end_sid) { - field_values.pop_back(); - } - } - - return s; -} - -storage::Status StreamStorage::DeleteStreamData(const std::string &key, const DB* db) { - assert(db); - std::string true_key = STREAM_DATA_HASH_PREFIX + key; - std::map type_status; - int64_t count = db->storage()->Del({true_key}, &type_status); - storage::Status s = type_status[storage::DataType::kHashes]; - return s; -} - -storage::Status StreamStorage::GetTreeNodeValue(const treeID tid, std::string &field, std::string &value, - const DB* db) { - assert(db); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - storage::Status s; - s = db->storage()->HGet(key, field, &value); - return s; -} - -storage::Status StreamStorage::InsertTreeNodeValue(const treeID tid, const std::string &filed, const std::string &value, - const DB* db) { - assert(db); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - - storage::Status s; - int res; - s = db->storage()->HSet(key, filed, value, &res); - (void)res; - return s; -} - -storage::Status StreamStorage::DeleteTreeNode(const treeID tid, const std::string &field, const DB* db) { - assert(db); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - - storage::Status s; - int res; - s = db->storage()->HDel(key, std::vector{field}, &res); - (void)res; - return s; -} - -storage::Status StreamStorage::GetAllTreeNode(const treeID tid, std::vector &field_values, - const DB* db) { - assert(db); - auto key = std::move(StreamUtils::TreeID2Key(tid)); - return db->storage()->PKHScanRange(key, "", "", "*", INT_MAX, &field_values, nullptr); -} - -storage::Status StreamStorage::DeleteTree(const treeID tid, const DB* db) { - assert(db); - assert(tid != kINVALID_TREE_ID); - storage::Status s; - auto key = std::move(StreamUtils::TreeID2Key(tid)); - std::map type_status; - int64_t count = db->storage()->Del({key}, &type_status); - s = type_status[storage::DataType::kStrings]; - return s; -} - -storage::Status StreamStorage::CreateConsumer(treeID consumer_tid, std::string &consumername, const DB* db) { - assert(db); - std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, db); - if (s.IsNotFound()) { - treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(db, pel_tid); - - StreamConsumerMetaValue consumer_meta; - consumer_meta.Init(pel_tid); - s = StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), db); - if (!s.ok()) { - LOG(ERROR) << "Insert consumer meta failed"; - return s; - } - } - // consumer meta already exists or other error - return s; -} - -storage::Status StreamStorage::GetOrCreateConsumer(treeID consumer_tid, std::string &consumername, const DB* db, - StreamConsumerMetaValue &consumer_meta) { - assert(db); - std::string consumer_meta_value; - auto s = StreamStorage::GetTreeNodeValue(consumer_tid, consumername, consumer_meta_value, db); - if (s.ok()) { - consumer_meta.ParseFrom(consumer_meta_value); - - } else if (s.IsNotFound()) { - treeID pel_tid; - GET_NEXT_TREE_ID_AND_CHECK(db, pel_tid); - consumer_meta.Init(pel_tid); - return StreamStorage::InsertTreeNodeValue(consumer_tid, consumername, consumer_meta.value(), db); - } - - return s; -} - -bool StreamUtils::StreamGenericParseID(const std::string &var, streamID &id, uint64_t missing_seq, bool strict, - bool *seq_given) { - char buf[128]; - if (var.size() > sizeof(buf) - 1) { - return false; - } - - memcpy(buf, var.data(), var.size()); - buf[var.size()] = '\0'; - - if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { - // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); - return false; - } - - if (seq_given != nullptr) { - *seq_given = true; - } - - if (buf[0] == '-' && buf[1] == '\0') { - id.ms = 0; - id.seq = 0; - return true; - } else if (buf[0] == '+' && buf[1] == '\0') { - id.ms = UINT64_MAX; - id.seq = UINT64_MAX; - return true; - } - - uint64_t ms; - uint64_t seq; - char *dot = strchr(buf, '-'); - if (dot) { - *dot = '\0'; - } - if (!StreamUtils::string2uint64(buf, ms)) { - return false; - }; - if (dot) { - size_t seqlen = strlen(dot + 1); - if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { - seq = 0; - *seq_given = false; - } else if (!StreamUtils::string2uint64(dot + 1, seq)) { - return false; - } - } else { - seq = missing_seq; - } - id.ms = ms; - id.seq = seq; - return true; -} - -bool StreamUtils::StreamParseID(const std::string &var, streamID &id, uint64_t missing_seq) { - return StreamGenericParseID(var, id, missing_seq, false, nullptr); -} - -bool StreamUtils::StreamParseStrictID(const std::string &var, streamID &id, uint64_t missing_seq, bool *seq_given) { - return StreamGenericParseID(var, id, missing_seq, true, seq_given); -} - -bool StreamUtils::StreamParseIntervalId(const std::string &var, streamID &id, bool *exclude, uint64_t missing_seq) { - if (exclude != nullptr) { - *exclude = (var.size() > 1 && var[0] == '('); - } - if (exclude != nullptr && *exclude) { - return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); - } else { - return StreamParseID(var, id, missing_seq); - } -} - -storage::Status StreamStorage::TrimByMaxlen(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const DB* db, const StreamAddTrimArgs &args) { - assert(db); - storage::Status s; - // we delete the message in batchs, prevent from using too much memory - while (stream_meta.length() - trim_ret.count > args.maxlen) { - auto cur_batch = - (std::min(static_cast(stream_meta.length() - trim_ret.count - args.maxlen), kDEFAULT_TRIM_BATCH_SIZE)); - std::vector filed_values; - - StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), kSTREAMID_MAX, cur_batch, false, false, - false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, db); - if (!s.ok() && !s.IsNotFound()) { - return s; - } - - assert(filed_values.size() == cur_batch); - trim_ret.count += cur_batch; - trim_ret.max_deleted_field = filed_values.back().field; - - // delete the message in batchs - std::vector fields_to_del; - fields_to_del.reserve(filed_values.size()); - for (auto &fv : filed_values) { - fields_to_del.emplace_back(std::move(fv.field)); - } - int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, db); - if (!s.ok()) { - return s; - } - assert(ret == fields_to_del.size()); - } - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::TrimByMinid(TrimRet &trim_ret, StreamMetaValue &stream_meta, const std::string &key, - const DB* db, const StreamAddTrimArgs &args) { - assert(db); - storage::Status s; - std::string serialized_min_id; - stream_meta.first_id().SerializeTo(trim_ret.next_field); - args.minid.SerializeTo(serialized_min_id); - - // we delete the message in batchs, prevent from using too much memory - while (trim_ret.next_field < serialized_min_id && stream_meta.length() - trim_ret.count > 0) { - auto cur_batch = static_cast(std::min(static_cast(stream_meta.length() - trim_ret.count), kDEFAULT_TRIM_BATCH_SIZE)); - std::vector filed_values; - - StreamStorage::ScanStreamOptions options(key, stream_meta.first_id(), args.minid, cur_batch, false, false, false); - s = StreamStorage::ScanStream(options, filed_values, trim_ret.next_field, db); - if (!s.ok() && !s.IsNotFound()) { - return s; - } - - if (!filed_values.empty()) { - if (filed_values.back().field == serialized_min_id) { - // we do not need to delete the message that it's id matches the minid - filed_values.pop_back(); - trim_ret.next_field = serialized_min_id; - } - // duble check - if (!filed_values.empty()) { - trim_ret.max_deleted_field = filed_values.back().field; - } - } - - assert(filed_values.size() <= cur_batch); - trim_ret.count += static_cast(filed_values.size()); - - // do the delete in batch - std::vector fields_to_del; - fields_to_del.reserve(filed_values.size()); - for (auto &fv : filed_values) { - fields_to_del.emplace_back(std::move(fv.field)); - } - int32_t ret; - s = StreamStorage::DeleteStreamMessage(key, fields_to_del, ret, db); - if (!s.ok()) { - return s; - } - assert(ret == fields_to_del.size()); - } - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::TrimStream(int32_t &count, StreamMetaValue &stream_meta, const std::string &key, - StreamAddTrimArgs &args, const DB* db) { - assert(db); - count = 0; - // 1 do the trim - TrimRet trim_ret; - storage::Status s; - if (args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MAXLEN) { - s = TrimByMaxlen(trim_ret, stream_meta, key, db, args); - } else { - assert(args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MINID); - s = TrimByMinid(trim_ret, stream_meta, key, db, args); - } - - if (!s.ok()) { - return s; - } - - if (trim_ret.count == 0) { - return s; - } - - // 2 update stream meta - streamID first_id; - streamID max_deleted_id; - if (stream_meta.length() == trim_ret.count) { - // all the message in the stream were deleted - first_id = kSTREAMID_MIN; - } else { - first_id.DeserializeFrom(trim_ret.next_field); - } - assert(!trim_ret.max_deleted_field.empty()); - max_deleted_id.DeserializeFrom(trim_ret.max_deleted_field); - - stream_meta.set_first_id(first_id); - if (max_deleted_id > stream_meta.max_deleted_entry_id()) { - stream_meta.set_max_deleted_entry_id(max_deleted_id); - } - stream_meta.set_length(stream_meta.length() - trim_ret.count); - - count = trim_ret.count; - - s = storage::Status::OK(); - return s; -} - -storage::Status StreamStorage::DestoryStreams(std::vector &keys, const DB* db) { - assert(db); - storage::Status s; - for (const auto &key : keys) { - // 1.try to get the stream meta - StreamMetaValue stream_meta; - s = StreamStorage::GetStreamMeta(stream_meta, key, db); - if (s.IsNotFound()) { - continue; - } else if (!s.ok()) { - LOG(ERROR) << "Get stream meta failed"; - return s; - } - - // 2 destroy all the cgroup - // 2.1 find all the cgroups' meta - auto cgroup_tid = stream_meta.groups_id(); - if (cgroup_tid != kINVALID_TREE_ID) { - // Korpse TODO: delete the cgroup, now we don't have groups. - } - - // 3 delete stream meta - s = StreamStorage::DeleteStreamMeta(key, db); - if (!s.ok()) { - return s; - } - - // 4 delete stream data - s = StreamStorage::DeleteStreamData(key, db); - if (!s.ok()) { - return s; - } - } - - s = storage::Status::OK(); - return s; -} - -bool StreamUtils::string2uint64(const char *s, uint64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - uint64_t tmp = strtoull(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int64(const char *s, int64_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - int64_t tmp = std::strtoll(s, &end, 10); - if (*end || errno == ERANGE) { - // Conversion either didn't consume the entire string, or overflow occurred - return false; - } - - value = tmp; - return true; -} - -bool StreamUtils::string2int32(const char *s, int32_t &value) { - if (!s || !*s) { - return false; - } - - char *end; - errno = 0; - long tmp = strtol(s, &end, 10); - if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { - // Conversion either didn't consume the entire string, - // or overflow or underflow occurred - return false; - } - - value = static_cast(tmp); - return true; -} - -std::string StreamUtils::TreeID2Key(const treeID &tid) { - std::string key; - key.reserve(STERAM_TREE_PREFIX.size() + sizeof(tid)); - key.append(STERAM_TREE_PREFIX); - key.append(reinterpret_cast(&tid), sizeof(tid)); - return key; -} - -bool StreamUtils::SerializeMessage(const std::vector &field_values, std::string &message, int field_pos) { - assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); - assert(message.empty()); - // count the size of serizlized message - size_t size = 0; - for (int i = field_pos; i < field_values.size(); i++) { - size += field_values[i].size() + sizeof(size_t); - } - message.reserve(size); - - // serialize message - for (int i = field_pos; i < field_values.size(); i++) { - size_t len = field_values[i].size(); - message.append(reinterpret_cast(&len), sizeof(len)); - message.append(field_values[i]); - } - - return true; -} - -bool StreamUtils::DeserializeMessage(const std::string &message, std::vector &parsed_message) { - size_t pos = 0; - while (pos < message.size()) { - // Read the length of the next field value from the message - size_t len = *reinterpret_cast(&message[pos]); - pos += sizeof(size_t); - - // Check if the calculated end of the string is still within the message bounds - if (pos + len > message.size()) { - LOG(ERROR) << "Invalid message format, failed to parse message"; - return false; // Error: not enough data in the message string - } - - // Extract the field value and add it to the vector - parsed_message.push_back(message.substr(pos, len)); - pos += len; - } - - return true; -} - -uint64_t StreamUtils::GetCurrentTimeMs() { - return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - .count(); -} diff --git a/src/storage/include/storage/storage.h b/src/storage/include/storage/storage.h index fa4b7f9604..cbd55ae3a3 100644 --- a/src/storage/include/storage/storage.h +++ b/src/storage/include/storage/storage.h @@ -39,6 +39,7 @@ inline const std::string HASHES_DB = "hashes"; inline const std::string LISTS_DB = "lists"; inline const std::string ZSETS_DB = "zsets"; inline const std::string SETS_DB = "sets"; +inline const std::string STREAMS_DB = "streams"; inline constexpr size_t BATCH_DELETE_LIMIT = 100; inline constexpr size_t COMPACT_THRESHOLD_COUNT = 2000; @@ -53,9 +54,16 @@ class RedisHashes; class RedisSets; class RedisLists; class RedisZSets; +class RedisStreams; class HyperLogLog; enum class OptionType; +struct StreamAddTrimArgs; +struct StreamReadGroupReadArgs; +struct StreamScanArgs; +struct streamID; +struct StreamInfoResult; + template class LRUCache; @@ -97,6 +105,12 @@ struct FieldValue { bool operator==(const FieldValue& fv) const { return (fv.field == field && fv.value == value); } }; +struct IdMessage { + std::string field; + std::string value; + bool operator==(const IdMessage& fv) const { return (fv.field == field && fv.value == value); } +}; + struct KeyVersion { std::string key; int32_t version; @@ -111,9 +125,9 @@ struct ScoreMember { enum BeforeOrAfter { Before, After }; -enum DataType { kAll, kStrings, kHashes, kLists, kZSets, kSets }; +enum DataType { kAll, kStrings, kHashes, kLists, kZSets, kSets, kStreams }; -const char DataTypeTag[] = {'a', 'k', 'h', 'l', 'z', 's'}; +const char DataTypeTag[] = {'a', 'k', 'h', 'l', 'z', 's', 'x'}; enum class OptionType { kDB, @@ -126,16 +140,26 @@ enum AGGREGATE { SUM, MIN, MAX }; enum BitOpType { kBitOpAnd = 1, kBitOpOr, kBitOpXor, kBitOpNot, kBitOpDefault }; -enum Operation { kNone = 0, kCleanAll, kCleanStrings, kCleanHashes, kCleanZSets, kCleanSets, kCleanLists, kCompactRange }; +enum Operation { + kNone = 0, + kCleanAll, + kCleanStrings, + kCleanHashes, + kCleanZSets, + kCleanSets, + kCleanLists, + kCleanStreams, + kCompactRange +}; struct BGTask { DataType type; Operation operation; std::vector argv; - BGTask(const DataType& _type = DataType::kAll, - const Operation& _opeation = Operation::kNone, - const std::vector& _argv = {}) : type(_type), operation(_opeation), argv(_argv) {} + BGTask(const DataType& _type = DataType::kAll, const Operation& _opeation = Operation::kNone, + const std::vector& _argv = {}) + : type(_type), operation(_opeation), argv(_argv) {} }; class Storage { @@ -908,6 +932,15 @@ class Storage { Status ZScan(const Slice& key, int64_t cursor, const std::string& pattern, int64_t count, std::vector* score_members, int64_t* next_cursor); + Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); + Status XDel(const Slice& key, const std::vector& ids, int32_t& ret); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count); + Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XLen(const Slice& key, int32_t& len); + Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys); + Status XInfo(const Slice& key, StreamInfoResult &result); // Keys Commands // Note: @@ -1067,6 +1100,7 @@ class Storage { std::unique_ptr sets_db_; std::unique_ptr zsets_db_; std::unique_ptr lists_db_; + std::unique_ptr streams_db_; std::atomic is_opened_ = false; std::unique_ptr> cursors_store_; diff --git a/src/storage/src/backupable.cc b/src/storage/src/backupable.cc index 3de6ae2ac6..9ff3ef6e07 100644 --- a/src/storage/src/backupable.cc +++ b/src/storage/src/backupable.cc @@ -7,6 +7,7 @@ #include #include "storage/backupable.h" +#include "storage/storage.h" namespace storage { @@ -36,7 +37,7 @@ Status BackupEngine::Open(storage::Storage* storage, std::shared_ptrGetDBByType(type))) { s = Status::Corruption("Error db type"); diff --git a/src/storage/src/base_data_key_format.h b/src/storage/src/base_data_key_format.h index 316f473070..ffac531046 100644 --- a/src/storage/src/base_data_key_format.h +++ b/src/storage/src/base_data_key_format.h @@ -113,9 +113,17 @@ class ParsedZSetsMemberKey : public ParsedBaseDataKey { Slice member() { return data_; } }; +class ParsedStreamDataKey : public ParsedBaseDataKey { + public: + explicit ParsedStreamDataKey(const std::string* key) : ParsedBaseDataKey(key) {} + explicit ParsedStreamDataKey(const Slice& key) : ParsedBaseDataKey(key) {} + Slice id() { return data_; } +}; + using HashesDataKey = BaseDataKey; using SetsMemberKey = BaseDataKey; using ZSetsMemberKey = BaseDataKey; +using StreamDataKey = BaseDataKey; } // namespace storage #endif // SRC_BASE_DATA_KEY_FORMAT_H_ diff --git a/src/storage/src/pika_stream_meta_value.h b/src/storage/src/pika_stream_meta_value.h new file mode 100644 index 0000000000..1a43f9dcbe --- /dev/null +++ b/src/storage/src/pika_stream_meta_value.h @@ -0,0 +1,497 @@ +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +#include +#include "glog/logging.h" +#include "pika_stream_types.h" +#include "src/coding.h" +#include "storage/storage.h" + +namespace storage { + +static const uint64_t kDefaultStreamValueLength = + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(int32_t) + sizeof(int32_t); +class StreamMetaValue { + public: + explicit StreamMetaValue() = default; + // used only when create a new stream + void InitMetaValue() { + groups_id_ = kINVALID_TREE_ID; + entries_added_ = 0; + first_id_ = streamID(); + last_id_ = streamID(); + max_deleted_entry_id_ = streamID(); + length_ = 0; + + // We do not reset version_ here, because we want to keep the version of the old stream meta. + // Each time we delete a stream, we will increase the version of the stream meta, so that the old stream date will + // not be seen by the new stream with the same key. + ++version_; + + uint64_t needed = kDefaultStreamValueLength; + value_.resize(needed); + + char* dst = &value_[0]; + + // Encode each member into the string + EncodeFixed64(dst, groups_id_); + dst += sizeof(tree_id_t); + + EncodeFixed64(dst, entries_added_); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, first_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, first_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, last_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, last_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed64(dst, max_deleted_entry_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, max_deleted_entry_id_.seq); + dst += sizeof(uint64_t); + + EncodeFixed32(dst, length_); + dst += sizeof(length_); + + EncodeFixed32(dst, version_); + } + + // used only when parse a existed stream meta + // value_ = std::move(value); + void ParseFrom(std::string& value) { + value_ = std::move(value); + assert(value_.size() == kDefaultStreamValueLength); + if (value_.size() != kDefaultStreamValueLength) { + LOG(ERROR) << "Invalid stream meta value length: "; + return; + } + char* pos = &value_[0]; + groups_id_ = DecodeFixed32(pos); + pos += sizeof(tree_id_t); + + entries_added_ = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + first_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + first_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + last_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + last_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + max_deleted_entry_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + max_deleted_entry_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + length_ = static_cast(DecodeFixed32(pos)); + pos += sizeof(length_); + + version_ = static_cast(DecodeFixed32(pos)); + } + + int32_t version() const { return version_; } + + tree_id_t groups_id() const { return groups_id_; } + + uint64_t entries_added() const { return entries_added_; } + + void ModifyEntriesAdded(uint64_t delta) { set_entries_added(entries_added_ + delta); } + + streamID first_id() const { return first_id_; } + + streamID last_id() const { return last_id_; } + + streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } + + int32_t length() const { return length_; } + + std::string& value() { return value_; } + + std::string ToString() { + return "stream_meta: " + std::string("groups_id: ") + std::to_string(groups_id_) + + std::string(", entries_added: ") + std::to_string(entries_added_) + std::string(", first_id: ") + + first_id_.ToString() + std::string(", last_id: ") + last_id_.ToString() + + std::string(", max_deleted_entry_id: ") + max_deleted_entry_id_.ToString() + std::string(", length: ") + + std::to_string(length_) + std::string(", version: ") + std::to_string(version_); + } + + void set_groups_id(tree_id_t groups_id) { + assert(value_.size() == kDefaultStreamValueLength); + groups_id_ = groups_id; + char* dst = const_cast(value_.data()); + EncodeFixed32(dst, groups_id_); + } + + void set_entries_added(uint64_t entries_added) { + assert(value_.size() == kDefaultStreamValueLength); + entries_added_ = entries_added; + char* dst = const_cast(value_.data()) + sizeof(tree_id_t); + EncodeFixed64(dst, entries_added_); + } + + void set_first_id(streamID first_id) { + assert(value_.size() == kDefaultStreamValueLength); + first_id_ = first_id; + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t); + EncodeFixed64(dst, first_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, first_id_.seq); + } + + void set_last_id(streamID last_id) { + assert(value_.size() == kDefaultStreamValueLength); + last_id_ = last_id; + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + sizeof(streamID); + EncodeFixed64(dst, last_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, last_id_.seq); + } + + void set_max_deleted_entry_id(streamID max_deleted_entry_id) { + assert(value_.size() == kDefaultStreamValueLength); + max_deleted_entry_id_ = max_deleted_entry_id; + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 2 * sizeof(streamID); + EncodeFixed64(dst, max_deleted_entry_id_.ms); + dst += sizeof(uint64_t); + EncodeFixed64(dst, max_deleted_entry_id_.seq); + } + + void set_length(int32_t length) { + assert(value_.size() == kDefaultStreamValueLength); + length_ = length; + char* dst = const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID); + EncodeFixed32(dst, length_); + } + + void set_version(int32_t version) { + assert(value_.size() == kDefaultStreamValueLength); + version_ = version; + char* dst = + const_cast(value_.data()) + sizeof(tree_id_t) + sizeof(uint64_t) + 3 * sizeof(streamID) + sizeof(length_); + EncodeFixed32(dst, version_); + } + + private: + tree_id_t groups_id_ = kINVALID_TREE_ID; + uint64_t entries_added_{0}; + streamID first_id_; + streamID last_id_; + streamID max_deleted_entry_id_; + int32_t length_{0}; // number of the messages in the stream + int32_t version_{0}; + + std::string value_{}; +}; + +// Used only for reading ! +class ParsedStreamMetaValue { + public: + ParsedStreamMetaValue(const Slice& value) { + assert(value.size() == kDefaultStreamValueLength); + if (value.size() != kDefaultStreamValueLength) { + LOG(ERROR) << "Invalid stream meta value length: "; + return; + } + char* pos = const_cast(value.data()); + groups_id_ = DecodeFixed32(pos); + pos += sizeof(tree_id_t); + + entries_added_ = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + first_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + first_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + last_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + last_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + max_deleted_entry_id_.ms = DecodeFixed64(pos); + pos += sizeof(uint64_t); + max_deleted_entry_id_.seq = DecodeFixed64(pos); + pos += sizeof(uint64_t); + + length_ = static_cast(DecodeFixed32(pos)); + pos += sizeof(length_); + + version_ = static_cast(DecodeFixed32(pos)); + } + + int32_t version() const { return version_; } + + tree_id_t groups_id() const { return groups_id_; } + + uint64_t entries_added() const { return entries_added_; } + + streamID first_id() const { return first_id_; } + + streamID last_id() const { return last_id_; } + + streamID max_deleted_entry_id() const { return max_deleted_entry_id_; } + + int32_t length() const { return length_; } + + std::string ToString() { + return "stream_meta: " + std::string("groups_id: ") + std::to_string(groups_id_) + + std::string(", entries_added: ") + std::to_string(entries_added_) + std::string(", first_id: ") + + first_id_.ToString() + std::string(", last_id: ") + last_id_.ToString() + + std::string(", max_deleted_entry_id: ") + max_deleted_entry_id_.ToString() + std::string(", length: ") + + std::to_string(length_) + std::string(", version: ") + std::to_string(version_); + } + + private: + tree_id_t groups_id_ = kINVALID_TREE_ID; + uint64_t entries_added_{0}; + streamID first_id_; + streamID last_id_; + streamID max_deleted_entry_id_; + int32_t length_{0}; // number of the messages in the stream + int32_t version_{0}; +}; + +static const uint64_t kDefaultStreamCGroupValueLength = sizeof(streamID) + sizeof(uint64_t) + 2 * sizeof(tree_id_t); +class StreamCGroupMetaValue { + public: + explicit StreamCGroupMetaValue() = default; + + // tid and consumers should be set at beginning + void Init(tree_id_t tid, tree_id_t consumers) { + pel_ = tid; + consumers_ = consumers; + uint64_t needed = kDefaultStreamCGroupValueLength; + assert(value_.size() == 0); + if (value_.size() != 0) { + LOG(FATAL) << "Init on a existed stream cgroup meta value!"; + return; + } + value_.resize(needed); + + char* dst = &value_[0]; + + memcpy(dst, &last_id_, sizeof(streamID)); + dst += sizeof(uint64_t); + memcpy(dst, &entries_read_, sizeof(uint64_t)); + dst += sizeof(uint64_t); + memcpy(dst, &pel_, sizeof(tree_id_t)); + dst += sizeof(tree_id_t); + memcpy(dst, &consumers_, sizeof(tree_id_t)); + } + + void ParseFrom(std::string& value) { + value_ = std::move(value); + assert(value_.size() == kDefaultStreamCGroupValueLength); + if (value_.size() != kDefaultStreamCGroupValueLength) { + LOG(FATAL) << "Invalid stream cgroup meta value length: "; + return; + } + if (value_.size() == kDefaultStreamCGroupValueLength) { + auto pos = value_.data(); + memcpy(&last_id_, pos, sizeof(streamID)); + pos += sizeof(streamID); + memcpy(&entries_read_, pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&pel_, pos, sizeof(tree_id_t)); + pos += sizeof(tree_id_t); + memcpy(&consumers_, pos, sizeof(tree_id_t)); + } + } + + streamID last_id() { return last_id_; } + + void set_last_id(streamID last_id) { + assert(value_.size() == kDefaultStreamCGroupValueLength); + last_id_ = last_id; + char* dst = const_cast(value_.data()); + memcpy(dst, &last_id_, sizeof(streamID)); + } + + uint64_t entries_read() { return entries_read_; } + + void set_entries_read(uint64_t entries_read) { + assert(value_.size() == kDefaultStreamCGroupValueLength); + entries_read_ = entries_read; + char* dst = const_cast(value_.data()) + sizeof(streamID); + memcpy(dst, &entries_read_, sizeof(uint64_t)); + } + + // pel and consumers were set in constructor, can't be modified + tree_id_t pel() { return pel_; } + + tree_id_t consumers() { return consumers_; } + + std::string& value() { return value_; } + + private: + std::string value_; + + streamID last_id_; + uint64_t entries_read_ = 0; + tree_id_t pel_ = 0; + tree_id_t consumers_ = 0; +}; + +static const uint64_t kDefaultStreamConsumerValueLength = sizeof(stream_ms_t) * 2 + sizeof(tree_id_t); +class StreamConsumerMetaValue { + public: + // pel must been set at beginning + StreamConsumerMetaValue() = default; + + void ParseFrom(std::string& value) { + value_ = std::move(value); + assert(value_.size() == kDefaultStreamConsumerValueLength); + if (value_.size() != kDefaultStreamConsumerValueLength) { + LOG(FATAL) << "Invalid stream consumer meta value length: " << value_.size() + << " expected: " << kDefaultStreamConsumerValueLength; + return; + } + if (value_.size() == kDefaultStreamConsumerValueLength) { + auto pos = value_.data(); + memcpy(&seen_time_, pos, sizeof(stream_ms_t)); + pos += sizeof(stream_ms_t); + memcpy(&active_time_, pos, sizeof(stream_ms_t)); + pos += sizeof(stream_ms_t); + memcpy(&pel_, pos, sizeof(tree_id_t)); + } + } + + void Init(tree_id_t pel) { + pel_ = pel; + assert(value_.size() == 0); + if (value_.size() != 0) { + LOG(FATAL) << "Invalid stream consumer meta value length: " << value_.size() << " expected: 0"; + return; + } + uint64_t needed = kDefaultStreamConsumerValueLength; + value_.resize(needed); + char* dst = &value_[0]; + + memcpy(dst, &seen_time_, sizeof(stream_ms_t)); + dst += sizeof(stream_ms_t); + memcpy(dst, &active_time_, sizeof(stream_ms_t)); + dst += sizeof(stream_ms_t); + memcpy(dst, &pel_, sizeof(tree_id_t)); + } + + stream_ms_t seen_time() { return seen_time_; } + + void set_seen_time(stream_ms_t seen_time) { + seen_time_ = seen_time; + assert(value_.size() == kDefaultStreamConsumerValueLength); + char* dst = const_cast(value_.data()); + memcpy(dst, &seen_time_, sizeof(stream_ms_t)); + } + + stream_ms_t active_time() { return active_time_; } + + void set_active_time(stream_ms_t active_time) { + active_time_ = active_time; + assert(value_.size() == kDefaultStreamConsumerValueLength); + char* dst = const_cast(value_.data()) + sizeof(stream_ms_t); + memcpy(dst, &active_time_, sizeof(stream_ms_t)); + } + + // pel was set in constructor, can't be modified + tree_id_t pel_tid() { return pel_; } + + std::string& value() { return value_; } + + private: + std::string value_; + + stream_ms_t seen_time_ = 0; + stream_ms_t active_time_ = 0; + tree_id_t pel_ = 0; +}; + +static const uint64_t kDefaultStreamPelMetaValueLength = sizeof(stream_ms_t) + sizeof(uint64_t) + sizeof(tree_id_t); +class StreamPelMeta { + public: + // consumer must been set at beginning + StreamPelMeta() = default; + + void Init(std::string consumer, stream_ms_t delivery_time) { + consumer_ = std::move(consumer); + delivery_time_ = delivery_time; + uint64_t needed = kDefaultStreamPelMetaValueLength; + assert(value_.size() == 0); + if (value_.size() != 0) { + LOG(ERROR) << "Init on a existed stream pel meta value!"; + return; + } + value_.resize(needed); + char* dst = &value_[0]; + + memcpy(dst, &delivery_time_, sizeof(stream_ms_t)); + dst += sizeof(stream_ms_t); + memcpy(dst, &delivery_count_, sizeof(uint64_t)); + dst += sizeof(uint64_t); + memcpy(dst, &cname_len_, sizeof(uint64_t)); + dst += sizeof(uint64_t); + memcpy(dst, consumer_.data(), cname_len_); + } + + void ParseFrom(std::string& value) { + value_ = std::move(value); + assert(value_.size() == kDefaultStreamPelMetaValueLength); + if (value_.size() != kDefaultStreamPelMetaValueLength) { + LOG(ERROR) << "Invalid stream pel meta value length: "; + return; + } + auto pos = value_.data(); + memcpy(&delivery_time_, pos, sizeof(stream_ms_t)); + pos += sizeof(stream_ms_t); + memcpy(&delivery_count_, pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + memcpy(&cname_len_, pos, sizeof(uint64_t)); + pos += sizeof(uint64_t); + consumer_.assign(pos, cname_len_); + } + + stream_ms_t delivery_time() { return delivery_time_; } + + void set_delivery_time(stream_ms_t delivery_time) { + assert(value_.size() == kDefaultStreamPelMetaValueLength); + delivery_time_ = delivery_time; + char* dst = const_cast(value_.data()); + memcpy(dst, &delivery_time_, sizeof(stream_ms_t)); + } + + uint64_t delivery_count() { return delivery_count_; } + + void set_delivery_count(uint64_t delivery_count) { + assert(value_.size() == kDefaultStreamPelMetaValueLength); + delivery_count_ = delivery_count; + char* dst = const_cast(value_.data()); + memcpy(dst + sizeof(stream_ms_t), &delivery_count_, sizeof(uint64_t)); + } + + std::string& consumer() { return consumer_; } + + std::string& value() { return value_; } + + private: + std::string value_; + + stream_ms_t delivery_time_ = 0; + uint64_t delivery_count_ = 1; + uint64_t cname_len_ = 0; + std::string consumer_; +}; + +} // namespace storage diff --git a/include/pika_stream_types.h b/src/storage/src/pika_stream_types.h similarity index 77% rename from include/pika_stream_types.h rename to src/storage/src/pika_stream_types.h index 0a76ba925d..69c4733334 100644 --- a/include/pika_stream_types.h +++ b/src/storage/src/pika_stream_types.h @@ -1,17 +1,20 @@ -// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. -#ifndef SRC_STREAM_TYPE_H_ -#define SRC_STREAM_TYPE_H_ +#pragma once #include #include #include #include #include -#include "src/storage/src/coding.h" +#include "src/coding.h" + +namespace storage { + +#define kINVALID_TREE_ID 0 using streamID = struct streamID { streamID(uint64_t _ms, uint64_t _seq) : ms(_ms), seq(_seq) {} @@ -25,7 +28,7 @@ using streamID = struct streamID { // We must store the streamID in memory in big-endian format. This way, our comparison of the serialized streamID byte // code will be equivalent to the comparison of the uint64_t numbers. inline void EncodeUint64InBigEndian(char* buf, uint64_t value) const { - if (storage::kLittleEndian) { + if (kLittleEndian) { // little endian, reverse the bytes for (int i = 7; i >= 0; --i) { buf[i] = static_cast(value & 0xff); @@ -39,7 +42,7 @@ using streamID = struct streamID { inline uint64_t DecodeUint64OfBigEndian(const char* ptr) { uint64_t value; - if (storage::kLittleEndian) { + if (kLittleEndian) { // little endian, reverse the bytes value = 0; for (int i = 0; i < 8; ++i) { @@ -53,10 +56,12 @@ using streamID = struct streamID { return value; } - void SerializeTo(std::string& dst) const { + std::string Serialize() const { + std::string dst; dst.resize(sizeof(ms) + sizeof(seq)); EncodeUint64InBigEndian(&dst[0], ms); EncodeUint64InBigEndian(&dst[0] + sizeof(ms), seq); + return dst; } void DeserializeFrom(std::string& src) { @@ -73,11 +78,10 @@ using streamID = struct streamID { static const streamID kSTREAMID_MAX = streamID(UINT64_MAX, UINT64_MAX); static const streamID kSTREAMID_MIN = streamID(0, 0); -enum class StreamTrimStrategy { TRIM_STRATEGY_NONE, TRIM_STRATEGY_MAXLEN, TRIM_STRATEGY_MINID }; +enum StreamTrimStrategy { TRIM_STRATEGY_NONE, TRIM_STRATEGY_MAXLEN, TRIM_STRATEGY_MINID }; -using treeID = int32_t; -static const treeID kINVALID_TREE_ID = -1; +using tree_id_t = uint32_t; using stream_ms_t = uint64_t; -#endif // SRC_STREAM_TYPE_H_ +} // namespace storage diff --git a/src/storage/src/redis_streams.cc b/src/storage/src/redis_streams.cc new file mode 100644 index 0000000000..84e11832be --- /dev/null +++ b/src/storage/src/redis_streams.cc @@ -0,0 +1,1248 @@ +// Copyright (c) 2017-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "src/redis_streams.h" +#include +#include +#include +#include +#include +#include +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "src/base_data_key_format.h" +#include "src/base_filter.h" +#include "src/debug.h" +#include "src/pika_stream_meta_value.h" +#include "src/scope_record_lock.h" +#include "src/scope_snapshot.h" +#include "storage/storage.h" +#include "storage/util.h" + +namespace storage { + +Status RedisStreams::XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args) { + // With the lock, we do not need snapshot for read. + // And it's bugy to use snapshot for read when we try to add message with trim. + // such as: XADD key 1-0 field value MINID 1-0 + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, default_read_options_); + if (s.IsNotFound() && args.no_mkstream) { + return Status::NotFound("no_mkstream"); + } else if (s.IsNotFound()) { + stream_meta.InitMetaValue(); + } else if (!s.ok()) { + return Status::Corruption("error from XADD, get stream meta failed: " + s.ToString()); + } + + if (stream_meta.length() == 0) { + if (args.no_mkstream) { + return Status::NotFound("no_mkstream"); + } + stream_meta.InitMetaValue(); + } + + if (stream_meta.last_id().ms == UINT64_MAX && stream_meta.last_id().seq == UINT64_MAX) { + return Status::Corruption("Fatal! Sequence number overflow !"); + } + + // 2 append the message to storage + s = GenerateStreamID(stream_meta, args); + if (!s.ok()) { + return s; + } + +#ifdef DEBUG + // check the serialized current id is larger than last_id + std::string serialized_last_id = stream_meta.last_id().Serialize(); + std::string current_id = args.id.Serialize(); + assert(current_id > serialized_last_id); +#endif + + StreamDataKey stream_data_key(key, stream_meta.version(), args.id.Serialize()); + s = db_->Put(default_write_options_, handles_[1], stream_data_key.Encode(), serialized_message); + if (!s.ok()) { + return Status::Corruption("error from XADD, insert stream message failed 1: " + s.ToString()); + } + + // 3 update stream meta + if (stream_meta.length() == 0) { + stream_meta.set_first_id(args.id); + } + stream_meta.set_entries_added(stream_meta.entries_added() + 1); + stream_meta.set_last_id(args.id); + stream_meta.set_length(stream_meta.length() + 1); + + // 4 trim the stream if needed + if (args.trim_strategy != StreamTrimStrategy::TRIM_STRATEGY_NONE) { + int32_t count{0}; + s = TrimStream(count, stream_meta, key, args, default_read_options_); + if (!s.ok()) { + return Status::Corruption("error from XADD, trim stream failed: " + s.ToString()); + } + (void)count; + } + + // 5 update stream meta + s = db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); + if (!s.ok()) { + return s; + } + + return Status::OK(); +} + +Status RedisStreams::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count) { + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, default_read_options_); + if (!s.ok()) { + return s; + } + + // 2 do the trim + count = 0; + s = TrimStream(count, stream_meta, key, args, default_read_options_); + if (!s.ok()) { + return s; + } + + // 3 update stream meta + s = db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); + if (!s.ok()) { + return s; + } + + return Status::OK(); +} + +Status RedisStreams::XDel(const Slice& key, const std::vector& ids, int32_t& count) { + + // 1 try to get stream meta + StreamMetaValue stream_meta; + auto s = GetStreamMeta(stream_meta, key, default_read_options_); + if (!s.ok()) { + return s; + } + + // 2 do the delete + if (ids.size() > INT32_MAX) { + return Status::InvalidArgument("Too many IDs specified"); + } + count = static_cast(ids.size()); + std::string unused; + for (auto id : ids) { + StreamDataKey stream_data_key(key, stream_meta.version(), id.Serialize()); + s = db_->Get(default_read_options_, handles_[1], stream_data_key.Encode(), &unused); + if (s.IsNotFound()) { + --count; + continue; + } else if (!s.ok()) { + return s; + } + } + s = DeleteStreamMessages(key, stream_meta, ids, default_read_options_); + if (!s.ok()) { + return s; + } + + // 3 update stream meta + stream_meta.set_length(stream_meta.length() - count); + for (const auto& id : ids) { + if (id > stream_meta.max_deleted_entry_id()) { + stream_meta.set_max_deleted_entry_id(id); + } + if (id == stream_meta.first_id()) { + s = SetFirstID(key, stream_meta, default_read_options_); + } else if (id == stream_meta.last_id()) { + s = SetLastID(key, stream_meta, default_read_options_); + } + if (!s.ok()) { + return s; + } + } + + return db_->Put(default_write_options_, handles_[0], key, stream_meta.value()); +} + +Status RedisStreams::XRange(const Slice& key, const StreamScanArgs& args, std::vector& field_values) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the scan + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), args.start_sid, args.end_sid, args.limit, args.start_ex, + args.end_ex, false); + s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + + return s; +} + +Status RedisStreams::XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& field_values) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + // 2 do the scan + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), args.start_sid, args.end_sid, args.limit, args.start_ex, + args.end_ex, true); + s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + + return s; +} + +Status RedisStreams::XLen(const Slice& key, int32_t& len) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, read_options); + if (!s.ok()) { + return s; + } + + len = stream_meta.length(); + return Status::OK(); +} + +Status RedisStreams::XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys) { + rocksdb::ReadOptions read_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + read_options.snapshot = snapshot; + + // 1 prepare stream_metas + rocksdb::Status s; + std::vector> streammeta_idx; + for (int i = 0; i < args.unparsed_ids.size(); i++) { + const auto& key = args.keys[i]; + + StreamMetaValue stream_meta; + auto s = GetStreamMeta(stream_meta, key, read_options); + if (s.IsNotFound()) { + continue; + } else if (!s.ok()) { + return s; + } + + streammeta_idx.emplace_back(std::move(stream_meta), i); + } + + if (streammeta_idx.empty()) { + return Status::OK(); + } + + // 2 do the scan + for (const auto& stream_meta_id : streammeta_idx) { + const auto& stream_meta = stream_meta_id.first; + const auto& idx = stream_meta_id.second; + const auto& unparsed_id = args.unparsed_ids[idx]; + const auto& key = args.keys[idx]; + + // 2.1 try to parse id + storage::streamID id; + if (unparsed_id == "<") { + return Status::Corruption( + "The > ID can be specified only when calling " + "XREADGROUP using the GROUP " + " option."); + } else if (unparsed_id == "$") { + id = stream_meta.last_id(); + } else { + if (!storage::StreamUtils::StreamParseStrictID(unparsed_id, id, 0, nullptr)) { + return Status::Corruption("Invalid stream ID specified as stream "); + } + } + + // 2.2 scan + std::vector field_values; + std::string next_field; + ScanStreamOptions options(key, stream_meta.version(), id, storage::kSTREAMID_MAX, args.count, true); + auto s = ScanStream(options, field_values, next_field, read_options); + (void)next_field; + if (!s.ok() && !s.IsNotFound()) { + return s; + } + results.emplace_back(std::move(field_values)); + reserved_keys.emplace_back(args.keys[idx]); + } + + return Status::OK(); +} + +Status RedisStreams::XInfo(const Slice& key, StreamInfoResult& result) { + // 1 get stream meta + rocksdb::Status s; + StreamMetaValue stream_meta; + s = GetStreamMeta(stream_meta, key, default_read_options_); + if (!s.ok()) { + return s; + } + + // 2 fill the result + result.length = stream_meta.length(); + result.last_id_str = stream_meta.last_id().ToString(); + result.max_deleted_entry_id_str = stream_meta.max_deleted_entry_id().ToString(); + result.entries_added = stream_meta.entries_added(); + result.first_id_str = stream_meta.first_id().ToString(); + + return Status::OK(); +} + +Status RedisStreams::Open(const StorageOptions& storage_options, const std::string& db_path) { + statistics_store_->SetCapacity(storage_options.statistics_max_size); + small_compaction_threshold_ = storage_options.small_compaction_threshold; + + rocksdb::Options ops(storage_options.options); + Status s = rocksdb::DB::Open(ops, db_path, &db_); + if (s.ok()) { + // create column family + rocksdb::ColumnFamilyHandle* cf; + s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "data_cf", &cf); + if (!s.ok()) { + return s; + } + // close DB + delete cf; + delete db_; + } + + // Open + rocksdb::DBOptions db_ops(storage_options.options); + rocksdb::ColumnFamilyOptions meta_cf_ops(storage_options.options); + rocksdb::ColumnFamilyOptions data_cf_ops(storage_options.options); + // Notice: Stream's Meta dose not have timestamp and version, so it does not need to be filtered. + + // use the bloom filter policy to reduce disk reads + rocksdb::BlockBasedTableOptions table_ops(storage_options.table_options); + table_ops.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true)); + rocksdb::BlockBasedTableOptions meta_cf_table_ops(table_ops); + rocksdb::BlockBasedTableOptions data_cf_table_ops(table_ops); + if (!storage_options.share_block_cache && storage_options.block_cache_size > 0) { + meta_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); + data_cf_table_ops.block_cache = rocksdb::NewLRUCache(storage_options.block_cache_size); + } + meta_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(meta_cf_table_ops)); + data_cf_ops.table_factory.reset(rocksdb::NewBlockBasedTableFactory(data_cf_table_ops)); + + std::vector column_families; + // Meta CF + column_families.emplace_back(rocksdb::kDefaultColumnFamilyName, meta_cf_ops); + // Data CF + column_families.emplace_back("data_cf", data_cf_ops); + return rocksdb::DB::Open(db_ops, db_path, column_families, &handles_, &db_); +} + +Status RedisStreams::CompactRange(const rocksdb::Slice* begin, const rocksdb::Slice* end, + const ColumnFamilyType& type) { + if (type == kMeta || type == kMetaAndData) { + db_->CompactRange(default_compact_range_options_, handles_[0], begin, end); + } + if (type == kData || type == kMetaAndData) { + db_->CompactRange(default_compact_range_options_, handles_[1], begin, end); + } + return Status::OK(); +} + +Status RedisStreams::GetProperty(const std::string& property, uint64_t* out) { + std::string value; + db_->GetProperty(handles_[0], property, &value); + *out = std::strtoull(value.c_str(), nullptr, 10); + db_->GetProperty(handles_[1], property, &value); + *out += std::strtoull(value.c_str(), nullptr, 10); + return Status::OK(); +} + +Status RedisStreams::ScanKeyNum(KeyInfo* key_info) { + uint64_t keys = 0; + uint64_t expires = 0; + uint64_t ttl_sum = 0; + uint64_t invaild_keys = 0; + + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + int64_t curtime; + rocksdb::Env::Default()->GetCurrentTime(&curtime); + + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); + if (parsed_stream_meta_value.length() == 0) { + invaild_keys++; + } else { + keys++; + } + } + delete iter; + + key_info->keys = keys; + key_info->invaild_keys = invaild_keys; + return Status::OK(); +} + +Status RedisStreams::ScanKeys(const std::string& pattern, std::vector* keys) { + std::string key; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + ParsedStreamMetaValue parsed_stream_meta_value(iter->value()); + if (parsed_stream_meta_value.length() != 0) { + key = iter->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + } + } + delete iter; + return Status::OK(); +} + +Status RedisStreams::PKPatternMatchDel(const std::string& pattern, int32_t* ret) { + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + std::string key; + std::string meta_value; + int32_t total_delete = 0; + Status s; + rocksdb::WriteBatch batch; + rocksdb::Iterator* iter = db_->NewIterator(iterator_options, handles_[0]); + iter->SeekToFirst(); + while (iter->Valid()) { + key = iter->key().ToString(); + meta_value = iter->value().ToString(); + StreamMetaValue stream_meta_value; + stream_meta_value.ParseFrom(meta_value); + if ((stream_meta_value.length() != 0) && + (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0)) { + stream_meta_value.InitMetaValue(); + batch.Put(handles_[0], key, stream_meta_value.value()); + } + if (static_cast(batch.Count()) >= BATCH_DELETE_LIMIT) { + s = db_->Write(default_write_options_, &batch); + if (s.ok()) { + total_delete += static_cast(batch.Count()); + batch.Clear(); + } else { + *ret = total_delete; + return s; + } + } + iter->Next(); + } + if (batch.Count() != 0U) { + s = db_->Write(default_write_options_, &batch); + if (s.ok()) { + total_delete += static_cast(batch.Count()); + batch.Clear(); + } + } + + *ret = total_delete; + return s; +} + +rocksdb::Status RedisStreams::PKScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, + int32_t limit, std::vector* keys, std::string* next_key) { + next_key->clear(); + + std::string key; + int32_t remain = limit; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + bool start_no_limit = key_start.compare("") == 0; + bool end_no_limit = key_end.compare("") == 0; + + if (!start_no_limit && !end_no_limit && (key_start.compare(key_end) > 0)) { + return rocksdb::Status::InvalidArgument("error in given range"); + } + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + if (start_no_limit) { + it->SeekToFirst(); + } else { + it->Seek(key_start); + } + + while (it->Valid() && remain > 0 && (end_no_limit || it->key().compare(key_end) <= 0)) { + ParsedStreamMetaValue parsed_meta_value(it->value()); + if (parsed_meta_value.length() == 0) { + it->Next(); + } else { + key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + remain--; + it->Next(); + } + } + + while (it->Valid() && (end_no_limit || it->key().compare(key_end) <= 0)) { + ParsedStreamMetaValue parsed_meta_value(it->value()); + if (parsed_meta_value.length() == 0) { + it->Next(); + } else { + *next_key = it->key().ToString(); + break; + } + } + delete it; + return rocksdb::Status::OK(); +} + +Status RedisStreams::PKRScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key) { + next_key->clear(); + + std::string key; + int32_t remain = limit; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + bool start_no_limit = key_start.compare("") == 0; + bool end_no_limit = key_end.compare("") == 0; + + if (!start_no_limit && !end_no_limit && (key_start.compare(key_end) < 0)) { + return Status::InvalidArgument("error in given range"); + } + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + if (start_no_limit) { + it->SeekToLast(); + } else { + it->SeekForPrev(key_start); + } + + while (it->Valid() && remain > 0 && (end_no_limit || it->key().compare(key_end) >= 0)) { + ParsedStreamMetaValue parsed_streams_meta_value(it->value()); + if (parsed_streams_meta_value.length() == 0) { + it->Prev(); + } else { + key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), key.data(), key.size(), 0) != 0) { + keys->push_back(key); + } + remain--; + it->Prev(); + } + } + + while (it->Valid() && (end_no_limit || it->key().compare(key_end) >= 0)) { + ParsedStreamMetaValue parsed_streams_meta_value(it->value()); + if (parsed_streams_meta_value.length() == 0) { + it->Prev(); + } else { + *next_key = it->key().ToString(); + break; + } + } + delete it; + return Status::OK(); +} + +Status RedisStreams::Del(const Slice& key) { + std::string meta_value; + Status s = db_->Get(default_read_options_, handles_[0], key, &meta_value); + if (s.ok()) { + StreamMetaValue stream_meta_value; + stream_meta_value.ParseFrom(meta_value); + if (stream_meta_value.length() == 0) { + return Status::NotFound(); + } else { + uint32_t statistic = stream_meta_value.length(); + stream_meta_value.InitMetaValue(); + s = db_->Put(default_write_options_, handles_[0], key, stream_meta_value.value()); + UpdateSpecificKeyStatistics(key.ToString(), statistic); + } + } + return s; +} + +bool RedisStreams::Scan(const std::string& start_key, const std::string& pattern, std::vector* keys, + int64_t* count, std::string* next_key) { + std::string meta_key; + bool is_finish = true; + rocksdb::ReadOptions iterator_options; + const rocksdb::Snapshot* snapshot; + ScopeSnapshot ss(db_, &snapshot); + iterator_options.snapshot = snapshot; + iterator_options.fill_cache = false; + + rocksdb::Iterator* it = db_->NewIterator(iterator_options, handles_[0]); + + it->Seek(start_key); + while (it->Valid() && (*count) > 0) { + ParsedStreamMetaValue parsed_stream_meta_value(it->value()); + if (parsed_stream_meta_value.length() == 0) { + it->Next(); + continue; + } else { + meta_key = it->key().ToString(); + if (StringMatch(pattern.data(), pattern.size(), meta_key.data(), meta_key.size(), 0) != 0) { + keys->push_back(meta_key); + } + (*count)--; + it->Next(); + } + } + + std::string prefix = isTailWildcard(pattern) ? pattern.substr(0, pattern.size() - 1) : ""; + if (it->Valid() && (it->key().compare(prefix) <= 0 || it->key().starts_with(prefix))) { + *next_key = it->key().ToString(); + is_finish = false; + } else { + *next_key = ""; + } + delete it; + return is_finish; +} + +Status RedisStreams::Expire(const Slice& key, int32_t ttl) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Expire not supported by stream")); + return Status::Corruption(s.ToString()); +} + +bool RedisStreams::PKExpireScan(const std::string& start_key, int32_t min_timestamp, int32_t max_timestamp, + std::vector* keys, int64_t* leftover_visits, std::string* next_key) { + TRACE("RedisStreams::PKExpireScan not supported by stream"); + return false; +} + +Status RedisStreams::Expireat(const Slice& key, int32_t timestamp) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Expireat not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::Persist(const Slice& key) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::Persist not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::TTL(const Slice& key, int64_t* timestamp) { + rocksdb::Status s(rocksdb::Status::NotSupported("RedisStreams::TTL not supported by stream")); + return Status::Corruption(s.ToString()); +} + +Status RedisStreams::GetStreamMeta(StreamMetaValue& stream_meta, const rocksdb::Slice& key, + rocksdb::ReadOptions& read_options) { + std::string value; + auto s = db_->Get(read_options, handles_[0], key, &value); + if (s.ok()) { + stream_meta.ParseFrom(value); + return Status::OK(); + } + return s; +} + +Status RedisStreams::TrimStream(int32_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + count = 0; + // 1 do the trim + TrimRet trim_ret; + Status s; + if (args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MAXLEN) { + s = TrimByMaxlen(trim_ret, stream_meta, key, args, read_options); + } else { + assert(args.trim_strategy == StreamTrimStrategy::TRIM_STRATEGY_MINID); + s = TrimByMinid(trim_ret, stream_meta, key, args, read_options); + } + + if (!s.ok()) { + return s; + } + + if (trim_ret.count == 0) { + return s; + } + + // 2 update stream meta + streamID first_id; + streamID max_deleted_id; + if (stream_meta.length() == trim_ret.count) { + // all the message in the stream were deleted + first_id = kSTREAMID_MIN; + } else { + first_id.DeserializeFrom(trim_ret.next_field); + } + assert(!trim_ret.max_deleted_field.empty()); + max_deleted_id.DeserializeFrom(trim_ret.max_deleted_field); + + stream_meta.set_first_id(first_id); + if (max_deleted_id > stream_meta.max_deleted_entry_id()) { + stream_meta.set_max_deleted_entry_id(max_deleted_id); + } + stream_meta.set_length(stream_meta.length() - trim_ret.count); + + count = trim_ret.count; + return Status::OK(); +} + +Status RedisStreams::ScanStream(const ScanStreamOptions& op, std::vector& field_values, + std::string& next_field, rocksdb::ReadOptions& read_options) { + std::string start_field; + std::string end_field; + Slice pattern = "*"; // match all the fields from start_field to end_field + Status s; + + // 1 do the scan + if (op.is_reverse) { + start_field = op.end_sid.Serialize(); + if (op.start_sid == kSTREAMID_MAX) { + start_field = ""; + } else { + start_field = op.start_sid.Serialize(); + } + s = ReScanRange(op.key, op.version, start_field, end_field, pattern, op.limit, field_values, next_field, + read_options); + } else { + start_field = op.start_sid.Serialize(); + if (op.end_sid == kSTREAMID_MAX) { + end_field = ""; + } else { + end_field = op.end_sid.Serialize(); + } + s = ScanRange(op.key, op.version, start_field, end_field, pattern, op.limit, field_values, next_field, + read_options); + } + + // 2 exclude the start_sid and end_sid if needed + if (op.start_ex && !field_values.empty()) { + streamID sid; + sid.DeserializeFrom(field_values.front().field); + if (sid == op.start_sid) { + field_values.erase(field_values.begin()); + } + } + + if (op.end_ex && !field_values.empty()) { + streamID sid; + sid.DeserializeFrom(field_values.back().field); + if (sid == op.end_sid) { + field_values.pop_back(); + } + } + + return s; +} + +Status RedisStreams::GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args) { + auto& id = args.id; + if (args.id_given && args.seq_given && id.ms == 0 && id.seq == 0) { + return Status::InvalidArgument("The ID specified in XADD must be greater than 0-0"); + } + + if (!args.id_given || !args.seq_given) { + // if id not given, generate one + if (!args.id_given) { + id.ms = StreamUtils::GetCurrentTimeMs(); + + if (id.ms < stream_meta.last_id().ms) { + id.ms = stream_meta.last_id().ms; + if (stream_meta.last_id().seq == UINT64_MAX) { + id.ms++; + id.seq = 0; + } else { + id.seq++; + } + return Status::OK(); + } + } + + // generate seq + auto last_id = stream_meta.last_id(); + if (id.ms < last_id.ms) { + return Status::InvalidArgument("The ID specified in XADD is equal or smaller"); + } else if (id.ms == last_id.ms) { + if (last_id.seq == UINT64_MAX) { + return Status::InvalidArgument("The ID specified in XADD is equal or smaller"); + } + id.seq = last_id.seq + 1; + } else { + id.seq = 0; + } + + } else { + // Full ID given, check id + auto last_id = stream_meta.last_id(); + if (id.ms < last_id.ms || (id.ms == last_id.ms && id.seq <= last_id.seq)) { + return Status::InvalidArgument("INVALID ID given"); + } + } + return Status::OK(); +} + +Status RedisStreams::TrimByMaxlen(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + Status s; + // we delete the message in batchs, prevent from using too much memory + while (stream_meta.length() - trim_ret.count > args.maxlen) { + auto cur_batch = + (std::min(static_cast(stream_meta.length() - trim_ret.count - args.maxlen), kDEFAULT_TRIM_BATCH_SIZE)); + std::vector id_messages; + + RedisStreams::ScanStreamOptions options(key, stream_meta.version(), stream_meta.first_id(), kSTREAMID_MAX, + cur_batch, false, false, false); + s = RedisStreams::ScanStream(options, id_messages, trim_ret.next_field, read_options); + if (!s.ok()) { + assert(!s.IsNotFound()); + return s; + } + + assert(id_messages.size() == cur_batch); + trim_ret.count += cur_batch; + trim_ret.max_deleted_field = id_messages.back().field; + + // delete the message in batchs + std::vector ids_to_del; + ids_to_del.reserve(id_messages.size()); + for (auto& fv : id_messages) { + ids_to_del.emplace_back(std::move(fv.field)); + } + s = DeleteStreamMessages(key, stream_meta, ids_to_del, read_options); + if (!s.ok()) { + return s; + } + } + + s = Status::OK(); + return s; +} + +Status RedisStreams::TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options) { + Status s; + std::string serialized_min_id; + trim_ret.next_field = stream_meta.first_id().Serialize(); + serialized_min_id = args.minid.Serialize(); + + // we delete the message in batchs, prevent from using too much memory + while (trim_ret.next_field < serialized_min_id && stream_meta.length() - trim_ret.count > 0) { + auto cur_batch = static_cast( + std::min(static_cast(stream_meta.length() - trim_ret.count), kDEFAULT_TRIM_BATCH_SIZE)); + std::vector id_messages; + + RedisStreams::ScanStreamOptions options(key, stream_meta.version(), stream_meta.first_id(), args.minid, cur_batch, + false, false, false); + s = RedisStreams::ScanStream(options, id_messages, trim_ret.next_field, read_options); + if (!s.ok()) { + assert(!s.IsNotFound()); + return s; + } + + if (!id_messages.empty()) { + if (id_messages.back().field == serialized_min_id) { + // we do not need to delete the message that it's id matches the minid + id_messages.pop_back(); + trim_ret.next_field = serialized_min_id; + } + // duble check + if (!id_messages.empty()) { + trim_ret.max_deleted_field = id_messages.back().field; + } + } + + assert(id_messages.size() <= cur_batch); + trim_ret.count += static_cast(id_messages.size()); + + // do the delete in batch + std::vector fields_to_del; + fields_to_del.reserve(id_messages.size()); + for (auto& fv : id_messages) { + fields_to_del.emplace_back(std::move(fv.field)); + } + + s = DeleteStreamMessages(key, stream_meta, fields_to_del, read_options); + if (!s.ok()) { + return s; + } + } + + s = Status::OK(); + return s; +} + +Status RedisStreams::ScanRange(const Slice& key, const int32_t version, const Slice& id_start, + const std::string& id_end, const Slice& pattern, int32_t limit, + std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options) { + next_id.clear(); + id_messages.clear(); + + auto remain = limit; + std::string meta_value; + + bool start_no_limit = id_start.compare("") == 0; + bool end_no_limit = id_end.empty(); + + if (!start_no_limit && !end_no_limit && (id_start.compare(id_end) > 0)) { + return Status::InvalidArgument("error in given range"); + } + + StreamDataKey streams_data_prefix(key, version, Slice()); + StreamDataKey streams_start_data_key(key, version, id_start); + std::string prefix = streams_data_prefix.Encode().ToString(); + rocksdb::Iterator* iter = db_->NewIterator(read_options, handles_[1]); + for (iter->Seek(start_no_limit ? prefix : streams_start_data_key.Encode()); + iter->Valid() && remain > 0 && iter->key().starts_with(prefix); iter->Next()) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + std::string id = parsed_streams_data_key.id().ToString(); + if (!end_no_limit && id.compare(id_end) > 0) { + break; + } + if (StringMatch(pattern.data(), pattern.size(), id.data(), id.size(), 0) != 0) { + id_messages.push_back({id, iter->value().ToString()}); + } + remain--; + } + + if (iter->Valid() && iter->key().starts_with(prefix)) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + if (end_no_limit || parsed_streams_data_key.id().compare(id_end) <= 0) { + next_id = parsed_streams_data_key.id().ToString(); + } + } + delete iter; + + return Status::OK(); +} + +Status RedisStreams::ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, + const std::string& id_end, const Slice& pattern, int32_t limit, + std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options) { + next_id.clear(); + id_messages.clear(); + + auto remain = limit; + std::string meta_value; + + bool start_no_limit = id_start.compare("") == 0; + bool end_no_limit = id_end.empty(); + + if (!start_no_limit && !end_no_limit && (id_start.compare(id_end) < 0)) { + return Status::InvalidArgument("error in given range"); + } + + int32_t start_key_version = start_no_limit ? version + 1 : version; + std::string start_key_id = start_no_limit ? "" : id_start.ToString(); + StreamDataKey streams_data_prefix(key, version, Slice()); + StreamDataKey streams_start_data_key(key, start_key_version, start_key_id); + std::string prefix = streams_data_prefix.Encode().ToString(); + rocksdb::Iterator* iter = db_->NewIterator(read_options, handles_[1]); + for (iter->SeekForPrev(streams_start_data_key.Encode().ToString()); + iter->Valid() && remain > 0 && iter->key().starts_with(prefix); iter->Prev()) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + std::string id = parsed_streams_data_key.id().ToString(); + if (!end_no_limit && id.compare(id_end) < 0) { + break; + } + if (StringMatch(pattern.data(), pattern.size(), id.data(), id.size(), 0) != 0) { + id_messages.push_back({id, iter->value().ToString()}); + } + remain--; + } + + if (iter->Valid() && iter->key().starts_with(prefix)) { + ParsedStreamDataKey parsed_streams_data_key(iter->key()); + if (end_no_limit || parsed_streams_data_key.id().compare(id_end) >= 0) { + next_id = parsed_streams_data_key.id().ToString(); + } + } + delete iter; + + return Status::OK(); +} + +Status RedisStreams::DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& ids, rocksdb::ReadOptions& read_options) { + std::vector serialized_ids; + serialized_ids.reserve(ids.size()); + for (const auto& id : ids) { + serialized_ids.emplace_back(id.Serialize()); + } + return DeleteStreamMessages(key, stream_meta, serialized_ids, read_options); +} + +Status RedisStreams::DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& serialized_ids, + rocksdb::ReadOptions& read_options) { + rocksdb::WriteBatch batch; + for (auto& sid : serialized_ids) { + StreamDataKey stream_data_key(key, stream_meta.version(), sid); + batch.Delete(handles_[1], stream_data_key.Encode()); + } + return db_->Write(default_write_options_, &batch); +} + +inline Status RedisStreams::SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, + rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, true, read_options); +} + +inline Status RedisStreams::SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, + rocksdb::ReadOptions& read_options) { + return SetFirstOrLastID(key, stream_meta, false, read_options); +} + +inline Status RedisStreams::SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, + rocksdb::ReadOptions& read_options) { + if (stream_meta.length() == 0) { + stream_meta.set_first_id(kSTREAMID_MIN); + return Status::OK(); + } + + std::vector id_messages; + std::string next_field; + + storage::Status s; + if (is_set_first) { + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MIN, kSTREAMID_MAX, 1); + s = ScanStream(option, id_messages, next_field, read_options); + } else { + bool is_reverse = true; + ScanStreamOptions option(key, stream_meta.version(), kSTREAMID_MAX, kSTREAMID_MIN, 1, false, false, is_reverse); + s = ScanStream(option, id_messages, next_field, read_options); + } + (void)next_field; + + if (!s.ok() && !s.IsNotFound()) { + LOG(ERROR) << "Internal error: scan stream failed: " << s.ToString(); + return Status::Corruption("Internal error: scan stream failed: " + s.ToString()); + } + + if (id_messages.empty()) { + LOG(ERROR) << "Internal error: no messages found but stream length is not 0"; + return Status::Corruption("Internal error: no messages found but stream length is not 0"); + } + + streamID id; + id.DeserializeFrom(id_messages[0].field); + stream_meta.set_first_id(id); + return Status::OK(); +} + +bool StreamUtils::StreamGenericParseID(const std::string& var, streamID& id, uint64_t missing_seq, bool strict, + bool* seq_given) { + char buf[128]; + if (var.size() > sizeof(buf) - 1) { + return false; + } + + memcpy(buf, var.data(), var.size()); + buf[var.size()] = '\0'; + + if (strict && (buf[0] == '-' || buf[0] == '+') && buf[1] == '\0') { + // res.SetRes(CmdRes::kInvalidParameter, "Invalid stream ID specified as stream "); + return false; + } + + if (seq_given != nullptr) { + *seq_given = true; + } + + if (buf[0] == '-' && buf[1] == '\0') { + id.ms = 0; + id.seq = 0; + return true; + } else if (buf[0] == '+' && buf[1] == '\0') { + id.ms = UINT64_MAX; + id.seq = UINT64_MAX; + return true; + } + + uint64_t ms; + uint64_t seq; + char* dot = strchr(buf, '-'); + if (dot) { + *dot = '\0'; + } + if (!StreamUtils::string2uint64(buf, ms)) { + return false; + }; + if (dot) { + auto seqlen = strlen(dot + 1); + if (seq_given != nullptr && seqlen == 1 && *(dot + 1) == '*') { + seq = 0; + *seq_given = false; + } else if (!StreamUtils::string2uint64(dot + 1, seq)) { + return false; + } + } else { + seq = missing_seq; + } + id.ms = ms; + id.seq = seq; + return true; +} + +bool StreamUtils::StreamParseID(const std::string& var, streamID& id, uint64_t missing_seq) { + return StreamGenericParseID(var, id, missing_seq, false, nullptr); +} + +bool StreamUtils::StreamParseStrictID(const std::string& var, streamID& id, uint64_t missing_seq, bool* seq_given) { + return StreamGenericParseID(var, id, missing_seq, true, seq_given); +} + +bool StreamUtils::StreamParseIntervalId(const std::string& var, streamID& id, bool* exclude, uint64_t missing_seq) { + if (exclude != nullptr) { + *exclude = (var.size() > 1 && var[0] == '('); + } + if (exclude != nullptr && *exclude) { + return StreamParseStrictID(var.substr(1), id, missing_seq, nullptr); + } else { + return StreamParseID(var, id, missing_seq); + } +} + +bool StreamUtils::string2uint64(const char* s, uint64_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + uint64_t tmp = strtoull(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int64(const char* s, int64_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + int64_t tmp = std::strtoll(s, &end, 10); + if (*end || errno == ERANGE) { + // Conversion either didn't consume the entire string, or overflow occurred + return false; + } + + value = tmp; + return true; +} + +bool StreamUtils::string2int32(const char* s, int32_t& value) { + if (!s || !*s) { + return false; + } + + char* end; + errno = 0; + long tmp = strtol(s, &end, 10); + if (*end || errno == ERANGE || tmp < INT_MIN || tmp > INT_MAX) { + // Conversion either didn't consume the entire string, + // or overflow or underflow occurred + return false; + } + + value = static_cast(tmp); + return true; +} + +bool StreamUtils::SerializeMessage(const std::vector& field_values, std::string& message, int field_pos) { + assert(field_values.size() - field_pos >= 2 && (field_values.size() - field_pos) % 2 == 0); + assert(message.empty()); + // count the size of serizlized message + uint32_t size = 0; + for (int i = field_pos; i < field_values.size(); i++) { + size += field_values[i].size() + sizeof(uint32_t); + } + message.reserve(size); + + // serialize message + for (int i = field_pos; i < field_values.size(); i++) { + uint32_t len = field_values[i].size(); + message.append(reinterpret_cast(&len), sizeof(len)); + message.append(field_values[i]); + } + + return true; +} + +bool StreamUtils::DeserializeMessage(const std::string& message, std::vector& parsed_message) { + uint32_t pos = 0; + while (pos < message.size()) { + // Read the length of the next field value from the message + uint32_t len = *reinterpret_cast(&message[pos]); + pos += sizeof(uint32_t); + + // Check if the calculated end of the string is still within the message bounds + if (pos + len > message.size()) { + LOG(ERROR) << "Invalid message format, failed to parse message"; + return false; // Error: not enough data in the message string + } + + // Extract the field value and add it to the vector + parsed_message.push_back(message.substr(pos, len)); + pos += len; + } + + return true; +} + +uint64_t StreamUtils::GetCurrentTimeMs() { + return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count(); +} +}; // namespace storage diff --git a/src/storage/src/redis_streams.h b/src/storage/src/redis_streams.h new file mode 100644 index 0000000000..c964efef7f --- /dev/null +++ b/src/storage/src/redis_streams.h @@ -0,0 +1,247 @@ +// Copyright (c) 2023-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#pragma once + +#include +#include +#include +#include +#include "pika_stream_meta_value.h" +#include "pika_stream_types.h" +#include "rocksdb/options.h" +#include "rocksdb/slice.h" +#include "rocksdb/status.h" +#include "src/redis.h" +#include "storage/storage.h" + +namespace storage { + +// the max number of each delete operation in XTRIM command，to avoid too much memory usage. +// eg. if a XTIRM command need to trim 10000 items, the implementation will use rocsDB's delete operation (10000 / +// kDEFAULT_TRIM_BATCH_SIZE) times +const static int32_t kDEFAULT_TRIM_BATCH_SIZE = 1000; +struct StreamAddTrimArgs { + // XADD options + streamID id; + bool id_given{false}; + bool seq_given{false}; + bool no_mkstream{false}; + + // XADD + XTRIM common options + StreamTrimStrategy trim_strategy{TRIM_STRATEGY_NONE}; + int trim_strategy_arg_idx{0}; + + // TRIM_STRATEGY_MAXLEN options + uint64_t maxlen{0}; + streamID minid; +}; + +struct StreamReadGroupReadArgs { + // XREAD + XREADGROUP common options + std::vector keys; + std::vector unparsed_ids; + int32_t count{INT32_MAX}; // The limit of read, in redis this is uint64_t, but PKHScanRange only support int32_t + uint64_t block{0}; // 0 means no block + + // XREADGROUP options + std::string group_name; + std::string consumer_name; + bool noack_{false}; +}; + +struct StreamScanArgs { + streamID start_sid; + streamID end_sid; + int32_t limit{INT32_MAX}; + bool start_ex{false}; // exclude first message + bool end_ex{false}; // exclude last message + bool is_reverse{false}; // scan in reverse order +}; + +struct StreamInfoResult { + int32_t length{0}; + std::string last_id_str; + std::string max_deleted_entry_id_str; + uint64_t entries_added{0}; + std::string first_id_str; +}; + +class StreamUtils { + public: + StreamUtils() = default; + ~StreamUtils() = default; + + static bool string2uint64(const char* s, uint64_t& value); + static bool string2int64(const char* s, int64_t& value); + static bool string2int32(const char* s, int32_t& value); + + static uint64_t GetCurrentTimeMs(); + + // serialize the message to a string. + // format: {field1.size, field1, value1.size, value1, field2.size, field2, ...} + static bool SerializeMessage(const std::vector& field_values, std::string& serialized_message, + int field_pos); + + // deserialize the message from a string with the format of SerializeMessage. + static bool DeserializeMessage(const std::string& message, std::vector& parsed_message); + + // Parse a stream ID in the format given by clients to Pika, that is + // -, and converts it into a streamID structure. The ID may be in incomplete + // form, just stating the milliseconds time part of the stream. In such a case + // the missing part is set according to the value of 'missing_seq' parameter. + // + // The IDs "-" and "+" specify respectively the minimum and maximum IDs + // that can be represented. If 'strict' is set to 1, "-" and "+" will be + // treated as an invalid ID. + // + // The ID form -* specifies a millisconds-only ID, leaving the sequence part + // to be autogenerated. When a non-NULL 'seq_given' argument is provided, this + // form is accepted and the argument is set to 0 unless the sequence part is + // specified. + static bool StreamGenericParseID(const std::string& var, streamID& id, uint64_t missing_seq, bool strict, + bool* seq_given); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 0, to be used when - and + are acceptable IDs. + static bool StreamParseID(const std::string& var, streamID& id, uint64_t missing_seq); + + // Wrapper for streamGenericParseID() with 'strict' argument set to + // 1, to be used when we want to return an error if the special IDs + or - + // are provided. + static bool StreamParseStrictID(const std::string& var, streamID& id, uint64_t missing_seq, bool* seq_given); + + // Helper for parsing a stream ID that is a range query interval. When the + // exclude argument is NULL, streamParseID() is called and the interval + // is treated as close (inclusive). Otherwise, the exclude argument is set if + // the interval is open (the "(" prefix) and streamParseStrictID() is + // called in that case. + static bool StreamParseIntervalId(const std::string& var, streamID& id, bool* exclude, uint64_t missing_seq); +}; + +class RedisStreams : public Redis { + public: + RedisStreams(Storage* const s, const DataType& type) : Redis(s, type) {} + ~RedisStreams() override = default; + + //===--------------------------------------------------------------------===// + // Commands + //===--------------------------------------------------------------------===// + Status XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args); + Status XDel(const Slice& key, const std::vector& ids, int32_t& count); + Status XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count); + Status XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages); + Status XLen(const Slice& key, int32_t& len); + Status XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys); + Status XInfo(const Slice& key, StreamInfoResult& result); + + //===--------------------------------------------------------------------===// + // Common Commands + //===--------------------------------------------------------------------===// + Status Open(const StorageOptions& storage_options, const std::string& db_path) override; + Status CompactRange(const rocksdb::Slice* begin, const rocksdb::Slice* end, + const ColumnFamilyType& type = kMetaAndData) override; + Status GetProperty(const std::string& property, uint64_t* out) override; + Status ScanKeyNum(KeyInfo* keyinfo) override; + Status ScanKeys(const std::string& pattern, std::vector* keys) override; + Status PKPatternMatchDel(const std::string& pattern, int32_t* ret) override; + Status PKScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key); + Status PKRScanRange(const Slice& key_start, const Slice& key_end, const Slice& pattern, int32_t limit, + std::vector* keys, std::string* next_key); + + //===--------------------------------------------------------------------===// + // Keys Commands + //===--------------------------------------------------------------------===// + Status Del(const Slice& key) override; + bool Scan(const std::string& start_key, const std::string& pattern, std::vector* keys, int64_t* count, + std::string* next_key) override; + + //===--------------------------------------------------------------------===// + // Not needed for streams + //===--------------------------------------------------------------------===// + Status Expire(const Slice& key, int32_t ttl) override; + bool PKExpireScan(const std::string& start_key, int32_t min_timestamp, int32_t max_timestamp, + std::vector* keys, int64_t* leftover_visits, std::string* next_key) override; + Status Expireat(const Slice& key, int32_t timestamp) override; + Status Persist(const Slice& key) override; + Status TTL(const Slice& key, int64_t* timestamp) override; + + //===--------------------------------------------------------------------===// + // Storage API + //===--------------------------------------------------------------------===// + struct ScanStreamOptions { + const rocksdb::Slice key; // the key of the stream + int32_t version; // the version of the stream + streamID start_sid; + streamID end_sid; + int32_t limit; + bool start_ex; // exclude first message + bool end_ex; // exclude last message + bool is_reverse; // scan in reverse order + ScanStreamOptions(const rocksdb::Slice skey, int32_t version, streamID start_sid, streamID end_sid, int32_t count, + bool start_ex = false, bool end_ex = false, bool is_reverse = false) + : key(skey), + version(version), + start_sid(start_sid), + end_sid(end_sid), + limit(count), + start_ex(start_ex), + end_ex(end_ex), + is_reverse(is_reverse) {} + }; + + Status ScanStream(const ScanStreamOptions& option, std::vector& id_messages, std::string& next_field, + rocksdb::ReadOptions& read_options); + // get and parse the stream meta if found + // @return ok only when the stream meta exists + Status GetStreamMeta(StreamMetaValue& tream_meta, const rocksdb::Slice& key, rocksdb::ReadOptions& read_options); + + // Before calling this function, the caller should ensure that the ids are valid + Status DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& ids, rocksdb::ReadOptions& read_options); + + // Before calling this function, the caller should ensure that the ids are valid + Status DeleteStreamMessages(const rocksdb::Slice& key, const StreamMetaValue& stream_meta, + const std::vector& serialized_ids, rocksdb::ReadOptions& read_options); + + Status TrimStream(int32_t& count, StreamMetaValue& stream_meta, const rocksdb::Slice& key, StreamAddTrimArgs& args, + rocksdb::ReadOptions& read_options); + + private: + Status GenerateStreamID(const StreamMetaValue& stream_meta, StreamAddTrimArgs& args); + + Status ScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, + const Slice& pattern, int32_t limit, std::vector& id_messages, std::string& next_id, + rocksdb::ReadOptions& read_options); + Status ReScanRange(const Slice& key, const int32_t version, const Slice& id_start, const std::string& id_end, + const Slice& pattern, int32_t limit, std::vector& id_values, std::string& next_id, + rocksdb::ReadOptions& read_options); + + struct TrimRet { + // the count of deleted messages + int32_t count{0}; + // the next field after trim + std::string next_field; + // the max deleted field, will be empty if no message is deleted + std::string max_deleted_field; + }; + + Status TrimByMaxlen(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); + + Status TrimByMinid(TrimRet& trim_ret, StreamMetaValue& stream_meta, const rocksdb::Slice& key, + const StreamAddTrimArgs& args, rocksdb::ReadOptions& read_options); + + inline Status SetFirstID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options); + + inline Status SetLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, rocksdb::ReadOptions& read_options); + + inline Status SetFirstOrLastID(const rocksdb::Slice& key, StreamMetaValue& stream_meta, bool is_set_first, + rocksdb::ReadOptions& read_options); +}; +} // namespace storage diff --git a/src/storage/src/storage.cc b/src/storage/src/storage.cc index df5a6cf4a1..3dbb6c29b4 100644 --- a/src/storage/src/storage.cc +++ b/src/storage/src/storage.cc @@ -18,6 +18,7 @@ #include "src/redis_hyperloglog.h" #include "src/redis_lists.h" #include "src/redis_sets.h" +#include "src/redis_streams.h" #include "src/redis_strings.h" #include "src/redis_zsets.h" @@ -69,6 +70,7 @@ Storage::~Storage() { rocksdb::CancelAllBackgroundWork(sets_db_->GetDB(), true); rocksdb::CancelAllBackgroundWork(lists_db_->GetDB(), true); rocksdb::CancelAllBackgroundWork(zsets_db_->GetDB(), true); + rocksdb::CancelAllBackgroundWork(streams_db_->GetDB(), true); } int ret = 0; @@ -117,6 +119,13 @@ Status Storage::Open(const StorageOptions& storage_options, const std::string& d if (!s.ok()) { LOG(FATAL) << "open zset db failed, " << s.ToString(); } + + streams_db_ = std::make_unique(this, kStreams); + s = streams_db_->Open(storage_options, AppendSubDirectory(db_path, "streams")); + if (!s.ok()) { + LOG(FATAL) << "open stream db failed, " << s.ToString(); + } + is_opened_.store(true); return Status::OK(); } @@ -530,6 +539,39 @@ Status Storage::ZScan(const Slice& key, int64_t cursor, const std::string& patte return zsets_db_->ZScan(key, cursor, pattern, count, score_members, next_cursor); } +Status Storage::XAdd(const Slice& key, const std::string& serialized_message, StreamAddTrimArgs& args) { + return streams_db_->XAdd(key, serialized_message, args); +} + +Status Storage::XDel(const Slice& key, const std::vector& ids, int32_t& ret) { + return streams_db_->XDel(key, ids, ret); +} + +Status Storage::XTrim(const Slice& key, StreamAddTrimArgs& args, int32_t& count) { + return streams_db_->XTrim(key, args, count); +} + +Status Storage::XRange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages) { + return streams_db_->XRange(key, args, id_messages); +} + +Status Storage::XRevrange(const Slice& key, const StreamScanArgs& args, std::vector& id_messages) { + return streams_db_->XRevrange(key, args, id_messages); +} + +Status Storage::XLen(const Slice& key, int32_t& len) { + return streams_db_->XLen(key, len); +} + +Status Storage::XRead(const StreamReadGroupReadArgs& args, std::vector>& results, + std::vector& reserved_keys) { + return streams_db_->XRead(args, results, reserved_keys); +} + +Status Storage::XInfo(const Slice& key, StreamInfoResult &result) { + return streams_db_->XInfo(key, result); +} + // Keys Commands int32_t Storage::Expire(const Slice& key, int32_t ttl, std::map* type_status) { int32_t ret = 0; @@ -637,6 +679,15 @@ int64_t Storage::Del(const std::vector& keys, std::mapDel(key); + if (s.ok()) { + count++; + } else if (!s.IsNotFound()) { + is_corruption = true; + (*type_status)[DataType::kStreams] = s; + } } if (is_corruption) { @@ -703,6 +754,16 @@ int64_t Storage::DelByType(const std::vector& keys, const DataType& } break; } + // Stream + case DataType::kStreams: { + s = streams_db_->Del(key); + if (s.ok()) { + count++; + } else if (!s.IsNotFound()) { + is_corruption = true; + } + break; + } case DataType::kAll: { return -1; } @@ -868,6 +929,22 @@ int64_t Storage::Scan(const DataType& dtype, int64_t cursor, const std::string& } } start_key = prefix; + case 'x': + is_finish = streams_db_->Scan(start_key, pattern, keys, &leftover_visits, &next_key); + if ((leftover_visits == 0) && !is_finish) { + cursor_ret = cursor + step_length; + StoreCursorStartKey(DataType::kStreams, cursor_ret, std::string("x") + next_key); + break; + } else if (is_finish) { + if (DataType::kStreams == dtype) { + cursor_ret = 0; + break; + } else if (leftover_visits == 0) { + cursor_ret = cursor + step_length; + StoreCursorStartKey(DataType::kStreams, cursor_ret, std::string("k") + prefix); + break; + } + } case 'z': is_finish = zsets_db_->Scan(start_key, pattern, keys, &leftover_visits, &next_key); if ((leftover_visits == 0) && !is_finish) { @@ -1018,6 +1095,9 @@ Status Storage::PKScanRange(const DataType& data_type, const Slice& key_start, c case DataType::kSets: s = sets_db_->PKScanRange(key_start, key_end, pattern, limit, keys, next_key); break; + case DataType::kStreams: + s = streams_db_->PKScanRange(key_start, key_end, pattern, limit, keys, next_key); + break; default: s = Status::Corruption("Unsupported data types"); break; @@ -1047,6 +1127,9 @@ Status Storage::PKRScanRange(const DataType& data_type, const Slice& key_start, case DataType::kSets: s = sets_db_->PKRScanRange(key_start, key_end, pattern, limit, keys, next_key); break; + case DataType::kStreams: + s = streams_db_->PKRScanRange(key_start, key_end, pattern, limit, keys, next_key); + break; default: s = Status::Corruption("Unsupported data types"); break; @@ -1100,6 +1183,9 @@ Status Storage::Scanx(const DataType& data_type, const std::string& start_key, c case DataType::kSets: sets_db_->Scan(start_key, pattern, keys, &count, next_key); break; + case DataType::kStreams: + streams_db_->Scan(start_key, pattern, keys, &count, next_key); + break; default: Status::Corruption("Unsupported data types"); break; @@ -1346,6 +1432,11 @@ Status Storage::Keys(const DataType& data_type, const std::string& pattern, std: if (!s.ok()) { return s; } + } else if (data_type == DataType::kStreams) { + s = streams_db_->ScanKeys(pattern, keys); + if (!s.ok()) { + return s; + } } else { s = strings_db_->ScanKeys(pattern, keys); if (!s.ok()) { @@ -1367,6 +1458,10 @@ Status Storage::Keys(const DataType& data_type, const std::string& pattern, std: if (!s.ok()) { return s; } + s = streams_db_->ScanKeys(pattern, keys); + if (!s.ok()) { + return s; + } } return s; } @@ -1587,6 +1682,9 @@ Status Storage::DoCompact(const DataType& type) { } else if (type == kLists) { current_task_type_ = Operation::kCleanLists; s = lists_db_->CompactRange(nullptr, nullptr); + } else if (type == kStreams) { + current_task_type_ = Operation::kCleanStreams; + s = streams_db_->CompactRange(nullptr, nullptr); } else { current_task_type_ = Operation::kCleanAll; s = strings_db_->CompactRange(nullptr, nullptr); @@ -1594,6 +1692,7 @@ Status Storage::DoCompact(const DataType& type) { s = sets_db_->CompactRange(nullptr, nullptr); s = zsets_db_->CompactRange(nullptr, nullptr); s = lists_db_->CompactRange(nullptr, nullptr); + s = streams_db_->CompactRange(nullptr, nullptr); } current_task_type_ = Operation::kNone; return s; @@ -1641,6 +1740,9 @@ Status Storage::DoCompactRange(const DataType& type, const std::string& start, c } else if (type == kLists) { s = lists_db_->CompactRange(&slice_meta_begin, &slice_meta_end, kMeta); s = lists_db_->CompactRange(&slice_data_begin, &slice_data_end, kData); + } else if (type == kStreams) { + s = streams_db_->CompactRange(&slice_meta_begin, &slice_meta_end, kMeta); + s = streams_db_->CompactRange(&slice_data_begin, &slice_data_end, kData); } return s; } @@ -1684,6 +1786,8 @@ std::string Storage::GetCurrentTaskType() { return "Set"; case kCleanLists: return "List"; + case kCleanStreams: + return "Stream"; case kNone: default: return "No"; @@ -1702,6 +1806,7 @@ Status Storage::GetUsage(const std::string& property, std::mapGetProperty(property, &out); result += out; } + if (db_type == ALL_DB || db_type == STREAMS_DB) { + streams_db_->GetProperty(property, &out); + result += out; + } return result; } @@ -1766,6 +1875,8 @@ rocksdb::DB* Storage::GetDBByType(const std::string& type) { return sets_db_->GetDB(); } else if (type == ZSETS_DB) { return zsets_db_->GetDB(); + } else if (type == STREAMS_DB) { + return streams_db_->GetDB(); } else { return nullptr; } @@ -1804,6 +1915,12 @@ Status Storage::SetOptions(const OptionType& option_type, const std::string& db_ return s; } } + if (db_type == ALL_DB || db_type == STREAMS_DB) { + s = streams_db_->SetOptions(option_type, options); + if (!s.ok()) { + return s; + } + } s = EnableDymayticOptions(option_type,db_type,options); return s; } diff --git a/tests/integration/stream_test.go b/tests/integration/stream_test.go index 82d47a8e1e..3c49ad9973 100644 --- a/tests/integration/stream_test.go +++ b/tests/integration/stream_test.go @@ -204,6 +204,32 @@ var _ = Describe("Stream Commands", func() { return atomic.LoadInt32(&messageCount) }).Should(Equal(int32(totalMessages))) }) + + It("should create a stream and find it using keys * command", func() { + Expect(client.Del(ctx, "mystream").Err()).NotTo(HaveOccurred()) + // Creating a stream and adding entries + _, err := client.XAdd(ctx, &redis.XAddArgs{ + Stream: "mystream", + ID: "*", + Values: map[string]interface{}{"key1": "value1", "key2": "value2"}, + }).Result() + Expect(err).NotTo(HaveOccurred()) + + // Using keys * to find all keys including the stream + keys, err := client.Keys(ctx, "*").Result() + Expect(err).NotTo(HaveOccurred()) + + // Checking if the stream 'mystream' exists in the returned keys + found := false + for _, key := range keys { + if key == "mystream" { + found = true + break + } + } + Expect(found).To(BeTrue(), "Stream 'mystream' should exist in keys") + }) + It("XADD wrong number of args", func() {