diff --git a/.github/workflows/operator.yml b/.github/workflows/operator.yml index c2dbf3b5ca..0a2e8eb3d1 100644 --- a/.github/workflows/operator.yml +++ b/.github/workflows/operator.yml @@ -29,12 +29,12 @@ jobs: - name: Build run: | - cd pika-tools/pika_operator && make + cd tools/pika_operator && make - name: Unit Test run: | - cd pika-tools/pika_operator && make test + cd tools/pika_operator && make test - name: E2E Test run: | - cd pika-tools/pika_operator && make e2e-test-local + cd tools/pika_operator && make e2e-test-local diff --git a/.github/workflows/pika.yml b/.github/workflows/pika.yml index 45b31e23cf..1946ce8c6c 100644 --- a/.github/workflows/pika.yml +++ b/.github/workflows/pika.yml @@ -35,7 +35,7 @@ jobs: - name: Configure CMake # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make. # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type - run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PIKA_TOOLS=ON - name: Build # Build your program with the given configuration diff --git a/.github/workflows/pika_exporter.yml b/.github/workflows/pika_exporter.yml index 220c7ec605..771b343b0d 100644 --- a/.github/workflows/pika_exporter.yml +++ b/.github/workflows/pika_exporter.yml @@ -6,7 +6,7 @@ on: pull_request: branches: [ "unstable" ] paths: - - 'pika-tools/pika_exporter/**' + - 'tools/pika_exporter/**' jobs: @@ -22,7 +22,7 @@ jobs: - name: Build run: | - cd pika-tools/pika_exporter && make -j + cd tools/pika_exporter && make -j - name: Test run: | - cd pika-tools/pika_exporter && make -j + cd tools/pika_exporter && make -j diff --git a/.gitignore b/.gitignore index a00b815232..6447601812 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ cmake-build-release make_config.mk log/ lib/ -tools/ output/ # DB diff --git a/CMakeLists.txt b/CMakeLists.txt index 0758a25cf1..5e41414a75 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -580,6 +580,65 @@ ExternalProject_Add(rocksdb make -j${CPU_CORE} ) +option(USE_PIKA_TOOLS "compile pika-tools" OFF) +if (USE_PIKA_TOOLS) + ExternalProject_Add(hiredis + URL + https://github.com/vipshop/hiredis-vip/archive/refs/tags/0.3.0.tar.gz + URL_HASH + MD5=e4d4a2928188b11f696d35ca1c52e9a4 + DOWNLOAD_NO_PROGRESS + 1 + UPDATE_COMMAND + "" + LOG_BUILD + 1 + LOG_INSTALL + 1 + BUILD_IN_SOURCE + 1 + SOURCE_SUBDIR + "" + BUILD_ALWAYS + 1 + CONFIGURE_COMMAND + "" + BUILD_COMMAND + make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} all + INSTALL_COMMAND + make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} install + ) + set(HIREDIS_LIBRARY ${INSTALL_LIBDIR}/libhiredis_vip.a) + + ExternalProject_Add(bz2 + URL + https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz + URL_HASH + MD5=67e051268d0c475ea773822f7500d0e5 + DOWNLOAD_NO_PROGRESS + 1 + UPDATE_COMMAND + "" + LOG_BUILD + 1 + LOG_INSTALL + 1 + BUILD_IN_SOURCE + 1 + SOURCE_SUBDIR + "" + BUILD_ALWAYS + 1 + CONFIGURE_COMMAND + "" + BUILD_COMMAND + make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} all + INSTALL_COMMAND + make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} install + ) + set(BZ2_LIBRARY ${INSTALL_LIBDIR}/libbz2.a) +endif() + if(${OS_VERSION} MATCHES "CentOS") set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR_64}/librocksdb.a) else() @@ -592,7 +651,9 @@ set(ROCKSDB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${EP_BASE_SUFFIX}/Source/rock add_subdirectory(src/pstd) add_subdirectory(src/net) add_subdirectory(src/storage) -add_subdirectory(pika-tools) +if (USE_PIKA_TOOLS) + add_subdirectory(tools) +endif() aux_source_directory(src DIR_SRCS) # # generate version diff --git a/build.sh b/build.sh index 9d696a3eea..bc851633f0 100755 --- a/build.sh +++ b/build.sh @@ -92,7 +92,12 @@ fi cd ${BUILD_DIR} -${CMAKE} .. . +use_pika_tools="" +if [ $1 = "tools" ]; then + use_pika_tools="-DUSE_PIKA_TOOLS=ON" +fi + +${CMAKE} ${use_pika_tools} .. . if [ $? -ne 0 ]; then echo -e "${C_RED} cmake execution error ${C_END}" diff --git a/pika-tools/CMakeLists.txt b/pika-tools/CMakeLists.txt deleted file mode 100644 index a89649ccea..0000000000 --- a/pika-tools/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_subdirectory(./aof_to_pika) \ No newline at end of file diff --git a/pika-tools/benchmark_client/Makefile b/pika-tools/benchmark_client/Makefile deleted file mode 100644 index cd3e1844eb..0000000000 --- a/pika-tools/benchmark_client/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -CXX = g++ - -LDFLAGS=-lpthread -lsnappy - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -THIRD_PATH=../third - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a - -HIREDIS_PATH=$(THIRD_PATH)/hiredis-vip -HIREDIS_LIBRARY=$(HIREDIS_PATH)/libhiredis_vip.a - -INCLUDE_PATH = -I./ \ - -I$(SLASH_PATH) \ - -I$(HIREDIS_PATH) \ - -OBJECT : SLASH HIREDIS benchmark_client - -SLASH : - make -C $(SLASH_PATH)/slash - -HIREDIS: - make -C $(HIREDIS_PATH) - -benchmark_client : benchmark_client.cc - $(CXX) $(CXXFLAGS) $(LDFLAGS) $^ -o $@ $(INCLUDE_PATH) $(SLASH_LIBRARY) $(HIREDIS_LIBRARY) - -clean: - rm -rf benchmark_client - -distclean: - make -C $(SLASH_PATH)/slash clean - make -C $(HIREDIS_PATH) clean - rm -rf benchmark_client diff --git a/pika-tools/binlog_sender/Makefile b/pika-tools/binlog_sender/Makefile deleted file mode 100755 index 90901525ab..0000000000 --- a/pika-tools/binlog_sender/Makefile +++ /dev/null @@ -1,60 +0,0 @@ -CXX = g++ - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -TARGET = binlog_sender - -SRC_PATH=. -THIRD_PATH=../third - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH=$(SLASH_PATH)/slash/lib/libslash.a - -PINK_PATH=$(THIRD_PATH)/net -PINK=$(PINK_PATH)/net/lib/libnet.a - - -INCLUDE_PATH = -I./ \ - -I$(SLASH_PATH) \ - -I$(PINK_PATH) \ - -LIB_PATH = -L$(SLASH_PATH)/slash/lib \ - -L$(PINK_PATH)/net/lib \ - -LIBS = -lnet \ - -lslash \ - -lpthread \ - -SOURCE := $(wildcard $(SRC_PATH)/*.cc) -OBJS := $(patsubst %.cc, %.o, $(SOURCE)) - -$(TARGET): $(OBJS) $(PINK) $(SLASH) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - - -$(PINK) : - make -C $(PINK_PATH)/net SLASH_PATH=../../slash - -$(SLASH) : - make -C $(SLASH_PATH)/slash - - -.PHONY: clean distclean - -clean: - rm -rf $(OBJS) - rm -rf $(TARGET) - -distclean: - rm -rf $(OBJS) - rm -rf $(TARGET) - make -C $(PINK_PATH)/net clean - make -C $(SLASH_PATH)/slash clean diff --git a/pika-tools/manifest_generator/Makefile b/pika-tools/manifest_generator/Makefile deleted file mode 100644 index fe419a1a2f..0000000000 --- a/pika-tools/manifest_generator/Makefile +++ /dev/null @@ -1,51 +0,0 @@ -CXX = g++ - -LDFLAGS=-lpthread - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -THIRD_PATH=$(CURDIR)/../third - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH_LIBRARY=$(SLASH_PATH)/slash/lib/libslash.a -PINK_PATH=$(THIRD_PATH)/net -PINK_LIBRARY=$(PINK_PATH)/net/lib/libnet.a - - -INCLUDE_PATH = -I./ \ - -I$(SLASH_PATH) \ - -I$(PINK_PATH) \ - -OBJECT : SLASH PINK manifest_generator - -PINK : $(SLASH) - make -C $(PINK_PATH)/net NO_PB=0 SLASH_PATH=$(SLASH_PATH) - -SLASH : - make -C $(SLASH_PATH)/slash - -pika_binlog.o : pika_binlog.cc - $(CXX) $(CXXFLAGS) -c $^ -o $@ $(INCLUDE_PATH) - -manifest_generator.o : manifest_generator.cc pika_binlog.o - $(CXX) $(CXXFLAGS) -c $^ -o $@ $(INCLUDE_PATH) - -manifest_generator : manifest_generator.o pika_binlog.o - $(CXX) $^ $(SLASH_LIBRARY) -o $@ $(LDFLAGS) - -clean: - rm -rf manifest_generator - rm -rf pika_binlog.o - rm -rf manifest_generator.o - -distclean: - make -C $(SLASH_PATH)/slash clean - make -C $(PINK_PATH)/net clean - rm -rf manifest_generator - rm -rf pika_binlog.o - rm -rf manifest_generator.o diff --git a/pika-tools/nemo_to_blackwidow/Makefile b/pika-tools/nemo_to_blackwidow/Makefile deleted file mode 100755 index f5ff6ef4f9..0000000000 --- a/pika-tools/nemo_to_blackwidow/Makefile +++ /dev/null @@ -1,105 +0,0 @@ -CXX = g++ - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -TARGET = nemo_to_blackwidow - -THIRD_PATH=../third -SRC_PATH=. - -ROCKSDB_PATH=$(THIRD_PATH)/rocksdb -ROCKSDB=$(ROCKSDB_PATH)/librocksdb.a - -NEMO_PATH=$(THIRD_PATH)/nemo -NEMO=$(NEMO_PATH)/lib/libnemo.a - -NEMO_ROCKSDB_PATH=$(THIRD_PATH)/nemo-rocksdb -NEMO_ROCKSDB=$(NEMO_ROCKSDB_PATH)/lib/libnemodb.a - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH=$(SLASH_PATH)/slash/lib/libslash.a - -PINK_PATH=$(THIRD_PATH)/net -PINK=$(PINK_PATH)/net/lib/libnet.a - -BLACKWIDOW_PATH=$(THIRD_PATH)/blackwidow -BLACKWIDOW=$(BLACKWIDOW_PATH)/lib/libblackwidow.a - -INCLUDE_PATH = -I./ \ - -I$(NEMO_PATH)/include \ - -I$(NEMO_ROCKSDB_PATH)/include \ - -I$(BLACKWIDOW_PATH)/include \ - -I$(SLASH_PATH) \ - -I$(PINK_PATH) \ - -I$(ROCKSDB_PATH)/include \ - -LIB_PATH = -L$(NEMO_PATH)/lib \ - -L$(NEMO_ROCKSDB_PATH)/lib \ - -L$(BLACKWIDOW_PATH)/lib \ - -L$(PINK_PATH)/net/lib \ - -L$(SLASH_PATH)/slash/lib \ - -L$(ROCKSDB_PATH)/ \ - -LIBS = -lnemo \ - -lnemodb \ - -lblackwidow \ - -lnet \ - -lslash \ - -lrocksdb \ - -lpthread \ - -lsnappy \ - -lz \ - -lbz2 \ - -lglog \ - -lgflags \ - -lrt \ - -SOURCE := $(wildcard $(SRC_PATH)/*.cc) -OBJS := $(patsubst %.cc, %.o, $(SOURCE)) - -$(TARGET): $(OBJS) $(NEMO) $(NEMO_ROCKSDB) $(BLACKWIDOW) $(PINK) $(SLASH) $(ROCKSDB) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - - -$(NEMO) : - make -C $(NEMO_PATH) ROCKSDB_PATH=../rocksdb NEMODB_PATH=../nemo-rocksdb - -$(NEMO_ROCKSDB) : - make -C $(NEMO_ROCKSDB_PATH) ROCKSDB_PATH=../rocksdb - -$(BLACKWIDOW) : - make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=../rocksdb SLASH_PATH=../slash - -$(PINK) : - make -C $(PINK_PATH)/net SLASH_PATH=../../slash - -$(SLASH) : - make -C $(SLASH_PATH)/slash - -$(ROCKSDB): - make -C $(ROCKSDB_PATH) -j 24 static_lib - - -.PHONY: clean distclean - -clean: - rm -rf $(OBJS) - rm -rf $(TARGET) - -distclean: - rm -rf $(OBJS) - rm -rf $(TARGET) - make -C $(NEMO_PATH) clean - make -C $(NEMO_ROCKSDB_PATH) clean - make -C $(BLACKWIDOW_PATH) clean - make -C $(PINK_PATH)/net clean - make -C $(SLASH_PATH)/slash clean - make -C $(ROCKSDB_PATH) clean diff --git a/pika-tools/nemo_to_blackwidow/classify_thread.cc b/pika-tools/nemo_to_blackwidow/classify_thread.cc deleted file mode 100644 index 4eb3131398..0000000000 --- a/pika-tools/nemo_to_blackwidow/classify_thread.cc +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "classify_thread.h" - -ClassifyThread::ClassifyThread(nemo::Nemo* nemo_db, std::vector migrators, const std::string& type) - : is_finish_(false), key_num_(0), consume_index_(0), nemo_db_(nemo_db), migrators_(migrators), type_(type) { - pthread_rwlock_init(&rwlock_, NULL); -} - -ClassifyThread::~ClassifyThread() { pthread_rwlock_destroy(&rwlock_); }; - -int64_t ClassifyThread::key_num() { - slash::RWLock l(&rwlock_, false); - return key_num_; -} - -void ClassifyThread::PlusProcessKeyNum() { - slash::RWLock l(&rwlock_, true); - key_num_++; -} - -void ClassifyThread::DispatchItem(const std::string& item) { - do { - consume_index_ = (consume_index_ + 1) % migrators_.size(); - } while (!migrators_[consume_index_]->LoadItem(item)); -} - -void* ClassifyThread::ThreadMain() { - std::string key; - std::string dst; - if (type_ == nemo::KV_DB) { - nemo::KIterator* iter = nemo_db_->KScan("", "", -1, false); - while (iter->Valid()) { - EncodeKeyValue(iter->key(), iter->value(), &dst); - DispatchItem(nemo::DataType::kKv + dst); - PlusProcessKeyNum(); - iter->Next(); - } - } else { - char c_type; - std::string key_start; - if (type_ == nemo::HASH_DB) { - c_type = 'h'; - key_start = nemo::DataType::kHSize; - } else if (type_ == nemo::LIST_DB) { - c_type = 'l'; - key_start = nemo::DataType::kLMeta; - } else if (type_ == nemo::ZSET_DB) { - c_type = 'z'; - key_start = nemo::DataType::kZSize; - } else if (type_ == nemo::SET_DB) { - c_type = 's'; - key_start = nemo::DataType::kSSize; - } else { - std::cout << "wrong type of db type in classify thread, exit..." << std::endl; - exit(-1); - } - rocksdb::Iterator* iter = nemo_db_->Scanbytype(c_type); - iter->Seek(key_start); - while (iter->Valid() && iter->key().starts_with(key_start)) { - key = iter->key().ToString(); - iter->Next(); - PlusProcessKeyNum(); - DispatchItem(key); - } - } - is_finish_ = true; - return NULL; -} diff --git a/pika-tools/nemo_to_blackwidow/classify_thread.h b/pika-tools/nemo_to_blackwidow/classify_thread.h deleted file mode 100644 index 5f18610e22..0000000000 --- a/pika-tools/nemo_to_blackwidow/classify_thread.h +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef INCLUDE_CLASSIFY_THREAD_H_ -#define INCLUDE_CLASSIFY_THREAD_H_ - -#include "iostream" -#include "vector" - -#include - -#include "nemo.h" -#include "net/include/net_thread.h" -#include "slash/include/slash_mutex.h" - -#include "migrator.h" -#include "utils.h" - -extern slash::Mutex mutex; - -class ClassifyThread : public net::Thread { - public: - ClassifyThread(nemo::Nemo* nemo_db, std::vector migrators, const std::string& type); - virtual ~ClassifyThread(); - int64_t key_num(); - bool is_finish() { return is_finish_; } - std::string type() { return type_; } - int64_t consume_index() { return consume_index_; } - - private: - void PlusProcessKeyNum(); - void DispatchItem(const std::string& item); - virtual void* ThreadMain(); - - bool is_finish_; - int64_t key_num_; - int64_t consume_index_; - pthread_rwlock_t rwlock_; - nemo::Nemo* nemo_db_; - std::vector migrators_; - std::string type_; -}; - -#endif // INCLUDE_CLASSIFY_THREAD_H_ diff --git a/pika-tools/nemo_to_blackwidow/migrator.cc b/pika-tools/nemo_to_blackwidow/migrator.cc deleted file mode 100644 index ea12d76299..0000000000 --- a/pika-tools/nemo_to_blackwidow/migrator.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "migrator.h" -#include "nemo.h" -#include "utils.h" - -extern int32_t need_write_log; -extern int32_t max_batch_limit; -int32_t Migrator::queue_size() { - slash::MutexLock l(&queue_mutex_); - return items_queue_.size(); -} - -void Migrator::PlusMigrateKey() { migrate_key_num_++; } - -void Migrator::SetShouldExit() { - queue_mutex_.Lock(); - should_exit_ = true; - queue_cond_.Signal(); - queue_mutex_.Unlock(); -} - -bool Migrator::LoadItem(const std::string& item) { - queue_mutex_.Lock(); - if (items_queue_.size() >= MAX_QUEUE_SIZE) { - queue_mutex_.Unlock(); - return false; - } else { - items_queue_.push(item); - queue_cond_.Signal(); - queue_mutex_.Unlock(); - return true; - } -} - -void* Migrator::ThreadMain() { - char prefix; - int32_t int32_ret; - uint64_t uint64_ret; - std::string item, dst, key, value; - - rocksdb::Status s; - std::map type_status; - while (items_queue_.size() || !should_exit_) { - queue_mutex_.Lock(); - while (items_queue_.empty() && !should_exit_) { - queue_cond_.Wait(); - } - queue_mutex_.Unlock(); - - if (queue_size() == 0 && should_exit_) { - return NULL; - } - - queue_mutex_.Lock(); - item = items_queue_.front(); - items_queue_.pop(); - queue_mutex_.Unlock(); - - prefix = item[0]; - if (prefix == nemo::DataType::kKv) { - dst = item.substr(1); - DecodeKeyValue(dst, &key, &value); - } else { - key = item.substr(1); - } - - if (need_write_log) { - LOG(INFO) << "migrator id: " << migrator_id_ << " queue size: " << queue_size() << " type : " << prefix - << " key: " << key; - } - - if (prefix == nemo::DataType::kKv) { - blackwidow_db_->Set(key, value); - } else if (prefix == nemo::DataType::kHSize) { - std::vector field_values; - nemo::HIterator* iter = nemo_db_->HScan(key, "", "", -1, false); - while (iter->Valid()) { - field_values.clear(); - for (int32_t idx = 0; idx < max_batch_limit && iter->Valid(); idx++, iter->Next()) { - field_values.push_back({iter->field(), iter->value()}); - } - blackwidow_db_->HMSet(iter->key(), field_values); - } - delete iter; - } else if (prefix == nemo::DataType::kLMeta) { - std::vector ivs; - std::vector values; - int64_t pos = 0; - nemo_db_->LRange(key, 0, pos + max_batch_limit - 1, ivs); - while (!ivs.empty()) { - for (const auto& node : ivs) { - values.push_back(node.val); - } - blackwidow_db_->RPush(key, values, &uint64_ret); - - pos += max_batch_limit; - ivs.clear(); - values.clear(); - nemo_db_->LRange(key, pos, pos + max_batch_limit - 1, ivs); - } - } else if (prefix == nemo::DataType::kZSize) { - std::vector score_members; - nemo::ZIterator* iter = nemo_db_->ZScan(key, nemo::ZSET_SCORE_MIN, nemo::ZSET_SCORE_MAX, -1, false); - while (iter->Valid()) { - score_members.clear(); - for (int32_t idx = 0; idx < max_batch_limit && iter->Valid(); idx++, iter->Next()) { - score_members.push_back({iter->score(), iter->member()}); - } - blackwidow_db_->ZAdd(iter->key(), score_members, &int32_ret); - } - delete iter; - - } else if (prefix == nemo::DataType::kSSize) { - std::vector members; - nemo::SIterator* iter = nemo_db_->SScan(key, -1, false); - while (iter->Valid()) { - members.clear(); - for (int32_t idx = 0; idx < max_batch_limit && iter->Valid(); idx++, iter->Next()) { - members.push_back(iter->member()); - } - blackwidow_db_->SAdd(iter->key(), members, &int32_ret); - } - delete iter; - - } else { - std::cout << "wrong type of db type in migrator, exit..." << std::endl; - exit(-1); - } - - int64_t ttl = -1; - nemo_db_->TTL(key, &ttl); - if (ttl > 0) { - int64_t timestamp = time(NULL) + ttl; - blackwidow_db_->Expireat(key, timestamp, &type_status); - } - PlusMigrateKey(); - } - slash::MutexLock l(&mutex); - std::cout << "Migrator " << migrator_id_ << " finish, keys num : " << migrate_key_num_ << " exit..." << std::endl; - return NULL; -} diff --git a/pika-tools/nemo_to_blackwidow/migrator.h b/pika-tools/nemo_to_blackwidow/migrator.h deleted file mode 100644 index 2c38eb5a1e..0000000000 --- a/pika-tools/nemo_to_blackwidow/migrator.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef INCLUDE_MIGRATOR_H_ -#define INCLUDE_MIGRATOR_H_ - -#define MAX_QUEUE_SIZE 10000 - -#include -#include "iostream" - -#include "blackwidow/blackwidow.h" -#include "net/include/net_thread.h" - -extern slash::Mutex mutex; - -class Migrator : public net::Thread { - public: - Migrator(int32_t migrator_id, nemo::Nemo* nemo_db, blackwidow::BlackWidow* blackwidow_db) - : nemo_db_(nemo_db), - blackwidow_db_(blackwidow_db), - migrator_id_(migrator_id), - migrate_key_num_(0), - should_exit_(false), - queue_cond_(&queue_mutex_) {} - virtual ~Migrator() {} - - int32_t queue_size(); - void PlusMigrateKey(); - void SetShouldExit(); - bool LoadItem(const std::string& item); - - private: - virtual void* ThreadMain(); - nemo::Nemo* nemo_db_; - blackwidow::BlackWidow* blackwidow_db_; - - int32_t migrator_id_; - int64_t migrate_key_num_; - std::atomic should_exit_; - slash::Mutex queue_mutex_; - slash::CondVar queue_cond_; - std::queue items_queue_; -}; - -#endif // INCLUDE_MIGRATOR_H_ diff --git a/pika-tools/nemo_to_blackwidow/nemo_to_blackwidow.cc b/pika-tools/nemo_to_blackwidow/nemo_to_blackwidow.cc deleted file mode 100644 index a9b9740aff..0000000000 --- a/pika-tools/nemo_to_blackwidow/nemo_to_blackwidow.cc +++ /dev/null @@ -1,180 +0,0 @@ -#include "chrono" -#include "ctime" -#include "iomanip" -#include "iostream" - -#include - -#include "blackwidow/blackwidow.h" -#include "nemo.h" -#include "slash/include/env.h" - -#include "classify_thread.h" -#include "migrator.h" -#include "progress_thread.h" - -int32_t thread_num = 6; -int32_t need_write_log = 0; -int32_t max_batch_limit = 512; -std::string nemo_db_path; -std::string blackwidow_db_path; - -slash::Mutex mutex; -ProgressThread* progress_thread; -std::vector migrators; -std::vector classify_threads; - -void PrintInfo(const std::time_t& now) { - std::cout << "================== Nemo To Blackwidow ==================" << std::endl; - std::cout << "Thread_num : " << thread_num << std::endl; - std::cout << "Need write log : " << (need_write_log ? "yes" : "no") << std::endl; - std::cout << "Max batch limit : " << max_batch_limit << std::endl; - std::cout << "Nemo_db_path : " << nemo_db_path << std::endl; - std::cout << "Blackwidow_db_path : " << blackwidow_db_path << std::endl; - std::cout << "Startup Time : " << asctime(localtime(&now)); - std::cout << "========================================================" << std::endl; -} - -void Usage() { - std::cout << "Usage: " << std::endl; - std::cout << "\tNemo_to_Blackwidow reads data from Nemo DB and send to Blackwidow DB" << std::endl; - std::cout << "\t-h -- displays this help information and exits" << std::endl; - std::cout << "\t-n -- numbers of migrator, default = 6" << std::endl; - std::cout << "\t-l -- whether write log, default = 0" << std::endl; - std::cout << "\t-b -- number of members in multiple data structures per migration, default = 512" << std::endl; - std::cout << "\texample: ./nemo_to_blackwidow ./nemo_db ./blackwidow_db -n 10 -l 0 -b 512" << std::endl; -} - -static void GlogInit() { - if (!slash::FileExists("./log")) { - slash::CreatePath("./log"); - } - - FLAGS_log_dir = "./log"; - FLAGS_max_log_size = 2048; // log file 2GB - ::google::InitGoogleLogging("nemo_to_blackwidow"); -} - -int main(int argc, char** argv) { - if (argc != 3 && argc != 5 && argc != 7 && argc != 9) { - Usage(); - exit(-1); - } - - nemo_db_path = std::string(argv[1]); - blackwidow_db_path = std::string(argv[2]); - - if (argc >= 5) { - if (std::string(argv[3]) == "-n") { - thread_num = atoi(argv[4]); - } else { - Usage(); - exit(-1); - } - } - if (argc >= 7) { - if (std::string(argv[5]) == "-l") { - need_write_log = atoi(argv[6]); - } else { - Usage(); - exit(-1); - } - } - if (argc >= 9) { - if (std::string(argv[7]) == "-b") { - max_batch_limit = atoi(argv[8]); - } else { - Usage(); - exit(-1); - } - } - - if (need_write_log) { - GlogInit(); - } - - std::chrono::system_clock::time_point start_time = std::chrono::system_clock::now(); - std::time_t now = std::chrono::system_clock::to_time_t(start_time); - PrintInfo(now); - - // Init nemo db - nemo::Options nemo_option; - nemo_option.create_if_missing = false; - nemo_option.write_buffer_size = 256 * 1024 * 1024; // 256M - nemo_option.target_file_size_base = 20 * 1024 * 1024; // 20M - nemo::Nemo* nemo_db = new nemo::Nemo(nemo_db_path, nemo_option); - if (nemo_db != NULL) { - std::cout << "Open Nemo db success..." << std::endl; - } else { - std::cout << "Open Nemo db failed..." << std::endl; - return -1; - } - - // Init blackwidow db - rocksdb::Status status; - blackwidow::BlackwidowOptions bw_option; - bw_option.options.create_if_missing = true; - bw_option.options.write_buffer_size = 256 * 1024 * 1024; // 256M - bw_option.options.target_file_size_base = 20 * 1024 * 1024; // 20M - blackwidow::BlackWidow* blackwidow_db = new blackwidow::BlackWidow(); - if (blackwidow_db != NULL && (status = blackwidow_db->Open(bw_option, blackwidow_db_path)).ok()) { - std::cout << "Open BlackWidow db success..." << std::endl; - } else { - std::cout << "Open BlackWidow db failed..." << std::endl; - return -1; - } - - for (size_t idx = 0; idx < static_cast(thread_num); ++idx) { - migrators.push_back(new Migrator(idx, nemo_db, blackwidow_db)); - } - - classify_threads.push_back(new ClassifyThread(nemo_db, migrators, nemo::KV_DB)); - classify_threads.push_back(new ClassifyThread(nemo_db, migrators, nemo::HASH_DB)); - classify_threads.push_back(new ClassifyThread(nemo_db, migrators, nemo::LIST_DB)); - classify_threads.push_back(new ClassifyThread(nemo_db, migrators, nemo::SET_DB)); - classify_threads.push_back(new ClassifyThread(nemo_db, migrators, nemo::ZSET_DB)); - - progress_thread = new ProgressThread(&classify_threads); - - std::cout << "Start migrating data from Nemo to Blackwidow..." << std::endl; - for (size_t idx = 0; idx < static_cast(thread_num); ++idx) { - migrators[idx]->StartThread(); - } - - for (size_t idx = 0; idx < classify_threads.size(); ++idx) { - classify_threads[idx]->StartThread(); - } - - progress_thread->StartThread(); - progress_thread->JoinThread(); - delete progress_thread; - - for (size_t idx = 0; idx < classify_threads.size(); ++idx) { - classify_threads[idx]->JoinThread(); - delete classify_threads[idx]; - } - - for (size_t idx = 0; idx < static_cast(thread_num); ++idx) { - migrators[idx]->SetShouldExit(); - } - - for (size_t idx = 0; idx < static_cast(thread_num); ++idx) { - migrators[idx]->JoinThread(); - delete migrators[idx]; - } - - delete nemo_db; - delete blackwidow_db; - - std::chrono::system_clock::time_point end_time = std::chrono::system_clock::now(); - now = std::chrono::system_clock::to_time_t(end_time); - std::cout << "Finish Time : " << asctime(localtime(&now)); - - auto hours = std::chrono::duration_cast(end_time - start_time).count(); - auto minutes = std::chrono::duration_cast(end_time - start_time).count(); - auto seconds = std::chrono::duration_cast(end_time - start_time).count(); - - std::cout << "Total Time Cost : " << hours << " hours " << minutes % 60 << " minutes " << seconds % 60 << " seconds " - << std::endl; - return 0; -} diff --git a/pika-tools/nemo_to_blackwidow/progress_thread.cc b/pika-tools/nemo_to_blackwidow/progress_thread.cc deleted file mode 100644 index 61a4ad0dbe..0000000000 --- a/pika-tools/nemo_to_blackwidow/progress_thread.cc +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "progress_thread.h" - -ProgressThread::ProgressThread(std::vector* classify_threads) : classify_threads_(classify_threads) {} - -bool ProgressThread::AllClassifyTreadFinish() { - for (const auto& classify_thread : *classify_threads_) { - if (!classify_thread->is_finish()) { - return false; - } - } - return true; -} - -void* ProgressThread::ThreadMain() { - while (true) { - slash::MutexLock l(&mutex); - bool is_finish = AllClassifyTreadFinish(); - printf("\rstring keys: %5ld, hashes keys: %5ld, lists keys: %5ld, sets keys: %5ld, zsets keys: %5ld ", - (*classify_threads_)[0]->key_num(), (*classify_threads_)[1]->key_num(), (*classify_threads_)[2]->key_num(), - (*classify_threads_)[3]->key_num(), (*classify_threads_)[4]->key_num()); - fflush(stdout); - if (is_finish) { - break; - } - sleep(1); - } - slash::MutexLock l(&mutex); - printf("\nClassify keys finished\n"); - return NULL; -} diff --git a/pika-tools/nemo_to_blackwidow/progress_thread.h b/pika-tools/nemo_to_blackwidow/progress_thread.h deleted file mode 100644 index 5fba197339..0000000000 --- a/pika-tools/nemo_to_blackwidow/progress_thread.h +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef INCLUDE_PROGRESS_THREAD_H_ -#define INCLUDE_PROGRESS_THREAD_H_ - -#include "iostream" -#include "vector" - -#include "net/include/net_thread.h" - -#include "classify_thread.h" - -extern slash::Mutex mutex; - -class ProgressThread : public net::Thread { - public: - ProgressThread(std::vector* classify_threads); - - private: - bool AllClassifyTreadFinish(); - virtual void* ThreadMain(); - std::vector* classify_threads_; -}; - -#endif // INCLUDE_PROGRESS_THREAD_H_ diff --git a/pika-tools/nemo_to_blackwidow/utils.cc b/pika-tools/nemo_to_blackwidow/utils.cc deleted file mode 100644 index 4f46ac5e03..0000000000 --- a/pika-tools/nemo_to_blackwidow/utils.cc +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "utils.h" -#include "iostream" - -void EncodeKeyValue(const std::string& key, const std::string& value, std::string* dst) { - dst->clear(); - int32_t key_size = key.size(); - int32_t value_size = value.size(); - dst->append(reinterpret_cast(&key_size), sizeof(int32_t)); - dst->append(key); - dst->append(reinterpret_cast(&value_size), sizeof(int32_t)); - dst->append(value); - return; -} - -void DecodeKeyValue(const std::string& dst, std::string* key, std::string* value) { - const char* p = dst.data(); - int32_t key_size = *(reinterpret_cast(p)); - p += sizeof(int32_t); - key->assign(p, key_size); - p += key_size; - - int32_t value_size = *(reinterpret_cast(p)); - p += sizeof(int32_t); - value->assign(p, value_size); - p += value_size; - return; -} diff --git a/pika-tools/nemo_to_blackwidow/utils.h b/pika-tools/nemo_to_blackwidow/utils.h deleted file mode 100644 index e1acaa9794..0000000000 --- a/pika-tools/nemo_to_blackwidow/utils.h +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) 2018-present The pika-tools Authors. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef INCLUDE_UTILS_H_ -#define INCLUDE_UTILS_H_ - -void EncodeKeyValue(const std::string& key, const std::string& value, std::string* dst); -void DecodeKeyValue(const std::string& dst, std::string* key, std::string* value); - -#endif // INCLUDE_UILTS_H_ diff --git a/pika-tools/pika-port/pika_port_2/Makefile b/pika-tools/pika-port/pika_port_2/Makefile deleted file mode 100644 index 644a596ae4..0000000000 --- a/pika-tools/pika-port/pika_port_2/Makefile +++ /dev/null @@ -1,106 +0,0 @@ -CXX = g++ - -ifeq ($(__REL), 1) - CXXFLAGS = -O2 -g -pipe -fPIC -W -DNDEBUG -Wwrite-strings -Wpointer-arith \ - -Wreorder -Wswitch -Wsign-promo -Wredundant-decls -Wformat -Wall -Wno-unused-parameter \ - -D_GNU_SOURCE -D__STDC_FORMAT_MACROS -std=c++11 -gdwarf-2 -Wno-redundant-decls -else - CXXFLAGS = -O0 -g -pg -pipe -fPIC -W -DDEBUG -Wwrite-strings -Wpointer-arith -Wreorder -Wswitch \ - -Wsign-promo -Wredundant-decls -Wformat -Wall -Wno-unused-parameter -D_GNU_SOURCE \ - -D__STDC_FORMAT_MACROS -std=c++11 -Wno-redundant-decls -endif - -TARGET = pika_port -SRC_DIR = . -THIRD_PATH = ../../third - -# ROCKSDB_LDFLAGS= -lsnappy -lz -lbz2 -llz4 -lzstd -ROCKSDB_LDFLAGS= -lsnappy -lgflags -lz -lbz2 -PROCESSOR_NUMS=2 - -NEMO_PATH=$(THIRD_PATH)/nemo -NEMO=$(NEMO_PATH)/lib/libnemo.a - -NEMO_ROCKSDB_PATH=$(THIRD_PATH)/nemo-rocksdb -NEMO_ROCKSDB=$(NEMO_ROCKSDB_PATH)/lib/libnemodb.a - -UNWIND_PATH=../deps/libunwind -GFLAGS_PATH=../deps/gflags -NEMO=$(NEMO_PATH)/lib/libnemo.a -PIKA_PATH=./pika2.3.6 - -INCLUDE_PATH = -I$(PIKA_PATH)/ \ - -I$(PIKA_PATH)/include \ - -I$(THIRD_PATH)/slash \ - -I$(THIRD_PATH)/net \ - -I$(THIRD_PATH)/nemo/include/ \ - -I$(THIRD_PATH)/nemo-rocksdb/include/ \ - -I$(THIRD_PATH)/rocksdb/include/ - -LIB_PATH = -L./ \ - -L$(GFLAGS_PATH)/lib/ \ - -L$(UNWIND_PATH)/lib/ \ - -L$(THIRD_PATH)/nemo-rocksdb/lib \ - -L$(THIRD_PATH)/rocksdb/ \ - -L$(THIRD_PATH)/slash/slash/lib \ - -L$(THIRD_PATH)/net/net/lib \ - -L$(THIRD_PATH)/nemo/lib/ - -LIBS = -Wl,-Bstatic -lnemo -lrocksdb\ - -Wl,-Bdynamic -lpthread\ - -lrt \ - -lnet \ - -lslash \ - -lnemodb \ - -lrocksdb \ - -lglog - -LIBS += $(ROCKSDB_LDFLAGS) - -PINK = $(THIRD_PATH)/net/net/lib/libnet.a -SLASH = $(THIRD_PATH)/slash/slash/lib/libslash.a - -.PHONY: all clean - -BASE_OBJS := $(wildcard $(SRC_DIR)/*.cc) -BASE_OBJS += $(wildcard $(SRC_DIR)/*.c) -BASE_OBJS += $(wildcard $(SRC_DIR)/*.cpp) -OBJS = $(patsubst %.cc,%.o,$(BASE_OBJS)) - -all: $(TARGET) - @echo "Success, go, go, go..." - -# $(TARGET): $(PINK) $(SLASH) $(OBJS) -$(TARGET): $(OBJS) $(NEMO) $(NEMO_ROCKSDB) $(GFLAGS) $(PINK) $(SLASH) $(GLOG) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LFLAGS) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - -$(TOBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - -$(NEMO) : - make -C $(NEMO_PATH) ROCKSDB_PATH=../rocksdb NEMODB_PATH=../nemo-rocksdb - -$(NEMO_ROCKSDB) : - make -C $(NEMO_ROCKSDB_PATH) ROCKSDB_PATH=../rocksdb - -$(PINK) : - make -C $(PINK_PATH)/net SLASH_PATH=../../slash - -$(SLASH) : - make -C $(SLASH_PATH)/slash - -$(GLOG): - cd $(THIRD_PATH)/glog; if [ ! -f ./Makefile ]; then ./configure --disable-shared; fi; make; echo '*' > $(CURDIR)/third/glog/.gitignore; - -clean: - rm -rf $(SRC_DIR)/*.o - rm -rf $(TARGET) - make -C $(NEMO_PATH) clean - make -C $(NEMO_ROCKSDB_PATH) clean - make -C $(BLACKWIDOW_PATH) clean - make -C $(PINK_PATH)/net clean - make -C $(SLASH_PATH)/slash clean - make -C $(ROCKSDB_PATH) clean diff --git a/pika-tools/pika-port/pika_port_2/binlog.cc b/pika-tools/pika-port/pika_port_2/binlog.cc deleted file mode 100644 index 529a7eb994..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog.cc +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "binlog.h" -#include "binlog_log.h" - -#include -#include -#include -#include -#include - -#include -#include - -#include "slash/include/slash_mutex.h" - -using slash::RWLock; - -std::string NewFileName(const std::string name, const uint32_t current) { - char buf[256]; - snprintf(buf, sizeof(buf), "%s%u", name.c_str(), current); - return std::string(buf); -} - -/* - * Version - */ -Version::Version(slash::RWFile* save) : pro_offset_(0), pro_num_(0), save_(save) { - assert(save_ != NULL); - - pthread_rwlock_init(&rwlock_, NULL); -} - -Version::~Version() { - StableSave(); - pthread_rwlock_destroy(&rwlock_); -} - -Status Version::StableSave() { - char* p = save_->GetData(); - memcpy(p, &pro_offset_, sizeof(uint64_t)); - p += 20; - // memcpy(p, &con_offset_, sizeof(uint64_t)); - // p += 8; - // memcpy(p, &item_num_, sizeof(uint32_t)); - // p += 4; - memcpy(p, &pro_num_, sizeof(uint32_t)); - // p += 4; - // memcpy(p, &con_num_, sizeof(uint32_t)); - // p += 4; - return Status::OK(); -} - -Status Version::Init() { - Status s; - if (save_->GetData() != NULL) { - memcpy((char*)(&pro_offset_), save_->GetData(), sizeof(uint64_t)); - // memcpy((char*)(&con_offset_), save_->GetData() + 8, sizeof(uint64_t)); - memcpy((char*)(&item_num_), save_->GetData() + 16, sizeof(uint32_t)); - memcpy((char*)(&pro_num_), save_->GetData() + 20, sizeof(uint32_t)); - // memcpy((char*)(&con_num_), save_->GetData() + 24, sizeof(uint32_t)); - // DLOG(INFO) << "Version Init pro_offset "<< pro_offset_ << " itemnum " << item_num << " pro_num " << pro_num_ << - // " con_num " << con_num_; - return Status::OK(); - } else { - return Status::Corruption("version init error"); - } -} - -/* - * Binlog - */ -Binlog::Binlog(const std::string& binlog_path, const int file_size) - : consumer_num_(0), - item_num_(0), - version_(NULL), - queue_(NULL), - versionfile_(NULL), - pro_num_(0), - pool_(NULL), - exit_all_consume_(false), - binlog_path_(binlog_path), - file_size_(file_size) { - // To intergrate with old version, we don't set mmap file size to 100M; - // slash::SetMmapBoundSize(file_size); - // slash::kMmapBoundSize = 1024 * 1024 * 100; - - Status s; - - slash::CreateDir(binlog_path_); - - filename = binlog_path_ + kBinlogPrefix; - const std::string manifest = binlog_path_ + kManifest; - std::string profile; - - if (!slash::FileExists(manifest)) { - DLOG(INFO) << "Binlog: Manifest file " << manifest << " not exist"; - - profile = NewFileName(filename, pro_num_); - s = slash::NewWritableFile(profile, &queue_); - if (!s.ok()) { - LOG(WARNING) << "Binlog: new " << filename << " " << s.ToString(); - } - - s = slash::NewRWFile(manifest, &versionfile_); - if (!s.ok()) { - LOG(WARNING) << "Binlog: new versionfile error " << s.ToString(); - } - LOG(WARNING) << "Binlog: new versionfile " << s.ToString(); - - version_ = new Version(versionfile_); - version_->StableSave(); - } else { - DLOG(INFO) << "Binlog: Find the exist file "; - - s = slash::NewRWFile(manifest, &versionfile_); - if (s.ok()) { - version_ = new Version(versionfile_); - version_->Init(); - pro_num_ = version_->pro_num_; - - // Debug - // version_->debug(); - } else { - LOG(WARNING) << "Binlog: open versionfile error"; - } - - profile = NewFileName(filename, pro_num_); - DLOG(INFO) << "Binlog: open profile " << profile; - slash::AppendWritableFile(profile, &queue_, version_->pro_offset_); - uint64_t filesize = queue_->Filesize(); - DLOG(INFO) << "Binlog: filesize is " << filesize; - } - - InitLogFile(); -} - -Binlog::~Binlog() { - delete version_; - delete versionfile_; - - delete queue_; -} - -void Binlog::InitLogFile() { - assert(queue_ != NULL); - - uint64_t filesize = queue_->Filesize(); - block_offset_ = filesize % kBlockSize; -} - -Status Binlog::GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset) { - slash::RWLock(&(version_->rwlock_), false); - - *filenum = version_->pro_num_; - *pro_offset = version_->pro_offset_; - //*filenum = version_->pro_num(); - //*pro_offset = version_->pro_offset(); - - return Status::OK(); -} - -// Note: mutex lock should be held -Status Binlog::Put(const std::string& item) { - Status s; - - /* Check to roll log file */ - uint64_t filesize = queue_->Filesize(); - if (filesize > file_size_) { - delete queue_; - queue_ = NULL; - - pro_num_++; - std::string profile = NewFileName(filename, pro_num_); - slash::NewWritableFile(profile, &queue_); - - { - slash::RWLock(&(version_->rwlock_), true); - version_->pro_offset_ = 0; - version_->pro_num_ = pro_num_; - // version_->set_pro_offset(0); - // version_->set_pro_num(pro_num_); - version_->StableSave(); - // version_->debug(); - } - InitLogFile(); - } - - int pro_offset; - s = Produce(Slice(item.data(), item.size()), &pro_offset); - if (s.ok()) { - slash::RWLock(&(version_->rwlock_), true); - // version_->plus_item_num(); - version_->pro_offset_ = pro_offset; - // version_->set_pro_offset(pro_offset); - version_->StableSave(); - } - - return s; -} - -// Note: mutex lock should be held -Status Binlog::Put(const char* item, int len) { - Status s; - - /* Check to roll log file */ - uint64_t filesize = queue_->Filesize(); - if (filesize > file_size_) { - delete queue_; - queue_ = NULL; - - pro_num_++; - std::string profile = NewFileName(filename, pro_num_); - slash::NewWritableFile(profile, &queue_); - - { - slash::RWLock(&(version_->rwlock_), true); - version_->pro_offset_ = 0; - version_->pro_num_ = pro_num_; - // version_->set_pro_offset(0); - // version_->set_pro_num(pro_num_); - version_->StableSave(); - } - // version_->debug(); - - InitLogFile(); - } - - int pro_offset; - s = Produce(Slice(item, len), &pro_offset); - if (s.ok()) { - slash::RWLock(&(version_->rwlock_), true); - version_->pro_offset_ = pro_offset; - // version_->plus_item_num(); - // version_->set_pro_offset(pro_offset); - version_->StableSave(); - } - - return s; -} - -Status Binlog::EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, int* temp_pro_offset) { - Status s; - assert(n <= 0xffffff); - assert(block_offset_ + kHeaderSize + n <= kBlockSize); - - char buf[kHeaderSize]; - - uint64_t now; - struct timeval tv; - gettimeofday(&tv, NULL); - now = tv.tv_sec; - buf[0] = static_cast(n & 0xff); - buf[1] = static_cast((n & 0xff00) >> 8); - buf[2] = static_cast(n >> 16); - buf[3] = static_cast(now & 0xff); - buf[4] = static_cast((now & 0xff00) >> 8); - buf[5] = static_cast((now & 0xff0000) >> 16); - buf[6] = static_cast((now & 0xff000000) >> 24); - buf[7] = static_cast(t); - - s = queue_->Append(Slice(buf, kHeaderSize)); - if (s.ok()) { - s = queue_->Append(Slice(ptr, n)); - if (s.ok()) { - s = queue_->Flush(); - } - } - block_offset_ += static_cast(kHeaderSize + n); - // log_info("block_offset %d", (kHeaderSize + n)); - - *temp_pro_offset += kHeaderSize + n; - // version_->rise_pro_offset((uint64_t)(kHeaderSize + n)); - // version_->StableSave(); - return s; -} - -Status Binlog::Produce(const Slice& item, int* temp_pro_offset) { - Status s; - const char* ptr = item.data(); - size_t left = item.size(); - bool begin = true; - - *temp_pro_offset = version_->pro_offset_; - do { - const int leftover = static_cast(kBlockSize) - block_offset_; - assert(leftover >= 0); - if (static_cast(leftover) < kHeaderSize) { - if (leftover > 0) { - queue_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover)); - // version_->rise_pro_offset(leftover); - *temp_pro_offset += leftover; - // version_->StableSave(); - } - block_offset_ = 0; - } - - const size_t avail = kBlockSize - block_offset_ - kHeaderSize; - const size_t fragment_length = (left < avail) ? left : avail; - RecordType type; - const bool end = (left == fragment_length); - if (begin && end) { - type = kFullType; - } else if (begin) { - type = kFirstType; - } else if (end) { - type = kLastType; - } else { - type = kMiddleType; - } - - s = EmitPhysicalRecord(type, ptr, fragment_length, temp_pro_offset); - ptr += fragment_length; - left -= fragment_length; - begin = false; - } while (s.ok() && left > 0); - - return s; -} - -Status Binlog::AppendBlank(slash::WritableFile* file, uint64_t len) { - if (len < kHeaderSize) { - return Status::OK(); - } - - uint64_t pos = 0; - - std::string blank(kBlockSize, ' '); - for (; pos + kBlockSize < len; pos += kBlockSize) { - file->Append(Slice(blank.data(), blank.size())); - } - - // Append a msg which occupy the remain part of the last block - // We simply increase the remain length to kHeaderSize when remain part < kHeaderSize - uint32_t n; - if (len % kBlockSize < kHeaderSize) { - n = 0; - } else { - n = (uint32_t)((len % kBlockSize) - kHeaderSize); - } - - char buf[kBlockSize]; - uint64_t now = slash::NowMicros(); - buf[0] = static_cast(n & 0xff); - buf[1] = static_cast((n & 0xff00) >> 8); - buf[2] = static_cast(n >> 16); - buf[3] = static_cast(now & 0xff); - buf[4] = static_cast((now & 0xff00) >> 8); - buf[5] = static_cast((now & 0xff0000) >> 16); - buf[6] = static_cast((now & 0xff000000) >> 24); - buf[7] = static_cast(kFullType); - - Status s = file->Append(Slice(buf, kHeaderSize)); - if (s.ok()) { - s = file->Append(Slice(blank.data(), n)); - if (s.ok()) { - s = file->Flush(); - } - } - return s; -} - -Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { - slash::MutexLock l(&mutex_); - - // offset smaller than the first header - if (pro_offset < 4) { - pro_offset = 0; - } - - delete queue_; - - std::string init_profile = NewFileName(filename, 0); - if (slash::FileExists(init_profile)) { - slash::DeleteFile(init_profile); - } - - std::string profile = NewFileName(filename, pro_num); - if (slash::FileExists(profile)) { - slash::DeleteFile(profile); - } - - slash::NewWritableFile(profile, &queue_); - Binlog::AppendBlank(queue_, pro_offset); - - pro_num_ = pro_num; - - { - slash::RWLock(&(version_->rwlock_), true); - version_->pro_num_ = pro_num; - version_->pro_offset_ = pro_offset; - // version_->set_pro_num(pro_num); - // version_->set_pro_offset(pro_offset); - version_->StableSave(); - } - - InitLogFile(); - return Status::OK(); -} diff --git a/pika-tools/pika-port/pika_port_2/binlog.h b/pika-tools/pika-port/pika_port_2/binlog.h deleted file mode 100644 index b4c7e613ac..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog.h +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef BINLOG_H_ -#define BINLOG_H_ - -#include -#include -#include -#include -#include - -#ifndef __STDC_FORMAT_MACROS -# define __STDC_FORMAT_MACROS -# include -#endif - -#include "pika_define.h" -#include "slash/include/env.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" - -using slash::Slice; -using slash::Status; - -std::string NewFileName(const std::string name, const uint32_t current); - -class Version; - -class Binlog { - public: - Binlog(const std::string& Binlog_path, const int file_size = 100 * 1024 * 1024); - ~Binlog(); - - void Lock() { mutex_.Lock(); } - void Unlock() { mutex_.Unlock(); } - - Status Put(const std::string& item); - Status Put(const char* item, int len); - - Status GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset); - /* - * Set Producer pro_num and pro_offset with lock - */ - Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset); - - static Status AppendBlank(slash::WritableFile* file, uint64_t len); - - slash::WritableFile* queue() { return queue_; } - - uint64_t file_size() { return file_size_; } - - std::string filename; - - private: - void InitLogFile(); - Status EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, int* temp_pro_offset); - - /* - * Produce - */ - Status Produce(const Slice& item, int* pro_offset); - - uint32_t consumer_num_; - uint64_t item_num_; - - Version* version_; - slash::WritableFile* queue_; - slash::RWFile* versionfile_; - - slash::Mutex mutex_; - - uint32_t pro_num_; - - int block_offset_; - - char* pool_; - bool exit_all_consume_; - const std::string binlog_path_; - - uint64_t file_size_; - - // Not use - // int32_t retry_; - - // No copying allowed - Binlog(const Binlog&); - void operator=(const Binlog&); -}; - -// We have to reserve the useless con_offset_, con_num_ and item_num, -// to be compatable with version 1.x . -class Version { - public: - Version(slash::RWFile* save); - ~Version(); - - Status Init(); - - // RWLock should be held when access members. - Status StableSave(); - - uint32_t item_num() { return item_num_; } - void set_item_num(uint32_t item_num) { item_num_ = item_num; } - void plus_item_num() { item_num_++; } - void minus_item_num() { item_num_--; } - - uint64_t pro_offset_; - uint32_t pro_num_; - pthread_rwlock_t rwlock_; - - void debug() { - slash::RWLock(&rwlock_, false); - printf("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); - } - - private: - slash::RWFile* save_; - - // Not used - uint64_t con_offset_; - uint32_t con_num_; - uint32_t item_num_; - - // No copying allowed; - Version(const Version&); - void operator=(const Version&); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/binlog_const.cc b/pika-tools/pika-port/pika_port_2/binlog_const.cc deleted file mode 100644 index 1bfb7d65b9..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog_const.cc +++ /dev/null @@ -1,55 +0,0 @@ -#include "binlog_const.h" - -#include "include/pika_define.h" - -std::string PikaState(int state) { - switch (state) { - case PIKA_REPL_NO_CONNECT: - return "PIKA_REPL_NO_CONNECT"; - - case PIKA_REPL_CONNECT: - return "PIKA_REPL_CONNECT"; - - case PIKA_REPL_CONNECTING: - return "PIKA_REPL_CONNECTING"; - - case PIKA_REPL_CONNECTED: - return "PIKA_REPL_CONNECTED"; - - case PIKA_REPL_WAIT_DBSYNC: - return "PIKA_REPL_WAIT_DBSYNC"; - - case PIKA_REPL_ERROR: - return "PIKA_REPL_ERROR"; - - default: - return "PIKA_REPL_UNKNOWN"; - } - - return "PIKA_REPL_UNKNOWN"; -} - -std::string PikaRole(int role) { - std::string roleStr = "|"; - if (role == PIKA_ROLE_SINGLE) { - roleStr += " PIKA_ROLE_SINGLE |"; - } - - if (role == PIKA_ROLE_SLAVE) { - roleStr += " PIKA_ROLE_SLAVE |"; - } - - if (role == PIKA_ROLE_MASTER) { - roleStr += " PIKA_ROLE_MASTER |"; - } - - if (role == PIKA_ROLE_DOUBLE_MASTER) { - roleStr += " PIKA_ROLE_DOUBLE_MASTER |"; - } - - if (role == PIKA_ROLE_PORT) { - roleStr += " PIKA_ROLE_PORT |"; - } - - return roleStr; -} diff --git a/pika-tools/pika-port/pika_port_2/binlog_const.h b/pika-tools/pika-port/pika_port_2/binlog_const.h deleted file mode 100644 index ea633bac19..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog_const.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef BINLOG_CONST_H_ -#define BINLOG_CONST_H_ - -// //role -// #define PIKA_ROLE_SINGLE 0 -// #define PIKA_ROLE_SLAVE 1 -// #define PIKA_ROLE_MASTER 2 -// #define PIKA_ROLE_DOUBLE_MASTER 3 -#define PIKA_ROLE_PORT 4 - -#include -#include - -const int64_t kTestPoint = 500000; -const int64_t kTestNum = LLONG_MAX; -const int64_t kDataSetNum = 5; - -#include -std::string PikaState(int state); -std::string PikaRole(int role); - -#endif diff --git a/pika-tools/pika-port/pika_port_2/binlog_log.h b/pika-tools/pika-port/pika_port_2/binlog_log.h deleted file mode 100644 index 43e1e1b72e..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog_log.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef BINLOG_LOG_H__ -#define BINLOG_LOG_H__ - -#define pline(fmt, ...) printf(fmt "\n", ##__VA_ARGS__) -#define pinfo(fmt, ...) \ - printf("\033[1;34;40m%s-%s-%d: " fmt "\033[0m\n", ((char*)__FILE__), (char*)__func__, (int)__LINE__, ##__VA_ARGS__) -#define perr(fmt, ...) \ - fprintf(stderr, "\033[1;31;40m%s-%s-%d: error: " fmt "\033[0m\n", (char*)__FILE__, (char*)__func__, (int)__LINE__, \ - ##__VA_ARGS__) - -#endif diff --git a/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.cc b/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.cc deleted file mode 100644 index ae316b219e..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.cc +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include - -#include "binlog_receiver_thread.h" -#include "master_conn.h" -#include "net/include/net_conn.h" -#include "pika_port.h" - -#include "port_conf.h" - -extern PikaPort* g_pika_port; - -BinlogReceiverThread::BinlogReceiverThread(std::string host, int port, int cron_interval) - : conn_factory_(this), handles_(this) { - // thread_rep_ = net::NewHolyThread(port, &conn_factory_, - // cron_interval, &handles_); - thread_rep_ = net::NewHolyThread(host, port, &conn_factory_, cron_interval, &handles_); - // to prevent HolyThread::DoCronTask close the pika sender connection - thread_rep_->set_keepalive_timeout(0); -} - -BinlogReceiverThread::~BinlogReceiverThread() { - thread_rep_->StopThread(); - DLOG(INFO) << "BinlogReceiver thread " << thread_rep_->thread_id() << " exit!!!"; - delete thread_rep_; -} - -int BinlogReceiverThread::StartThread() { return thread_rep_->StartThread(); } - -bool BinlogReceiverThread::Handles::AccessHandle(std::string& ip) const { - if (ip == "127.0.0.1") { - ip = g_port_conf.local_ip; - } - if (binlog_receiver_->thread_rep_->conn_num() != 0 || !g_pika_port->ShouldAccessConnAsMaster(ip)) { - DLOG(INFO) << "BinlogReceiverThread AccessHandle failed"; - return false; - } - g_pika_port->PlusMasterConnection(); - return true; -} - -void BinlogReceiverThread::KillBinlogSender() { thread_rep_->KillAllConns(); } diff --git a/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.h b/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.h deleted file mode 100644 index ca88eb7829..0000000000 --- a/pika-tools/pika-port/pika_port_2/binlog_receiver_thread.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef BINLOG_RECEIVER_THREAD_H_ -#define BINLOG_RECEIVER_THREAD_H_ - -#include -#include - -#include "include/pika_define.h" -#include "master_conn.h" -#include "net/include/server_thread.h" -#include "slash/include/slash_mutex.h" - -class BinlogReceiverThread { - public: - BinlogReceiverThread(std::string host, int port, int cron_interval = 0); - virtual ~BinlogReceiverThread(); - int StartThread(); - - void KillBinlogSender(); - - private: - class MasterConnFactory : public net::ConnFactory { - public: - explicit MasterConnFactory(BinlogReceiverThread* binlog_receiver) : binlog_receiver_(binlog_receiver) {} - - virtual net::PinkConn* NewPinkConn(int connfd, const std::string& ip_port, net::ServerThread* thread, - void* worker_specific_data) const override { - return new MasterConn(connfd, ip_port, binlog_receiver_); - } - - private: - BinlogReceiverThread* binlog_receiver_; - }; - - class Handles : public net::ServerHandle { - public: - explicit Handles(BinlogReceiverThread* binlog_receiver) : binlog_receiver_(binlog_receiver) {} - - bool AccessHandle(std::string& ip) const override; - - private: - BinlogReceiverThread* binlog_receiver_; - }; - - MasterConnFactory conn_factory_; - Handles handles_; - net::ServerThread* thread_rep_; -}; -#endif diff --git a/pika-tools/pika-port/pika_port_2/main.cc b/pika-tools/pika-port/pika_port_2/main.cc deleted file mode 100644 index 571b82f94e..0000000000 --- a/pika-tools/pika-port/pika_port_2/main.cc +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include -#include -#include -#include -#include -#include - -#include "pika_port.h" -#include "port_conf.h" - -PortConf g_port_conf; - -PikaPort* g_pika_port; - -static void GlogInit(std::string& log_path, bool is_daemon) { - if (!slash::FileExists(log_path)) { - slash::CreatePath(log_path); - } - - if (!is_daemon) { - FLAGS_alsologtostderr = true; - } - FLAGS_log_dir = log_path; - FLAGS_minloglevel = 0; - FLAGS_max_log_size = 1800; - FLAGS_logbufsecs = 0; - ::google::InitGoogleLogging("PikaPort"); -} - -static void daemonize() { - if (fork() != 0) exit(0); /* parent exits */ - setsid(); /* create a new session */ -} - -static void close_std() { - int fd; - if ((fd = open("/dev/null", O_RDWR, 0)) != -1) { - dup2(fd, STDIN_FILENO); - dup2(fd, STDOUT_FILENO); - dup2(fd, STDERR_FILENO); - close(fd); - } -} - -#include - -static void IntSigHandle(const int sig) { - DLOG(INFO) << "Catch Signal " << sig << ", cleanup..."; - g_pika_port->Stop(); -} - -static void SignalSetup() { - signal(SIGHUP, SIG_IGN); - signal(SIGPIPE, SIG_IGN); - signal(SIGINT, &IntSigHandle); - signal(SIGQUIT, &IntSigHandle); - signal(SIGTERM, &IntSigHandle); -} - -static void Usage() { - fprintf(stderr, - "Usage: pika_port [-h] [-t local_ip -p local_port -i master_ip -o master_port " - "-m forward_ip -n forward_port -x forward_thread_num -y forward_passwd]\n" - "-f filenum -s offset -w password -r rsync_dump_path -l log_path " - "\t-h -- show this help\n" - "\t-t -- local host ip(OPTIONAL default: 127.0.0.1) \n" - "\t-p -- local port(OPTIONAL) \n" - "\t-i -- master ip(OPTIONAL default: 127.0.0.1) \n" - "\t-o -- master port(REQUIRED) \n" - "\t-m -- forward ip(OPTIONAL default: 127.0.0.1) \n" - "\t-n -- forward port(REQUIRED) \n" - "\t-x -- forward thread num(OPTIONAL default: 1) \n" - "\t-y -- forward password(OPTIONAL) \n" - "\t-f -- binlog filenum(OPTIONAL default: local offset) \n" - "\t-s -- binlog offset(OPTIONAL default: local offset) \n" - "\t-w -- password for master(OPTIONAL) \n" - "\t-r -- rsync dump data path(OPTIONAL default: ./rsync_dump) \n" - "\t-l -- local log path(OPTIONAL default: ./log) \n" - "\t-d -- daemonize(OPTIONAL) \n" - " example: ./pika_port -t 127.0.0.1 -p 12345 -i 127.0.0.1 -o 9221 " - "-m 127.0.0.1 -n 6379 -x 7 -f 0 -s 0 -w abc -l ./log -r ./rsync_dump -d\n"); -} - -int main(int argc, char* argv[]) { - if (argc < 2) { - Usage(); - exit(-1); - } - - char c; - char buf[1024]; - bool is_daemon = false; - long num = 0; - while (-1 != (c = getopt(argc, argv, "t:p:i:o:f:s:w:r:l:m:n:x:y:dh"))) { - switch (c) { - case 't': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.local_ip = std::string(buf); - break; - case 'p': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.local_port = int(num); - break; - case 'i': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.master_ip = std::string(buf); - break; - case 'o': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.master_port = int(num); - break; - case 'm': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.forward_ip = std::string(buf); - break; - case 'n': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.forward_port = int(num); - break; - case 'x': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.forward_thread_num = int(num); - break; - case 'y': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.forward_passwd = std::string(buf); - break; - - case 'f': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.filenum = (size_t)(num); - break; - case 's': - snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); - g_port_conf.offset = (size_t)(num); - break; - case 'w': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.passwd = std::string(buf); - break; - - case 'r': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.dump_path = std::string(buf); - if (g_port_conf.dump_path[g_port_conf.dump_path.length() - 1] != '/') { - g_port_conf.dump_path.append("/"); - } - break; - case 'l': - snprintf(buf, 1024, "%s", optarg); - g_port_conf.log_path = std::string(buf); - if (g_port_conf.log_path[g_port_conf.log_path.length() - 1] != '/') { - g_port_conf.log_path.append("/"); - } - break; - case 'd': - is_daemon = true; - break; - case 'h': - Usage(); - return 0; - default: - Usage(); - return 0; - } - } - - if (g_port_conf.local_port == 0) { - std::random_device rd; - std::mt19937 mt(rd()); - std::uniform_int_distribution di(10000, 40000); - g_port_conf.local_port = di(mt); - LOG(INFO) << "Use random port: " << g_port_conf.local_port; - } - - std::cout << "local_ip:" << g_port_conf.local_ip << " " - << "local_port:" << g_port_conf.local_port << " " - << "master_ip:" << g_port_conf.master_ip << " " - << "master_port:" << g_port_conf.master_port << " " - << "forward_ip:" << g_port_conf.forward_ip << " " - << "forward_port:" << g_port_conf.forward_port << " " - << "forward_passwd:" << g_port_conf.forward_passwd << " " - << "forward_thread_num:" << g_port_conf.forward_thread_num << " " - << "log_path:" << g_port_conf.log_path << " " - << "dump_path:" << g_port_conf.dump_path << " " - << "filenum:" << g_port_conf.filenum << " " - << "offset:" << g_port_conf.offset << " " - << "passwd:" << g_port_conf.passwd << std::endl; - if (g_port_conf.master_port == 0 || g_port_conf.forward_port == 0) { - fprintf(stderr, "Invalid Arguments\n"); - Usage(); - exit(-1); - } - - // daemonize if needed - if (is_daemon) { - daemonize(); - } - - GlogInit(g_port_conf.log_path, is_daemon); - SignalSetup(); - - g_pika_port = new PikaPort(g_port_conf.master_ip, g_port_conf.master_port, g_port_conf.passwd); - if (is_daemon) { - close_std(); - } - - g_pika_port->Start(); - - return 0; -} diff --git a/pika-tools/pika-port/pika_port_2/master_conn.cc b/pika-tools/pika-port/pika_port_2/master_conn.cc deleted file mode 100644 index 2601585e9a..0000000000 --- a/pika-tools/pika-port/pika_port_2/master_conn.cc +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "master_conn.h" -#include -#include "binlog_receiver_thread.h" -#include "include/pika_command.h" -#include "include/pika_slot.h" - -#include "pika_port.h" - -extern PikaPort* g_pika_port; - -MasterConn::MasterConn(int fd, std::string ip_port, BinlogReceiverThread* binlog_receiver) - : RedisConn(fd, ip_port, NULL), self_thread_(binlog_receiver) {} - -void MasterConn::RestoreArgs(net::RedisCmdArgsType& argv) { - raw_args_.clear(); - size_t num = argv.size(); - if (argv.size() > 4 && *(argv.end() - 4) == kPikaBinlogMagic) { - num = argv.size() - 4; - } - if (argv[1].find(SlotKeyPrefix) != std::string::npos) { - return; - } - RedisAppendLen(raw_args_, num, "*"); - PikaCmdArgsType::const_iterator it = argv.begin(); - for (size_t idx = 0; idx < num && it != argv.end(); ++it, ++idx) { - RedisAppendLen(raw_args_, (*it).size(), "$"); - RedisAppendContent(raw_args_, *it); - } -} - -int MasterConn::DealMessage(net::RedisCmdArgsType& argv, std::string* response) { - // no reply - // eq set_is_reply(false); - - // if (argv.empty()) { - if (argv.size() < 5) { // special chars: __PIKA_X#$SKGI\r\n1\r\n[\r\n - // return -2; - return 0; - } - - // if (argv[0] == "auth") { - // return 0; - // } - - RestoreArgs(argv); - if (raw_args_.size() == 0) { - return 0; - } - - // //g_pika_port->logger_->Lock(); - // g_pika_port->logger()->Put(raw_args_); - // //g_pika_port->logger_->Unlock(); - - std::string key(" "); - if (1 < argv.size()) { - key = argv[1]; - } - int ret = g_pika_port->SendRedisCommand(raw_args_, key); - if (ret != 0) { - DLOG(WARNING) << "send redis command:" << raw_args_ << ", ret:%d" << ret; - } - - return 0; -} diff --git a/pika-tools/pika-port/pika_port_2/master_conn.h b/pika-tools/pika-port/pika_port_2/master_conn.h deleted file mode 100644 index 6e0058f064..0000000000 --- a/pika-tools/pika-port/pika_port_2/master_conn.h +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef MASTER_CONN_H_ -#define MASTER_CONN_H_ - -#include "include/pika_command.h" -#include "net/include/net_thread.h" -#include "net/include/redis_conn.h" - -class BinlogReceiverThread; - -class MasterConn : public net::RedisConn { - public: - MasterConn(int fd, std::string ip_port, BinlogReceiverThread* binlog_receiver); - virtual ~MasterConn() {} - virtual int DealMessage(net::RedisCmdArgsType& argv, std::string* response); - - private: - BinlogReceiverThread* self_thread_; - void RestoreArgs(net::RedisCmdArgsType&); - std::string raw_args_; -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/migrator_thread.cc b/pika-tools/pika-port/pika_port_2/migrator_thread.cc deleted file mode 100644 index 27386ed3ca..0000000000 --- a/pika-tools/pika-port/pika_port_2/migrator_thread.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "migrator_thread.h" - -MigratorThread::~MigratorThread() {} - -void MigratorThread::MigrateDB(const char type) { - if (type == nemo::DataType::kKv) { - nemo::KIterator* it = db_->KScan("", "", -1, false); - std::string key, value; - - while (it->Valid()) { - key = it->key(); - value = it->value(); - net::RedisCmdArgsType argv; - std::string cmd; - - int64_t ttl; - db_->TTL(key, &ttl); - - argv.push_back("SET"); - argv.push_back(key); - argv.push_back(value); - if (ttl > 0) { - argv.push_back("EX"); - argv.push_back(std::to_string(ttl)); - } - - it->Next(); - net::SerializeRedisCommand(argv, &cmd); - PlusNum(); - cmd = 'k' + cmd; - DispatchKey(cmd); - } - delete it; - } else { - char c_type = 'a'; - switch (type) { - case nemo::DataType::kHSize: - c_type = 'h'; - break; - case nemo::DataType::kSSize: - c_type = 's'; - break; - case nemo::DataType::kLMeta: - c_type = 'l'; - break; - case nemo::DataType::kZSize: - c_type = 'z'; - break; - } - rocksdb::Iterator* it = db_->Scanbytype(c_type); - std::string key_start = "a"; - key_start[0] = type; - it->Seek(key_start); - for (; it->Valid(); it->Next()) { - PlusNum(); - DispatchKey(it->key().ToString()); - } - } -} - -void MigratorThread::DispatchKey(const std::string& key) { - (*senders_)[thread_index_]->LoadKey(key); - thread_index_ = (thread_index_ + 1) % thread_num_; -} - -void* MigratorThread::ThreadMain() { - MigrateDB(type_); - should_exit_ = true; - log_info("%c keys have been dispatched completly", static_cast(type_)); - return NULL; -} diff --git a/pika-tools/pika-port/pika_port_2/migrator_thread.h b/pika-tools/pika-port/pika_port_2/migrator_thread.h deleted file mode 100644 index a13480aea6..0000000000 --- a/pika-tools/pika-port/pika_port_2/migrator_thread.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef MIGRATOR_THREAD_H_ -#define MIGRATOR_THREAD_H_ - -#include -#include "nemo.h" -#include "net/include/redis_cli.h" -#include "pika_sender.h" - -class MigratorThread : public net::Thread { - public: - MigratorThread(nemo::Nemo* db, std::vector* senders, char type, int thread_num) - : db_(db), senders_(senders), type_(type), thread_num_(thread_num), thread_index_(0), num_(0) {} - - int64_t num() { - slash::MutexLock l(&num_mutex_); - return num_; - } - - virtual ~MigratorThread(); - bool should_exit_; - - private: - nemo::Nemo* db_; - std::vector* senders_; - char type_; - int thread_num_; - int thread_index_; - - void MigrateDB(const char type); - void DispatchKey(const std::string& key); - - int64_t num_; - slash::Mutex num_mutex_; - - void PlusNum() { - slash::MutexLock l(&num_mutex_); - ++num_; - } - virtual void* ThreadMain(); -}; -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_binlog.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_binlog.h deleted file mode 100644 index 0bf0e2238e..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_binlog.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_BINLOG_H_ -#define PIKA_BINLOG_H_ - -#include -#include -#include -#include -#include - -#ifndef __STDC_FORMAT_MACROS -# define __STDC_FORMAT_MACROS -# include -#endif - -#include "include/pika_define.h" -#include "slash/include/env.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" - -using slash::Slice; -using slash::Status; - -std::string NewFileName(const std::string name, const uint32_t current); - -class Version; - -class Binlog { - public: - Binlog(const std::string& Binlog_path, const int file_size = 100 * 1024 * 1024); - ~Binlog(); - - void Lock() { mutex_.Lock(); } - void Unlock() { mutex_.Unlock(); } - - Status Put(const std::string& item); - Status Put(const char* item, int len); - - Status GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset); - /* - * Set Producer pro_num and pro_offset with lock - */ - Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset); - - // Double master used - Status GetDoubleRecvInfo(uint32_t* double_filenum, uint64_t* double_offset); - - Status SetDoubleRecvInfo(uint32_t double_filenum, uint64_t double_offset); - - static Status AppendBlank(slash::WritableFile* file, uint64_t len); - - slash::WritableFile* queue() { return queue_; } - - uint64_t file_size() { return file_size_; } - - std::string filename; - - private: - void InitLogFile(); - Status EmitPhysicalRecord(RecordType t, const char* ptr, size_t n, int* temp_pro_offset); - - /* - * Produce - */ - Status Produce(const Slice& item, int* pro_offset); - - uint32_t consumer_num_; - uint64_t item_num_; - - Version* version_; - slash::WritableFile* queue_; - slash::RWFile* versionfile_; - - slash::Mutex mutex_; - - uint32_t pro_num_; - - int block_offset_; - - char* pool_; - bool exit_all_consume_; - const std::string binlog_path_; - - uint64_t file_size_; - - // Not use - // int32_t retry_; - - // No copying allowed - Binlog(const Binlog&); - void operator=(const Binlog&); -}; - -// We have to reserve the useless con_offset_, con_num_ and item_num, -// to be compatable with version 1.x . -class Version { - public: - Version(slash::RWFile* save); - ~Version(); - - Status Init(); - - // RWLock should be held when access members. - Status StableSave(); - - uint32_t item_num() { return item_num_; } - void set_item_num(uint32_t item_num) { item_num_ = item_num; } - void plus_item_num() { item_num_++; } - void minus_item_num() { item_num_--; } - - uint64_t pro_offset_; - uint32_t pro_num_; - - // Double master used - uint64_t double_master_recv_offset_; - uint32_t double_master_recv_num_; - - pthread_rwlock_t rwlock_; - - void debug() { - slash::RWLock(&rwlock_, false); - printf("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); - } - - private: - slash::RWFile* save_; - - // Not used - uint64_t con_offset_; - uint32_t con_num_; - uint32_t item_num_; - - // No copying allowed; - Version(const Version&); - void operator=(const Version&); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_client_conn.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_client_conn.h deleted file mode 100644 index da2cc0649c..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_client_conn.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_CLIENT_CONN_H_ -#define PIKA_CLIENT_CONN_H_ - -#include -#include - -#include "include/pika_command.h" -#include "net/include/net_thread.h" -#include "net/include/redis_conn.h" - -class PikaWorkerSpecificData; - -class PikaClientConn : public net::RedisConn { - public: - PikaClientConn(int fd, std::string ip_port, net::ServerThread* server_thread, void* worker_specific_data); - virtual ~PikaClientConn() {} - int DealMessage(PikaCmdArgsType& argv, std::string* response) override; - - bool IsPubSub() { return is_pubsub_; } - void SetIsPubSub(bool is_pubsub) { is_pubsub_ = is_pubsub; } - - private: - net::ServerThread* const server_thread_; - CmdTable* const cmds_table_; - bool is_pubsub_; - - std::string DoCmd(PikaCmdArgsType& argv, const std::string& opt); - - // Auth related - class AuthStat { - public: - void Init(); - bool IsAuthed(const CmdInfo* const cinfo_ptr); - bool ChecknUpdate(const std::string& arg); - - private: - enum StatType { - kNoAuthed = 0, - kAdminAuthed, - kLimitAuthed, - }; - StatType stat_; - }; - AuthStat auth_stat_; -}; - -struct ClientInfo { - int fd; - std::string ip_port; - int64_t last_interaction; - PikaClientConn* conn; -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h deleted file mode 100644 index 07a8b21fa6..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_command.h +++ /dev/null @@ -1,463 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_COMMAND_H_ -#define PIKA_COMMAND_H_ - -#include -#include -#include -#include - -#include "net/include/redis_conn.h" -#include "slash/include/slash_string.h" - -const std::string kPikaBinlogMagic = "__PIKA_X#$SKGI"; - -// Constant for command name -// Admin -const std::string kCmdNameSlaveof = "slaveof"; -const std::string kCmdNameTrysync = "trysync"; -const std::string kCmdNameInternalTrysync = "internaltrysync"; -const std::string kCmdNameAuth = "auth"; -const std::string kCmdNameBgsave = "bgsave"; -const std::string kCmdNameBgsaveoff = "bgsaveoff"; -const std::string kCmdNameCompact = "compact"; -const std::string kCmdNamePurgelogsto = "purgelogsto"; -const std::string kCmdNamePing = "ping"; -const std::string kCmdNameSelect = "select"; -const std::string kCmdNameFlushall = "flushall"; -const std::string kCmdNameFlushdb = "flushdb"; -const std::string kCmdNameReadonly = "readonly"; -const std::string kCmdNameClient = "client"; -const std::string kCmdNameShutdown = "shutdown"; -const std::string kCmdNameInfo = "info"; -const std::string kCmdNameConfig = "config"; -const std::string kCmdNameMonitor = "monitor"; -const std::string kCmdNameDbsize = "dbsize"; -const std::string kCmdNameTime = "time"; -const std::string kCmdNameDelbackup = "delbackup"; -const std::string kCmdNameEcho = "echo"; - -// Migrate slot -const std::string kCmdNameSlotsMgrtSlot = "slotsmgrtslot"; -const std::string kCmdNameSlotsMgrtTagSlot = "slotsmgrttagslot"; -const std::string kCmdNameSlotsMgrtOne = "slotsmgrtone"; -const std::string kCmdNameSlotsMgrtTagOne = "slotsmgrttagone"; -const std::string kCmdNameSlotsInfo = "slotsinfo"; -const std::string kCmdNameSlotsHashKey = "slotshashkey"; -const std::string kCmdNameSlotsReload = "slotsreload"; -const std::string kCmdNameSlotsReloadOff = "slotsreloadoff"; -const std::string kCmdNameSlotsDel = "slotsdel"; -const std::string kCmdNameSlotsScan = "slotsscan"; -const std::string kCmdNameSlotsCleanup = "slotscleanup"; -const std::string kCmdNameSlotsCleanupOff = "slotscleanupoff"; -const std::string kCmdNameSlotsMgrtTagSlotAsync = "slotsmgrttagslot-async"; -const std::string kCmdNameSlotsMgrtSlotAsync = "slotsmgrtslot-async"; -const std::string kCmdNameSlotsMgrtExecWrapper = "slotsmgrt-exec-wrapper"; -const std::string kCmdNameSlotsMgrtAsyncStatus = "slotsmgrt-async-status"; -const std::string kCmdNameSlotsMgrtAsyncCancel = "slotsmgrt-async-cancel"; - -// Kv -const std::string kCmdNameSet = "set"; -const std::string kCmdNameGet = "get"; -const std::string kCmdNameDel = "del"; -const std::string kCmdNameIncr = "incr"; -const std::string kCmdNameIncrby = "incrby"; -const std::string kCmdNameIncrbyfloat = "incrbyfloat"; -const std::string kCmdNameDecr = "decr"; -const std::string kCmdNameDecrby = "decrby"; -const std::string kCmdNameGetset = "getset"; -const std::string kCmdNameAppend = "append"; -const std::string kCmdNameMget = "mget"; -const std::string kCmdNameKeys = "keys"; -const std::string kCmdNameSetnx = "setnx"; -const std::string kCmdNameSetex = "setex"; -const std::string kCmdNameMset = "mset"; -const std::string kCmdNameMsetnx = "msetnx"; -const std::string kCmdNameGetrange = "getrange"; -const std::string kCmdNameSetrange = "setrange"; -const std::string kCmdNameStrlen = "strlen"; -const std::string kCmdNameExists = "exists"; -const std::string kCmdNameExpire = "expire"; -const std::string kCmdNamePexpire = "pexpire"; -const std::string kCmdNameExpireat = "expireat"; -const std::string kCmdNamePexpireat = "pexpireat"; -const std::string kCmdNameTtl = "ttl"; -const std::string kCmdNamePttl = "pttl"; -const std::string kCmdNamePersist = "persist"; -const std::string kCmdNameType = "type"; -const std::string kCmdNameScan = "scan"; -// Hash -const std::string kCmdNameHDel = "hdel"; -const std::string kCmdNameHSet = "hset"; -const std::string kCmdNameHGet = "hget"; -const std::string kCmdNameHGetall = "hgetall"; -const std::string kCmdNameHExists = "hexists"; -const std::string kCmdNameHIncrby = "hincrby"; -const std::string kCmdNameHIncrbyfloat = "hincrbyfloat"; -const std::string kCmdNameHKeys = "hkeys"; -const std::string kCmdNameHLen = "hlen"; -const std::string kCmdNameHMget = "hmget"; -const std::string kCmdNameHMset = "hmset"; -const std::string kCmdNameHSetnx = "hsetnx"; -const std::string kCmdNameHStrlen = "hstrlen"; -const std::string kCmdNameHVals = "hvals"; -const std::string kCmdNameHScan = "hscan"; - -// List -const std::string kCmdNameLIndex = "lindex"; -const std::string kCmdNameLInsert = "linsert"; -const std::string kCmdNameLLen = "llen"; -const std::string kCmdNameLPop = "lpop"; -const std::string kCmdNameLPush = "lpush"; -const std::string kCmdNameLPushx = "lpushx"; -const std::string kCmdNameLRange = "lrange"; -const std::string kCmdNameLRem = "lrem"; -const std::string kCmdNameLSet = "lset"; -const std::string kCmdNameLTrim = "ltrim"; -const std::string kCmdNameRPop = "rpop"; -const std::string kCmdNameRPopLPush = "rpoplpush"; -const std::string kCmdNameRPush = "rpush"; -const std::string kCmdNameRPushx = "rpushx"; - -// BitMap -const std::string kCmdNameBitSet = "setbit"; -const std::string kCmdNameBitGet = "getbit"; -const std::string kCmdNameBitPos = "bitpos"; -const std::string kCmdNameBitOp = "bitop"; -const std::string kCmdNameBitCount = "bitcount"; - -// Zset -const std::string kCmdNameZAdd = "zadd"; -const std::string kCmdNameZCard = "zcard"; -const std::string kCmdNameZScan = "zscan"; -const std::string kCmdNameZIncrby = "zincrby"; -const std::string kCmdNameZRange = "zrange"; -const std::string kCmdNameZRangebyscore = "zrangebyscore"; -const std::string kCmdNameZCount = "zcount"; -const std::string kCmdNameZRem = "zrem"; -const std::string kCmdNameZUnionstore = "zunionstore"; -const std::string kCmdNameZInterstore = "zinterstore"; -const std::string kCmdNameZRank = "zrank"; -const std::string kCmdNameZRevrank = "zrevrank"; -const std::string kCmdNameZScore = "zscore"; -const std::string kCmdNameZRevrange = "zrevrange"; -const std::string kCmdNameZRevrangebyscore = "zrevrangebyscore"; -const std::string kCmdNameZRangebylex = "zrangebylex"; -const std::string kCmdNameZRevrangebylex = "zrevrangebylex"; -const std::string kCmdNameZLexcount = "zlexcount"; -const std::string kCmdNameZRemrangebyrank = "zremrangebyrank"; -const std::string kCmdNameZRemrangebylex = "zremrangebylex"; -const std::string kCmdNameZRemrangebyscore = "zremrangebyscore"; - -// Set -const std::string kCmdNameSAdd = "sadd"; -const std::string kCmdNameSPop = "spop"; -const std::string kCmdNameSCard = "scard"; -const std::string kCmdNameSMembers = "smembers"; -const std::string kCmdNameSScan = "sscan"; -const std::string kCmdNameSRem = "srem"; -const std::string kCmdNameSUnion = "sunion"; -const std::string kCmdNameSUnionstore = "sunionstore"; -const std::string kCmdNameSInter = "sinter"; -const std::string kCmdNameSInterstore = "sinterstore"; -const std::string kCmdNameSIsmember = "sismember"; -const std::string kCmdNameSDiff = "sdiff"; -const std::string kCmdNameSDiffstore = "sdiffstore"; -const std::string kCmdNameSMove = "smove"; -const std::string kCmdNameSRandmember = "srandmember"; - -// HyperLogLog -const std::string kCmdNamePfAdd = "pfadd"; -const std::string kCmdNamePfCount = "pfcount"; -const std::string kCmdNamePfMerge = "pfmerge"; - -// GEO -const std::string kCmdNameGeoAdd = "geoadd"; -const std::string kCmdNameGeoPos = "geopos"; -const std::string kCmdNameGeoDist = "geodist"; -const std::string kCmdNameGeoHash = "geohash"; -const std::string kCmdNameGeoRadius = "georadius"; -const std::string kCmdNameGeoRadiusByMember = "georadiusbymember"; - -// Pub/Sub -const std::string kCmdNamePublish = "publish"; -const std::string kCmdNameSubscribe = "subscribe"; -const std::string kCmdNameUnSubscribe = "unsubscribe"; -const std::string kCmdNamePubSub = "pubsub"; -const std::string kCmdNamePSubscribe = "psubscribe"; -const std::string kCmdNamePUnSubscribe = "punsubscribe"; - -typedef net::RedisCmdArgsType PikaCmdArgsType; -static const int RAW_ARGS_LEN = 1024 * 1024; - -enum CmdFlagsMask { - kCmdFlagsMaskRW = 1, - kCmdFlagsMaskType = 30, - kCmdFlagsMaskLocal = 32, - kCmdFlagsMaskSuspend = 64, - kCmdFlagsMaskPrior = 128, - kCmdFlagsMaskAdminRequire = 256 -}; - -enum CmdFlags { - kCmdFlagsRead = 0, // default rw - kCmdFlagsWrite = 1, - kCmdFlagsAdmin = 0, // default type - kCmdFlagsKv = 2, - kCmdFlagsHash = 4, - kCmdFlagsList = 6, - kCmdFlagsSet = 8, - kCmdFlagsZset = 10, - kCmdFlagsBit = 12, - kCmdFlagsHyperLogLog = 14, - kCmdFlagsGeo = 16, - kCmdFlagsPubSub = 18, - kCmdFlagsNoLocal = 0, // default nolocal - kCmdFlagsLocal = 32, - kCmdFlagsNoSuspend = 0, // default nosuspend - kCmdFlagsSuspend = 64, - kCmdFlagsNoPrior = 0, // default noprior - kCmdFlagsPrior = 128, - kCmdFlagsNoAdminRequire = 0, // default no need admin - kCmdFlagsAdminRequire = 256 -}; - -class CmdInfo { - public: - CmdInfo(const std::string _name, int _num, uint16_t _flag) : name_(_name), arity_(_num), flag_(_flag) {} - bool CheckArg(int num) const { - if ((arity_ > 0 && num != arity_) || (arity_ < 0 && num < -arity_)) { - return false; - } - return true; - } - bool is_write() const { return ((flag_ & kCmdFlagsMaskRW) == kCmdFlagsWrite); } - uint16_t flag_type() const { return flag_ & kCmdFlagsMaskType; } - bool is_local() const { return ((flag_ & kCmdFlagsMaskLocal) == kCmdFlagsLocal); } - // Others need to be suspended when a suspend command run - bool is_suspend() const { return ((flag_ & kCmdFlagsMaskSuspend) == kCmdFlagsSuspend); } - bool is_prior() const { return ((flag_ & kCmdFlagsMaskPrior) == kCmdFlagsPrior); } - // Must with admin auth - bool is_admin_require() const { return ((flag_ & kCmdFlagsMaskAdminRequire) == kCmdFlagsAdminRequire); } - std::string name() const { return name_; } - - private: - std::string name_; - int arity_; - uint16_t flag_; - - CmdInfo(const CmdInfo&); - CmdInfo& operator=(const CmdInfo&); -}; - -void inline RedisAppendContent(std::string& str, const std::string& value); -void inline RedisAppendLen(std::string& str, int ori, const std::string& prefix); - -const std::string kNewLine = "\r\n"; - -class CmdRes { - public: - enum CmdRet { - kNone = 0, - kOk, - kPong, - kSyntaxErr, - kInvalidInt, - kInvalidBitInt, - kInvalidBitOffsetInt, - kInvalidBitPosArgument, - kWrongBitOpNotNum, - kInvalidFloat, - kOverFlow, - kNotFound, - kOutOfRange, - kInvalidPwd, - kNoneBgsave, - kPurgeExist, - kInvalidParameter, - kWrongNum, - kInvalidIndex, - kInvalidDbType, - kErrOther, - }; - - CmdRes() : ret_(kNone) {} - - bool none() const { return ret_ == kNone && message_.empty(); } - bool ok() const { return ret_ == kOk || ret_ == kNone; } - void clear() { - message_.clear(); - ret_ = kNone; - } - std::string raw_message() const { return message_; } - std::string message() const { - std::string result; - switch (ret_) { - case kNone: - return message_; - case kOk: - return "+OK\r\n"; - case kPong: - return "+PONG\r\n"; - case kSyntaxErr: - return "-ERR syntax error\r\n"; - case kInvalidInt: - return "-ERR value is not an integer or out of range\r\n"; - case kInvalidBitInt: - return "-ERR bit is not an integer or out of range\r\n"; - case kInvalidBitOffsetInt: - return "-ERR bit offset is not an integer or out of range\r\n"; - case kWrongBitOpNotNum: - return "-ERR BITOP NOT must be called with a single source key.\r\n"; - - case kInvalidBitPosArgument: - return "-ERR The bit argument must be 1 or 0.\r\n"; - case kInvalidFloat: - return "-ERR value is not an float\r\n"; - case kOverFlow: - return "-ERR increment or decrement would overflow\r\n"; - case kNotFound: - return "-ERR no such key\r\n"; - case kOutOfRange: - return "-ERR index out of range\r\n"; - case kInvalidPwd: - return "-ERR invalid password\r\n"; - case kNoneBgsave: - return "-ERR No BGSave Works now\r\n"; - case kPurgeExist: - return "-ERR binlog already in purging...\r\n"; - case kInvalidParameter: - return "-ERR Invalid Argument\r\n"; - case kWrongNum: - result = "-ERR wrong number of arguments for '"; - result.append(message_); - result.append("' command\r\n"); - break; - case kInvalidIndex: - result = "-ERR invalid DB index\r\n"; - break; - case kInvalidDbType: - result = "-ERR invalid DB type\r\n"; - break; - case kErrOther: - result = "-ERR "; - result.append(message_); - result.append(kNewLine); - break; - default: - break; - } - return result; - } - - // Inline functions for Create Redis protocol - void AppendStringLen(int ori) { RedisAppendLen(message_, ori, "$"); } - void AppendArrayLen(int ori) { RedisAppendLen(message_, ori, "*"); } - void AppendInteger(int ori) { RedisAppendLen(message_, ori, ":"); } - void AppendContent(const std::string& value) { RedisAppendContent(message_, value); } - void AppendString(const std::string& value) { - AppendStringLen(value.size()); - AppendContent(value); - } - void AppendStringRaw(std::string& value) { message_.append(value); } - void SetRes(CmdRet _ret, const std::string content = "") { - ret_ = _ret; - if (!content.empty()) { - message_ = content; - } - } - - private: - std::string message_; - CmdRet ret_; -}; - -class Cmd { - public: - Cmd() {} - virtual ~Cmd() {} - - virtual void Do() = 0; - - void Initial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info) { - res_.clear(); // Clear res content - Clear(); // Clear cmd, Derived class can has own implement - DoInitial(argvs, ptr_info); - }; - - CmdRes& res() { return res_; } - - virtual std::string ToBinlog(const PikaCmdArgsType& argv, const std::string& server_id, - const std::string& binlog_info, bool need_send_to_hub) { - std::string res; - res.reserve(RAW_ARGS_LEN); - RedisAppendLen(res, argv.size() + 4, "*"); - - for (auto& v : argv) { - RedisAppendLen(res, v.size(), "$"); - RedisAppendContent(res, v); - } - - AppendAffiliatedInfo(res, server_id, binlog_info, need_send_to_hub); - - return res; - } - - protected: - CmdRes res_; - - void AppendAffiliatedInfo(std::string& res, const std::string& server_id, const std::string& binlog_info, - bool need_send_to_hub) { - // kPikaBinlogMagic - RedisAppendLen(res, kPikaBinlogMagic.size(), "$"); - RedisAppendContent(res, kPikaBinlogMagic); - // server_id - RedisAppendLen(res, server_id.size(), "$"); - RedisAppendContent(res, server_id); - // binlog_info - RedisAppendLen(res, binlog_info.size(), "$"); - RedisAppendContent(res, binlog_info); - // need_send_to_hub - std::string v = need_send_to_hub ? "1" : "0"; - RedisAppendLen(res, v.size(), "$"); - RedisAppendContent(res, v); - } - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info) = 0; - virtual void Clear(){}; - - Cmd(const Cmd&); - Cmd& operator=(const Cmd&); -}; - -typedef std::unordered_map CmdTable; - -// Method for CmdInfo Table -void InitCmdInfoTable(); -const CmdInfo* GetCmdInfo(const std::string& opt); -void DestoryCmdInfoTable(); - -// Method for Cmd Table -void InitCmdTable(CmdTable* cmd_table); -Cmd* GetCmdFromTable(const std::string& opt, const CmdTable& cmd_table); -void DestoryCmdTable(CmdTable* cmd_table); - -void RedisAppendContent(std::string& str, const std::string& value) { - str.append(value.data(), value.size()); - str.append(kNewLine); -} - -void RedisAppendLen(std::string& str, int ori, const std::string& prefix) { - char buf[32]; - slash::ll2string(buf, 32, static_cast(ori)); - str.append(prefix); - str.append(buf); - str.append(kNewLine); -} - -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_define.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_define.h deleted file mode 100644 index c98449c11a..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_define.h +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef PIKA_DEFINE_H_ -#define PIKA_DEFINE_H_ - -#define PIKA_MAX_WORKER_THREAD_NUM 24 - -const std::string kPikaPidFile = "pika.pid"; - -struct WorkerCronTask { - int task; - std::string ip_port; -}; -typedef WorkerCronTask MonitorCronTask; -// task define -#define TASK_KILL 0 -#define TASK_KILLALL 1 - -// slave item -struct SlaveItem { - int64_t sid; - std::string ip_port; - int port; - pthread_t sender_tid; - int hb_fd; - int stage; - void* sender; - struct timeval create_time; -}; - -#define PIKA_MIN_RESERVED_FDS 5000 - -#define SLAVE_ITEM_STAGE_ONE 1 -#define SLAVE_ITEM_STAGE_TWO 2 - -// repl_state_ -#define PIKA_REPL_NO_CONNECT 0 -#define PIKA_REPL_CONNECT 1 -#define PIKA_REPL_CONNECTING 2 -#define PIKA_REPL_CONNECTED 3 -#define PIKA_REPL_WAIT_DBSYNC 4 -#define PIKA_REPL_ERROR 5 - -// role -#define PIKA_ROLE_SINGLE 0 -#define PIKA_ROLE_SLAVE 1 -#define PIKA_ROLE_MASTER 2 -#define PIKA_ROLE_DOUBLE_MASTER 3 - -/* - * The size of Binlogfile - */ -// static uint64_t kBinlogSize = 128; -// static const uint64_t kBinlogSize = 1024 * 1024 * 100; - -enum RecordType { - kZeroType = 0, - kFullType = 1, - kFirstType = 2, - kMiddleType = 3, - kLastType = 4, - kEof = 5, - kBadRecord = 6, - kOldRecord = 7 -}; - -/* - * the block size that we read and write from write2file - * the default size is 64KB - */ -static const size_t kBlockSize = 64 * 1024; - -/* - * Header is Type(1 byte), length (3 bytes), time (4 bytes) - */ -static const size_t kHeaderSize = 1 + 3 + 4; - -/* - * the size of memory when we use memory mode - * the default memory size is 2GB - */ -const int64_t kPoolSize = 1073741824; - -const std::string kBinlogPrefix = "write2file"; -const size_t kBinlogPrefixLen = 10; - -const std::string kManifest = "manifest"; - -/* - * define common character - * - */ -#define COMMA ',' - -/* - * define reply between master and slave - * - */ -const std::string kInnerReplOk = "ok"; -const std::string kInnerReplWait = "wait"; - -const unsigned int kMaxBitOpInputKey = 12800; -const int kMaxBitOpInputBit = 21; -/* - * db sync - */ -const uint32_t kDBSyncMaxGap = 50; -const std::string kDBSyncModule = "document"; - -const std::string kBgsaveInfoFile = "info"; -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_slot.h b/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_slot.h deleted file mode 100644 index b7cbefb07f..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika2.3.6/include/pika_slot.h +++ /dev/null @@ -1,229 +0,0 @@ -#ifndef PIKA_SLOT_H_ -#define PIKA_SLOT_H_ - -#include "include/pika_client_conn.h" -#include "include/pika_command.h" -#include "net/include/net_cli.h" -#include "net/include/net_thread.h" -#include "strings.h" - -const std::string SlotKeyPrefix = "_internal:slotkey:4migrate:"; -const size_t MaxKeySendSize = 10 * 1024; -// crc 32 -#define HASH_SLOTS_MASK 0x000003ff -#define HASH_SLOTS_SIZE (HASH_SLOTS_MASK + 1) - -const uint32_t IEEE_POLY = 0xedb88320; -extern uint32_t crc32tab[256]; - -void CRC32TableInit(uint32_t poly); - -extern void InitCRC32Table(); - -extern uint32_t CRC32Update(uint32_t crc, const char* buf, int len); - -extern int SlotNum(const std::string& str); -extern int KeyType(const std::string key, std::string& key_type); - -extern void SlotKeyAdd(const std::string type, const std::string key); -extern void SlotKeyRem(const std::string key); -extern void KeyNotExistsRem(const std::string type, const std::string key); -extern int KeyDelete(const std::string key, const char key_type); - -class SlotsMgrtTagSlotCmd : public Cmd { - public: - SlotsMgrtTagSlotCmd() {} - virtual void Do(); - - private: - std::string dest_ip_; - int64_t dest_port_; - int64_t timeout_ms_; - int64_t slot_num_; - std::string key_; - char key_type_; - - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); - int SlotKeyPop(); -}; - -class SlotsMgrtTagOneCmd : public Cmd { - public: - SlotsMgrtTagOneCmd() {} - virtual void Do(); - - private: - std::string dest_ip_; - int64_t dest_port_; - int64_t timeout_ms_; - std::string key_; - int64_t slot_num_; - char key_type_; - - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); - int KeyTypeCheck(); - int SlotKeyRemCheck(); -}; - -class SlotsInfoCmd : public Cmd { - public: - SlotsInfoCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsHashKeyCmd : public Cmd { - public: - SlotsHashKeyCmd() {} - virtual void Do(); - - private: - std::vector keys_; - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsReloadCmd : public Cmd { - public: - SlotsReloadCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsReloadOffCmd : public Cmd { - public: - SlotsReloadOffCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsDelCmd : public Cmd { - public: - SlotsDelCmd() {} - virtual void Do(); - - private: - std::vector slots_; - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsScanCmd : public Cmd { - public: - SlotsScanCmd() : pattern_("*"), count_(10) {} - virtual void Do(); - - private: - std::string key_, pattern_; - int64_t cursor_, count_; - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); - virtual void Clear() { - pattern_ = "*"; - count_ = 10; - } -}; - -class SlotsCleanupCmd : public Cmd { - public: - SlotsCleanupCmd() {} - virtual void Do(); - std::vector cleanup_slots_; - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsCleanupOffCmd : public Cmd { - public: - SlotsCleanupOffCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsMgrtTagSlotAsyncCmd : public Cmd { - public: - SlotsMgrtTagSlotAsyncCmd() {} - virtual void Do(); - - private: - std::string dest_ip_; - int64_t dest_port_; - int64_t timeout_ms_; - int64_t max_bulks_; - int64_t max_bytes_; - int64_t slot_num_; - int64_t keys_num_; - - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsMgrtExecWrapperCmd : public Cmd { - public: - SlotsMgrtExecWrapperCmd() {} - virtual void Do(); - - private: - std::string key_; - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsMgrtAsyncStatusCmd : public Cmd { - public: - SlotsMgrtAsyncStatusCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsMgrtAsyncCancelCmd : public Cmd { - public: - SlotsMgrtAsyncCancelCmd() {} - virtual void Do(); - - private: - virtual void DoInitial(PikaCmdArgsType& argvs, const CmdInfo* const ptr_info); -}; - -class SlotsMgrtSenderThread : public net::Thread { - public: - SlotsMgrtSenderThread(); - virtual ~SlotsMgrtSenderThread(); - int SlotsMigrateOne(const std::string& key); - bool SlotsMigrateBatch(const std::string& ip, int64_t port, int64_t time_out, int64_t slot, int64_t keys_num); - bool GetSlotsMigrateResul(int64_t* moved, int64_t* remained); - void GetSlotsMgrtSenderStatus(std::string* ip, int64_t* port, int64_t* slot, bool* migrating, int64_t* moved, - int64_t* remained); - bool SlotsMigrateAsyncCancel(); - - private: - std::string dest_ip_; - int64_t dest_port_; - int64_t timeout_ms_; - int64_t slot_num_; - int64_t keys_num_; - int64_t moved_keys_num_; // during one batch moved - int64_t moved_keys_all_; // all keys moved in the slot - int64_t remained_keys_num_; - std::vector> migrating_batch_; - std::vector> migrating_ones_; - net::PinkCli* cli_; - pthread_rwlock_t rwlock_db_; - pthread_rwlock_t rwlock_batch_; - pthread_rwlock_t rwlock_ones_; - slash::CondVar slotsmgrt_cond_; - slash::Mutex slotsmgrt_cond_mutex_; - std::atomic is_migrating_; - std::atomic should_exit_; - - bool ElectMigrateKeys(); - virtual void* ThreadMain(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika_port.cc b/pika-tools/pika-port/pika_port_2/pika_port.cc deleted file mode 100644 index 5ca75e1495..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika_port.cc +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "binlog_const.h" -#include "pika_port.h" -#include "port_conf.h" -#include "slash/include/env.h" -#include "slash/include/rsync.h" -#include "slash/include/slash_string.h" - -PikaPort::PikaPort(std::string& master_ip, int master_port, std::string& passwd) - : sid_(0), - ping_thread_(NULL), - master_ip_(master_ip), - master_port_(master_port), - master_connection_(0), - role_(PIKA_ROLE_PORT), - repl_state_(PIKA_REPL_NO_CONNECT), - requirepass_(passwd), - // cli_(NULL), - should_exit_(false) { - pthread_rwlockattr_t attr; - pthread_rwlockattr_init(&attr); - pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); - pthread_rwlock_init(&rwlock_, &attr); - - // Init ip host - if (!Init()) { - LOG(FATAL) << "Init iotcl error"; - } - - // Create redis sender - size_t thread_num = g_port_conf.forward_thread_num; - for (size_t i = 0; i < thread_num; i++) { - senders_.emplace_back( - new RedisSender(int(i), g_port_conf.forward_ip, g_port_conf.forward_port, g_port_conf.forward_passwd)); - } - - // Create thread - binlog_receiver_thread_ = new BinlogReceiverThread(g_port_conf.local_ip, g_port_conf.local_port + 1000, 1000); - trysync_thread_ = new TrysyncThread(); - - pthread_rwlock_init(&state_protector_, NULL); - logger_ = new Binlog(g_port_conf.log_path, 104857600); -} - -PikaPort::~PikaPort() { - LOG(INFO) << "Ending..."; - delete trysync_thread_; - delete ping_thread_; - sleep(1); - delete binlog_receiver_thread_; - - delete logger_; - - pthread_rwlock_destroy(&state_protector_); - pthread_rwlock_destroy(&rwlock_); - - DLOG(INFO) << "PikaPort " << pthread_self() << " exit!!!"; -} - -bool PikaPort::Init() { - // DLOG(INFO) << "host: " << g_port_conf.local_ip << " port: " << g_port_conf.local_port; - return true; -} - -void PikaPort::Cleanup() { - // shutdown server - // if (g_port_conf->daemonize()) { - // unlink(g_port_conf->pidfile().c_str()); - // } - - // sender_->Stop(); - // sender_->JoinThread(); - // delete cli_; - // delete sender_; - size_t thread_num = g_port_conf.forward_thread_num; - for (size_t i = 0; i < thread_num; i++) { - senders_[i]->Stop(); - } - for (size_t i = 0; i < thread_num; i++) { - senders_[i]->JoinThread(); - } - int64_t replies = 0; - for (size_t i = 0; i < thread_num; i++) { - replies += senders_[i]->elements(); - delete senders_[i]; - } - DLOG(INFO) << "=============== Syncing =====================" << std::endl; - DLOG(INFO) << "Total replies : " << replies << " received from redis server"; - - delete this; // PikaPort is a global object - ::google::ShutdownGoogleLogging(); -} - -void PikaPort::Start() { - // start redis sender threads - size_t thread_num = g_port_conf.forward_thread_num; - for (size_t i = 0; i < thread_num; i++) { - senders_[i]->StartThread(); - } - - // sender_->StartThread(); - trysync_thread_->StartThread(); - binlog_receiver_thread_->StartThread(); - - // if (g_port_conf.filenum >= 0 && g_port_conf.filenum != UINT32_MAX && g_port_conf.offset >= 0) { - if (g_port_conf.filenum != UINT32_MAX) { - logger_->SetProducerStatus(g_port_conf.filenum, g_port_conf.offset); - } - SetMaster(master_ip_, master_port_); - - mutex_.Lock(); - mutex_.Lock(); - mutex_.Unlock(); - DLOG(INFO) << "Goodbye..."; - Cleanup(); -} - -void PikaPort::Stop() { mutex_.Unlock(); } - -int PikaPort::SendRedisCommand(std::string& command, std::string& key) { - // Send command - size_t idx = std::hash()(key) % g_port_conf.forward_thread_num; - senders_[idx]->SendRedisCommand(command); - - return 0; -} - -bool PikaPort::SetMaster(std::string& master_ip, int master_port) { - slash::RWLock l(&state_protector_, true); - if ((role_ ^ PIKA_ROLE_SLAVE) && repl_state_ == PIKA_REPL_NO_CONNECT) { - master_ip_ = master_ip; - master_port_ = master_port; - // role_ |= PIKA_ROLE_SLAVE; - role_ = PIKA_ROLE_PORT; - repl_state_ = PIKA_REPL_CONNECT; - DLOG(INFO) << "set role_ = PIKA_ROLE_PORT, repl_state_ = PIKA_REPL_CONNECT"; - return true; - } - return false; -} - -bool PikaPort::ShouldConnectMaster() { - slash::RWLock l(&state_protector_, false); - // DLOG(INFO) << "repl_state: " << PikaState(repl_state_) - // << " role: " << PikaRole(role_) - // << " master_connection: " << master_connection_; - if (repl_state_ == PIKA_REPL_CONNECT) { - return true; - } - return false; -} - -void PikaPort::ConnectMasterDone() { - slash::RWLock l(&state_protector_, true); - if (repl_state_ == PIKA_REPL_CONNECT) { - repl_state_ = PIKA_REPL_CONNECTING; - } -} - -bool PikaPort::ShouldStartPingMaster() { - slash::RWLock l(&state_protector_, false); - DLOG(INFO) << "ShouldStartPingMaster: master_connection " << master_connection_ << " repl_state " - << PikaState(repl_state_); - if (repl_state_ == PIKA_REPL_CONNECTING && master_connection_ < 2) { - return true; - } - - return false; -} - -void PikaPort::MinusMasterConnection() { - slash::RWLock l(&state_protector_, true); - if (master_connection_ > 0) { - if ((--master_connection_) <= 0) { - // two connection with master has been deleted - if ((role_ & PIKA_ROLE_SLAVE) || (role_ & PIKA_ROLE_PORT)) { - // not change by slaveof no one, so set repl_state = PIKA_REPL_CONNECT, continue to connect master - repl_state_ = PIKA_REPL_CONNECT; - } else { - // change by slaveof no one, so set repl_state = PIKA_REPL_NO_CONNECT, reset to SINGLE state - repl_state_ = PIKA_REPL_NO_CONNECT; - } - master_connection_ = 0; - } - } -} - -void PikaPort::PlusMasterConnection() { - slash::RWLock l(&state_protector_, true); - if (master_connection_ < 2) { - if ((++master_connection_) >= 2) { - // two connection with master has been established - repl_state_ = PIKA_REPL_CONNECTED; - LOG(INFO) << "Start Sync..."; - master_connection_ = 2; - } - } -} - -bool PikaPort::ShouldAccessConnAsMaster(const std::string& ip) { - slash::RWLock l(&state_protector_, false); - DLOG(INFO) << "ShouldAccessConnAsMaster, repl_state_: " << PikaState(repl_state_) << " ip: " << ip - << " master_ip: " << master_ip_; - if (repl_state_ != PIKA_REPL_NO_CONNECT && ip == master_ip_) { - return true; - } - return false; -} - -void PikaPort::RemoveMaster() { - { - slash::RWLock l(&state_protector_, true); - repl_state_ = PIKA_REPL_NO_CONNECT; - role_ &= ~PIKA_ROLE_SLAVE; - master_ip_ = ""; - master_port_ = -1; - } - if (ping_thread_ != NULL) { - int err = ping_thread_->StopThread(); - if (err != 0) { - std::string msg = "can't join thread " + std::string(strerror(err)); - LOG(WARNING) << msg; - } - delete ping_thread_; - ping_thread_ = NULL; - } -} - -bool PikaPort::IsWaitingDBSync() { - slash::RWLock l(&state_protector_, false); - // DLOG(INFO) << "repl_state: " << PikaState(repl_state_) - // << " role: " << PikaRole(role_) << " master_connection: " << master_connection_; - if (repl_state_ == PIKA_REPL_WAIT_DBSYNC) { - return true; - } - return false; -} - -void PikaPort::NeedWaitDBSync() { - slash::RWLock l(&state_protector_, true); - repl_state_ = PIKA_REPL_WAIT_DBSYNC; -} - -void PikaPort::WaitDBSyncFinish() { - slash::RWLock l(&state_protector_, true); - if (repl_state_ == PIKA_REPL_WAIT_DBSYNC) { - repl_state_ = PIKA_REPL_CONNECT; - } -} diff --git a/pika-tools/pika-port/pika_port_2/pika_port.h b/pika-tools/pika-port/pika_port_2/pika_port.h deleted file mode 100644 index 3494394133..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika_port.h +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef BINLOG_SYNC_H_ -#define BINLOG_SYNC_H_ - -#include "binlog_receiver_thread.h" -#include "include/pika_binlog.h" -#include "include/pika_define.h" -#include "redis_sender.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" -#include "slaveping_thread.h" -#include "trysync_thread.h" - -#include - -using slash::Slice; -using slash::Status; - -class PikaPort { - public: - PikaPort(std::string& master_ip, int master_port, std::string& passwd); - ~PikaPort(); - - /* - * Get & Set - */ - std::string& master_ip() { return master_ip_; } - int master_port() { return master_port_; } - - int64_t sid() { return sid_; } - - void SetSid(int64_t sid) { sid_ = sid; } - - int role() { - slash::RWLock(&state_protector_, false); - return role_; - } - int repl_state() { - slash::RWLock(&state_protector_, false); - return repl_state_; - } - std::string requirepass() { return requirepass_; } - pthread_rwlock_t* rwlock() { return &rwlock_; } - BinlogReceiverThread* binlog_receiver_thread() { return binlog_receiver_thread_; } - TrysyncThread* trysync_thread() { return trysync_thread_; } - Binlog* logger() { return logger_; } - - int SendRedisCommand(std::string& command, std::string& key); - - bool SetMaster(std::string& master_ip, int master_port); - bool ShouldConnectMaster(); - void ConnectMasterDone(); - bool ShouldStartPingMaster(); - void MinusMasterConnection(); - void PlusMasterConnection(); - bool ShouldAccessConnAsMaster(const std::string& ip); - void RemoveMaster(); - bool IsWaitingDBSync(); - void NeedWaitDBSync(); - void WaitDBSyncFinish(); - - void Start(); - void Stop(); - void Cleanup(); - - bool Init(); - SlavepingThread* ping_thread_; - - private: - std::string master_ip_; - int master_port_; - int master_connection_; - int role_; - int repl_state_; - std::string requirepass_; - std::string log_path_; - std::string dump_path_; - pthread_rwlock_t rwlock_; - - slash::Mutex mutex_; // double lock to block main thread - - // redis client - // net::PinkCli *cli_; - // RedisSender *sender_; - std::vector senders_; - - bool should_exit_; - - // Master use - int64_t sid_; - - BinlogReceiverThread* binlog_receiver_thread_; - TrysyncThread* trysync_thread_; - - Binlog* logger_; - - pthread_rwlock_t state_protector_; // protect below, use for master-slave mode - - PikaPort(PikaPort& bs); - void operator=(const PikaPort& bs); - void ConnectRedis(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/pika_sender.cc b/pika-tools/pika-port/pika_port_2/pika_sender.cc deleted file mode 100644 index 2031985e6b..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika_sender.cc +++ /dev/null @@ -1,301 +0,0 @@ -#include "pika_sender.h" - -#include "include/pika_slot.h" - -PikaSender::PikaSender(nemo::Nemo* db, std::string ip, int64_t port, std::string password) - : cli_(NULL), - rsignal_(&keys_mutex_), - wsignal_(&keys_mutex_), - db_(db), - ip_(ip), - port_(port), - password_(password), - should_exit_(false), - elements_(0) {} - -PikaSender::~PikaSender() {} - -void PikaSender::ConnectRedis() { - while (cli_ == NULL) { - // Connect to redis - cli_ = net::NewRedisCli(); - cli_->set_connect_timeout(1000); - slash::Status s = cli_->Connect(ip_, port_); - if (!s.ok()) { - delete cli_; - cli_ = NULL; - log_info("Can not connect to %s:%d: %s", ip_.data(), port_, s.ToString().data()); - continue; - } else { - // Connect success - log_info("Connect to %s:%d:%s", ip_.data(), port_, s.ToString().data()); - - // Authentication - if (!password_.empty()) { - net::RedisCmdArgsType argv, resp; - std::string cmd; - - argv.push_back("AUTH"); - argv.push_back(password_); - net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); - - if (s.ok()) { - s = cli_->Recv(&resp); - if (resp[0] == "OK") { - log_info("Authentic success"); - } else { - cli_->Close(); - log_warn("Invalid password"); - cli_ = NULL; - should_exit_ = true; - return; - } - } else { - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - continue; - } - } else { - // If forget to input password - net::RedisCmdArgsType argv, resp; - std::string cmd; - - argv.push_back("PING"); - net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); - - if (s.ok()) { - s = cli_->Recv(&resp); - if (s.ok()) { - if (resp[0] == "NOAUTH Authentication required.") { - cli_->Close(); - log_warn("Authentication required"); - cli_ = NULL; - should_exit_ = true; - return; - } - } else { - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - } - } - } - } - } -} - -void PikaSender::LoadKey(const std::string& key) { - keys_mutex_.Lock(); - if (keys_queue_.size() < 100000) { - keys_queue_.push(key); - rsignal_.Signal(); - keys_mutex_.Unlock(); - } else { - while (keys_queue_.size() > 100000) { - wsignal_.Wait(); - } - keys_queue_.push(key); - rsignal_.Signal(); - keys_mutex_.Unlock(); - } -} - -void PikaSender::SendCommand(std::string& command, const std::string& key) { - // Send command - slash::Status s = cli_->Send(&command); - if (!s.ok()) { - elements_--; - LoadKey(key); - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - ConnectRedis(); - } -} - -void* PikaSender::ThreadMain() { - log_info("Start sender thread..."); - expire_command_.clear(); - int cnt = 0; - - while (!should_exit_ || QueueSize() != 0 || !expire_command_.empty()) { - std::string command; - // Resend expire command - if (!expire_command_.empty() && cli_ != NULL) { - slash::Status s = cli_->Send(&expire_command_); - std::cout << expire_command_ << std::endl; - if (!s.ok()) { - cli_->Close(); - cli_ = NULL; - log_info("%s", s.ToString().data()); - ConnectRedis(); - } else { - expire_command_.clear(); - } - } - - if (expire_command_.empty()) { - keys_mutex_.Lock(); - while (keys_queue_.size() == 0 && !should_exit_) { - rsignal_.Wait(); - } - keys_mutex_.Unlock(); - if (QueueSize() == 0 && should_exit_) { - return NULL; - } - } - - if (cli_ == NULL) { - ConnectRedis(); - if (QueueSize() == 0 && should_exit_) { - return NULL; - } - } else { - if (QueueSize() == 0) { - continue; - } - // Parse keys - std::string key; - - keys_mutex_.Lock(); - key = keys_queue_.front(); - elements_++; - keys_queue_.pop(); - wsignal_.Signal(); - keys_mutex_.Unlock(); - - char type = key[0]; - if (type == nemo::DataType::kHSize) { // Hash - std::string h_key = key.substr(1); - nemo::HIterator* iter = db_->HScan(h_key, "", "", -1, false); - for (; iter->Valid(); iter->Next()) { - net::RedisCmdArgsType argv; - - argv.push_back("HSET"); - argv.push_back(iter->key()); - argv.push_back(iter->field()); - argv.push_back(iter->value()); - - net::SerializeRedisCommand(argv, &command); - SendCommand(command, key); - cnt++; - } - delete iter; - } else if (type == nemo::DataType::kSSize) { // Set - std::string s_key = key.substr(1); - if (s_key.find(SlotKeyPrefix) != std::string::npos) { - continue; - } - - nemo::SIterator* iter = db_->SScan(s_key, -1, false); - for (; iter->Valid(); iter->Next()) { - net::RedisCmdArgsType argv; - - argv.push_back("SADD"); - argv.push_back(iter->key()); - argv.push_back(iter->member()); - - net::SerializeRedisCommand(argv, &command); - SendCommand(command, key); - cnt++; - } - delete iter; - } else if (type == nemo::DataType::kLMeta) { // List - std::string l_key = key.substr(1); - std::vector ivs; - std::vector::const_iterator it; - int64_t pos = 0; - int64_t len = 512; - - db_->LRange(l_key, pos, pos + len - 1, ivs); - - while (!ivs.empty()) { - net::RedisCmdArgsType argv; - std::string cmd; - - argv.push_back("RPUSH"); - argv.push_back(l_key); - - for (it = ivs.begin(); it != ivs.end(); ++it) { - argv.push_back(it->val); - } - net::SerializeRedisCommand(argv, &command); - SendCommand(command, key); - cnt++; - - pos += len; - ivs.clear(); - db_->LRange(l_key, pos, pos + len - 1, ivs); - } - } else if (type == nemo::DataType::kZSize) { // Zset - std::string z_key = key.substr(1); - nemo::ZIterator* iter = db_->ZScan(z_key, nemo::ZSET_SCORE_MIN, nemo::ZSET_SCORE_MAX, -1, false); - for (; iter->Valid(); iter->Next()) { - net::RedisCmdArgsType argv; - - std::string score = std::to_string(iter->score()); - - argv.push_back("ZADD"); - argv.push_back(iter->key()); - argv.push_back(score); - argv.push_back(iter->member()); - - net::SerializeRedisCommand(argv, &command); - SendCommand(command, key); - cnt++; - } - delete iter; - } else if (type == nemo::DataType::kKv) { // Kv - std::string k_key = key.substr(1); - command = k_key; - SendCommand(command, key); - cnt++; - } - - if (cnt >= 200) { - for (; cnt > 0; cnt--) { - cli_->Recv(NULL); - } - } - - continue; - - // expire command - if (type != nemo::DataType::kKv) { - int64_t ttl = -1; - std::string e_key = key.substr(1); - db_->TTL(key, &ttl); - - if (ttl >= 0) { - net::RedisCmdArgsType argv; - - argv.push_back("EXPIRE"); - argv.push_back(e_key); - argv.push_back(std::to_string(ttl)); - - net::SerializeRedisCommand(argv, &expire_command_); - slash::Status s = cli_->Send(&expire_command_); - if (!s.ok()) { - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - ConnectRedis(); - } else { - expire_command_.clear(); - } - } - } - } - } - for (; cnt > 0; cnt--) { - cli_->Recv(NULL); - } - - delete cli_; - cli_ = NULL; - log_info("PikaSender thread complete"); - return NULL; -} diff --git a/pika-tools/pika-port/pika_port_2/pika_sender.h b/pika-tools/pika-port/pika_port_2/pika_sender.h deleted file mode 100644 index cd8daef64c..0000000000 --- a/pika-tools/pika-port/pika_port_2/pika_sender.h +++ /dev/null @@ -1,51 +0,0 @@ -#ifndef PIKA_SENDER_H_ -#define PIKA_SENDER_H_ - -#include -#include -#include -#include -#include "nemo.h" -#include "net/include/bg_thread.h" -#include "net/include/net_cli.h" -#include "net/include/redis_cli.h" - -class PikaSender : public net::Thread { - public: - PikaSender(nemo::Nemo* db, std::string ip, int64_t port, std::string password); - virtual ~PikaSender(); - void LoadKey(const std::string& cmd); - void Stop() { - should_exit_ = true; - keys_mutex_.Lock(); - rsignal_.Signal(); - keys_mutex_.Unlock(); - } - int64_t elements() { return elements_; } - - void SendCommand(std::string& command, const std::string& key); - int QueueSize() { - slash::MutexLock l(&keys_mutex_); - int len = keys_queue_.size(); - return len; - } - void ConnectRedis(); - - private: - net::PinkCli* cli_; - slash::CondVar rsignal_; - slash::CondVar wsignal_; - nemo::Nemo* db_; - slash::Mutex keys_mutex_; - std::queue keys_queue_; - std::string expire_command_; - std::string ip_; - int port_; - std::string password_; - bool should_exit_; - int64_t elements_; - - virtual void* ThreadMain(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/port_conf.h b/pika-tools/pika-port/pika_port_2/port_conf.h deleted file mode 100644 index ec2e2c1271..0000000000 --- a/pika-tools/pika-port/pika_port_2/port_conf.h +++ /dev/null @@ -1,40 +0,0 @@ -#ifndef BINLOG_CONF_H_ -#define BINLOG_CONF_H_ - -#include - -class PortConf { - public: - PortConf() { - local_ip = "127.0.0.1"; - local_port = 0; - master_ip = "127.0.0.1"; - master_port = 0; - forward_ip = "127.0.0.1"; - forward_port = 0; - forward_thread_num = 1; - filenum = size_t(UINT32_MAX); // src/pika_trysync_thread.cc:48 - offset = 0; - log_path = "./log/"; - dump_path = "./rsync_dump/"; - } - - public: - size_t filenum; - size_t offset; - std::string local_ip; - int local_port; - std::string master_ip; - int master_port; - std::string forward_ip; - int forward_port; - std::string forward_passwd; - int forward_thread_num; - std::string passwd; - std::string log_path; - std::string dump_path; -}; - -extern PortConf g_port_conf; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/readme.md b/pika-tools/pika-port/pika_port_2/readme.md deleted file mode 100644 index 1e6993589e..0000000000 --- a/pika-tools/pika-port/pika_port_2/readme.md +++ /dev/null @@ -1,20 +0,0 @@ -## pika_port -**pika port is like redis-port. it copy a snapshot of pika to pika/redis/codis and then transfer delta data to them.** - -## Version list - -> V1.4 - * Bug Fix: filter out SlotKeyPrefix when sync snapshot data - -> V1.3 - * send redis data asynchronously by RedisSender in multiple thread mode - -> V1.2 - * send redis data asynchronously by RedisSender - -> V1.1 - * filter out SlotKeyPrefix - * disable ping-pong log - -> V1.0 - * Init diff --git a/pika-tools/pika-port/pika_port_2/redis_sender.cc b/pika-tools/pika-port/pika_port_2/redis_sender.cc deleted file mode 100644 index f88a4bc2c3..0000000000 --- a/pika-tools/pika-port/pika_port_2/redis_sender.cc +++ /dev/null @@ -1,202 +0,0 @@ -#include "redis_sender.h" -#include -#include -#include -// #include "binlog_log.h" - -static time_t kCheckDiff = 1; - -RedisSender::RedisSender(int id, std::string ip, int64_t port, std::string password) - : id_(id), - cli_(NULL), - rsignal_(&commands_mutex_), - wsignal_(&commands_mutex_), - ip_(ip), - port_(port), - password_(password), - should_exit_(false), - elements_(0) { - last_write_time_ = ::time(NULL); -} - -RedisSender::~RedisSender() { DLOG(INFO) << "RedisSender thread " << id_ << " exit!!!"; } - -void RedisSender::ConnectRedis() { - while (cli_ == NULL) { - // Connect to redis - cli_ = net::NewRedisCli(); - cli_->set_connect_timeout(1000); - slash::Status s = cli_->Connect(ip_, port_); - if (!s.ok()) { - delete cli_; - cli_ = NULL; - log_info("Can not connect to %s:%d: %s", ip_.data(), port_, s.ToString().data()); - continue; - } else { - // Connect success - log_info("Connect to %s:%d:%s", ip_.data(), port_, s.ToString().data()); - - // Authentication - if (!password_.empty()) { - net::RedisCmdArgsType argv, resp; - std::string cmd; - - argv.push_back("AUTH"); - argv.push_back(password_); - net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); - - if (s.ok()) { - s = cli_->Recv(&resp); - if (resp[0] == "OK") { - log_info("Authentic success"); - } else { - cli_->Close(); - log_warn("Invalid password"); - cli_ = NULL; - should_exit_ = true; - return; - } - } else { - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - continue; - } - } else { - // If forget to input password - net::RedisCmdArgsType argv, resp; - std::string cmd; - - argv.push_back("PING"); - net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); - - if (s.ok()) { - s = cli_->Recv(&resp); - if (s.ok()) { - if (resp[0] == "NOAUTH Authentication required.") { - cli_->Close(); - log_warn("Authentication required"); - cli_ = NULL; - should_exit_ = true; - return; - } - } else { - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - } - } - } - } - } -} - -void RedisSender::Stop() { - should_exit_ = true; - commands_mutex_.Lock(); - rsignal_.Signal(); - commands_mutex_.Unlock(); -} - -void RedisSender::SendRedisCommand(const std::string& command) { - commands_mutex_.Lock(); - if (commands_queue_.size() < 100000) { - commands_queue_.push(command); - rsignal_.Signal(); - commands_mutex_.Unlock(); - return; - } - - DLOG(WARNING) << id_ << " commands queue size is beyond 100000"; - while (commands_queue_.size() > 100000) { - wsignal_.Wait(); - } - commands_queue_.push(command); - rsignal_.Signal(); - commands_mutex_.Unlock(); -} - -int RedisSender::SendCommand(std::string& command) { - time_t now = ::time(NULL); - if (kCheckDiff < now - last_write_time_) { - int ret = cli_->CheckAliveness(); - if (ret < 0) { - ConnectRedis(); - } - last_write_time_ = now; - } - - // Send command - int idx = 0; - do { - slash::Status s = cli_->Send(&command); - if (s.ok()) { - return 0; - } - - DLOG(WARNING) << "RedisSender " << id_ << " fails to send redis command " << command << ", times:" << (idx + 1); - cli_->Close(); - log_info("%s", s.ToString().data()); - cli_ = NULL; - ConnectRedis(); - } while (++idx < 3); - - return -1; -} - -void* RedisSender::ThreadMain() { - DLOG(INFO) << "Start sender " << id_ << " thread..."; - // sleep(15); - int cnt = 0; - int ret = 0; - - ConnectRedis(); - - while (!should_exit_) { - commands_mutex_.Lock(); - while (commands_queue_.size() == 0 && !should_exit_) { - // rsignal_.TimedWait(100); - rsignal_.Wait(); - } - if (commands_queue_.size() == 0 && should_exit_) { - commands_mutex_.Unlock(); - break; - } - - if (commands_queue_.size() == 0) { - commands_mutex_.Unlock(); - continue; - } - commands_mutex_.Unlock(); - - // get redis command - std::string command; - commands_mutex_.Lock(); - command = commands_queue_.front(); - // printf("%d, command %s\n", id_, command.c_str()); - elements_++; - commands_queue_.pop(); - wsignal_.Signal(); - commands_mutex_.Unlock(); - ret = SendCommand(command); - if (ret == 0) { - cnt++; - } - - if (cnt >= 200) { - for (; cnt > 0; cnt--) { - cli_->Recv(NULL); - } - } - } - for (; cnt > 0; cnt--) { - cli_->Recv(NULL); - } - - delete cli_; - cli_ = NULL; - DLOG(INFO) << "RedisSender thread " << id_ << " complete"; - return NULL; -} diff --git a/pika-tools/pika-port/pika_port_2/redis_sender.h b/pika-tools/pika-port/pika_port_2/redis_sender.h deleted file mode 100644 index 410391e80c..0000000000 --- a/pika-tools/pika-port/pika_port_2/redis_sender.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef REDIS_SENDER_H_ -#define REDIS_SENDER_H_ - -#include -#include -#include -#include -#include - -#include "nemo.h" -#include "net/include/bg_thread.h" -#include "net/include/net_cli.h" -#include "net/include/redis_cli.h" - -class RedisSender : public net::Thread { - public: - RedisSender(int id, std::string ip, int64_t port, std::string password); - virtual ~RedisSender(); - void Stop(void); - int64_t elements() { return elements_; } - - void SendRedisCommand(const std::string& command); - - private: - int SendCommand(std::string& command); - void ConnectRedis(); - - private: - int id_; - net::PinkCli* cli_; - slash::CondVar rsignal_; - slash::CondVar wsignal_; - slash::Mutex commands_mutex_; - std::queue commands_queue_; - std::string ip_; - int port_; - std::string password_; - bool should_exit_; - int64_t elements_; - std::atomic last_write_time_; - - virtual void* ThreadMain(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/slaveping_thread.cc b/pika-tools/pika-port/pika_port_2/slaveping_thread.cc deleted file mode 100644 index e95f24d246..0000000000 --- a/pika-tools/pika-port/pika_port_2/slaveping_thread.cc +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include "slaveping_thread.h" -#include -#include -#include "pika_port.h" - -extern PikaPort* g_pika_port; - -Status SlavepingThread::Send() { - std::string wbuf_str; - if (!is_first_send_) { - net::SerializeRedisCommand(&wbuf_str, "ping"); // reply == pong - } else { - net::RedisCmdArgsType argv; - argv.push_back("spci"); // reply == pong - argv.push_back(std::to_string(sid_)); - net::SerializeRedisCommand(argv, &wbuf_str); - is_first_send_ = false; - DLOG(INFO) << wbuf_str; - } - - return cli_->Send(&wbuf_str); -} - -Status SlavepingThread::RecvProc() { - net::RedisCmdArgsType argv; - Status s = cli_->Recv(&argv); - if (s.ok()) { - slash::StringToLower(argv[0]); - // DLOG(INFO) << "Reply from master after ping: " << argv[0]; - if (argv[0] == "pong" || argv[0] == "ok") { - } else { - s = Status::Corruption(""); - } - } else { - DLOG(INFO) << "RecvProc, recv error: " << s.ToString(); - } - return s; -} - -void* SlavepingThread::ThreadMain() { - struct timeval last_interaction; - struct timeval now; - gettimeofday(&now, NULL); - last_interaction = now; - Status s; - int connect_retry_times = 0; - while (!should_stop() && g_pika_port->ShouldStartPingMaster()) { - if (!should_stop() && (cli_->Connect(g_pika_port->master_ip(), g_pika_port->master_port() + 2000)).ok()) { - cli_->set_send_timeout(1000); - cli_->set_recv_timeout(1000); - connect_retry_times = 0; - g_pika_port->PlusMasterConnection(); - while (true) { - if (should_stop()) { - DLOG(INFO) << "Close Slaveping Thread now"; - close(cli_->fd()); - g_pika_port->binlog_receiver_thread()->KillBinlogSender(); - break; - } - - s = Send(); - if (s.ok()) { - s = RecvProc(); - } - if (s.ok()) { - // DLOG(INFO) << "Ping master success"; - gettimeofday(&last_interaction, NULL); - } else if (s.IsTimeout()) { - DLOG(INFO) << "Slaveping timeout once"; - gettimeofday(&now, NULL); - if (now.tv_sec - last_interaction.tv_sec > 30) { - // timeout; - DLOG(INFO) << "Ping master timeout"; - close(cli_->fd()); - g_pika_port->binlog_receiver_thread()->KillBinlogSender(); - break; - } - } else { - DLOG(INFO) << "Ping master error"; - close(cli_->fd()); - g_pika_port->binlog_receiver_thread()->KillBinlogSender(); - break; - } - sleep(1); - } - g_pika_port->MinusMasterConnection(); - } else if (!should_stop()) { - DLOG(INFO) << "Slaveping, Connect timeout"; - if ((++connect_retry_times) >= 30) { - DLOG(INFO) << "Slaveping, Connect timeout 10 times, disconnect with master"; - close(cli_->fd()); - g_pika_port->binlog_receiver_thread()->KillBinlogSender(); - connect_retry_times = 0; - } - } - } - return NULL; -} diff --git a/pika-tools/pika-port/pika_port_2/slaveping_thread.h b/pika-tools/pika-port/pika_port_2/slaveping_thread.h deleted file mode 100644 index 07afb5ee0d..0000000000 --- a/pika-tools/pika-port/pika_port_2/slaveping_thread.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef SLAVEPING_THREAD_H_ -#define SLAVEPING_THREAD_H_ - -#include "net/include/net_cli.h" -#include "net/include/net_thread.h" -#include "net/include/redis_cli.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" - -using slash::Status; - -class SlavepingThread : public net::Thread { - public: - SlavepingThread(int64_t sid) : sid_(sid), is_first_send_(true) { - cli_ = net::NewRedisCli(); - cli_->set_connect_timeout(1500); - }; - virtual ~SlavepingThread() { - StopThread(); - delete cli_; - DLOG(INFO) << " Slaveping thread " << pthread_self() << " exit!!!"; - }; - - Status Send(); - Status RecvProc(); - - private: - int64_t sid_; - bool is_first_send_; - - int sockfd_; - net::PinkCli* cli_; - - virtual void* ThreadMain(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_2/sync.sh b/pika-tools/pika-port/pika_port_2/sync.sh deleted file mode 100644 index 981a91b8b9..0000000000 --- a/pika-tools/pika-port/pika_port_2/sync.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh - -if [[ -f ~/pika_port ]]; then - mv ~/pika_port bin -fi -export LD_LIBRARY_PATH=./bin - -# Usage: pika_port [-h] [-t local_ip -p local_port -i master_ip -o master_port -m forward_ip -n forward_port -x forward_thread_num -y forward_passwd] -# -f filenum -s offset -w password -r rsync_dump_path -l log_path -h -- show this help -# -t -- local host ip(OPTIONAL default: 127.0.0.1) -# -p -- local port(OPTIONAL) -# -i -- master ip(OPTIONAL default: 127.0.0.1) -# -o -- master port(REQUIRED) -# -m -- forward ip(OPTIONAL default: 127.0.0.1) -# -n -- forward port(REQUIRED) -# -x -- forward thread num(OPTIONAL default: 1) -# -y -- forward password(OPTIONAL) -# -f -- binlog filenum(OPTIONAL default: local offset) -# -s -- binlog offset(OPTIONAL default: local offset) -# -w -- password for master(OPTIONAL) -# -r -- rsync dump data path(OPTIONAL default: ./rsync_dump) -# -l -- local log path(OPTIONAL default: ./log) -# -d -- daemonize(OPTIONAL) -# example: ./pika_port -t 127.0.0.1 -p 12345 -i 127.0.0.1 -o 9221 -m 127.0.0.1 -n 6379 -x 7 -f 0 -s 0 -w abc -l ./log -r ./rsync_dump -d - -rm -rf ./rsync_dump/ -rm -rf ./sync_log/ -./bin/pika_port -t 10.33.80.155 -i 10.33.80.155 -o 13333 -m 10.33.80.155 -n 16379 -x 10 -l ./sync_log/ -r ./rsync_dump diff --git a/pika-tools/pika-port/pika_port_2/trysync_thread.cc b/pika-tools/pika-port/pika_port_2/trysync_thread.cc deleted file mode 100644 index eeda17e759..0000000000 --- a/pika-tools/pika-port/pika_port_2/trysync_thread.cc +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#include -#include -#include -#include -#include -#include - -#include "slash/include/rsync.h" -#include "slash/include/slash_status.h" -#include "slaveping_thread.h" - -#include "include/pika_define.h" - -#include "binlog_const.h" -#include "pika_port.h" -#include "port_conf.h" -#include "trysync_thread.h" - -extern PikaPort* g_pika_port; - -TrysyncThread::~TrysyncThread() { - StopThread(); - delete cli_; - DLOG(INFO) << " Trysync thread " << pthread_self() << " exit!!!"; -} - -void TrysyncThread::PrepareRsync() { - std::string db_sync_path = g_port_conf.dump_path; - slash::StopRsync(db_sync_path); - slash::CreatePath(db_sync_path); - slash::CreatePath(db_sync_path + "kv"); - slash::CreatePath(db_sync_path + "hash"); - slash::CreatePath(db_sync_path + "list"); - slash::CreatePath(db_sync_path + "set"); - slash::CreatePath(db_sync_path + "zset"); -} - -bool TrysyncThread::Send() { - net::RedisCmdArgsType argv; - std::string wbuf_str; - std::string requirepass = g_pika_port->requirepass(); - if (requirepass != "") { - argv.push_back("auth"); - argv.push_back(requirepass); - net::SerializeRedisCommand(argv, &wbuf_str); - } - - argv.clear(); - std::string tbuf_str; - argv.push_back("trysync"); - // argv.push_back(g_pika_port->host()); - // argv.push_back(std::to_string(g_pika_port->port())); - argv.push_back(g_port_conf.local_ip); - argv.push_back(std::to_string(g_port_conf.local_port)); - uint32_t filenum; - uint64_t pro_offset; - g_pika_port->logger()->GetProducerStatus(&filenum, &pro_offset); - LOG(WARNING) << "producer filenum: " << filenum << ", producer offset:" << pro_offset; - - argv.push_back(std::to_string(filenum)); - argv.push_back(std::to_string(pro_offset)); - - net::SerializeRedisCommand(argv, &tbuf_str); - - wbuf_str.append(tbuf_str); - DLOG(INFO) << "redis command: trysync " << g_port_conf.local_ip << " " << g_port_conf.local_port << " " << filenum - << " " << pro_offset; - - slash::Status s; - s = cli_->Send(&wbuf_str); - if (!s.ok()) { - LOG(WARNING) << "Connect master, Send, error: " << strerror(errno); - return false; - } - return true; -} - -// if send command {trysync slaveip slaveport 0 0}, the reply = wait. -// if send command {trysync slaveip slaveport 11 38709514}, the reply = "sid:. -// it means that slave sid is allocated by master. -bool TrysyncThread::RecvProc() { - bool should_auth = g_pika_port->requirepass() == "" ? false : true; - bool is_authed = false; - slash::Status s; - std::string reply; - - net::RedisCmdArgsType argv; - while (1) { - s = cli_->Recv(&argv); - if (!s.ok()) { - LOG(WARNING) << "Connect master, Recv, error: " << strerror(errno); - return false; - } - - reply = argv[0]; - DLOG(INFO) << "Reply from master after trysync: " << reply; - if (!is_authed && should_auth) { - if (kInnerReplOk != slash::StringToLower(reply)) { - g_pika_port->RemoveMaster(); - return false; - } - is_authed = true; - } else { - // pinfo("xxxxxx argv size %zu, reply %s", argv.size(), reply.data()); - if (argv.size() == 1 && slash::string2l(reply.data(), reply.size(), &sid_)) { - // Luckly, I got your point, the sync is comming - DLOG(INFO) << "Recv sid from master: " << sid_; - g_pika_port->SetSid(sid_); - break; - } - - // Failed - if (reply == kInnerReplWait) { - // You can't sync this time, but may be different next time, - // This may happened when - // 1, Master do bgsave first. - // 2, Master waiting for an existing bgsaving process - // 3, Master do dbsyncing - LOG(INFO) << "Need wait to sync"; - g_pika_port->NeedWaitDBSync(); - // break; - } else { - LOG(INFO) << "Sync Error, Quit"; - kill(getpid(), SIGQUIT); - g_pika_port->RemoveMaster(); - } - return false; - } - } - - return true; -} - -// Try to update master offset -// This may happend when dbsync from master finished -// Here we do: -// 1, Check dbsync finished, got the new binlog offset -// 2, Replace the old db -// 3, Update master offset, and the PikaTrysyncThread cron will connect and do slaveof task with master -bool TrysyncThread::TryUpdateMasterOffset() { - // Check dbsync finished - std::string db_sync_path = g_port_conf.dump_path; - std::string info_path = db_sync_path + kBgsaveInfoFile; - if (!slash::FileExists(info_path)) { - return false; - } - - // Got new binlog offset - std::ifstream is(info_path); - if (!is) { - LOG(WARNING) << "Failed to open info file after db sync"; - return false; - } - std::string line, master_ip; - int lineno = 0; - int64_t filenum = 0, offset = 0, tmp = 0, master_port = 0; - while (std::getline(is, line)) { - lineno++; - if (lineno == 2) { - master_ip = line; - } else if (lineno > 2 && lineno < 6) { - if (!slash::string2l(line.data(), line.size(), &tmp) || tmp < 0) { - LOG(WARNING) << "Format of info file after db sync error, line : " << line; - is.close(); - return false; - } - if (lineno == 3) { - master_port = tmp; - } else if (lineno == 4) { - filenum = tmp; - } else { - offset = tmp; - } - } else if (lineno > 5) { - LOG(WARNING) << "Format of info file after db sync error, line : " << line; - is.close(); - return false; - } - } - is.close(); - LOG(INFO) << "Information from dbsync info. master_ip: " << master_ip << ", master_port: " << master_port - << ", filenum: " << filenum << ", offset: " << offset; - - // Sanity check - if (master_ip != g_port_conf.master_ip || master_port != g_port_conf.master_port) { - LOG(WARNING) << "Error master ip port: " << master_ip << ":" << master_port; - return false; - } - - // Replace the old db - slash::StopRsync(db_sync_path); - slash::DeleteFile(info_path); - - // Update master offset - g_pika_port->logger()->SetProducerStatus(filenum, offset); - Retransmit(); - g_pika_port->WaitDBSyncFinish(); - // g_pika_server->SetForceFullSync(false); - - return true; -} - -#include -#include -#include -#include - -#include "migrator_thread.h" -#include "nemo.h" -#include "pika_sender.h" - -using std::chrono::high_resolution_clock; -using std::chrono::milliseconds; - -int TrysyncThread::Retransmit() { - std::string db_path = g_port_conf.dump_path; - std::string ip = g_port_conf.forward_ip; - int port = g_port_conf.forward_port; - size_t thread_num = g_port_conf.forward_thread_num; - std::string password = g_port_conf.forward_passwd; - - std::vector senders; - std::vector> migrators; - std::unique_ptr db; - - high_resolution_clock::time_point start = high_resolution_clock::now(); - if (db_path[db_path.length() - 1] != '/') { - db_path.append("/"); - } - - // Init db - nemo::Options option; - option.write_buffer_size = 512 * 1024 * 1024; // 512M - option.target_file_size_base = 40 * 1024 * 1024; // 40M - db = std::unique_ptr(new nemo::Nemo(db_path, option)); - - // Init SenderThread - for (size_t i = 0; i < thread_num; i++) { - senders.emplace_back(new PikaSender(db.get(), ip, port, password)); - } - - migrators.emplace_back(new MigratorThread(db.get(), &senders, nemo::DataType::kKv, thread_num)); - migrators.emplace_back(new MigratorThread(db.get(), &senders, nemo::DataType::kHSize, thread_num)); - migrators.emplace_back(new MigratorThread(db.get(), &senders, nemo::DataType::kSSize, thread_num)); - migrators.emplace_back(new MigratorThread(db.get(), &senders, nemo::DataType::kLMeta, thread_num)); - migrators.emplace_back(new MigratorThread(db.get(), &senders, nemo::DataType::kZSize, thread_num)); - - // start threads - for (size_t i = 0; i < kDataSetNum; i++) { - migrators[i]->StartThread(); - } - for (size_t i = 0; i < thread_num; i++) { - senders[i]->StartThread(); - } - - for (size_t i = 0; i < kDataSetNum; i++) { - migrators[i]->JoinThread(); - } - for (size_t i = 0; i < thread_num; i++) { - senders[i]->Stop(); - } - for (size_t i = 0; i < thread_num; i++) { - senders[i]->JoinThread(); - } - - int64_t replies = 0, records = 0; - // for (size_t i = 0; i < kDataSetNum; i++) { - // records += migrators[i]->num(); - // delete migrators[i]; - // } - for (size_t i = 0; i < thread_num; i++) { - replies += senders[i]->elements(); - delete senders[i]; - } - - high_resolution_clock::time_point end = high_resolution_clock::now(); - std::chrono::hours h = std::chrono::duration_cast(end - start); - std::chrono::minutes m = std::chrono::duration_cast(end - start); - std::chrono::seconds s = std::chrono::duration_cast(end - start); - - DLOG(INFO) << "=============== Retransmitting =====================" << std::endl; - DLOG(INFO) << "Running time :"; - DLOG(INFO) << h.count() << " hour " << m.count() - h.count() * 60 << " min " << s.count() - h.count() * 60 * 60 - << " s"; - DLOG(INFO) << "Total records : " << records << " have been Scaned"; - DLOG(INFO) << "Total replies : " << replies << " received from redis server"; - // delete db - - return 0; -} - -void* TrysyncThread::ThreadMain() { - while (!should_stop()) { - sleep(1); - - if (g_pika_port->IsWaitingDBSync()) { - LOG(INFO) << "Waiting db sync"; - // Try to update offset by db sync - if (TryUpdateMasterOffset()) { - LOG(INFO) << "Success Update Master Offset"; - } - } - - if (!g_pika_port->ShouldConnectMaster()) { - continue; - } - sleep(2); - DLOG(INFO) << "Should connect master"; - - std::string master_ip = g_port_conf.master_ip; - int master_port = g_port_conf.master_port; - std::string dbsync_path = g_port_conf.dump_path; - - // Start rsync service - PrepareRsync(); - std::string ip_port = slash::IpPortString(g_port_conf.master_ip, g_port_conf.master_port); - int ret = slash::StartRsync(dbsync_path, kDBSyncModule + "_" + ip_port, g_port_conf.local_ip, - g_port_conf.local_port + 3000); - if (0 != ret) { - LOG(WARNING) << "Failed to start rsync, path:" << dbsync_path << " error : " << ret; - return false; - } - LOG(INFO) << "Finish to start rsync, path:" << dbsync_path; - - if ((cli_->Connect(master_ip, master_port, g_port_conf.local_ip)).ok()) { - LOG(INFO) << "Connect to master{ip:" << master_ip << ", port: " << master_port << "}"; - cli_->set_send_timeout(5000); - cli_->set_recv_timeout(5000); - if (Send() && RecvProc()) { - g_pika_port->ConnectMasterDone(); - // Stop rsync, binlog sync with master is begin - slash::StopRsync(dbsync_path); - - delete g_pika_port->ping_thread_; - g_pika_port->ping_thread_ = new SlavepingThread(sid_); - g_pika_port->ping_thread_->StartThread(); - DLOG(INFO) << "Trysync success"; - } - cli_->Close(); - } else { - LOG(WARNING) << "Failed to connect to master, " << master_ip << ":" << master_port; - } - } - return NULL; -} diff --git a/pika-tools/pika-port/pika_port_2/trysync_thread.h b/pika-tools/pika-port/pika_port_2/trysync_thread.h deleted file mode 100644 index 4492bec870..0000000000 --- a/pika-tools/pika-port/pika_port_2/trysync_thread.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. - -#ifndef TRYSYNC_THREAD_H_ -#define TRYSYNC_THREAD_H_ - -#include "net/include/net_thread.h" -#include "net/include/redis_cli.h" - -class TrysyncThread : public net::Thread { - public: - TrysyncThread() { - cli_ = net::NewRedisCli(); - cli_->set_connect_timeout(1500); - }; - virtual ~TrysyncThread(); - - private: - int sockfd_; - int64_t sid_; - net::PinkCli* cli_; - - bool Send(); - bool RecvProc(); - void PrepareRsync(); - bool TryUpdateMasterOffset(); - int Retransmit(); - - virtual void* ThreadMain(); -}; - -#endif diff --git a/pika-tools/pika-port/pika_port_3/Makefile b/pika-tools/pika-port/pika_port_3/Makefile deleted file mode 100644 index a4f0c42ab9..0000000000 --- a/pika-tools/pika-port/pika_port_3/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -CXX = g++ - -CXXFLAGS = -O2 -pipe -fPIC -W -DNDEBUG -Wwrite-strings -Wpointer-arith \ - -Wreorder -Wswitch -Wsign-promo -Wredundant-decls -Wformat -Wall -Wno-unused-parameter \ - -D_GNU_SOURCE -D__STDC_FORMAT_MACROS -std=c++11 -gdwarf-2 -Wno-redundant-decls - -# CXXFLAGS = -O0 -g -pg -pipe -fPIC -W -DDEBUG -Wwrite-strings -Wpointer-arith -Wreorder -Wswitch \ -# -Wsign-promo -Wredundant-decls -Wformat -Wall -Wno-unused-parameter -D_GNU_SOURCE \ -# -D__STDC_FORMAT_MACROS -std=c++11 -Wno-redundant-decls - -OBJECT = pika_port -SRC_DIR = . -THIRD_PATH = ../../third - -ROCKSDB_PATH=$(THIRD_PATH)/rocksdb -ROCKSDB=$(ROCKSDB_PATH)/librocksdb.a - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH=$(SLASH_PATH)/slash/lib/libslash.a - -PINK_PATH=$(THIRD_PATH)/net -PINK=$(PINK_PATH)/net/lib/libnet.a - -BLACKWIDOW_PATH=$(THIRD_PATH)/blackwidow -BLACKWIDOW=$(BLACKWIDOW_PATH)/lib/libblackwidow.a - -INCLUDE_PATH = -I./ \ - -I$(THIRD_PATH)/slash/ \ - -I$(THIRD_PATH)/net \ - -I$(THIRD_PATH)/blackwidow/include/ \ - -I$(THIRD_PATH)/blackwidow/ \ - -I$(THIRD_PATH)/rocksdb/include/ - -LIB_PATH = -L./ \ - -L$(THIRD_PATH)/rocksdb/ \ - -L$(THIRD_PATH)/slash/slash/lib \ - -L$(THIRD_PATH)/net/net/lib \ - -L$(THIRD_PATH)/blackwidow/lib - -LIBS = -Wl,-Bstatic -lrocksdb -Wl,-Bdynamic -lpthread -lrt -lnet -lslash -lblackwidow -lrocksdb -lglog -lgflags -lsnappy -lz -lbz2 -llz4 -lzstd - -LIBS += $(ROCKSDB_LDFLAGS) - -.PHONY: all clean - - -BASE_OBJS := $(wildcard $(SRC_DIR)/*.cc) -BASE_OBJS += $(wildcard $(SRC_DIR)/*.c) -BASE_OBJS += $(wildcard $(SRC_DIR)/*.cpp) -OBJS = $(patsubst %.cc,%.o,$(BASE_OBJS)) - - -all: $(OBJECT) - @echo "Success, go, go, go..." - - -$(OBJECT): $(ROCKSDB) $(SLASH) $(PINK) $(BLACKWIDOW) $(OBJS) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LFLAGS) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - -$(TOBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - -$(ROCKSDB): - make -C $(ROCKSDB_PATH) -j 24 static_lib - -$(SLASH) : - make -C $(SLASH_PATH)/slash - -$(PINK) : - make -C $(PINK_PATH)/net NO_PB=1 SLASH_PATH=../../slash - -$(BLACKWIDOW) : - make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=../rocksdb SLASH_PATH=../slash - -clean: - rm -rf $(SRC_DIR)/*.o - rm -rf $(OBJECT) diff --git a/pika-tools/pika_to_txt/Makefile b/pika-tools/pika_to_txt/Makefile deleted file mode 100644 index 103b7f616d..0000000000 --- a/pika-tools/pika_to_txt/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -CXX = g++ - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -TARGET = pika_to_txt - -THIRD_PATH=../third -SRC_PATH=. - -ROCKSDB_PATH=$(THIRD_PATH)/rocksdb -ROCKSDB=$(ROCKSDB_PATH)/librocksdb.a - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH=$(SLASH_PATH)/slash/lib/libslash.a - -PINK_PATH=$(THIRD_PATH)/net -PINK=$(PINK_PATH)/net/lib/libnet.a - -BLACKWIDOW_PATH=$(THIRD_PATH)/blackwidow -BLACKWIDOW=$(BLACKWIDOW_PATH)/lib/libblackwidow.a - -INCLUDE_PATH = -I./ \ - -I$(BLACKWIDOW_PATH)/include \ - -I$(SLASH_PATH) \ - -I$(PINK_PATH) \ - -I$(ROCKSDB_PATH)/include \ - -LIB_PATH = -L$(BLACKWIDOW_PATH)/lib \ - -L$(PINK_PATH)/net/lib \ - -L$(SLASH_PATH)/slash/lib \ - -L$(ROCKSDB_PATH)/ \ - -LIBS = -lblackwidow \ - -lnet \ - -lslash \ - -lrocksdb \ - -lpthread \ - -lsnappy \ - -lz \ - -lbz2 \ - -lglog \ - -lgflags \ - -lrt \ - -SOURCE := $(wildcard $(SRC_PATH)/*.cc) -OBJS := $(patsubst %.cc, %.o, $(SOURCE)) - -$(TARGET): $(OBJS) $(BLACKWIDOW) $(PINK) $(SLASH) $(ROCKSDB) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - - -$(BLACKWIDOW) : - make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=../rocksdb SLASH_PATH=../slash - -$(PINK) : - make -C $(PINK_PATH)/net SLASH_PATH=../../slash - -$(SLASH) : - make -C $(SLASH_PATH)/slash - -$(ROCKSDB): - make -C $(ROCKSDB_PATH) -j 24 static_lib - - -.PHONY: clean distclean - -clean: - rm -rf $(OBJS) - rm -rf $(TARGET) - -distclean: - rm -rf $(OBJS) - rm -rf $(TARGET) - make -C $(BLACKWIDOW_PATH) clean - make -C $(PINK_PATH)/net clean - make -C $(SLASH_PATH)/slash clean - make -C $(ROCKSDB_PATH) clean diff --git a/pika-tools/rdb_to_pika/Makefile b/pika-tools/rdb_to_pika/Makefile deleted file mode 100644 index 4d3d3275a3..0000000000 --- a/pika-tools/rdb_to_pika/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -CXX=g++ - -CXXFLAGS = -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++11 - - -SRC_DIR=./ - -OUT_PUT=./ - -TARGET=redisConn - -THIRD=./../third - -HIREDIS_PATH=$(THIRD)/hiredis-vip -LIB_HIREDIS=$(HIREDIS_PATH)/libhiredis_vip.a - -INCLUDE_PATH=-I./ \ - -I$(HIREDIS_PATH) - -BASE_OBJ := $(wildcard $(SRC_DIR)/*.cc) -BASE_OBJ += $(wildcard $(SRC_DIR)/*.cpp) -BASE_OBJ += $(wildcard $(SRC_DIR)/*.c) -OBJ=$(patsubst %.cc, %.o, $(BASE_OBJ)) - -all:$(TARGET) - echo "successful" - -$(TARGET) : $(OBJ) $(LIB_HIREDIS) - $(CXX) $(OBJ) $(LIB_HIREDIS) -o $(TARGET) - -$(LIB_HIREDIS): - make -C $(HIREDIS_PATH) - -$(OBJ): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) -.PHONEY:clean -clean: - rm -rf $(TARGET) *.o diff --git a/pika-tools/rdb_to_pika/install_rdb_tools.sh b/pika-tools/rdb_to_pika/install_rdb_tools.sh deleted file mode 100755 index 1f0eab50c9..0000000000 --- a/pika-tools/rdb_to_pika/install_rdb_tools.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -#author justforfun -cd ../third/redis-rdb-tools -sudo python setup.py install -cd ../../rdb_to_pika - - diff --git a/pika-tools/txt_to_pika/Makefile b/pika-tools/txt_to_pika/Makefile deleted file mode 100644 index 6cc0b5a75d..0000000000 --- a/pika-tools/txt_to_pika/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -CXX = g++ - -WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ - -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ - -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ - -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers - -CXXFLAGS = $(WARNING_FLAGS) -std=c++11 -g - -TARGET = txt_to_pika - -THIRD_PATH=../third -SRC_PATH=. - -ROCKSDB_PATH=$(THIRD_PATH)/rocksdb -ROCKSDB=$(ROCKSDB_PATH)/librocksdb.a - -SLASH_PATH=$(THIRD_PATH)/slash -SLASH=$(SLASH_PATH)/slash/lib/libslash.a - -PINK_PATH=$(THIRD_PATH)/net -PINK=$(PINK_PATH)/net/lib/libnet.a - -BLACKWIDOW_PATH=$(THIRD_PATH)/blackwidow -BLACKWIDOW=$(BLACKWIDOW_PATH)/lib/libblackwidow.a - -INCLUDE_PATH = -I./ \ - -I$(BLACKWIDOW_PATH)/include \ - -I$(SLASH_PATH) \ - -I$(PINK_PATH) \ - -I$(ROCKSDB_PATH)/include \ - -LIB_PATH = -L$(BLACKWIDOW_PATH)/lib \ - -L$(PINK_PATH)/net/lib \ - -L$(SLASH_PATH)/slash/lib \ - -L$(ROCKSDB_PATH)/ \ - -LIBS = -lblackwidow \ - -lnet \ - -lslash \ - -lrocksdb \ - -lpthread \ - -lsnappy \ - -lz \ - -lbz2 \ - -lglog \ - -lgflags \ - -lrt \ - -SOURCE := $(wildcard $(SRC_PATH)/*.cc) -OBJS := $(patsubst %.cc, %.o, $(SOURCE)) - -$(TARGET): $(OBJS) $(BLACKWIDOW) $(PINK) $(SLASH) $(ROCKSDB) - $(CXX) $(CXXFLAGS) -o $@ $(OBJS) $(INCLUDE_PATH) $(LIB_PATH) $(LIBS) - -$(OBJS): %.o : %.cc - $(CXX) $(CXXFLAGS) -c $< -o $@ $(INCLUDE_PATH) - - -$(BLACKWIDOW) : - make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=../rocksdb SLASH_PATH=../slash - -$(PINK) : - make -C $(PINK_PATH)/net SLASH_PATH=../../slash - -$(SLASH) : - make -C $(SLASH_PATH)/slash - -$(ROCKSDB): - make -C $(ROCKSDB_PATH) -j 24 static_lib - - -.PHONY: clean distclean - -clean: - rm -rf $(OBJS) - rm -rf $(TARGET) - -distclean: - rm -rf $(OBJS) - rm -rf $(TARGET) - make -C $(BLACKWIDOW_PATH) clean - make -C $(PINK_PATH)/net clean - make -C $(SLASH_PATH)/slash clean - make -C $(ROCKSDB_PATH) clean diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..4c90e9745c --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,8 @@ +add_subdirectory(./aof_to_pika) +add_subdirectory(./benchmark_client) +add_subdirectory(./binlog_sender) +add_subdirectory(./manifest_generator) +add_subdirectory(./rdb_to_pika) +add_subdirectory(./pika_to_txt) +add_subdirectory(./txt_to_pika) +add_subdirectory(./pika-port/pika_port_3) \ No newline at end of file diff --git a/pika-tools/aof_to_pika/CMakeLists.txt b/tools/aof_to_pika/CMakeLists.txt similarity index 88% rename from pika-tools/aof_to_pika/CMakeLists.txt rename to tools/aof_to_pika/CMakeLists.txt index 7b024e5b67..b3c9c9d4b1 100644 --- a/pika-tools/aof_to_pika/CMakeLists.txt +++ b/tools/aof_to_pika/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories(aof_to_pika PRIVATE ./include PRIVATE ../..) target_link_libraries(aof_to_pika pthread) set_target_properties(aof_to_pika PROPERTIES - RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tools + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} CMAKE_COMPILER_IS_GNUCXX TRUE COMPILE_FLAGS ${CXXFLAGS}) diff --git a/pika-tools/aof_to_pika/README.md b/tools/aof_to_pika/README.md similarity index 87% rename from pika-tools/aof_to_pika/README.md rename to tools/aof_to_pika/README.md index d1f735fc22..88a6cefc00 100644 --- a/pika-tools/aof_to_pika/README.md +++ b/tools/aof_to_pika/README.md @@ -13,5 +13,5 @@ A tool to transfer data for redis from one to another, it also support any redis ``` shell ../../build.sh -cd ../../output/tools && ./aof_to_pika -h # for more information +cd ../../output && ./aof_to_pika -h # for more information ``` diff --git a/pika-tools/aof_to_pika/include/aof_info.h b/tools/aof_to_pika/include/aof_info.h similarity index 100% rename from pika-tools/aof_to_pika/include/aof_info.h rename to tools/aof_to_pika/include/aof_info.h diff --git a/pika-tools/aof_to_pika/include/aof_lock.h b/tools/aof_to_pika/include/aof_lock.h similarity index 100% rename from pika-tools/aof_to_pika/include/aof_lock.h rename to tools/aof_to_pika/include/aof_lock.h diff --git a/pika-tools/aof_to_pika/include/aof_sender.h b/tools/aof_to_pika/include/aof_sender.h similarity index 100% rename from pika-tools/aof_to_pika/include/aof_sender.h rename to tools/aof_to_pika/include/aof_sender.h diff --git a/pika-tools/aof_to_pika/src/aof_info.cc b/tools/aof_to_pika/src/aof_info.cc similarity index 100% rename from pika-tools/aof_to_pika/src/aof_info.cc rename to tools/aof_to_pika/src/aof_info.cc diff --git a/pika-tools/aof_to_pika/src/aof_lock.cc b/tools/aof_to_pika/src/aof_lock.cc similarity index 100% rename from pika-tools/aof_to_pika/src/aof_lock.cc rename to tools/aof_to_pika/src/aof_lock.cc diff --git a/pika-tools/aof_to_pika/src/aof_reader.cc b/tools/aof_to_pika/src/aof_reader.cc similarity index 100% rename from pika-tools/aof_to_pika/src/aof_reader.cc rename to tools/aof_to_pika/src/aof_reader.cc diff --git a/pika-tools/aof_to_pika/src/aof_sender.cc b/tools/aof_to_pika/src/aof_sender.cc similarity index 100% rename from pika-tools/aof_to_pika/src/aof_sender.cc rename to tools/aof_to_pika/src/aof_sender.cc diff --git a/tools/benchmark_client/CMakeLists.txt b/tools/benchmark_client/CMakeLists.txt new file mode 100644 index 0000000000..15aac2abc2 --- /dev/null +++ b/tools/benchmark_client/CMakeLists.txt @@ -0,0 +1,22 @@ +set(WARNING_FLAGS "-W -Wextra -Wall -Wsign-compare \ +-Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ +-Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ +-Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers") + +set(CXXFLAGS "$(WARNING_FLAGS) -std=c++17 -g") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(benchmark_client ${BASE_OBJS}) + +target_include_directories(benchmark_client PRIVATE ${INSTALL_INCLUDEDIR} PRIVATE ${PROJECT_SOURCE_DIR}) + +target_link_libraries(benchmark_client pthread) +target_link_libraries(benchmark_client ${SNAPPY_LIBRARY}) +target_link_libraries(benchmark_client pstd) +target_link_libraries(benchmark_client ${HIREDIS_LIBRARY}) +set_target_properties(benchmark_client PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/benchmark_client/benchmark_client.cc b/tools/benchmark_client/benchmark_client.cc similarity index 98% rename from pika-tools/benchmark_client/benchmark_client.cc rename to tools/benchmark_client/benchmark_client.cc index b0c26fbbaa..84dba996c0 100644 --- a/pika-tools/benchmark_client/benchmark_client.cc +++ b/tools/benchmark_client/benchmark_client.cc @@ -10,13 +10,13 @@ #include #include -#include "hiredis.h" -#include "slash/include/slash_status.h" -#include "slash/include/slash_string.h" +#include "hiredis-vip/hiredis.h" +#include "pstd/include/pstd_status.h" +#include "pstd/include/pstd_string.h" #define TIME_OF_LOOP 1000000 -using slash::Status; +using pstd::Status; using std::default_random_engine; Status RunSetCommand(redisContext* c); @@ -321,7 +321,7 @@ int main(int argc, char* argv[]) { } } - slash::StringSplit(tables_str, ',', tables); + pstd::StringSplit(tables_str, ',', tables); if (tables.empty()) { Usage(); diff --git a/tools/binlog_sender/CMakeLists.txt b/tools/binlog_sender/CMakeLists.txt new file mode 100644 index 0000000000..5d4eafc70d --- /dev/null +++ b/tools/binlog_sender/CMakeLists.txt @@ -0,0 +1,19 @@ +set(WARNING_FLAGS "-W -Wextra -Wall -Wsign-compare \ +-Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ +-Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ +-Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers") + +set(CXXFLAGS "$(WARNING_FLAGS) -std=c++17 -g") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(binlog_sender ${BASE_OBJS}) + +target_include_directories(binlog_sender PRIVATE ${PROJECT_SOURCE_DIR}) + +target_link_libraries(binlog_sender net pstd pthread) +set_target_properties(binlog_sender PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/binlog_sender/binlog_consumer.cc b/tools/binlog_sender/binlog_consumer.cc similarity index 83% rename from pika-tools/binlog_sender/binlog_consumer.cc rename to tools/binlog_sender/binlog_consumer.cc index 3fbcfc0d5e..88680f50f8 100644 --- a/pika-tools/binlog_sender/binlog_consumer.cc +++ b/tools/binlog_sender/binlog_consumer.cc @@ -29,7 +29,7 @@ bool BinlogConsumer::Init() { std::string profile; for (size_t idx = first_filenum_; idx <= last_filenum_; ++idx) { profile = NewFileName(filename_, idx); - if (!slash::FileExists(profile)) { + if (!pstd::FileExists(profile)) { fprintf(stderr, "Binlog %s not exists\n", profile.c_str()); return false; } @@ -37,7 +37,7 @@ bool BinlogConsumer::Init() { current_filenum_ = first_filenum_; profile = NewFileName(filename_, current_filenum_); - slash::Status s = slash::NewSequentialFile(profile, &queue_); + pstd::Status s = pstd::NewSequentialFile(profile, &queue_); if (!s.ok()) { return false; } else { @@ -46,7 +46,7 @@ bool BinlogConsumer::Init() { } bool BinlogConsumer::trim() { - slash::Status s; + pstd::Status s; uint64_t start_block = (current_offset_ / kBlockSize) * kBlockSize; s = queue_->Skip((current_offset_ / kBlockSize) * kBlockSize); if (!s.ok()) { @@ -79,7 +79,7 @@ uint64_t BinlogConsumer::current_offset() { return current_offset_; } uint64_t BinlogConsumer::get_next(bool* is_error) { uint64_t offset = 0; - slash::Status s; + pstd::Status s; *is_error = false; while (true) { @@ -119,8 +119,8 @@ uint64_t BinlogConsumer::get_next(bool* is_error) { return offset; } -uint32_t BinlogConsumer::ReadPhysicalRecord(slash::Slice* result) { - slash::Status s; +uint32_t BinlogConsumer::ReadPhysicalRecord(pstd::Slice* result) { + pstd::Status s; if (kBlockSize - last_record_offset_ <= kHeaderSize) { queue_->Skip(kBlockSize - last_record_offset_); current_offset_ += (kBlockSize - last_record_offset_); @@ -147,7 +147,7 @@ uint32_t BinlogConsumer::ReadPhysicalRecord(slash::Slice* result) { buffer_.clear(); s = queue_->Read(length, &buffer_, backing_store_); - *result = slash::Slice(buffer_.data(), buffer_.size()); + *result = pstd::Slice(buffer_.data(), buffer_.size()); last_record_offset_ += kHeaderSize + length; if (s.ok()) { current_offset_ += (kHeaderSize + length); @@ -155,73 +155,73 @@ uint32_t BinlogConsumer::ReadPhysicalRecord(slash::Slice* result) { return type; } -slash::Status BinlogConsumer::Consume(std::string* scratch) { - slash::Status s; +pstd::Status BinlogConsumer::Consume(std::string* scratch) { + pstd::Status s; - slash::Slice fragment; + pstd::Slice fragment; while (true) { const uint32_t record_type = ReadPhysicalRecord(&fragment); switch (record_type) { case kFullType: *scratch = std::string(fragment.data(), fragment.size()); - s = slash::Status::OK(); + s = pstd::Status::OK(); break; case kFirstType: scratch->assign(fragment.data(), fragment.size()); - s = slash::Status::NotFound("Middle Status"); + s = pstd::Status::NotFound("Middle Status"); break; case kMiddleType: scratch->append(fragment.data(), fragment.size()); - s = slash::Status::NotFound("Middle Status"); + s = pstd::Status::NotFound("Middle Status"); break; case kLastType: scratch->append(fragment.data(), fragment.size()); - s = slash::Status::OK(); + s = pstd::Status::OK(); break; case kEof: - return slash::Status::EndFile("Eof"); + return pstd::Status::EndFile("Eof"); case kBadRecord: - return slash::Status::IOError("Data Corruption"); + return pstd::Status::IOError("Data Corruption"); case kOldRecord: - return slash::Status::EndFile("Eof"); + return pstd::Status::EndFile("Eof"); default: - return slash::Status::IOError("Unknow reason"); + return pstd::Status::IOError("Unknow reason"); } if (s.ok()) { break; } } - return slash::Status::OK(); + return pstd::Status::OK(); } // Get a whole message; // the status will be OK, IOError or Corruption; -slash::Status BinlogConsumer::Parse(std::string* scratch) { - slash::Status s; +pstd::Status BinlogConsumer::Parse(std::string* scratch) { + pstd::Status s; scratch->clear(); while (true) { s = Consume(scratch); if (s.IsEndFile()) { if (current_filenum_ == last_filenum_) { - return slash::Status::Complete("finish"); + return pstd::Status::Complete("finish"); } else { std::string confile = NewFileName(filename_, current_filenum_ + 1); // Roll to next File - if (slash::FileExists(confile)) { + if (pstd::FileExists(confile)) { // DLOG(INFO) << "BinlogSender roll to new binlog" << confile; delete queue_; queue_ = NULL; - slash::NewSequentialFile(confile, &queue_); + pstd::NewSequentialFile(confile, &queue_); current_filenum_++; current_offset_ = 0; last_record_offset_ = 0; } else { - return slash::Status::NotFound("not found"); + return pstd::Status::NotFound("not found"); } } } else { diff --git a/pika-tools/binlog_sender/binlog_consumer.h b/tools/binlog_sender/binlog_consumer.h similarity index 80% rename from pika-tools/binlog_sender/binlog_consumer.h rename to tools/binlog_sender/binlog_consumer.h index 748be06fde..992cd1b9e1 100644 --- a/pika-tools/binlog_sender/binlog_consumer.h +++ b/tools/binlog_sender/binlog_consumer.h @@ -9,10 +9,10 @@ #include "stddef.h" #include "stdint.h" -#include "slash/include/env.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_slice.h" -#include "slash/include/slash_status.h" +#include "pstd/include/env.h" +#include "pstd/include/pstd_mutex.h" +#include "pstd/include/pstd_slice.h" +#include "pstd/include/pstd_status.h" enum RecordType { kZeroType = 0, @@ -42,12 +42,12 @@ class BinlogConsumer { bool trim(); uint32_t current_filenum(); uint64_t current_offset(); - slash::Status Parse(std::string* scratch); + pstd::Status Parse(std::string* scratch); private: uint64_t get_next(bool* is_error); - uint32_t ReadPhysicalRecord(slash::Slice* result); - slash::Status Consume(std::string* scratch); + uint32_t ReadPhysicalRecord(pstd::Slice* result); + pstd::Status Consume(std::string* scratch); std::string filename_; uint32_t first_filenum_; uint32_t last_filenum_; @@ -56,9 +56,9 @@ class BinlogConsumer { uint64_t current_offset_; uint64_t last_record_offset_; - slash::Slice buffer_; + pstd::Slice buffer_; char* const backing_store_; - slash::SequentialFile* queue_; + pstd::SequentialFile* queue_; }; #endif // INCLUDE_BINLOG_Consumber_H_ diff --git a/pika-tools/binlog_sender/binlog_sender.cc b/tools/binlog_sender/binlog_sender.cc similarity index 99% rename from pika-tools/binlog_sender/binlog_sender.cc rename to tools/binlog_sender/binlog_sender.cc index e1a60a8cc6..347326f3a0 100644 --- a/pika-tools/binlog_sender/binlog_sender.cc +++ b/tools/binlog_sender/binlog_sender.cc @@ -30,7 +30,7 @@ std::string end_time_str = "2100-01-01 00:00:01"; bool need_auth = false; std::string pass_wd; -net::PinkCli* cli = nullptr; +net::NetCli* cli = nullptr; BinlogConsumer* binlog_consumer = nullptr; ProgressThread* progress_thread = nullptr; @@ -201,7 +201,7 @@ int main(int argc, char* argv[]) { BinlogItem binlog_item; while (true) { std::string scratch; - slash::Status s = binlog_consumer->Parse(&scratch); + pstd::Status s = binlog_consumer->Parse(&scratch); if (s.ok()) { if (PikaBinlogTransverter::BinlogDecode(TypeFirst, scratch, &binlog_item)) { std::string redis_cmd = binlog_item.content(); diff --git a/pika-tools/binlog_sender/binlog_transverter.cc b/tools/binlog_sender/binlog_transverter.cc similarity index 79% rename from pika-tools/binlog_sender/binlog_transverter.cc rename to tools/binlog_sender/binlog_transverter.cc index f5c7d8779e..dae9dc1120 100644 --- a/pika-tools/binlog_sender/binlog_transverter.cc +++ b/tools/binlog_sender/binlog_transverter.cc @@ -52,14 +52,14 @@ std::string PikaBinlogTransverter::BinlogEncode(BinlogType type, uint32_t exec_t uint64_t logic_id, uint32_t filenum, uint64_t offset, const std::string& content, const std::vector& extends) { std::string binlog; - slash::PutFixed16(&binlog, type); - slash::PutFixed32(&binlog, exec_time); - slash::PutFixed32(&binlog, server_id); - slash::PutFixed64(&binlog, logic_id); - slash::PutFixed32(&binlog, filenum); - slash::PutFixed64(&binlog, offset); + pstd::PutFixed16(&binlog, type); + pstd::PutFixed32(&binlog, exec_time); + pstd::PutFixed32(&binlog, server_id); + pstd::PutFixed64(&binlog, logic_id); + pstd::PutFixed32(&binlog, filenum); + pstd::PutFixed64(&binlog, offset); uint32_t content_length = content.size(); - slash::PutFixed32(&binlog, content_length); + pstd::PutFixed32(&binlog, content_length); binlog.append(content); return binlog; } @@ -68,16 +68,16 @@ bool PikaBinlogTransverter::BinlogDecode(BinlogType type, const std::string& bin uint16_t binlog_type = 0; uint32_t content_length = 0; std::string binlog_str = binlog; - slash::GetFixed16(&binlog_str, &binlog_type); + pstd::GetFixed16(&binlog_str, &binlog_type); if (binlog_type != type) { return false; } - slash::GetFixed32(&binlog_str, &binlog_item->exec_time_); - slash::GetFixed32(&binlog_str, &binlog_item->server_id_); - slash::GetFixed64(&binlog_str, &binlog_item->logic_id_); - slash::GetFixed32(&binlog_str, &binlog_item->filenum_); - slash::GetFixed64(&binlog_str, &binlog_item->offset_); - slash::GetFixed32(&binlog_str, &content_length); + pstd::GetFixed32(&binlog_str, &binlog_item->exec_time_); + pstd::GetFixed32(&binlog_str, &binlog_item->server_id_); + pstd::GetFixed64(&binlog_str, &binlog_item->logic_id_); + pstd::GetFixed32(&binlog_str, &binlog_item->filenum_); + pstd::GetFixed64(&binlog_str, &binlog_item->offset_); + pstd::GetFixed32(&binlog_str, &content_length); if (binlog_str.size() >= content_length) { binlog_item->content_.assign(binlog_str.data(), content_length); } else { diff --git a/pika-tools/binlog_sender/binlog_transverter.h b/tools/binlog_sender/binlog_transverter.h similarity index 98% rename from pika-tools/binlog_sender/binlog_transverter.h rename to tools/binlog_sender/binlog_transverter.h index 49b46f9f90..ffcfdf4af4 100644 --- a/pika-tools/binlog_sender/binlog_transverter.h +++ b/tools/binlog_sender/binlog_transverter.h @@ -9,7 +9,7 @@ #include #include -#include "slash/include/slash_coding.h" +#include "pstd/include/pstd_coding.h" /* * ***********************************************Type First Binlog Item diff --git a/pika-tools/binlog_sender/progress_thread.cc b/tools/binlog_sender/progress_thread.cc similarity index 100% rename from pika-tools/binlog_sender/progress_thread.cc rename to tools/binlog_sender/progress_thread.cc diff --git a/pika-tools/binlog_sender/progress_thread.h b/tools/binlog_sender/progress_thread.h similarity index 96% rename from pika-tools/binlog_sender/progress_thread.h rename to tools/binlog_sender/progress_thread.h index d4cf8928f2..1842a4fff3 100644 --- a/pika-tools/binlog_sender/progress_thread.h +++ b/tools/binlog_sender/progress_thread.h @@ -13,7 +13,7 @@ #include "binlog_consumer.h" -extern slash::Mutex mutex; +extern pstd::Mutex mutex; class ProgressThread : public net::Thread { public: diff --git a/pika-tools/binlog_sender/utils.cc b/tools/binlog_sender/utils.cc similarity index 97% rename from pika-tools/binlog_sender/utils.cc rename to tools/binlog_sender/utils.cc index e1d8c3cb0d..27eac19abd 100644 --- a/pika-tools/binlog_sender/utils.cc +++ b/tools/binlog_sender/utils.cc @@ -48,7 +48,7 @@ bool CheckBinlogExists(const std::string& binlog_path, const std::vector #include "include/pika_binlog.h" -#include "slash/include/slash_status.h" -#include "slash/include/slash_string.h" +#include "pstd/include/pstd_status.h" +#include "pstd/include/pstd_string.h" std::string db_dump_path; int32_t db_dump_filenum; @@ -20,7 +20,7 @@ std::string new_pika_log_path; void ParseInfoFile(const std::string& path) { std::string info_file = path + kBgsaveInfoFile; - if (!slash::FileExists(info_file)) { + if (!pstd::FileExists(info_file)) { std::cout << "Info file " << info_file << " does not exist" << std::endl; exit(-1); } @@ -39,7 +39,7 @@ void ParseInfoFile(const std::string& path) { if (lineno == 2) { master_ip = line; } else if (lineno > 2 && lineno < 6) { - if (!slash::string2l(line.data(), line.size(), &tmp) || tmp < 0) { + if (!pstd::string2int(line.data(), line.size(), &tmp) || tmp < 0) { std::cout << "Format of info file " << info_file << " error, line : " << line; is.close(); exit(-1); @@ -112,7 +112,7 @@ int main(int argc, char* argv[]) { new_pika_log_path.push_back('/'); } // if this dir exist - if (slash::IsDir(new_pika_log_path) == 0) { + if (pstd::IsDir(new_pika_log_path) == 0) { std::cout << "Dir " << new_pika_log_path << "exist, please delete it!" << std::endl; exit(-1); } @@ -124,7 +124,7 @@ int main(int argc, char* argv[]) { std::cout << std::endl << "Step 2, Generate manifest file to " << new_pika_log_path << std::endl; // generate manifest and newest binlog Binlog binlog(new_pika_log_path); - slash::Status s = binlog.SetProducerStatus(db_dump_filenum, db_dump_offset); + pstd::Status s = binlog.SetProducerStatus(db_dump_filenum, db_dump_offset); if (!s.ok()) { std::cout << s.ToString() << std::endl; exit(-1); diff --git a/pika-tools/manifest_generator/pika_binlog.cc b/tools/manifest_generator/pika_binlog.cc similarity index 88% rename from pika-tools/manifest_generator/pika_binlog.cc rename to tools/manifest_generator/pika_binlog.cc index 30ba654801..492cac33a6 100644 --- a/pika-tools/manifest_generator/pika_binlog.cc +++ b/tools/manifest_generator/pika_binlog.cc @@ -9,7 +9,7 @@ #include -using slash::RWLock; +using pstd::RWLock; std::string NewFileName(const std::string name, const uint32_t current) { char buf[256]; @@ -20,7 +20,7 @@ std::string NewFileName(const std::string name, const uint32_t current) { /* * Version */ -Version::Version(slash::RWFile* save) : pro_num_(0), pro_offset_(0), logic_id_(0), save_(save) { +Version::Version(pstd::RWFile* save) : pro_num_(0), pro_offset_(0), logic_id_(0), save_(save) { assert(save_ != NULL); pthread_rwlock_init(&rwlock_, NULL); @@ -68,28 +68,28 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) binlog_path_(binlog_path), file_size_(file_size) { // To intergrate with old version, we don't set mmap file size to 100M; - // slash::SetMmapBoundSize(file_size); - // slash::kMmapBoundSize = 1024 * 1024 * 100; + // pstd::SetMmapBoundSize(file_size); + // pstd::kMmapBoundSize = 1024 * 1024 * 100; Status s; - slash::CreateDir(binlog_path_); + pstd::CreateDir(binlog_path_); filename = binlog_path_ + kBinlogPrefix; const std::string manifest = binlog_path_ + kManifest; std::string profile; - if (!slash::FileExists(manifest)) { + if (!pstd::FileExists(manifest)) { std::cout << "Binlog: Manifest file not exist, we create a new one."; profile = NewFileName(filename, pro_num_); - s = slash::NewWritableFile(profile, &queue_); + s = pstd::NewWritableFile(profile, &queue_); if (!s.ok()) { std::cout << "Binlog: new " << filename << " " << s.ToString(); exit(-1); } - s = slash::NewRWFile(manifest, &versionfile_); + s = pstd::NewRWFile(manifest, &versionfile_); if (!s.ok()) { std::cout << "Binlog: new versionfile error " << s.ToString(); exit(-1); @@ -100,7 +100,7 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) } else { std::cout << "Binlog: Find the exist file."; - s = slash::NewRWFile(manifest, &versionfile_); + s = pstd::NewRWFile(manifest, &versionfile_); if (s.ok()) { version_ = new Version(versionfile_); version_->Init(); @@ -114,7 +114,7 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) } profile = NewFileName(filename, pro_num_); - s = slash::AppendWritableFile(profile, &queue_, version_->pro_offset_); + s = pstd::AppendWritableFile(profile, &queue_, version_->pro_offset_); if (!s.ok()) { std::cout << "Binlog: Open file " << profile << " error " << s.ToString(); exit(-1); @@ -139,7 +139,7 @@ void Binlog::InitLogFile() { } Status Binlog::GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset, uint64_t* logic_id) { - slash::RWLock(&(version_->rwlock_), false); + pstd::RWLock(&(version_->rwlock_), false); *filenum = version_->pro_num_; *pro_offset = version_->pro_offset_; @@ -165,10 +165,10 @@ Status Binlog::Put(const char* item, int len) { pro_num_++; std::string profile = NewFileName(filename, pro_num_); - slash::NewWritableFile(profile, &queue_); + pstd::NewWritableFile(profile, &queue_); { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_offset_ = 0; version_->pro_num_ = pro_num_; version_->StableSave(); @@ -179,7 +179,7 @@ Status Binlog::Put(const char* item, int len) { int pro_offset; s = Produce(Slice(item, len), &pro_offset); if (s.ok()) { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_offset_ = pro_offset; version_->logic_id_++; version_->StableSave(); @@ -265,7 +265,7 @@ Status Binlog::Produce(const Slice& item, int* temp_pro_offset) { return s; } -Status Binlog::AppendBlank(slash::WritableFile* file, uint64_t len) { +Status Binlog::AppendBlank(pstd::WritableFile* file, uint64_t len) { if (len < kHeaderSize) { return Status::OK(); } @@ -311,7 +311,7 @@ Status Binlog::AppendBlank(slash::WritableFile* file, uint64_t len) { } Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { - slash::MutexLock l(&mutex_); + pstd::MutexLock l(&mutex_); // offset smaller than the first header if (pro_offset < 4) { @@ -321,22 +321,22 @@ Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { delete queue_; std::string init_profile = NewFileName(filename, 0); - if (slash::FileExists(init_profile)) { - slash::DeleteFile(init_profile); + if (pstd::FileExists(init_profile)) { + pstd::DeleteFile(init_profile); } std::string profile = NewFileName(filename, pro_num); - if (slash::FileExists(profile)) { - slash::DeleteFile(profile); + if (pstd::FileExists(profile)) { + pstd::DeleteFile(profile); } - slash::NewWritableFile(profile, &queue_); + pstd::NewWritableFile(profile, &queue_); Binlog::AppendBlank(queue_, pro_offset); pro_num_ = pro_num; { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_num_ = pro_num; version_->pro_offset_ = pro_offset; version_->StableSave(); diff --git a/pika-tools/manifest_generator/static_pack.sh b/tools/manifest_generator/static_pack.sh similarity index 100% rename from pika-tools/manifest_generator/static_pack.sh rename to tools/manifest_generator/static_pack.sh diff --git a/pika-tools/pika-port/README.md b/tools/pika-port/README.md similarity index 100% rename from pika-tools/pika-port/README.md rename to tools/pika-port/README.md diff --git a/pika-tools/pika-port/build2.sh b/tools/pika-port/build2.sh similarity index 100% rename from pika-tools/pika-port/build2.sh rename to tools/pika-port/build2.sh diff --git a/pika-tools/pika-port/build3.sh b/tools/pika-port/build3.sh similarity index 100% rename from pika-tools/pika-port/build3.sh rename to tools/pika-port/build3.sh diff --git a/pika-tools/pika-port/glog.sh b/tools/pika-port/glog.sh similarity index 100% rename from pika-tools/pika-port/glog.sh rename to tools/pika-port/glog.sh diff --git a/tools/pika-port/pika_port_3/CMakeLists.txt b/tools/pika-port/pika_port_3/CMakeLists.txt new file mode 100644 index 0000000000..76515d00e6 --- /dev/null +++ b/tools/pika-port/pika_port_3/CMakeLists.txt @@ -0,0 +1,19 @@ +set(CXXFLAGS "-O2 -pipe -fPIC -W -DNDEBUG -Wwrite-strings -Wpointer-arith \ +-Wreorder -Wswitch -Wsign-promo -Wredundant-decls -Wformat -Wall -Wno-unused-parameter \ +-D_GNU_SOURCE -D__STDC_FORMAT_MACROS -std=c++17 -gdwarf-2 -Wno-redundant-decls") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(pika_port ${BASE_OBJS}) + +target_include_directories(pika_port PRIVATE ${INSTALL_INCLUDEDIR} + PRIVATE ${PROJECT_SOURCE_DIR}) + +target_link_libraries(pika_port ${ROCKSDB_LIBRARY} pthread rt net pstd storage ${GLOG_LIBRARY} + ${GFLAGS_LIBRARY} ${SNAPPY_LIBRARY} ${ZLIB_LIBRARY} ${BZ2_LIBRARY} + ${LZ4_LIBRARY} ${ZSTD_LIBRARY}) +set_target_properties(pika_port PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/pika-port/pika_port_3/binlog_receiver_thread.cc b/tools/pika-port/pika_port_3/binlog_receiver_thread.cc similarity index 100% rename from pika-tools/pika-port/pika_port_3/binlog_receiver_thread.cc rename to tools/pika-port/pika_port_3/binlog_receiver_thread.cc diff --git a/pika-tools/pika-port/pika_port_3/binlog_receiver_thread.h b/tools/pika-port/pika_port_3/binlog_receiver_thread.h similarity index 77% rename from pika-tools/pika-port/pika_port_3/binlog_receiver_thread.h rename to tools/pika-port/pika_port_3/binlog_receiver_thread.h index a05c48fefb..40560b5961 100644 --- a/pika-tools/pika-port/pika_port_3/binlog_receiver_thread.h +++ b/tools/pika-port/pika_port_3/binlog_receiver_thread.h @@ -12,7 +12,7 @@ #include "master_conn.h" #include "net/include/server_thread.h" #include "pika_define.h" -#include "slash/include/slash_mutex.h" +#include "pstd/include/pstd_mutex.h" class BinlogReceiverThread { public: @@ -35,9 +35,10 @@ class BinlogReceiverThread { public: explicit MasterConnFactory(BinlogReceiverThread* binlog_receiver) : binlog_receiver_(binlog_receiver) {} - virtual net::PinkConn* NewPinkConn(int connfd, const std::string& ip_port, net::ServerThread* thread, - void* worker_specific_data) const override { - return new MasterConn(connfd, ip_port, binlog_receiver_); + virtual std::shared_ptr NewNetConn(int connfd, const std::string& ip_port, net::Thread* thread, + void* worker_specific_data, + net::NetMultiplexer* net_mpx = nullptr) const override { + return std::static_pointer_cast(std::make_shared(connfd, ip_port, binlog_receiver_)); } private: diff --git a/pika-tools/pika-port/pika_port_3/binlog_transverter.cc b/tools/pika-port/pika_port_3/binlog_transverter.cc similarity index 82% rename from pika-tools/pika-port/pika_port_3/binlog_transverter.cc rename to tools/pika-port/pika_port_3/binlog_transverter.cc index 8c6a4d86ae..fc2a95e5c3 100644 --- a/pika-tools/pika-port/pika_port_3/binlog_transverter.cc +++ b/tools/pika-port/pika_port_3/binlog_transverter.cc @@ -54,14 +54,14 @@ std::string PortBinlogTransverter::PortBinlogEncode(PortBinlogType type, uint32_ const std::string& content, const std::vector& extends) { std::string binlog; - slash::PutFixed16(&binlog, type); - slash::PutFixed32(&binlog, exec_time); - slash::PutFixed32(&binlog, server_id); - slash::PutFixed64(&binlog, logic_id); - slash::PutFixed32(&binlog, filenum); - slash::PutFixed64(&binlog, offset); + pstd::PutFixed16(&binlog, type); + pstd::PutFixed32(&binlog, exec_time); + pstd::PutFixed32(&binlog, server_id); + pstd::PutFixed64(&binlog, logic_id); + pstd::PutFixed32(&binlog, filenum); + pstd::PutFixed64(&binlog, offset); uint32_t content_length = content.size(); - slash::PutFixed32(&binlog, content_length); + pstd::PutFixed32(&binlog, content_length); binlog.append(content); return binlog; } @@ -71,18 +71,18 @@ bool PortBinlogTransverter::PortBinlogDecode(PortBinlogType type, const std::str uint16_t binlog_type = 0; uint32_t content_length = 0; std::string binlog_str = binlog; - slash::GetFixed16(&binlog_str, &binlog_type); + pstd::GetFixed16(&binlog_str, &binlog_type); if (binlog_type != type) { LOG(WARNING) << "PortBinlog Item type error, expect type: " << static_cast(type) << " actualy type: " << binlog_type; return false; } - slash::GetFixed32(&binlog_str, &binlog_item->exec_time_); - slash::GetFixed32(&binlog_str, &binlog_item->server_id_); - slash::GetFixed64(&binlog_str, &binlog_item->logic_id_); - slash::GetFixed32(&binlog_str, &binlog_item->filenum_); - slash::GetFixed64(&binlog_str, &binlog_item->offset_); - slash::GetFixed32(&binlog_str, &content_length); + pstd::GetFixed32(&binlog_str, &binlog_item->exec_time_); + pstd::GetFixed32(&binlog_str, &binlog_item->server_id_); + pstd::GetFixed64(&binlog_str, &binlog_item->logic_id_); + pstd::GetFixed32(&binlog_str, &binlog_item->filenum_); + pstd::GetFixed64(&binlog_str, &binlog_item->offset_); + pstd::GetFixed32(&binlog_str, &content_length); if (binlog_str.size() >= content_length) { binlog_item->content_.assign(binlog_str.data(), content_length); } else { diff --git a/pika-tools/pika-port/pika_port_3/binlog_transverter.h b/tools/pika-port/pika_port_3/binlog_transverter.h similarity index 98% rename from pika-tools/pika-port/pika_port_3/binlog_transverter.h rename to tools/pika-port/pika_port_3/binlog_transverter.h index a058745fbb..5ebcfecf99 100644 --- a/pika-tools/pika-port/pika_port_3/binlog_transverter.h +++ b/tools/pika-port/pika_port_3/binlog_transverter.h @@ -9,7 +9,7 @@ #include #include -#include "slash/include/slash_coding.h" +#include "pstd/include/pstd_coding.h" /* * *****************Type First PortBinlog Item Format***************** diff --git a/pika-tools/pika-port/pika_port_3/conf.h b/tools/pika-port/pika_port_3/conf.h similarity index 100% rename from pika-tools/pika-port/pika_port_3/conf.h rename to tools/pika-port/pika_port_3/conf.h diff --git a/pika-tools/pika-port/pika_port_3/const.cc b/tools/pika-port/pika_port_3/const.cc similarity index 73% rename from pika-tools/pika-port/pika_port_3/const.cc rename to tools/pika-port/pika_port_3/const.cc index dafc7d9261..85d8ce0dbc 100644 --- a/pika-tools/pika-port/pika_port_3/const.cc +++ b/tools/pika-port/pika_port_3/const.cc @@ -1,6 +1,6 @@ #include "const.h" -#include "blackwidow/blackwidow.h" +#include "storage/storage.h" #include "pika_define.h" @@ -58,28 +58,28 @@ std::string PikaRole(int role) { const char* GetDBTypeString(int type) { switch (type) { - case int(blackwidow::kStrings): { - return "blackwidow::kStrings"; + case int(storage::kStrings): { + return "storage::kStrings"; } - case int(blackwidow::kLists): { - return "blackwidow::kLists"; + case int(storage::kLists): { + return "storage::kLists"; } - case int(blackwidow::kHashes): { - return "blackwidow::kHashes"; + case int(storage::kHashes): { + return "storage::kHashes"; } - case int(blackwidow::kSets): { - return "blackwidow::kSets"; + case int(storage::kSets): { + return "storage::kSets"; } - case int(blackwidow::kZSets): { - return "blackwidow::kZSets"; + case int(storage::kZSets): { + return "storage::kZSets"; } default: { - return "blackwidow::Unknown"; + return "storage::Unknown"; } } } diff --git a/pika-tools/pika-port/pika_port_3/const.h b/tools/pika-port/pika_port_3/const.h similarity index 100% rename from pika-tools/pika-port/pika_port_3/const.h rename to tools/pika-port/pika_port_3/const.h diff --git a/pika-tools/pika-port/pika_port_3/main.cc b/tools/pika-port/pika_port_3/main.cc similarity index 94% rename from pika-tools/pika-port/pika_port_3/main.cc rename to tools/pika-port/pika_port_3/main.cc index 03beff3dba..c4f137869d 100644 --- a/pika-tools/pika-port/pika_port_3/main.cc +++ b/tools/pika-port/pika_port_3/main.cc @@ -78,8 +78,8 @@ static void SignalSetup() { } static void GlogInit(const std::string& log_path, bool is_daemon) { - if (!slash::FileExists(log_path)) { - slash::CreatePath(log_path); + if (!pstd::FileExists(log_path)) { + pstd::CreatePath(log_path); } if (!is_daemon) { @@ -157,7 +157,7 @@ int main(int argc, char* argv[]) { break; case 'p': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.local_port = int(num); break; case 'i': @@ -166,7 +166,7 @@ int main(int argc, char* argv[]) { break; case 'o': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.master_port = int(num); break; case 'm': @@ -175,12 +175,12 @@ int main(int argc, char* argv[]) { break; case 'n': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.forward_port = int(num); break; case 'x': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.forward_thread_num = int(num); break; case 'y': @@ -189,18 +189,18 @@ int main(int argc, char* argv[]) { break; case 'z': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.wait_bgsave_timeout = time_t(num); break; case 'f': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.filenum = (size_t)(num); break; case 's': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.offset = (size_t)(num); break; case 'w': @@ -224,7 +224,7 @@ int main(int argc, char* argv[]) { break; case 'b': snprintf(buf, 1024, "%s", optarg); - slash::string2l(buf, strlen(buf), &(num)); + pstd::string2int(buf, strlen(buf), &(num)); g_conf.sync_batch_num = (size_t)(num); break; case 'e': diff --git a/pika-tools/pika-port/pika_port_3/master_conn.cc b/tools/pika-port/pika_port_3/master_conn.cc similarity index 96% rename from pika-tools/pika-port/pika_port_3/master_conn.cc rename to tools/pika-port/pika_port_3/master_conn.cc index ff68ecd966..bba284a606 100644 --- a/pika-tools/pika-port/pika_port_3/master_conn.cc +++ b/tools/pika-port/pika_port_3/master_conn.cc @@ -5,8 +5,8 @@ #include -#include "slash/include/slash_coding.h" -#include "slash/include/slash_string.h" +#include "pstd/include/pstd_coding.h" +#include "pstd/include/pstd_string.h" #include "binlog_receiver_thread.h" #include "binlog_transverter.h" @@ -17,7 +17,7 @@ extern PikaPort* g_pika_port; MasterConn::MasterConn(int fd, std::string ip_port, void* worker_specific_data) - : PinkConn(fd, ip_port, NULL), + : NetConn(fd, ip_port, NULL), rbuf_(nullptr), rbuf_len_(0), rbuf_size_(REDIS_IOBUF_LEN), @@ -102,7 +102,7 @@ int32_t MasterConn::GetNextNum(const std::string& content, int32_t left_pos, int // 012 3 // num range [left_pos + 1, right_pos - 2] assert(left_pos < right_pos); - if (slash::string2l(content.data() + left_pos + 1, right_pos - left_pos - 2, value)) { + if (pstd::string2int(content.data() + left_pos + 1, right_pos - left_pos - 2, value)) { return 0; } return -1; @@ -179,8 +179,8 @@ net::ReadStatus MasterConn::GetRequest() { uint16_t type = 0; uint32_t body_length = 0; std::string header(rbuf_, HEADER_LEN); - slash::GetFixed16(&header, &type); - slash::GetFixed32(&header, &body_length); + pstd::GetFixed16(&header, &type); + pstd::GetFixed32(&header, &body_length); if (type != kTypePortAuth && type != kTypePortBinlog) { LOG(INFO) << "Unrecognizable Type: " << type << " maybe identify binlog type error"; diff --git a/pika-tools/pika-port/pika_port_3/master_conn.h b/tools/pika-port/pika_port_3/master_conn.h similarity index 97% rename from pika-tools/pika-port/pika_port_3/master_conn.h rename to tools/pika-port/pika_port_3/master_conn.h index d264e750cd..c0bf37e14f 100644 --- a/pika-tools/pika-port/pika_port_3/master_conn.h +++ b/tools/pika-port/pika_port_3/master_conn.h @@ -22,7 +22,7 @@ class BinlogReceiverThread; enum PortTransferOperate { kTypePortAuth = 1, kTypePortBinlog = 2 }; -class MasterConn : public net::PinkConn { +class MasterConn : public net::NetConn { public: MasterConn(int fd, std::string ip_port, void* worker_specific_data); virtual ~MasterConn(); diff --git a/pika-tools/pika-port/pika_port_3/migrator_thread.cc b/tools/pika-port/pika_port_3/migrator_thread.cc similarity index 90% rename from pika-tools/pika-port/pika_port_3/migrator_thread.cc rename to tools/pika-port/pika_port_3/migrator_thread.cc index 833bb273de..3dbb27c62b 100644 --- a/pika-tools/pika-port/pika_port_3/migrator_thread.cc +++ b/tools/pika-port/pika_port_3/migrator_thread.cc @@ -7,7 +7,7 @@ #include #include -#include "blackwidow/blackwidow.h" +#include "storage/storage.h" #include "src/redis_hashes.h" #include "src/redis_lists.h" #include "src/redis_sets.h" @@ -21,12 +21,12 @@ const int64_t MAX_BATCH_NUM = 30000; MigratorThread::~MigratorThread() {} void MigratorThread::MigrateStringsDB() { - blackwidow::RedisStrings* db = (blackwidow::RedisStrings*)(db_); + storage::RedisStrings* db = (storage::RedisStrings*)(db_); rocksdb::ReadOptions iterator_options; const rocksdb::Snapshot* snapshot; rocksdb::DB* rocksDB = db->GetDB(); - blackwidow::ScopeSnapshot ss(rocksDB, &snapshot); + storage::ScopeSnapshot ss(rocksDB, &snapshot); iterator_options.snapshot = snapshot; iterator_options.fill_cache = false; int64_t curtime; @@ -37,7 +37,7 @@ void MigratorThread::MigrateStringsDB() { auto iter = rocksDB->NewIterator(iterator_options); for (iter->SeekToFirst(); !should_exit_ && iter->Valid(); iter->Next()) { - blackwidow::ParsedStringsValue parsed_strings_value(iter->value()); + storage::ParsedStringsValue parsed_strings_value(iter->value()); int32_t ttl = 0; int64_t ts = (int64_t)(parsed_strings_value.timestamp()); if (ts != 0) { @@ -76,7 +76,7 @@ void MigratorThread::MigrateStringsDB() { } void MigratorThread::MigrateListsDB() { - blackwidow::RedisLists* db = (blackwidow::RedisLists*)(db_); + storage::RedisLists* db = (storage::RedisLists*)(db_); std::string start_key; std::string next_key; @@ -109,7 +109,7 @@ void MigratorThread::MigrateListsDB() { int64_t pos = 0; std::vector list; - blackwidow::Status s = db->LRange(k, pos, pos + g_conf.sync_batch_num - 1, &list); + storage::Status s = db->LRange(k, pos, pos + g_conf.sync_batch_num - 1, &list); if (!s.ok()) { LOG(WARNING) << "db->LRange(key:" << k << ", pos:" << pos << ", batch size: " << g_conf.sync_batch_num << ") = " << s.ToString(); @@ -159,7 +159,7 @@ void MigratorThread::MigrateListsDB() { } void MigratorThread::MigrateHashesDB() { - blackwidow::RedisHashes* db = (blackwidow::RedisHashes*)(db_); + storage::RedisHashes* db = (storage::RedisHashes*)(db_); std::string start_key; std::string next_key; @@ -186,8 +186,8 @@ void MigratorThread::MigrateHashesDB() { if (should_exit_) { break; } - std::vector fvs; - blackwidow::Status s = db->HGetall(k, &fvs); + std::vector fvs; + storage::Status s = db->HGetall(k, &fvs); if (!s.ok()) { LOG(WARNING) << "db->HGetall(key:" << k << ") = " << s.ToString(); continue; @@ -230,7 +230,7 @@ void MigratorThread::MigrateHashesDB() { } void MigratorThread::MigrateSetsDB() { - blackwidow::RedisSets* db = (blackwidow::RedisSets*)(db_); + storage::RedisSets* db = (storage::RedisSets*)(db_); std::string start_key; std::string next_key; @@ -258,7 +258,7 @@ void MigratorThread::MigrateSetsDB() { break; } std::vector members; - blackwidow::Status s = db->SMembers(k, &members); + storage::Status s = db->SMembers(k, &members); if (!s.ok()) { LOG(WARNING) << "db->SMembers(key:" << k << ") = " << s.ToString(); continue; @@ -298,7 +298,7 @@ void MigratorThread::MigrateSetsDB() { } void MigratorThread::MigrateZsetsDB() { - blackwidow::RedisZSets* db = (blackwidow::RedisZSets*)(db_); + storage::RedisZSets* db = (storage::RedisZSets*)(db_); std::string start_key; std::string next_key; @@ -325,8 +325,8 @@ void MigratorThread::MigrateZsetsDB() { if (should_exit_) { break; } - std::vector score_members; - blackwidow::Status s = db->ZRange(k, 0, -1, &score_members); + std::vector score_members; + storage::Status s = db->ZRange(k, 0, -1, &score_members); if (!s.ok()) { LOG(WARNING) << "db->ZRange(key:" << k << ") = " << s.ToString(); continue; @@ -369,27 +369,27 @@ void MigratorThread::MigrateZsetsDB() { void MigratorThread::MigrateDB() { switch (int(type_)) { - case int(blackwidow::kStrings): { + case int(storage::kStrings): { MigrateStringsDB(); break; } - case int(blackwidow::kLists): { + case int(storage::kLists): { MigrateListsDB(); break; } - case int(blackwidow::kHashes): { + case int(storage::kHashes): { MigrateHashesDB(); break; } - case int(blackwidow::kSets): { + case int(storage::kSets): { MigrateSetsDB(); break; } - case int(blackwidow::kZSets): { + case int(storage::kZSets): { MigrateZsetsDB(); break; } diff --git a/pika-tools/pika-port/pika_port_3/migrator_thread.h b/tools/pika-port/pika_port_3/migrator_thread.h similarity index 91% rename from pika-tools/pika-port/pika_port_3/migrator_thread.h rename to tools/pika-port/pika_port_3/migrator_thread.h index 67a09f8f3c..8c4483fc3d 100644 --- a/pika-tools/pika-port/pika_port_3/migrator_thread.h +++ b/tools/pika-port/pika_port_3/migrator_thread.h @@ -22,7 +22,7 @@ class MigratorThread : public net::Thread { virtual ~MigratorThread(); int64_t num() { - slash::MutexLock l(&num_mutex_); + pstd::MutexLock l(&num_mutex_); return num_; } @@ -30,7 +30,7 @@ class MigratorThread : public net::Thread { private: void PlusNum() { - slash::MutexLock l(&num_mutex_); + pstd::MutexLock l(&num_mutex_); ++num_; } @@ -55,7 +55,7 @@ class MigratorThread : public net::Thread { int thread_index_; int64_t num_; - slash::Mutex num_mutex_; + pstd::Mutex num_mutex_; }; #endif diff --git a/pika-tools/pika-port/pika_port_3/pika_binlog.cc b/tools/pika-port/pika_port_3/pika_binlog.cc similarity index 88% rename from pika-tools/pika-port/pika_port_3/pika_binlog.cc rename to tools/pika-port/pika_port_3/pika_binlog.cc index 7cb71f20ac..00c43aab14 100644 --- a/pika-tools/pika-port/pika_port_3/pika_binlog.cc +++ b/tools/pika-port/pika_port_3/pika_binlog.cc @@ -14,9 +14,9 @@ #include -#include "slash/include/slash_mutex.h" +#include "pstd/include/pstd_mutex.h" -using slash::RWLock; +using pstd::RWLock; std::string NewFileName(const std::string name, const uint32_t current) { char buf[256]; @@ -27,7 +27,7 @@ std::string NewFileName(const std::string name, const uint32_t current) { /* * Version */ -Version::Version(slash::RWFile* save) : pro_num_(0), pro_offset_(0), logic_id_(0), save_(save) { +Version::Version(pstd::RWFile* save) : pro_num_(0), pro_offset_(0), logic_id_(0), save_(save) { assert(save_ != NULL); pthread_rwlock_init(&rwlock_, NULL); @@ -81,27 +81,27 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) binlog_path_(binlog_path), file_size_(file_size) { // To intergrate with old version, we don't set mmap file size to 100M; - // slash::SetMmapBoundSize(file_size); - // slash::kMmapBoundSize = 1024 * 1024 * 100; + // pstd::SetMmapBoundSize(file_size); + // pstd::kMmapBoundSize = 1024 * 1024 * 100; Status s; - slash::CreateDir(binlog_path_); + pstd::CreateDir(binlog_path_); filename = binlog_path_ + kBinlogPrefix; const std::string manifest = binlog_path_ + kManifest; std::string profile; - if (!slash::FileExists(manifest)) { + if (!pstd::FileExists(manifest)) { LOG(INFO) << "Binlog: Manifest file not exist, we create a new one."; profile = NewFileName(filename, pro_num_); - s = slash::NewWritableFile(profile, &queue_); + s = pstd::NewWritableFile(profile, &queue_); if (!s.ok()) { LOG(FATAL) << "Binlog: NewWritableFile(" << filename << ") = " << s.ToString(); } - s = slash::NewRWFile(manifest, &versionfile_); + s = pstd::NewRWFile(manifest, &versionfile_); if (!s.ok()) { LOG(FATAL) << "Binlog: new versionfile error " << s.ToString(); } @@ -111,7 +111,7 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) } else { LOG(INFO) << "Binlog: Find the exist file."; - s = slash::NewRWFile(manifest, &versionfile_); + s = pstd::NewRWFile(manifest, &versionfile_); if (s.ok()) { version_ = new Version(versionfile_); version_->Init(); @@ -125,7 +125,7 @@ Binlog::Binlog(const std::string& binlog_path, const int file_size) profile = NewFileName(filename, pro_num_); LOG(INFO) << "Binlog: open profile " << profile; - s = slash::AppendWritableFile(profile, &queue_, version_->pro_offset_); + s = pstd::AppendWritableFile(profile, &queue_, version_->pro_offset_); if (!s.ok()) { LOG(FATAL) << "Binlog: Open file " << profile << " error " << s.ToString(); } @@ -152,7 +152,7 @@ void Binlog::InitLogFile() { } Status Binlog::GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset, uint64_t* logic_id) { - slash::RWLock(&(version_->rwlock_), false); + pstd::RWLock(&(version_->rwlock_), false); *filenum = version_->pro_num_; *pro_offset = version_->pro_offset_; @@ -178,10 +178,10 @@ Status Binlog::Put(const char* item, int len) { pro_num_++; std::string profile = NewFileName(filename, pro_num_); - slash::NewWritableFile(profile, &queue_); + pstd::NewWritableFile(profile, &queue_); { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_offset_ = 0; version_->pro_num_ = pro_num_; version_->StableSave(); @@ -192,7 +192,7 @@ Status Binlog::Put(const char* item, int len) { int pro_offset; s = Produce(Slice(item, len), &pro_offset); if (s.ok()) { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_offset_ = pro_offset; version_->logic_id_++; version_->StableSave(); @@ -278,7 +278,7 @@ Status Binlog::Produce(const Slice& item, int* temp_pro_offset) { return s; } -Status Binlog::AppendBlank(slash::WritableFile* file, uint64_t len) { +Status Binlog::AppendBlank(pstd::WritableFile* file, uint64_t len) { if (len < kHeaderSize) { return Status::OK(); } @@ -324,7 +324,7 @@ Status Binlog::AppendBlank(slash::WritableFile* file, uint64_t len) { } Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { - slash::MutexLock l(&mutex_); + pstd::MutexLock l(&mutex_); // offset smaller than the first header if (pro_offset < 4) { @@ -334,22 +334,22 @@ Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { delete queue_; std::string init_profile = NewFileName(filename, 0); - if (slash::FileExists(init_profile)) { - slash::DeleteFile(init_profile); + if (pstd::FileExists(init_profile)) { + pstd::DeleteFile(init_profile); } std::string profile = NewFileName(filename, pro_num); - if (slash::FileExists(profile)) { - slash::DeleteFile(profile); + if (pstd::FileExists(profile)) { + pstd::DeleteFile(profile); } - slash::NewWritableFile(profile, &queue_); + pstd::NewWritableFile(profile, &queue_); Binlog::AppendBlank(queue_, pro_offset); pro_num_ = pro_num; { - slash::RWLock(&(version_->rwlock_), true); + pstd::RWLock(&(version_->rwlock_), true); version_->pro_num_ = pro_num; version_->pro_offset_ = pro_offset; version_->StableSave(); diff --git a/pika-tools/pika-port/pika_port_3/pika_binlog.h b/tools/pika-port/pika_port_3/pika_binlog.h similarity index 82% rename from pika-tools/pika-port/pika_port_3/pika_binlog.h rename to tools/pika-port/pika_port_3/pika_binlog.h index 302c2f222a..f33d640d2b 100644 --- a/pika-tools/pika-port/pika_port_3/pika_binlog.h +++ b/tools/pika-port/pika_port_3/pika_binlog.h @@ -18,12 +18,12 @@ #endif #include "pika_define.h" -#include "slash/include/env.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" +#include "pstd/include/env.h" +#include "pstd/include/pstd_mutex.h" +#include "pstd/include/pstd_status.h" -using slash::Slice; -using slash::Status; +using pstd::Slice; +using pstd::Status; std::string NewFileName(const std::string name, const uint32_t current); @@ -46,9 +46,9 @@ class Binlog { */ Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset); - static Status AppendBlank(slash::WritableFile* file, uint64_t len); + static Status AppendBlank(pstd::WritableFile* file, uint64_t len); - slash::WritableFile* queue() { return queue_; } + pstd::WritableFile* queue() { return queue_; } uint64_t file_size() { return file_size_; } @@ -67,10 +67,10 @@ class Binlog { uint64_t item_num_; Version* version_; - slash::WritableFile* queue_; - slash::RWFile* versionfile_; + pstd::WritableFile* queue_; + pstd::RWFile* versionfile_; - slash::Mutex mutex_; + pstd::Mutex mutex_; uint32_t pro_num_; @@ -89,7 +89,7 @@ class Binlog { class Version { public: - Version(slash::RWFile* save); + Version(pstd::RWFile* save); ~Version(); Status Init(); @@ -104,12 +104,12 @@ class Version { pthread_rwlock_t rwlock_; void debug() { - slash::RWLock(&rwlock_, false); + pstd::RWLock(&rwlock_, false); printf("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); } private: - slash::RWFile* save_; + pstd::RWFile* save_; // No copying allowed; Version(const Version&); diff --git a/pika-tools/pika-port/pika_port_3/pika_command.h b/tools/pika-port/pika_port_3/pika_command.h similarity index 99% rename from pika-tools/pika-port/pika_port_3/pika_command.h rename to tools/pika-port/pika_port_3/pika_command.h index 84f5c3ef14..dcb6ffc789 100644 --- a/pika-tools/pika-port/pika_port_3/pika_command.h +++ b/tools/pika-port/pika_port_3/pika_command.h @@ -12,7 +12,7 @@ #include #include "net/include/redis_conn.h" -#include "slash/include/slash_string.h" +#include "pstd/include/pstd_string.h" #include "binlog_transverter.h" @@ -445,7 +445,7 @@ void RedisAppendContent(std::string& str, const std::string& value) { void RedisAppendLen(std::string& str, int64_t ori, const std::string& prefix) { char buf[32]; - slash::ll2string(buf, 32, static_cast(ori)); + pstd::ll2string(buf, 32, static_cast(ori)); str.append(prefix); str.append(buf); str.append(kNewLine); diff --git a/pika-tools/pika-port/pika_port_3/pika_define.h b/tools/pika-port/pika_port_3/pika_define.h similarity index 100% rename from pika-tools/pika-port/pika_port_3/pika_define.h rename to tools/pika-port/pika_port_3/pika_define.h diff --git a/pika-tools/pika-port/pika_port_3/pika_port.cc b/tools/pika-port/pika_port_3/pika_port.cc similarity index 91% rename from pika-tools/pika-port/pika_port_3/pika_port.cc rename to tools/pika-port/pika_port_3/pika_port.cc index 90dea95d60..c0755b064c 100644 --- a/pika-tools/pika-port/pika_port_3/pika_port.cc +++ b/tools/pika-port/pika_port_3/pika_port.cc @@ -16,9 +16,9 @@ #include "conf.h" #include "const.h" #include "pika_port.h" -#include "slash/include/env.h" -#include "slash/include/rsync.h" -#include "slash/include/slash_string.h" +#include "pstd/include/env.h" +#include "pstd/include/rsync.h" +#include "pstd/include/pstd_string.h" PikaPort::PikaPort(std::string& master_ip, int master_port, std::string& passwd) : ping_thread_(NULL), @@ -136,7 +136,7 @@ int PikaPort::SendRedisCommand(const std::string& command, const std::string& ke } bool PikaPort::SetMaster(std::string& master_ip, int master_port) { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); if ((role_ ^ PIKA_ROLE_SLAVE) && repl_state_ == PIKA_REPL_NO_CONNECT) { master_ip_ = master_ip; master_port_ = master_port; @@ -151,7 +151,7 @@ bool PikaPort::SetMaster(std::string& master_ip, int master_port) { } bool PikaPort::ShouldConnectMaster() { - slash::RWLock l(&state_protector_, false); + pstd::RWLock l(&state_protector_, false); // LOG(INFO) << "repl_state: " << PikaState(repl_state_) // << " role: " << PikaRole(role_) // << " master_connection: " << master_connection_; @@ -162,14 +162,14 @@ bool PikaPort::ShouldConnectMaster() { } void PikaPort::ConnectMasterDone() { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); if (repl_state_ == PIKA_REPL_CONNECT) { repl_state_ = PIKA_REPL_CONNECTING; } } bool PikaPort::ShouldStartPingMaster() { - slash::RWLock l(&state_protector_, false); + pstd::RWLock l(&state_protector_, false); LOG(INFO) << "ShouldStartPingMaster: master_connection " << master_connection_ << ", repl_state " << PikaState(repl_state_); if (repl_state_ == PIKA_REPL_CONNECTING && master_connection_ < 2) { @@ -180,7 +180,7 @@ bool PikaPort::ShouldStartPingMaster() { } void PikaPort::MinusMasterConnection() { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); if (master_connection_ > 0) { if ((--master_connection_) <= 0) { // two connection with master has been deleted @@ -197,7 +197,7 @@ void PikaPort::MinusMasterConnection() { } void PikaPort::PlusMasterConnection() { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); if (master_connection_ < 2) { if ((++master_connection_) >= 2) { // two connection with master has been established @@ -209,7 +209,7 @@ void PikaPort::PlusMasterConnection() { } bool PikaPort::ShouldAccessConnAsMaster(const std::string& ip) { - slash::RWLock l(&state_protector_, false); + pstd::RWLock l(&state_protector_, false); LOG(INFO) << "ShouldAccessConnAsMaster, repl_state_: " << PikaState(repl_state_) << ", ip: " << ip << ", master_ip: " << master_ip_; if (repl_state_ != PIKA_REPL_NO_CONNECT && ip == master_ip_) { @@ -220,7 +220,7 @@ bool PikaPort::ShouldAccessConnAsMaster(const std::string& ip) { void PikaPort::RemoveMaster() { { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); repl_state_ = PIKA_REPL_NO_CONNECT; role_ &= ~PIKA_ROLE_SLAVE; master_ip_ = ""; @@ -237,7 +237,7 @@ void PikaPort::RemoveMaster() { } bool PikaPort::IsWaitingDBSync() { - slash::RWLock l(&state_protector_, false); + pstd::RWLock l(&state_protector_, false); if (repl_state_ == PIKA_REPL_WAIT_DBSYNC) { return true; } @@ -245,12 +245,12 @@ bool PikaPort::IsWaitingDBSync() { } void PikaPort::NeedWaitDBSync() { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); repl_state_ = PIKA_REPL_WAIT_DBSYNC; } void PikaPort::WaitDBSyncFinish() { - slash::RWLock l(&state_protector_, true); + pstd::RWLock l(&state_protector_, true); if (repl_state_ == PIKA_REPL_WAIT_DBSYNC) { repl_state_ = PIKA_REPL_CONNECT; } diff --git a/pika-tools/pika-port/pika_port_3/pika_port.h b/tools/pika-port/pika_port_3/pika_port.h similarity index 89% rename from pika-tools/pika-port/pika_port_3/pika_port.h rename to tools/pika-port/pika_port_3/pika_port.h index c43e2030eb..3ea1f9acb1 100644 --- a/pika-tools/pika-port/pika_port_3/pika_port.h +++ b/tools/pika-port/pika_port_3/pika_port.h @@ -10,15 +10,15 @@ #include "pika_binlog.h" #include "pika_define.h" #include "redis_sender.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" +#include "pstd/include/pstd_mutex.h" +#include "pstd/include/pstd_status.h" #include "slaveping_thread.h" #include "trysync_thread.h" #include -using slash::Slice; -using slash::Status; +using pstd::Slice; +using pstd::Status; class PikaPort { public: @@ -36,11 +36,11 @@ class PikaPort { void SetSid(int64_t sid) { sid_ = sid; } int role() { - slash::RWLock(&state_protector_, false); + pstd::RWLock(&state_protector_, false); return role_; } int repl_state() { - slash::RWLock(&state_protector_, false); + pstd::RWLock(&state_protector_, false); return repl_state_; } std::string requirepass() { return requirepass_; } @@ -81,10 +81,10 @@ class PikaPort { std::string dump_path_; pthread_rwlock_t rwlock_; - slash::Mutex mutex_; // double lock to block main thread + pstd::Mutex mutex_; // double lock to block main thread // redis client - // net::PinkCli *cli_; + // net::NetCli *cli_; // RedisSender *sender_; std::vector senders_; diff --git a/pika-tools/pika-port/pika_port_3/pika_sender.cc b/tools/pika-port/pika_port_3/pika_sender.cc similarity index 93% rename from pika-tools/pika-port/pika_port_3/pika_sender.cc rename to tools/pika-port/pika_port_3/pika_sender.cc index 4b1a60ef56..9d8ef2bc73 100644 --- a/pika-tools/pika-port/pika_port_3/pika_sender.cc +++ b/tools/pika-port/pika_port_3/pika_sender.cc @@ -2,7 +2,7 @@ #include "const.h" #include "pika_sender.h" -#include "slash/include/xdebug.h" +#include "pstd/include/xdebug.h" PikaSender::PikaSender(std::string ip, int64_t port, std::string password) : cli_(NULL), @@ -17,7 +17,7 @@ PikaSender::PikaSender(std::string ip, int64_t port, std::string password) PikaSender::~PikaSender() {} int PikaSender::QueueSize() { - slash::MutexLock l(&keys_mutex_); + pstd::MutexLock l(&keys_mutex_); return keys_queue_.size(); } @@ -33,7 +33,7 @@ void PikaSender::ConnectRedis() { // Connect to redis cli_ = net::NewRedisCli(); cli_->set_connect_timeout(1000); - slash::Status s = cli_->Connect(ip_, port_); + pstd::Status s = cli_->Connect(ip_, port_); if (!s.ok()) { delete cli_; cli_ = NULL; @@ -52,7 +52,7 @@ void PikaSender::ConnectRedis() { argv.push_back("AUTH"); argv.push_back(password_); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -78,7 +78,7 @@ void PikaSender::ConnectRedis() { argv.push_back("PING"); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -119,7 +119,7 @@ void PikaSender::LoadKey(const std::string& key) { void PikaSender::SendCommand(std::string& command, const std::string& key) { // Send command - slash::Status s = cli_->Send(&command); + pstd::Status s = cli_->Send(&command); if (!s.ok()) { elements_--; LoadKey(key); diff --git a/pika-tools/pika-port/pika_port_3/pika_sender.h b/tools/pika-port/pika_port_3/pika_sender.h similarity index 91% rename from pika-tools/pika-port/pika_port_3/pika_sender.h rename to tools/pika-port/pika_port_3/pika_sender.h index e58b392af8..fd1251eff1 100644 --- a/pika-tools/pika-port/pika_port_3/pika_sender.h +++ b/tools/pika-port/pika_port_3/pika_sender.h @@ -27,9 +27,9 @@ class PikaSender : public net::Thread { void ConnectRedis(); private: - net::PinkCli* cli_; - slash::CondVar signal_; - slash::Mutex keys_mutex_; + net::NetCli* cli_; + pstd::CondVar signal_; + pstd::Mutex keys_mutex_; std::queue keys_queue_; std::string ip_; int port_; diff --git a/pika-tools/pika-port/pika_port_3/readme.md b/tools/pika-port/pika_port_3/readme.md similarity index 100% rename from pika-tools/pika-port/pika_port_3/readme.md rename to tools/pika-port/pika_port_3/readme.md diff --git a/pika-tools/pika-port/pika_port_3/redis_sender.cc b/tools/pika-port/pika_port_3/redis_sender.cc similarity index 95% rename from pika-tools/pika-port/pika_port_3/redis_sender.cc rename to tools/pika-port/pika_port_3/redis_sender.cc index 812607df02..e18fb14ea4 100644 --- a/pika-tools/pika-port/pika_port_3/redis_sender.cc +++ b/tools/pika-port/pika_port_3/redis_sender.cc @@ -6,7 +6,7 @@ #include -#include "slash/include/xdebug.h" +#include "pstd/include/xdebug.h" static time_t kCheckDiff = 1; @@ -33,7 +33,7 @@ void RedisSender::ConnectRedis() { cli_->set_connect_timeout(1000); cli_->set_recv_timeout(10000); cli_->set_send_timeout(10000); - slash::Status s = cli_->Connect(ip_, port_); + pstd::Status s = cli_->Connect(ip_, port_); if (!s.ok()) { delete cli_; cli_ = NULL; @@ -52,7 +52,7 @@ void RedisSender::ConnectRedis() { argv.push_back("AUTH"); argv.push_back(password_); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -78,7 +78,7 @@ void RedisSender::ConnectRedis() { argv.push_back("PING"); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -140,7 +140,7 @@ int RedisSender::SendCommand(std::string& command) { // Send command int idx = 0; do { - slash::Status s = cli_->Send(&command); + pstd::Status s = cli_->Send(&command); if (s.ok()) { return 0; } diff --git a/pika-tools/pika-port/pika_port_3/redis_sender.h b/tools/pika-port/pika_port_3/redis_sender.h similarity index 88% rename from pika-tools/pika-port/pika_port_3/redis_sender.h rename to tools/pika-port/pika_port_3/redis_sender.h index 4807eed6e6..fadede997a 100644 --- a/pika-tools/pika-port/pika_port_3/redis_sender.h +++ b/tools/pika-port/pika_port_3/redis_sender.h @@ -26,10 +26,10 @@ class RedisSender : public net::Thread { private: int id_; - net::PinkCli* cli_; - slash::CondVar rsignal_; - slash::CondVar wsignal_; - slash::Mutex commands_mutex_; + net::NetCli* cli_; + pstd::CondVar rsignal_; + pstd::CondVar wsignal_; + pstd::Mutex commands_mutex_; std::queue commands_queue_; std::string ip_; int port_; diff --git a/pika-tools/pika-port/pika_port_3/slaveping_thread.cc b/tools/pika-port/pika_port_3/slaveping_thread.cc similarity index 98% rename from pika-tools/pika-port/pika_port_3/slaveping_thread.cc rename to tools/pika-port/pika_port_3/slaveping_thread.cc index 26712409ee..a7c77f5e94 100644 --- a/pika-tools/pika-port/pika_port_3/slaveping_thread.cc +++ b/tools/pika-port/pika_port_3/slaveping_thread.cc @@ -30,7 +30,7 @@ Status SlavepingThread::RecvProc() { net::RedisCmdArgsType argv; Status s = cli_->Recv(&argv); if (s.ok()) { - slash::StringToLower(argv[0]); + pstd::StringToLower(argv[0]); // LOG(INFO) << "Reply from master after ping: " << argv[0]; if (argv[0] == "pong" || argv[0] == "ok") { } else { diff --git a/pika-tools/pika-port/pika_port_3/slaveping_thread.h b/tools/pika-port/pika_port_3/slaveping_thread.h similarity index 88% rename from pika-tools/pika-port/pika_port_3/slaveping_thread.h rename to tools/pika-port/pika_port_3/slaveping_thread.h index 81992c1a4b..4da5b9756e 100644 --- a/pika-tools/pika-port/pika_port_3/slaveping_thread.h +++ b/tools/pika-port/pika_port_3/slaveping_thread.h @@ -9,12 +9,12 @@ #include "net/include/net_cli.h" #include "net/include/net_thread.h" #include "net/include/redis_cli.h" -#include "slash/include/slash_mutex.h" -#include "slash/include/slash_status.h" +#include "pstd/include/pstd_mutex.h" +#include "pstd/include/pstd_status.h" #include -using slash::Status; +using pstd::Status; class SlavepingThread : public net::Thread { public: @@ -36,7 +36,7 @@ class SlavepingThread : public net::Thread { bool is_first_send_; int sockfd_; - net::PinkCli* cli_; + net::NetCli* cli_; virtual void* ThreadMain(); }; diff --git a/pika-tools/pika-port/pika_port_3/static_pack.sh b/tools/pika-port/pika_port_3/static_pack.sh similarity index 100% rename from pika-tools/pika-port/pika_port_3/static_pack.sh rename to tools/pika-port/pika_port_3/static_pack.sh diff --git a/pika-tools/pika-port/pika_port_3/trysync_thread.cc b/tools/pika-port/pika_port_3/trysync_thread.cc similarity index 88% rename from pika-tools/pika-port/pika_port_3/trysync_thread.cc rename to tools/pika-port/pika_port_3/trysync_thread.cc index 2d62bd8478..853c22f42f 100644 --- a/pika-tools/pika-port/pika_port_3/trysync_thread.cc +++ b/tools/pika-port/pika_port_3/trysync_thread.cc @@ -18,11 +18,11 @@ #include #include -#include "slash/include/rsync.h" -#include "slash/include/slash_status.h" +#include "pstd/include/rsync.h" +#include "pstd/include/pstd_status.h" #include "slaveping_thread.h" -#include "blackwidow/blackwidow.h" +#include "storage/storage.h" #include "pika_define.h" extern Conf g_conf; @@ -30,7 +30,7 @@ extern PikaPort* g_pika_port; TrysyncThread::~TrysyncThread() { StopThread(); - slash::StopRsync(g_conf.dump_path); + pstd::StopRsync(g_conf.dump_path); delete cli_; LOG(INFO) << " Trysync thread " << pthread_self() << " exit!!!"; } @@ -54,14 +54,14 @@ void TrysyncThread::Stop() { void TrysyncThread::PrepareRsync() { std::string db_sync_path = g_conf.dump_path; - slash::StopRsync(db_sync_path); - slash::CreatePath(db_sync_path); - - slash::CreatePath(db_sync_path + "strings"); - slash::CreatePath(db_sync_path + "hashes"); - slash::CreatePath(db_sync_path + "lists"); - slash::CreatePath(db_sync_path + "sets"); - slash::CreatePath(db_sync_path + "zsets"); + pstd::StopRsync(db_sync_path); + pstd::CreatePath(db_sync_path); + + pstd::CreatePath(db_sync_path + "strings"); + pstd::CreatePath(db_sync_path + "hashes"); + pstd::CreatePath(db_sync_path + "lists"); + pstd::CreatePath(db_sync_path + "sets"); + pstd::CreatePath(db_sync_path + "zsets"); } bool TrysyncThread::Send(std::string lip) { @@ -92,7 +92,7 @@ bool TrysyncThread::Send(std::string lip) { LOG(INFO) << "redis command: trysync " << g_conf.local_ip.c_str() << " " << g_conf.local_port << " " << filenum << " " << pro_offset; - slash::Status s; + pstd::Status s; s = cli_->Send(&wbuf_str); if (!s.ok()) { LOG(WARNING) << "Connect master, Send: " << wbuf_str << ", status: " << s.ToString(); @@ -108,7 +108,7 @@ bool TrysyncThread::Send(std::string lip) { bool TrysyncThread::RecvProc() { bool should_auth = g_pika_port->requirepass() == "" ? false : true; bool is_authed = false; - slash::Status s; + pstd::Status s; std::string reply; net::RedisCmdArgsType argv; @@ -122,13 +122,13 @@ bool TrysyncThread::RecvProc() { reply = argv[0]; LOG(INFO) << "Reply from master after trysync: " << reply.c_str(); if (!is_authed && should_auth) { - if (kInnerReplOk != slash::StringToLower(reply)) { + if (kInnerReplOk != pstd::StringToLower(reply)) { g_pika_port->RemoveMaster(); return false; } is_authed = true; } else { - if (argv.size() == 1 && slash::string2l(reply.data(), reply.size(), &sid_)) { + if (argv.size() == 1 && pstd::string2int(reply.data(), reply.size(), &sid_)) { // Luckily, I got your point, the sync is comming LOG(INFO) << "Recv sid from master: " << sid_; g_pika_port->SetSid(sid_); @@ -171,7 +171,7 @@ bool TrysyncThread::TryUpdateMasterOffset() { // Check dbsync finished std::string db_sync_path = g_conf.dump_path; std::string info_path = db_sync_path + kBgsaveInfoFile; - if (!slash::FileExists(info_path)) { + if (!pstd::FileExists(info_path)) { return false; } @@ -189,7 +189,7 @@ bool TrysyncThread::TryUpdateMasterOffset() { if (lineno == 2) { master_ip = line; } else if (lineno > 2 && lineno < 6) { - if (!slash::string2l(line.data(), line.size(), &tmp) || tmp < 0) { + if (!pstd::string2int(line.data(), line.size(), &tmp) || tmp < 0) { LOG(WARNING) << "Format of info file after db sync error, line: " << line; is.close(); return false; @@ -220,8 +220,8 @@ bool TrysyncThread::TryUpdateMasterOffset() { } // Replace the old db - slash::StopRsync(db_sync_path); - slash::DeleteFile(info_path); + pstd::StopRsync(db_sync_path); + pstd::DeleteFile(info_path); // Update master offset g_pika_port->logger()->SetProducerStatus(filenum, offset); @@ -275,49 +275,49 @@ int TrysyncThread::Retransmit() { options.write_buffer_size = 512 * 1024 * 1024; // 512M options.target_file_size_base = 40 * 1024 * 1024; // 40M - blackwidow::BlackwidowOptions bwOptions; + storage::StorageOptions bwOptions; bwOptions.options = options; - blackwidow::BlackWidow bw; + storage::Storage bw; - blackwidow::RedisStrings stringsDB(&bw, blackwidow::kStrings); + storage::RedisStrings stringsDB(&bw, storage::kStrings); std::string path = db_path + "strings"; s = stringsDB.Open(bwOptions, path); LOG(INFO) << "Open strings DB " << path << " result " << s.ToString(); if (s.ok()) { - migrators_.emplace_back(new MigratorThread((void*)(&stringsDB), &senders_, blackwidow::kStrings, thread_num)); + migrators_.emplace_back(new MigratorThread((void*)(&stringsDB), &senders_, storage::kStrings, thread_num)); } - blackwidow::RedisLists listsDB(&bw, blackwidow::kLists); + storage::RedisLists listsDB(&bw, storage::kLists); path = db_path + "lists"; s = listsDB.Open(bwOptions, path); LOG(INFO) << "Open lists DB " << path << " result " << s.ToString(); if (s.ok()) { - migrators_.emplace_back(new MigratorThread((void*)(&listsDB), &senders_, blackwidow::kLists, thread_num)); + migrators_.emplace_back(new MigratorThread((void*)(&listsDB), &senders_, storage::kLists, thread_num)); } - blackwidow::RedisHashes hashesDB(&bw, blackwidow::kHashes); + storage::RedisHashes hashesDB(&bw, storage::kHashes); path = db_path + "hashes"; s = hashesDB.Open(bwOptions, path); LOG(INFO) << "Open hashes DB " << path << " result " << s.ToString(); if (s.ok()) { - migrators_.emplace_back(new MigratorThread((void*)(&hashesDB), &senders_, blackwidow::kHashes, thread_num)); + migrators_.emplace_back(new MigratorThread((void*)(&hashesDB), &senders_, storage::kHashes, thread_num)); } - blackwidow::RedisSets setsDB(&bw, blackwidow::kSets); + storage::RedisSets setsDB(&bw, storage::kSets); path = db_path + "sets"; s = setsDB.Open(bwOptions, path); LOG(INFO) << "Open sets DB " << path << " result " << s.ToString(); if (s.ok()) { - migrators_.emplace_back(new MigratorThread((void*)(&setsDB), &senders_, blackwidow::kSets, thread_num)); + migrators_.emplace_back(new MigratorThread((void*)(&setsDB), &senders_, storage::kSets, thread_num)); } - blackwidow::RedisZSets zsetsDB(&bw, blackwidow::kZSets); + storage::RedisZSets zsetsDB(&bw, storage::kZSets); path = db_path + "zsets"; s = zsetsDB.Open(bwOptions, path); LOG(INFO) << "Open zsets DB " << path << " result " << s.ToString(); if (s.ok()) { - migrators_.emplace_back(new MigratorThread((void*)(&zsetsDB), &senders_, blackwidow::kZSets, thread_num)); + migrators_.emplace_back(new MigratorThread((void*)(&zsetsDB), &senders_, storage::kZSets, thread_num)); } retransmit_mutex_.Lock(); @@ -428,15 +428,15 @@ void* TrysyncThread::ThreadMain() { // the pika master module name rule is: document_${slave_ip}:master_port // // document_${slave_ip}:master_port - // std::string ip_port = slash::IpPortString(lip, master_port); + // std::string ip_port = pstd::IpPortString(lip, master_port); // pika 3.0.0-3.0.15 uses document_${master_ip}:${master_port} // document_${master_ip}:${master_port} - std::string ip_port = slash::IpPortString(master_ip, master_port); + std::string ip_port = pstd::IpPortString(master_ip, master_port); // We append the master ip port after module name // To make sure only data from current master is received int rsync_port = g_conf.local_port + 3000; - int ret = slash::StartRsync(dbsync_path, kDBSyncModule + "_" + ip_port, lip, rsync_port); + int ret = pstd::StartRsync(dbsync_path, kDBSyncModule + "_" + ip_port, lip, rsync_port, g_conf.passwd); if (0 != ret) { LOG(WARNING) << "Failed to start rsync, path: " << dbsync_path << ", error: " << ret; } @@ -445,7 +445,7 @@ void* TrysyncThread::ThreadMain() { // Make sure the listening addr of rsyncd is accessible, to avoid the corner case // that "rsync --daemon" process has started but can not bind its port which is // used by other process. - net::PinkCli* rsync = net::NewRedisCli(); + net::NetCli* rsync = net::NewRedisCli(); int retry_times; for (retry_times = 0; retry_times < 5; retry_times++) { if (rsync->Connect(lip, rsync_port, "").ok()) { @@ -463,7 +463,7 @@ void* TrysyncThread::ThreadMain() { if (Send(lip) && RecvProc()) { g_pika_port->ConnectMasterDone(); // Stop rsync, binlog sync with master is begin - slash::StopRsync(dbsync_path); + pstd::StopRsync(dbsync_path); delete g_pika_port->ping_thread_; g_pika_port->ping_thread_ = new SlavepingThread(sid_); diff --git a/pika-tools/pika-port/pika_port_3/trysync_thread.h b/tools/pika-port/pika_port_3/trysync_thread.h similarity index 95% rename from pika-tools/pika-port/pika_port_3/trysync_thread.h rename to tools/pika-port/pika_port_3/trysync_thread.h index bff556c70f..005b5bed34 100644 --- a/pika-tools/pika-port/pika_port_3/trysync_thread.h +++ b/tools/pika-port/pika_port_3/trysync_thread.h @@ -41,9 +41,9 @@ class TrysyncThread : public net::Thread { private: long sid_; int sockfd_; - net::PinkCli* cli_; + net::NetCli* cli_; - slash::Mutex retransmit_mutex_; + pstd::Mutex retransmit_mutex_; bool retransmit_flag_; std::vector senders_; diff --git a/pika-tools/pika-port/tool/bench.py b/tools/pika-port/tool/bench.py similarity index 100% rename from pika-tools/pika-port/tool/bench.py rename to tools/pika-port/tool/bench.py diff --git a/pika-tools/pika-port/tool/get_missing_keys.py b/tools/pika-port/tool/get_missing_keys.py similarity index 100% rename from pika-tools/pika-port/tool/get_missing_keys.py rename to tools/pika-port/tool/get_missing_keys.py diff --git a/pika-tools/pika-port/tool/sync.sh b/tools/pika-port/tool/sync.sh similarity index 100% rename from pika-tools/pika-port/tool/sync.sh rename to tools/pika-port/tool/sync.sh diff --git a/pika-tools/pika_exporter/LICENSE b/tools/pika_exporter/LICENSE similarity index 100% rename from pika-tools/pika_exporter/LICENSE rename to tools/pika_exporter/LICENSE diff --git a/pika-tools/pika_exporter/Makefile b/tools/pika_exporter/Makefile similarity index 100% rename from pika-tools/pika_exporter/Makefile rename to tools/pika_exporter/Makefile diff --git a/pika-tools/pika_exporter/README.md b/tools/pika_exporter/README.md similarity index 100% rename from pika-tools/pika_exporter/README.md rename to tools/pika_exporter/README.md diff --git a/pika-tools/pika_exporter/contrib/base_info.png b/tools/pika_exporter/contrib/base_info.png similarity index 100% rename from pika-tools/pika_exporter/contrib/base_info.png rename to tools/pika_exporter/contrib/base_info.png diff --git a/pika-tools/pika_exporter/contrib/grafana_prometheus_pika_dashboard.json b/tools/pika_exporter/contrib/grafana_prometheus_pika_dashboard.json similarity index 100% rename from pika-tools/pika_exporter/contrib/grafana_prometheus_pika_dashboard.json rename to tools/pika_exporter/contrib/grafana_prometheus_pika_dashboard.json diff --git a/pika-tools/pika_exporter/contrib/keys_metrics.png b/tools/pika_exporter/contrib/keys_metrics.png similarity index 100% rename from pika-tools/pika_exporter/contrib/keys_metrics.png rename to tools/pika_exporter/contrib/keys_metrics.png diff --git a/pika-tools/pika_exporter/contrib/overview.png b/tools/pika_exporter/contrib/overview.png similarity index 100% rename from pika-tools/pika_exporter/contrib/overview.png rename to tools/pika_exporter/contrib/overview.png diff --git a/pika-tools/pika_exporter/contrib/replication.png b/tools/pika_exporter/contrib/replication.png similarity index 100% rename from pika-tools/pika_exporter/contrib/replication.png rename to tools/pika_exporter/contrib/replication.png diff --git a/pika-tools/pika_exporter/contrib/sample_pika_hosts_file.txt b/tools/pika_exporter/contrib/sample_pika_hosts_file.txt similarity index 100% rename from pika-tools/pika_exporter/contrib/sample_pika_hosts_file.txt rename to tools/pika_exporter/contrib/sample_pika_hosts_file.txt diff --git a/pika-tools/pika_exporter/contrib/time_consuming_operation.png b/tools/pika_exporter/contrib/time_consuming_operation.png similarity index 100% rename from pika-tools/pika_exporter/contrib/time_consuming_operation.png rename to tools/pika_exporter/contrib/time_consuming_operation.png diff --git a/pika-tools/pika_exporter/discovery/discovery.go b/tools/pika_exporter/discovery/discovery.go similarity index 100% rename from pika-tools/pika_exporter/discovery/discovery.go rename to tools/pika_exporter/discovery/discovery.go diff --git a/pika-tools/pika_exporter/exporter/client.go b/tools/pika_exporter/exporter/client.go similarity index 100% rename from pika-tools/pika_exporter/exporter/client.go rename to tools/pika_exporter/exporter/client.go diff --git a/pika-tools/pika_exporter/exporter/future.go b/tools/pika_exporter/exporter/future.go similarity index 100% rename from pika-tools/pika_exporter/exporter/future.go rename to tools/pika_exporter/exporter/future.go diff --git a/pika-tools/pika_exporter/exporter/metrics/binlog.go b/tools/pika_exporter/exporter/metrics/binlog.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/binlog.go rename to tools/pika_exporter/exporter/metrics/binlog.go diff --git a/pika-tools/pika_exporter/exporter/metrics/clients.go b/tools/pika_exporter/exporter/metrics/clients.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/clients.go rename to tools/pika_exporter/exporter/metrics/clients.go diff --git a/pika-tools/pika_exporter/exporter/metrics/command_exec_count.go b/tools/pika_exporter/exporter/metrics/command_exec_count.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/command_exec_count.go rename to tools/pika_exporter/exporter/metrics/command_exec_count.go diff --git a/pika-tools/pika_exporter/exporter/metrics/cpu.go b/tools/pika_exporter/exporter/metrics/cpu.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/cpu.go rename to tools/pika_exporter/exporter/metrics/cpu.go diff --git a/pika-tools/pika_exporter/exporter/metrics/data.go b/tools/pika_exporter/exporter/metrics/data.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/data.go rename to tools/pika_exporter/exporter/metrics/data.go diff --git a/pika-tools/pika_exporter/exporter/metrics/keyspace.go b/tools/pika_exporter/exporter/metrics/keyspace.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/keyspace.go rename to tools/pika_exporter/exporter/metrics/keyspace.go diff --git a/pika-tools/pika_exporter/exporter/metrics/metrics.go b/tools/pika_exporter/exporter/metrics/metrics.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/metrics.go rename to tools/pika_exporter/exporter/metrics/metrics.go diff --git a/pika-tools/pika_exporter/exporter/metrics/parser.go b/tools/pika_exporter/exporter/metrics/parser.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/parser.go rename to tools/pika_exporter/exporter/metrics/parser.go diff --git a/pika-tools/pika_exporter/exporter/metrics/replication.go b/tools/pika_exporter/exporter/metrics/replication.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/replication.go rename to tools/pika_exporter/exporter/metrics/replication.go diff --git a/pika-tools/pika_exporter/exporter/metrics/server.go b/tools/pika_exporter/exporter/metrics/server.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/server.go rename to tools/pika_exporter/exporter/metrics/server.go diff --git a/pika-tools/pika_exporter/exporter/metrics/stats.go b/tools/pika_exporter/exporter/metrics/stats.go similarity index 100% rename from pika-tools/pika_exporter/exporter/metrics/stats.go rename to tools/pika_exporter/exporter/metrics/stats.go diff --git a/pika-tools/pika_exporter/exporter/parser.go b/tools/pika_exporter/exporter/parser.go similarity index 100% rename from pika-tools/pika_exporter/exporter/parser.go rename to tools/pika_exporter/exporter/parser.go diff --git a/pika-tools/pika_exporter/exporter/parser_test.go b/tools/pika_exporter/exporter/parser_test.go similarity index 100% rename from pika-tools/pika_exporter/exporter/parser_test.go rename to tools/pika_exporter/exporter/parser_test.go diff --git a/pika-tools/pika_exporter/exporter/pika.go b/tools/pika_exporter/exporter/pika.go similarity index 100% rename from pika-tools/pika_exporter/exporter/pika.go rename to tools/pika_exporter/exporter/pika.go diff --git a/pika-tools/pika_exporter/exporter/pika_test.go b/tools/pika_exporter/exporter/pika_test.go similarity index 100% rename from pika-tools/pika_exporter/exporter/pika_test.go rename to tools/pika_exporter/exporter/pika_test.go diff --git a/pika-tools/pika_exporter/exporter/test/test.go b/tools/pika_exporter/exporter/test/test.go similarity index 100% rename from pika-tools/pika_exporter/exporter/test/test.go rename to tools/pika_exporter/exporter/test/test.go diff --git a/pika-tools/pika_exporter/exporter/test/v3.3.5_master.go b/tools/pika_exporter/exporter/test/v3.3.5_master.go similarity index 100% rename from pika-tools/pika_exporter/exporter/test/v3.3.5_master.go rename to tools/pika_exporter/exporter/test/v3.3.5_master.go diff --git a/pika-tools/pika_exporter/exporter/test/v3.3.5_slave.go b/tools/pika_exporter/exporter/test/v3.3.5_slave.go similarity index 100% rename from pika-tools/pika_exporter/exporter/test/v3.3.5_slave.go rename to tools/pika_exporter/exporter/test/v3.3.5_slave.go diff --git a/pika-tools/pika_exporter/exporter/test/v3.4.2_master.go b/tools/pika_exporter/exporter/test/v3.4.2_master.go similarity index 100% rename from pika-tools/pika_exporter/exporter/test/v3.4.2_master.go rename to tools/pika_exporter/exporter/test/v3.4.2_master.go diff --git a/pika-tools/pika_exporter/exporter/test/v3.4.2_slave.go b/tools/pika_exporter/exporter/test/v3.4.2_slave.go similarity index 100% rename from pika-tools/pika_exporter/exporter/test/v3.4.2_slave.go rename to tools/pika_exporter/exporter/test/v3.4.2_slave.go diff --git a/pika-tools/pika_exporter/go.mod b/tools/pika_exporter/go.mod similarity index 100% rename from pika-tools/pika_exporter/go.mod rename to tools/pika_exporter/go.mod diff --git a/pika-tools/pika_exporter/go.sum b/tools/pika_exporter/go.sum similarity index 100% rename from pika-tools/pika_exporter/go.sum rename to tools/pika_exporter/go.sum diff --git a/pika-tools/pika_exporter/main.go b/tools/pika_exporter/main.go similarity index 100% rename from pika-tools/pika_exporter/main.go rename to tools/pika_exporter/main.go diff --git a/pika-tools/pika_exporter/version.go b/tools/pika_exporter/version.go similarity index 100% rename from pika-tools/pika_exporter/version.go rename to tools/pika_exporter/version.go diff --git a/pika-tools/pika_operator/.dockerignore b/tools/pika_operator/.dockerignore similarity index 100% rename from pika-tools/pika_operator/.dockerignore rename to tools/pika_operator/.dockerignore diff --git a/pika-tools/pika_operator/.gitignore b/tools/pika_operator/.gitignore similarity index 100% rename from pika-tools/pika_operator/.gitignore rename to tools/pika_operator/.gitignore diff --git a/pika-tools/pika_operator/Dockerfile b/tools/pika_operator/Dockerfile similarity index 100% rename from pika-tools/pika_operator/Dockerfile rename to tools/pika_operator/Dockerfile diff --git a/pika-tools/pika_operator/Makefile b/tools/pika_operator/Makefile similarity index 100% rename from pika-tools/pika_operator/Makefile rename to tools/pika_operator/Makefile diff --git a/pika-tools/pika_operator/PROJECT b/tools/pika_operator/PROJECT similarity index 100% rename from pika-tools/pika_operator/PROJECT rename to tools/pika_operator/PROJECT diff --git a/pika-tools/pika_operator/README.md b/tools/pika_operator/README.md similarity index 100% rename from pika-tools/pika_operator/README.md rename to tools/pika_operator/README.md diff --git a/pika-tools/pika_operator/api/v1alpha1/additional.go b/tools/pika_operator/api/v1alpha1/additional.go similarity index 100% rename from pika-tools/pika_operator/api/v1alpha1/additional.go rename to tools/pika_operator/api/v1alpha1/additional.go diff --git a/pika-tools/pika_operator/api/v1alpha1/groupversion_info.go b/tools/pika_operator/api/v1alpha1/groupversion_info.go similarity index 100% rename from pika-tools/pika_operator/api/v1alpha1/groupversion_info.go rename to tools/pika_operator/api/v1alpha1/groupversion_info.go diff --git a/pika-tools/pika_operator/api/v1alpha1/pika_types.go b/tools/pika_operator/api/v1alpha1/pika_types.go similarity index 100% rename from pika-tools/pika_operator/api/v1alpha1/pika_types.go rename to tools/pika_operator/api/v1alpha1/pika_types.go diff --git a/pika-tools/pika_operator/api/v1alpha1/zz_generated.deepcopy.go b/tools/pika_operator/api/v1alpha1/zz_generated.deepcopy.go similarity index 100% rename from pika-tools/pika_operator/api/v1alpha1/zz_generated.deepcopy.go rename to tools/pika_operator/api/v1alpha1/zz_generated.deepcopy.go diff --git a/pika-tools/pika_operator/config/crd/bases/pika.openatom.org_pikas.yaml b/tools/pika_operator/config/crd/bases/pika.openatom.org_pikas.yaml similarity index 100% rename from pika-tools/pika_operator/config/crd/bases/pika.openatom.org_pikas.yaml rename to tools/pika_operator/config/crd/bases/pika.openatom.org_pikas.yaml diff --git a/pika-tools/pika_operator/config/crd/kustomization.yaml b/tools/pika_operator/config/crd/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/crd/kustomization.yaml rename to tools/pika_operator/config/crd/kustomization.yaml diff --git a/pika-tools/pika_operator/config/crd/kustomizeconfig.yaml b/tools/pika_operator/config/crd/kustomizeconfig.yaml similarity index 100% rename from pika-tools/pika_operator/config/crd/kustomizeconfig.yaml rename to tools/pika_operator/config/crd/kustomizeconfig.yaml diff --git a/pika-tools/pika_operator/config/crd/patches/cainjection_in_pikas.yaml b/tools/pika_operator/config/crd/patches/cainjection_in_pikas.yaml similarity index 100% rename from pika-tools/pika_operator/config/crd/patches/cainjection_in_pikas.yaml rename to tools/pika_operator/config/crd/patches/cainjection_in_pikas.yaml diff --git a/pika-tools/pika_operator/config/crd/patches/webhook_in_pikas.yaml b/tools/pika_operator/config/crd/patches/webhook_in_pikas.yaml similarity index 100% rename from pika-tools/pika_operator/config/crd/patches/webhook_in_pikas.yaml rename to tools/pika_operator/config/crd/patches/webhook_in_pikas.yaml diff --git a/pika-tools/pika_operator/config/default/kustomization.yaml b/tools/pika_operator/config/default/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/default/kustomization.yaml rename to tools/pika_operator/config/default/kustomization.yaml diff --git a/pika-tools/pika_operator/config/default/manager_auth_proxy_patch.yaml b/tools/pika_operator/config/default/manager_auth_proxy_patch.yaml similarity index 100% rename from pika-tools/pika_operator/config/default/manager_auth_proxy_patch.yaml rename to tools/pika_operator/config/default/manager_auth_proxy_patch.yaml diff --git a/pika-tools/pika_operator/config/default/manager_config_patch.yaml b/tools/pika_operator/config/default/manager_config_patch.yaml similarity index 100% rename from pika-tools/pika_operator/config/default/manager_config_patch.yaml rename to tools/pika_operator/config/default/manager_config_patch.yaml diff --git a/pika-tools/pika_operator/config/manager/kustomization.yaml b/tools/pika_operator/config/manager/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/manager/kustomization.yaml rename to tools/pika_operator/config/manager/kustomization.yaml diff --git a/pika-tools/pika_operator/config/manager/manager.yaml b/tools/pika_operator/config/manager/manager.yaml similarity index 100% rename from pika-tools/pika_operator/config/manager/manager.yaml rename to tools/pika_operator/config/manager/manager.yaml diff --git a/pika-tools/pika_operator/config/manifests/kustomization.yaml b/tools/pika_operator/config/manifests/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/manifests/kustomization.yaml rename to tools/pika_operator/config/manifests/kustomization.yaml diff --git a/pika-tools/pika_operator/config/prometheus/kustomization.yaml b/tools/pika_operator/config/prometheus/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/prometheus/kustomization.yaml rename to tools/pika_operator/config/prometheus/kustomization.yaml diff --git a/pika-tools/pika_operator/config/prometheus/monitor.yaml b/tools/pika_operator/config/prometheus/monitor.yaml similarity index 100% rename from pika-tools/pika_operator/config/prometheus/monitor.yaml rename to tools/pika_operator/config/prometheus/monitor.yaml diff --git a/pika-tools/pika_operator/config/rbac/auth_proxy_client_clusterrole.yaml b/tools/pika_operator/config/rbac/auth_proxy_client_clusterrole.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/auth_proxy_client_clusterrole.yaml rename to tools/pika_operator/config/rbac/auth_proxy_client_clusterrole.yaml diff --git a/pika-tools/pika_operator/config/rbac/auth_proxy_role.yaml b/tools/pika_operator/config/rbac/auth_proxy_role.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/auth_proxy_role.yaml rename to tools/pika_operator/config/rbac/auth_proxy_role.yaml diff --git a/pika-tools/pika_operator/config/rbac/auth_proxy_role_binding.yaml b/tools/pika_operator/config/rbac/auth_proxy_role_binding.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/auth_proxy_role_binding.yaml rename to tools/pika_operator/config/rbac/auth_proxy_role_binding.yaml diff --git a/pika-tools/pika_operator/config/rbac/auth_proxy_service.yaml b/tools/pika_operator/config/rbac/auth_proxy_service.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/auth_proxy_service.yaml rename to tools/pika_operator/config/rbac/auth_proxy_service.yaml diff --git a/pika-tools/pika_operator/config/rbac/kustomization.yaml b/tools/pika_operator/config/rbac/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/kustomization.yaml rename to tools/pika_operator/config/rbac/kustomization.yaml diff --git a/pika-tools/pika_operator/config/rbac/leader_election_role.yaml b/tools/pika_operator/config/rbac/leader_election_role.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/leader_election_role.yaml rename to tools/pika_operator/config/rbac/leader_election_role.yaml diff --git a/pika-tools/pika_operator/config/rbac/leader_election_role_binding.yaml b/tools/pika_operator/config/rbac/leader_election_role_binding.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/leader_election_role_binding.yaml rename to tools/pika_operator/config/rbac/leader_election_role_binding.yaml diff --git a/pika-tools/pika_operator/config/rbac/pika_editor_role.yaml b/tools/pika_operator/config/rbac/pika_editor_role.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/pika_editor_role.yaml rename to tools/pika_operator/config/rbac/pika_editor_role.yaml diff --git a/pika-tools/pika_operator/config/rbac/pika_viewer_role.yaml b/tools/pika_operator/config/rbac/pika_viewer_role.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/pika_viewer_role.yaml rename to tools/pika_operator/config/rbac/pika_viewer_role.yaml diff --git a/pika-tools/pika_operator/config/rbac/role.yaml b/tools/pika_operator/config/rbac/role.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/role.yaml rename to tools/pika_operator/config/rbac/role.yaml diff --git a/pika-tools/pika_operator/config/rbac/role_binding.yaml b/tools/pika_operator/config/rbac/role_binding.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/role_binding.yaml rename to tools/pika_operator/config/rbac/role_binding.yaml diff --git a/pika-tools/pika_operator/config/rbac/service_account.yaml b/tools/pika_operator/config/rbac/service_account.yaml similarity index 100% rename from pika-tools/pika_operator/config/rbac/service_account.yaml rename to tools/pika_operator/config/rbac/service_account.yaml diff --git a/pika-tools/pika_operator/config/samples/kustomization.yaml b/tools/pika_operator/config/samples/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/samples/kustomization.yaml rename to tools/pika_operator/config/samples/kustomization.yaml diff --git a/pika-tools/pika_operator/config/samples/pika_v1alpha1_pika.yaml b/tools/pika_operator/config/samples/pika_v1alpha1_pika.yaml similarity index 100% rename from pika-tools/pika_operator/config/samples/pika_v1alpha1_pika.yaml rename to tools/pika_operator/config/samples/pika_v1alpha1_pika.yaml diff --git a/pika-tools/pika_operator/config/scorecard/bases/config.yaml b/tools/pika_operator/config/scorecard/bases/config.yaml similarity index 100% rename from pika-tools/pika_operator/config/scorecard/bases/config.yaml rename to tools/pika_operator/config/scorecard/bases/config.yaml diff --git a/pika-tools/pika_operator/config/scorecard/kustomization.yaml b/tools/pika_operator/config/scorecard/kustomization.yaml similarity index 100% rename from pika-tools/pika_operator/config/scorecard/kustomization.yaml rename to tools/pika_operator/config/scorecard/kustomization.yaml diff --git a/pika-tools/pika_operator/config/scorecard/patches/basic.config.yaml b/tools/pika_operator/config/scorecard/patches/basic.config.yaml similarity index 100% rename from pika-tools/pika_operator/config/scorecard/patches/basic.config.yaml rename to tools/pika_operator/config/scorecard/patches/basic.config.yaml diff --git a/pika-tools/pika_operator/config/scorecard/patches/olm.config.yaml b/tools/pika_operator/config/scorecard/patches/olm.config.yaml similarity index 100% rename from pika-tools/pika_operator/config/scorecard/patches/olm.config.yaml rename to tools/pika_operator/config/scorecard/patches/olm.config.yaml diff --git a/pika-tools/pika_operator/controllers/factory/default.go b/tools/pika_operator/controllers/factory/default.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/default.go rename to tools/pika_operator/controllers/factory/default.go diff --git a/pika-tools/pika_operator/controllers/factory/finalize/common.go b/tools/pika_operator/controllers/factory/finalize/common.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/finalize/common.go rename to tools/pika_operator/controllers/factory/finalize/common.go diff --git a/pika-tools/pika_operator/controllers/factory/k8stools/annotations.go b/tools/pika_operator/controllers/factory/k8stools/annotations.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/k8stools/annotations.go rename to tools/pika_operator/controllers/factory/k8stools/annotations.go diff --git a/pika-tools/pika_operator/controllers/factory/k8stools/service.go b/tools/pika_operator/controllers/factory/k8stools/service.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/k8stools/service.go rename to tools/pika_operator/controllers/factory/k8stools/service.go diff --git a/pika-tools/pika_operator/controllers/factory/k8stools/statefulset.go b/tools/pika_operator/controllers/factory/k8stools/statefulset.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/k8stools/statefulset.go rename to tools/pika_operator/controllers/factory/k8stools/statefulset.go diff --git a/pika-tools/pika_operator/controllers/factory/pika.go b/tools/pika_operator/controllers/factory/pika.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/pika.go rename to tools/pika_operator/controllers/factory/pika.go diff --git a/pika-tools/pika_operator/controllers/factory/pika_test.go b/tools/pika_operator/controllers/factory/pika_test.go similarity index 100% rename from pika-tools/pika_operator/controllers/factory/pika_test.go rename to tools/pika_operator/controllers/factory/pika_test.go diff --git a/pika-tools/pika_operator/controllers/pika_controller.go b/tools/pika_operator/controllers/pika_controller.go similarity index 100% rename from pika-tools/pika_operator/controllers/pika_controller.go rename to tools/pika_operator/controllers/pika_controller.go diff --git a/pika-tools/pika_operator/controllers/pika_controller_test.go b/tools/pika_operator/controllers/pika_controller_test.go similarity index 100% rename from pika-tools/pika_operator/controllers/pika_controller_test.go rename to tools/pika_operator/controllers/pika_controller_test.go diff --git a/pika-tools/pika_operator/controllers/suite_test.go b/tools/pika_operator/controllers/suite_test.go similarity index 100% rename from pika-tools/pika_operator/controllers/suite_test.go rename to tools/pika_operator/controllers/suite_test.go diff --git a/pika-tools/pika_operator/examples/pika-minikube/pika-cm.yaml b/tools/pika_operator/examples/pika-minikube/pika-cm.yaml similarity index 100% rename from pika-tools/pika_operator/examples/pika-minikube/pika-cm.yaml rename to tools/pika_operator/examples/pika-minikube/pika-cm.yaml diff --git a/pika-tools/pika_operator/examples/pika-minikube/pika-pika.yaml b/tools/pika_operator/examples/pika-minikube/pika-pika.yaml similarity index 100% rename from pika-tools/pika_operator/examples/pika-minikube/pika-pika.yaml rename to tools/pika_operator/examples/pika-minikube/pika-pika.yaml diff --git a/pika-tools/pika_operator/examples/pika-pvc/pika-cm.yaml b/tools/pika_operator/examples/pika-pvc/pika-cm.yaml similarity index 100% rename from pika-tools/pika_operator/examples/pika-pvc/pika-cm.yaml rename to tools/pika_operator/examples/pika-pvc/pika-cm.yaml diff --git a/pika-tools/pika_operator/examples/pika-pvc/pika-pika.yaml b/tools/pika_operator/examples/pika-pvc/pika-pika.yaml similarity index 100% rename from pika-tools/pika_operator/examples/pika-pvc/pika-pika.yaml rename to tools/pika_operator/examples/pika-pvc/pika-pika.yaml diff --git a/pika-tools/pika_operator/examples/pika-sample/pika-pika.yaml b/tools/pika_operator/examples/pika-sample/pika-pika.yaml similarity index 100% rename from pika-tools/pika_operator/examples/pika-sample/pika-pika.yaml rename to tools/pika_operator/examples/pika-sample/pika-pika.yaml diff --git a/pika-tools/pika_operator/go.mod b/tools/pika_operator/go.mod similarity index 100% rename from pika-tools/pika_operator/go.mod rename to tools/pika_operator/go.mod diff --git a/pika-tools/pika_operator/go.sum b/tools/pika_operator/go.sum similarity index 100% rename from pika-tools/pika_operator/go.sum rename to tools/pika_operator/go.sum diff --git a/pika-tools/pika_operator/hack/boilerplate.go.txt b/tools/pika_operator/hack/boilerplate.go.txt similarity index 100% rename from pika-tools/pika_operator/hack/boilerplate.go.txt rename to tools/pika_operator/hack/boilerplate.go.txt diff --git a/pika-tools/pika_operator/integration.mk b/tools/pika_operator/integration.mk similarity index 100% rename from pika-tools/pika_operator/integration.mk rename to tools/pika_operator/integration.mk diff --git a/pika-tools/pika_operator/main.go b/tools/pika_operator/main.go similarity index 100% rename from pika-tools/pika_operator/main.go rename to tools/pika_operator/main.go diff --git a/pika-tools/pika_operator/test/e2e/e2e.go b/tools/pika_operator/test/e2e/e2e.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/e2e.go rename to tools/pika_operator/test/e2e/e2e.go diff --git a/pika-tools/pika_operator/test/e2e/e2e_test.go b/tools/pika_operator/test/e2e/e2e_test.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/e2e_test.go rename to tools/pika_operator/test/e2e/e2e_test.go diff --git a/pika-tools/pika_operator/test/e2e/scaffold/k8s.go b/tools/pika_operator/test/e2e/scaffold/k8s.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/scaffold/k8s.go rename to tools/pika_operator/test/e2e/scaffold/k8s.go diff --git a/pika-tools/pika_operator/test/e2e/scaffold/kubectl.go b/tools/pika_operator/test/e2e/scaffold/kubectl.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/scaffold/kubectl.go rename to tools/pika_operator/test/e2e/scaffold/kubectl.go diff --git a/pika-tools/pika_operator/test/e2e/scaffold/scaffold.go b/tools/pika_operator/test/e2e/scaffold/scaffold.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/scaffold/scaffold.go rename to tools/pika_operator/test/e2e/scaffold/scaffold.go diff --git a/pika-tools/pika_operator/test/e2e/test-standalone/standalone.go b/tools/pika_operator/test/e2e/test-standalone/standalone.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/test-standalone/standalone.go rename to tools/pika_operator/test/e2e/test-standalone/standalone.go diff --git a/pika-tools/pika_operator/test/e2e/utils/cmd.go b/tools/pika_operator/test/e2e/utils/cmd.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/utils/cmd.go rename to tools/pika_operator/test/e2e/utils/cmd.go diff --git a/pika-tools/pika_operator/test/e2e/utils/cmd_test.go b/tools/pika_operator/test/e2e/utils/cmd_test.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/utils/cmd_test.go rename to tools/pika_operator/test/e2e/utils/cmd_test.go diff --git a/pika-tools/pika_operator/test/e2e/utils/tmp_file.go b/tools/pika_operator/test/e2e/utils/tmp_file.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/utils/tmp_file.go rename to tools/pika_operator/test/e2e/utils/tmp_file.go diff --git a/pika-tools/pika_operator/test/e2e/utils/tmp_file_test.go b/tools/pika_operator/test/e2e/utils/tmp_file_test.go similarity index 100% rename from pika-tools/pika_operator/test/e2e/utils/tmp_file_test.go rename to tools/pika_operator/test/e2e/utils/tmp_file_test.go diff --git a/tools/pika_to_txt/CMakeLists.txt b/tools/pika_to_txt/CMakeLists.txt new file mode 100644 index 0000000000..ed71b7d1a4 --- /dev/null +++ b/tools/pika_to_txt/CMakeLists.txt @@ -0,0 +1,21 @@ +set(WARNING_FLAGS "-W -Wextra -Wall -Wsign-compare \ +-Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ +-Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ +-Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers") + +set(CXXFLAGS "$(WARNING_FLAGS) -std=c++17 -g") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(pika_to_txt ${BASE_OBJS}) + +target_include_directories(pika_to_txt PRIVATE ${INSTALL_INCLUDEDIR} + PRIVATE ${PROJECT_SOURCE_DIR}) + +target_link_libraries(pika_to_txt storage net pstd ${ROCKSDB_LIBRARY} pthread ${SNAPPY_LIBRARY} + ${ZLIB_LIBRARY} ${BZ2_LIBRARY} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} rt) +set_target_properties(pika_to_txt PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/pika_to_txt/pika_to_txt.cc b/tools/pika_to_txt/pika_to_txt.cc similarity index 82% rename from pika-tools/pika_to_txt/pika_to_txt.cc rename to tools/pika_to_txt/pika_to_txt.cc index ef21635e1c..29bb33295b 100644 --- a/pika-tools/pika_to_txt/pika_to_txt.cc +++ b/tools/pika_to_txt/pika_to_txt.cc @@ -7,21 +7,21 @@ #include "iostream" #include "unistd.h" -#include "blackwidow/blackwidow.h" +#include "storage/storage.h" #include "progress_thread.h" #include "scan_thread.h" #include "write_thread.h" int32_t scan_batch_limit = 256; -std::string blackwidow_db_path; +std::string storage_db_path; std::string target_file; using std::chrono::high_resolution_clock; void PrintInfo(const std::time_t& now) { std::cout << "===================== Pika To Txt ======================" << std::endl; - std::cout << "Blackwidow_db_path : " << blackwidow_db_path << std::endl; + std::cout << "Blackwidow_db_path : " << storage_db_path << std::endl; std::cout << "Target_file_path : " << target_file << std::endl; std::cout << "Scan_batch_limit : " << scan_batch_limit << std::endl; std::cout << "Startup Time : " << asctime(localtime(&now)); @@ -33,7 +33,7 @@ void Usage() { std::cout << "\tPika_To_Txt reads kv data from Blackwidow DB and write to file" << std::endl; std::cout << "\t-h -- displays this help information and exits" << std::endl; std::cout << "\t-b -- the upper limit for each scan, default = 256" << std::endl; - std::cout << "\texample: ./pika_to_txt ./blackwidow_db ./data.txt" << std::endl; + std::cout << "\texample: ./pika_to_txt ./storage_db ./data.txt" << std::endl; } int main(int argc, char** argv) { @@ -42,7 +42,7 @@ int main(int argc, char** argv) { exit(-1); } - blackwidow_db_path = std::string(argv[1]); + storage_db_path = std::string(argv[1]); target_file = std::string(argv[2]); if (argc >= 5) { @@ -60,22 +60,22 @@ int main(int argc, char** argv) { // Init db rocksdb::Status status; - blackwidow::BlackwidowOptions bw_option; + storage::StorageOptions bw_option; bw_option.options.create_if_missing = true; bw_option.options.write_buffer_size = 256 * 1024 * 1024; // 256M bw_option.options.target_file_size_base = 20 * 1024 * 1024; // 20M - blackwidow::BlackWidow* blackwidow_db = new blackwidow::BlackWidow(); - if (blackwidow_db != NULL && (status = blackwidow_db->Open(bw_option, blackwidow_db_path)).ok()) { - std::cout << "Open BlackWidow db success..." << std::endl; + storage::Storage* storage_db = new storage::Storage(); + if (storage_db != NULL && (status = storage_db->Open(bw_option, storage_db_path)).ok()) { + std::cout << "Open Storage db success..." << std::endl; } else { - std::cout << "Open BlackWidow db failed..." << std::endl; + std::cout << "Open Storage db failed..." << std::endl; return -1; } std::cout << "Start migrating data from Blackwidow db to " << target_file << "..." << std::endl; WriteThread* write_thread = new WriteThread(target_file); - ScanThread* scan_thread = new ScanThread(write_thread, blackwidow_db); + ScanThread* scan_thread = new ScanThread(write_thread, storage_db); ProgressThread* progress_thread = new ProgressThread(scan_thread); write_thread->StartThread(); @@ -89,7 +89,7 @@ int main(int argc, char** argv) { write_thread->Stop(); write_thread->JoinThread(); - delete blackwidow_db; + delete storage_db; delete write_thread; delete scan_thread; delete progress_thread; diff --git a/pika-tools/pika_to_txt/progress_thread.cc b/tools/pika_to_txt/progress_thread.cc similarity index 100% rename from pika-tools/pika_to_txt/progress_thread.cc rename to tools/pika_to_txt/progress_thread.cc diff --git a/pika-tools/pika_to_txt/progress_thread.h b/tools/pika_to_txt/progress_thread.h similarity index 100% rename from pika-tools/pika_to_txt/progress_thread.h rename to tools/pika_to_txt/progress_thread.h diff --git a/pika-tools/pika_to_txt/scan_thread.cc b/tools/pika_to_txt/scan_thread.cc similarity index 80% rename from pika-tools/pika_to_txt/scan_thread.cc rename to tools/pika_to_txt/scan_thread.cc index 690e70cb8e..b5e0974d1e 100644 --- a/pika-tools/pika_to_txt/scan_thread.cc +++ b/tools/pika_to_txt/scan_thread.cc @@ -17,18 +17,18 @@ void* ScanThread::ThreadMain() { std::string pattern = "*"; std::string next_key; std::vector keys; - std::vector kvs; + std::vector kvs; do { - blackwidow_db_->PKScanRange(blackwidow::DataType::kStrings, key_start, key_end, "*", scan_batch_limit, &keys, &kvs, + storage_db_->PKScanRange(storage::DataType::kStrings, key_start, key_end, "*", scan_batch_limit, &keys, &kvs, &next_key); if (!kvs.empty()) { scan_number_ += kvs.size(); std::string data; for (const auto& kv : kvs) { - slash::PutFixed32(&data, kv.key.size()); + pstd::PutFixed32(&data, kv.key.size()); data.append(kv.key); - slash::PutFixed32(&data, kv.value.size()); + pstd::PutFixed32(&data, kv.value.size()); data.append(kv.value); } kvs.clear(); diff --git a/pika-tools/pika_to_txt/scan_thread.h b/tools/pika_to_txt/scan_thread.h similarity index 76% rename from pika-tools/pika_to_txt/scan_thread.h rename to tools/pika_to_txt/scan_thread.h index 5c8f676fff..b01e532173 100644 --- a/pika-tools/pika_to_txt/scan_thread.h +++ b/tools/pika_to_txt/scan_thread.h @@ -9,16 +9,16 @@ #include "iostream" #include "vector" -#include "blackwidow/blackwidow.h" +#include "storage/storage.h" #include "net/include/net_thread.h" -#include "slash/include/slash_coding.h" +#include "pstd/include/pstd_coding.h" #include "write_thread.h" class ScanThread : public net::Thread { public: - ScanThread(WriteThread* write_thread, blackwidow::BlackWidow* blackwidow_db) - : is_finish_(false), scan_number_(0), write_thread_(write_thread), blackwidow_db_(blackwidow_db) {} + ScanThread(WriteThread* write_thread, storage::Storage* storage_db) + : is_finish_(false), scan_number_(0), write_thread_(write_thread), storage_db_(storage_db) {} bool is_finish(); int32_t scan_number(); @@ -27,7 +27,7 @@ class ScanThread : public net::Thread { bool is_finish_; int32_t scan_number_; WriteThread* write_thread_; - blackwidow::BlackWidow* blackwidow_db_; + storage::Storage* storage_db_; }; #endif // INCLUDE_SCAN_THREAD_H_ diff --git a/pika-tools/pika_to_txt/write_thread.cc b/tools/pika_to_txt/write_thread.cc similarity index 100% rename from pika-tools/pika_to_txt/write_thread.cc rename to tools/pika_to_txt/write_thread.cc diff --git a/pika-tools/pika_to_txt/write_thread.h b/tools/pika_to_txt/write_thread.h similarity index 85% rename from pika-tools/pika_to_txt/write_thread.h rename to tools/pika_to_txt/write_thread.h index 84cda71889..24e5f95b20 100644 --- a/pika-tools/pika_to_txt/write_thread.h +++ b/tools/pika_to_txt/write_thread.h @@ -9,7 +9,7 @@ #include #include "net/include/net_thread.h" -#include "slash/include/slash_mutex.h" +#include "pstd/include/pstd_mutex.h" class WriteThread : public net::Thread { public: @@ -21,9 +21,9 @@ class WriteThread : public net::Thread { void* ThreadMain() override; bool should_stop_; std::string file_name_; - slash::CondVar rsignal_; - slash::CondVar wsignal_; - slash::Mutex data_queue_mutex_; + pstd::CondVar rsignal_; + pstd::CondVar wsignal_; + pstd::Mutex data_queue_mutex_; std::queue data_queue_; }; diff --git a/tools/rdb_to_pika/CMakeLists.txt b/tools/rdb_to_pika/CMakeLists.txt new file mode 100644 index 0000000000..0dddd89aa9 --- /dev/null +++ b/tools/rdb_to_pika/CMakeLists.txt @@ -0,0 +1,14 @@ +set(CXXFLAGS "-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DOS_LINUX -Wall -W -DDEBUG -g -O2 -D__XDEBUG__ -fPIC -Wno-unused-function -std=c++17") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(redisConn ${BASE_OBJS}) + +target_include_directories(redisConn PRIVATE ${INSTALL_INCLUDEDIR}) + +target_link_libraries(redisConn ${HIREDIS_LIBRARY}) +set_target_properties(redisConn PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/rdb_to_pika/README.md b/tools/rdb_to_pika/README.md similarity index 100% rename from pika-tools/rdb_to_pika/README.md rename to tools/rdb_to_pika/README.md diff --git a/tools/rdb_to_pika/install_rdb_tools.sh b/tools/rdb_to_pika/install_rdb_tools.sh new file mode 100755 index 0000000000..211dacdc91 --- /dev/null +++ b/tools/rdb_to_pika/install_rdb_tools.sh @@ -0,0 +1,5 @@ +#!/bin/bash +#author justforfun + +# redis-rdb-tools only support python2 +python2 -m pip install rdbtools python-lzf \ No newline at end of file diff --git a/pika-tools/rdb_to_pika/protocoltopika.cc b/tools/rdb_to_pika/protocoltopika.cc similarity index 98% rename from pika-tools/rdb_to_pika/protocoltopika.cc rename to tools/rdb_to_pika/protocoltopika.cc index 955a133f4a..1a19d2b88d 100644 --- a/pika-tools/rdb_to_pika/protocoltopika.cc +++ b/tools/rdb_to_pika/protocoltopika.cc @@ -3,7 +3,7 @@ #include #include #include -#include "hiredis.h" +#include "hiredis-vip/hiredis.h" void Usage() { std::cout << "Usage:" << std::endl; diff --git a/pika-tools/rdb_to_pika/trans_rdb_to_pro.sh b/tools/rdb_to_pika/trans_rdb_to_pro.sh similarity index 100% rename from pika-tools/rdb_to_pika/trans_rdb_to_pro.sh rename to tools/rdb_to_pika/trans_rdb_to_pro.sh diff --git a/pika-tools/readme.md b/tools/readme.md similarity index 100% rename from pika-tools/readme.md rename to tools/readme.md diff --git a/tools/txt_to_pika/CMakeLists.txt b/tools/txt_to_pika/CMakeLists.txt new file mode 100644 index 0000000000..dbceff0249 --- /dev/null +++ b/tools/txt_to_pika/CMakeLists.txt @@ -0,0 +1,21 @@ +set(WARNING_FLAGS "-W -Wextra -Wall -Wsign-compare \ +-Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ +-Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ +-Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers") + +set(CXXFLAGS "$(WARNING_FLAGS) -std=c++17 -g") + +set(SRC_DIR .) +aux_source_directory(${SRC_DIR} BASE_OBJS) + +add_executable(txt_to_pika ${BASE_OBJS}) + +target_include_directories(txt_to_pika PRIVATE ${INSTALL_INCLUDEDIR} + PRIVATE ${PROJECT_SOURCE_DIR}) + +target_link_libraries(txt_to_pika storage net pstd ${ROCKSDB_LIBRARY} pthread ${SNAPPY_LIBRARY} + ${ZLIB_LIBRARY} ${BZ2_LIBRARY} ${GLOG_LIBRARY} ${GFLAGS_LIBRARY} rt) +set_target_properties(txt_to_pika PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} + CMAKE_COMPILER_IS_GNUCXX TRUE + COMPILE_FLAGS ${CXXFLAGS}) \ No newline at end of file diff --git a/pika-tools/txt_to_pika/scan.cc b/tools/txt_to_pika/scan.cc similarity index 100% rename from pika-tools/txt_to_pika/scan.cc rename to tools/txt_to_pika/scan.cc diff --git a/pika-tools/txt_to_pika/scan.h b/tools/txt_to_pika/scan.h similarity index 100% rename from pika-tools/txt_to_pika/scan.h rename to tools/txt_to_pika/scan.h diff --git a/pika-tools/txt_to_pika/sender.cc b/tools/txt_to_pika/sender.cc similarity index 95% rename from pika-tools/txt_to_pika/sender.cc rename to tools/txt_to_pika/sender.cc index ad699a1a72..9e6818332f 100644 --- a/pika-tools/txt_to_pika/sender.cc +++ b/tools/txt_to_pika/sender.cc @@ -17,7 +17,7 @@ void SenderThread::ConnectPika() { // Connect to redis cli_ = net::NewRedisCli(); cli_->set_connect_timeout(1000); - slash::Status s = cli_->Connect(ip_, port_); + pstd::Status s = cli_->Connect(ip_, port_); if (!s.ok()) { cli_ = NULL; log_info("Can not connect to %s:%d: %s", ip_.data(), port_, s.ToString().data()); @@ -34,7 +34,7 @@ void SenderThread::ConnectPika() { argv.push_back("AUTH"); argv.push_back(password_); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -60,7 +60,7 @@ void SenderThread::ConnectPika() { argv.push_back("PING"); net::SerializeRedisCommand(argv, &cmd); - slash::Status s = cli_->Send(&cmd); + pstd::Status s = cli_->Send(&cmd); if (s.ok()) { s = cli_->Recv(&resp); @@ -101,7 +101,7 @@ void SenderThread::LoadCmd(const std::string& cmd) { void SenderThread::SendCommand(std::string& command) { // Send command - slash::Status s = cli_->Send(&command); + pstd::Status s = cli_->Send(&command); if (!s.ok()) { elements_--; LoadCmd(command); diff --git a/pika-tools/txt_to_pika/sender.h b/tools/txt_to_pika/sender.h similarity index 83% rename from pika-tools/txt_to_pika/sender.h rename to tools/txt_to_pika/sender.h index 66d670d41d..a5b99737ac 100644 --- a/pika-tools/txt_to_pika/sender.h +++ b/tools/txt_to_pika/sender.h @@ -8,7 +8,7 @@ #include "net/include/bg_thread.h" #include "net/include/net_cli.h" #include "net/include/redis_cli.h" -#include "slash/include/xdebug.h" +#include "pstd/include/xdebug.h" class SenderThread : public net::Thread { public: @@ -26,17 +26,17 @@ class SenderThread : public net::Thread { void SendCommand(std::string& command); int QueueSize() { - slash::MutexLock l(&cmd_mutex_); + pstd::MutexLock l(&cmd_mutex_); int len = cmd_queue_.size(); return len; } void ConnectPika(); private: - net::PinkCli* cli_; - slash::CondVar rsignal_; - slash::CondVar wsignal_; - slash::Mutex cmd_mutex_; + net::NetCli* cli_; + pstd::CondVar rsignal_; + pstd::CondVar wsignal_; + pstd::Mutex cmd_mutex_; std::queue cmd_queue_; std::string ip_; int port_; diff --git a/pika-tools/txt_to_pika/txt_to_pika.cc b/tools/txt_to_pika/txt_to_pika.cc similarity index 100% rename from pika-tools/txt_to_pika/txt_to_pika.cc rename to tools/txt_to_pika/txt_to_pika.cc