diff --git a/.github/workflows/pika.yml b/.github/workflows/pika.yml index b8fddf5919..007f73aeed 100644 --- a/.github/workflows/pika.yml +++ b/.github/workflows/pika.yml @@ -84,51 +84,32 @@ jobs: chmod +x integrate_test.sh sh integrate_test.sh - build_on_rocky: + + build_on_centos: runs-on: ubuntu-latest container: - image: rockylinux:9 + image: cheniujh/pika-centos7-ci:v5 steps: - - name: Install deps - run: | - dnf update -y - dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.19 - - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v1 with: fetch-depth: 0 - name: Configure CMake run: | - source /opt/rh/gcc-toolset-13/enable - cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address . - - - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/deps - key: ${{ runner.os }}-rocky-deps-${{ hashFiles('**/CMakeLists.txt') }} - - - uses: actions/cache@v3 - with: - path: ${{ github.workspace }}/buildtrees - key: ${{ runner.os }}-rocky-buildtrees-${{ hashFiles('**/CMakeLists.txt') }} + source /opt/rh/devtoolset-10/enable + cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address - name: Build run: | - source /opt/rh/gcc-toolset-13/enable + source /opt/rh/devtoolset-10/enable cmake --build build --config ${{ env.BUILD_TYPE }} - name: Cleanup run: | rm -rf ./deps - rm -rf ./buildtrees + rm -rf ./buildtrees - name: Test working-directory: ${{ github.workspace }}/build @@ -145,7 +126,6 @@ jobs: ../tests/integration/start_master_and_slave.sh chmod +x ../tests/integration/start_codis.sh ../tests/integration/start_codis.sh - - name: Run Go E2E Tests working-directory: ${{ github.workspace }}/build run: | @@ -155,8 +135,10 @@ jobs: chmod +x integrate_test.sh sh integrate_test.sh + build_on_macos: - runs-on: macos-12 + + runs-on: macos-13 steps: - uses: actions/checkout@v4 @@ -169,7 +151,7 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@v1.2.13 with: - key: macos-12 + key: macos-13 - name: Install Deps run: | @@ -188,8 +170,10 @@ jobs: - name: Cleanup run: | - rm -rf ./deps - rm -rf ./buildtrees + cp deps/lib/libz.1.dylib . + cp deps/lib/libz.1.dylib tests/integration/ + rm -rf ./deps + rm -rf ./buildtree - name: Test working-directory: ${{ github.workspace }}/build @@ -201,21 +185,21 @@ jobs: ./pikatests.sh all clean - name: Start codis, pika master and pika slave - working-directory: ${{ github.workspace }}/build + working-directory: ${{ github.workspace }} run: | - chmod +x ../tests/integration/start_master_and_slave.sh - ../tests/integration/start_master_and_slave.sh - chmod +x ../tests/integration/start_codis.sh - ../tests/integration/start_codis.sh + cd tests/integration/ + chmod +x start_master_and_slave.sh + ./start_master_and_slave.sh + chmod +x start_codis.sh + ./start_codis.sh + - name: Run Go E2E Tests - working-directory: ${{ github.workspace }}/build + working-directory: ${{ github.workspace }} run: | - cd ../tools/pika_keys_analysis/ - go test -v ./... - cd ../../tests/integration/ + cd tests/integration/ chmod +x integrate_test.sh - sh integrate_test.sh + # sh integrate_test.sh build_pika_image: name: Build Pika Docker image diff --git a/.gitmodule b/.gitmodule new file mode 100644 index 0000000000..425491b8f6 --- /dev/null +++ b/.gitmodule @@ -0,0 +1,11 @@ +[submodule "tools/pika_migrate/third/blackwidow"] + url = https://github.com/Qihoo360/blackwidow.git +[submodule "tools/pika_migrate/third/slash"] + url = https://github.com/Qihoo360/slash.git +[submodule "tools/pika_migrate/third/pink"] + url = https://github.com/Qihoo360/pink.git +[submodule "tools/pika_migrate/third/glog"] + url = https://github.com/Qihoo360/glog.git +[submodule "tools/pika_migrate/third/rocksdb"] + url = https://github.com/facebook/rocksdb.git + diff --git a/codis/config/proxy.toml b/codis/config/proxy.toml index 5f46885413..96269d16b5 100644 --- a/codis/config/proxy.toml +++ b/codis/config/proxy.toml @@ -105,10 +105,10 @@ session_break_on_failure = false # Slowlog-log-slower-than(us), from receive command to send response, 0 is allways print slow log slowlog_log_slower_than = 100000 -# quick command list e.g. get, set -quick_cmd_list = "" -# slow command list e.g. hgetall, mset -slow_cmd_list = "" +# quick command list +quick_cmd_list = "get,set" +# slow command list +slow_cmd_list = "mget, mset" # Set metrics server (such as http://localhost:28000), proxy will report json formatted metrics to specified server in a predefined period. metrics_report_server = "" diff --git a/pikatests.sh b/pikatests.sh index 7d3163c40e..21b7c864fa 100755 --- a/pikatests.sh +++ b/pikatests.sh @@ -45,6 +45,7 @@ function setup_pika_bin { exit 1 fi cp $PIKA_BIN src/redis-server + cp $PIKA_BIN tests/integration/pika cp tests/conf/pika.conf tests/assets/default.conf } diff --git a/src/pika_repl_bgworker.cc b/src/pika_repl_bgworker.cc index 05a7915a0f..1e12ffdf0a 100644 --- a/src/pika_repl_bgworker.cc +++ b/src/pika_repl_bgworker.cc @@ -267,7 +267,7 @@ void PikaReplBgWorker::WriteDBInSyncWay(const std::shared_ptr& c_ptr) { if (duration > g_pika_conf->slowlog_slower_than()) { g_pika_server->SlowlogPushEntry(argv, start_time, duration); if (g_pika_conf->slowlog_write_errorlog()) { - LOG(ERROR) << "command: " << argv[0] << ", start_time(s): " << start_time << ", duration(us): " << duration; + LOG(INFO) << "command: " << argv[0] << ", start_time(s): " << start_time << ", duration(us): " << duration; } } } diff --git a/tools/pika_migrate/.gitattributes b/tools/pika_migrate/.gitattributes new file mode 100644 index 0000000000..3ff2dd9c7b --- /dev/null +++ b/tools/pika_migrate/.gitattributes @@ -0,0 +1 @@ +tests/* linguist-vendored diff --git a/tools/pika_migrate/.gitignore b/tools/pika_migrate/.gitignore new file mode 100644 index 0000000000..5d21ed9c5a --- /dev/null +++ b/tools/pika_migrate/.gitignore @@ -0,0 +1,49 @@ +# Compiled Object files +*.slo +*.lo +*.o +*.obj +*pb.cc +*pb.h + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Log path +make_config.mk +log/ +lib/ +tools/ +output/ + +# DB +db/ +dump/ + +# third party +gdb.txt +tags + +make_config.mk +src/*.d +src/build_version.cc diff --git a/tools/pika_migrate/.travis.yml b/tools/pika_migrate/.travis.yml new file mode 100644 index 0000000000..cdc94a458c --- /dev/null +++ b/tools/pika_migrate/.travis.yml @@ -0,0 +1,26 @@ +sudo: required +dist: trusty +language: cpp + +os: + - linux + +env: + global: + - PROTOBUF_VERSION=2.5.0 + +install: + - wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_VERSION/protobuf-$PROTOBUF_VERSION.tar.bz2 + - tar xvf protobuf-$PROTOBUF_VERSION.tar.bz2 + - ( cd protobuf-$PROTOBUF_VERSION && ./configure --prefix=/usr && make && sudo make install ) + +addons: + apt: + packages: ['libsnappy-dev', 'libprotobuf-dev', 'libgoogle-glog-dev'] + +compiler: + - gcc + +language: cpp + +script: make diff --git a/tools/pika_migrate/CODE_OF_CONDUCT.md b/tools/pika_migrate/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..f50b192489 --- /dev/null +++ b/tools/pika_migrate/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at g-infra-bada@360.cn. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/tools/pika_migrate/CONTRIBUTING.md b/tools/pika_migrate/CONTRIBUTING.md new file mode 100644 index 0000000000..4cf487071f --- /dev/null +++ b/tools/pika_migrate/CONTRIBUTING.md @@ -0,0 +1 @@ +### Contributing to pika diff --git a/tools/pika_migrate/Dockerfile b/tools/pika_migrate/Dockerfile new file mode 100644 index 0000000000..3fc690c3e7 --- /dev/null +++ b/tools/pika_migrate/Dockerfile @@ -0,0 +1,21 @@ +FROM centos:latest +MAINTAINER left2right + +RUN rpm -ivh https://mirrors.ustc.edu.cn/epel/epel-release-latest-7.noarch.rpm && \ + yum -y update && \ + yum -y install snappy-devel && \ + yum -y install protobuf-devel && \ + yum -y install gflags-devel && \ + yum -y install glog-devel && \ + yum -y install gcc-c++ && \ + yum -y install make && \ + yum -y install which && \ + yum -y install git + +ENV PIKA /pika +COPY . ${PIKA} +WORKDIR ${PIKA} +RUN make +ENV PATH ${PIKA}/output/bin:${PATH} + +WORKDIR ${PIKA}/output diff --git a/tools/pika_migrate/LICENSE b/tools/pika_migrate/LICENSE new file mode 100644 index 0000000000..93ce6ffc0b --- /dev/null +++ b/tools/pika_migrate/LICENSE @@ -0,0 +1,10 @@ + The MIT License (MIT) + +Copyright © 2018 + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tools/pika_migrate/Makefile b/tools/pika_migrate/Makefile new file mode 100644 index 0000000000..be7e8191f5 --- /dev/null +++ b/tools/pika_migrate/Makefile @@ -0,0 +1,245 @@ +CLEAN_FILES = # deliberately empty, so we can append below. +CXX=g++ +PLATFORM_LDFLAGS= -lpthread -lrt +PLATFORM_CXXFLAGS= -std=c++11 -fno-builtin-memcmp -msse -msse4.2 +PROFILING_FLAGS=-pg +OPT= +LDFLAGS += -Wl,-rpath=$(RPATH) + +# DEBUG_LEVEL can have two values: +# * DEBUG_LEVEL=2; this is the ultimate debug mode. It will compile pika +# without any optimizations. To compile with level 2, issue `make dbg` +# * DEBUG_LEVEL=0; this is the debug level we use for release. If you're +# running pika in production you most definitely want to compile pika +# with debug level 0. To compile with level 0, run `make`, + +# Set the default DEBUG_LEVEL to 0 +DEBUG_LEVEL?=0 + +ifeq ($(MAKECMDGOALS),dbg) + DEBUG_LEVEL=2 +endif + +ifneq ($(DISABLE_UPDATE_SB), 1) +$(info updating submodule) +dummy := $(shell (git submodule init && git submodule update)) +endif + +# compile with -O2 if debug level is not 2 +ifneq ($(DEBUG_LEVEL), 2) +OPT += -O2 -fno-omit-frame-pointer +# if we're compiling for release, compile without debug code (-DNDEBUG) and +# don't treat warnings as errors +OPT += -DNDEBUG +DISABLE_WARNING_AS_ERROR=1 +# Skip for archs that don't support -momit-leaf-frame-pointer +ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1)) +OPT += -momit-leaf-frame-pointer +endif +else +$(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) +OPT += $(PROFILING_FLAGS) +DEBUG_SUFFIX = "_debug" +endif + +# Link tcmalloc if exist +dummy := $(shell ("$(CURDIR)/detect_environment" "$(CURDIR)/make_config.mk")) +include make_config.mk +CLEAN_FILES += $(CURDIR)/make_config.mk +PLATFORM_LDFLAGS += $(TCMALLOC_LDFLAGS) +PLATFORM_LDFLAGS += $(ROCKSDB_LDFLAGS) +PLATFORM_CXXFLAGS += $(TCMALLOC_EXTENSION_FLAGS) + +# ---------------------------------------------- +OUTPUT = $(CURDIR)/output +THIRD_PATH = $(CURDIR)/third +SRC_PATH = $(CURDIR)/src + +# ----------------Dependences------------------- + +ifndef SLASH_PATH +SLASH_PATH = $(THIRD_PATH)/slash +endif +SLASH = $(SLASH_PATH)/slash/lib/libslash$(DEBUG_SUFFIX).a + +ifndef PINK_PATH +PINK_PATH = $(THIRD_PATH)/pink +endif +PINK = $(PINK_PATH)/pink/lib/libpink$(DEBUG_SUFFIX).a + +ifndef ROCKSDB_PATH +ROCKSDB_PATH = $(THIRD_PATH)/rocksdb +endif +ROCKSDB = $(ROCKSDB_PATH)/librocksdb$(DEBUG_SUFFIX).a + +ifndef GLOG_PATH +GLOG_PATH = $(THIRD_PATH)/glog +endif + +ifndef BLACKWIDOW_PATH +BLACKWIDOW_PATH = $(THIRD_PATH)/blackwidow +endif +BLACKWIDOW = $(BLACKWIDOW_PATH)/lib/libblackwidow$(DEBUG_SUFFIX).a + + +ifeq ($(360), 1) +GLOG := $(GLOG_PATH)/.libs/libglog.a +endif + +INCLUDE_PATH = -I. \ + -I$(SLASH_PATH) \ + -I$(PINK_PATH) \ + -I$(BLACKWIDOW_PATH)/include \ + -I$(BLACKWIDOW_PATH)\ + -I$(ROCKSDB_PATH) \ + -I$(ROCKSDB_PATH)/include \ + -I$(GLOG_PATH)/src \ + +LIB_PATH = -L./ \ + -L$(SLASH_PATH)/slash/lib \ + -L$(PINK_PATH)/pink/lib \ + -L$(BLACKWIDOW_PATH)/lib \ + -L$(ROCKSDB_PATH) \ + -L$(GLOG_PATH)/.libs \ + +LDFLAGS += $(LIB_PATH) \ + -lpink$(DEBUG_SUFFIX) \ + -lslash$(DEBUG_SUFFIX) \ + -lblackwidow$(DEBUG_SUFFIX) \ + -lrocksdb$(DEBUG_SUFFIX) \ + -lglog \ + -lprotobuf \ + -static-libstdc++ \ + +# ---------------End Dependences---------------- + +VERSION_CC=$(SRC_PATH)/build_version.cc +LIB_SOURCES := $(VERSION_CC) \ + $(filter-out $(VERSION_CC), $(wildcard $(SRC_PATH)/*.cc)) + +PIKA_PROTO := $(wildcard $(SRC_PATH)/*.proto) +PIKA_PROTO_GENS:= $(PIKA_PROTO:%.proto=%.pb.h) $(PIKA_PROTO:%.proto=%.pb.cc) + + +#----------------------------------------------- + +AM_DEFAULT_VERBOSITY = 0 + +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $(notdir $@); +am__v_GEN_1 = +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +am__v_at_1 = + +AM_V_CC = $(am__v_CC_$(V)) +am__v_CC_ = $(am__v_CC_$(AM_DEFAULT_VERBOSITY)) +am__v_CC_0 = @echo " CC " $(notdir $@); +am__v_CC_1 = +CCLD = $(CC) +LINK = $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_$(V)) +am__v_CCLD_ = $(am__v_CCLD_$(AM_DEFAULT_VERBOSITY)) +am__v_CCLD_0 = @echo " CCLD " $(notdir $@); +am__v_CCLD_1 = + +AM_LINK = $(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + +CXXFLAGS += -g + +# This (the first rule) must depend on "all". +default: all + +WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ + -Wno-unused-parameter -Woverloaded-virtual \ + -Wnon-virtual-dtor -Wno-missing-field-initializers + +ifndef DISABLE_WARNING_AS_ERROR + WARNING_FLAGS += -Werror +endif + +CXXFLAGS += $(WARNING_FLAGS) $(INCLUDE_PATH) $(PLATFORM_CXXFLAGS) $(OPT) + +LDFLAGS += $(PLATFORM_LDFLAGS) + +date := $(shell date +%F) +git_sha := $(shell git rev-parse HEAD 2>/dev/null) +gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ src/build_version.cc.in +# Record the version of the source that we are compiling. +# We keep a record of the git revision in this file. It is then built +# as a regular source file as part of the compilation process. +# One can run "strings executable_filename | grep _build_" to find +# the version of the source that we used to build the executable file. +CLEAN_FILES += $(SRC_PATH)/build_version.cc + +$(SRC_PATH)/build_version.cc: FORCE + $(AM_V_GEN)rm -f $@-t + $(AM_V_at)$(gen_build_version) > $@-t + $(AM_V_at)if test -f $@; then \ + cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \ + else mv -f $@-t $@; fi +FORCE: + +LIBOBJECTS = $(LIB_SOURCES:.cc=.o) +PROTOOBJECTS = $(PIKA_PROTO:.proto=.pb.o) + +# if user didn't config LIBNAME, set the default +ifeq ($(BINNAME),) +# we should only run pika in production with DEBUG_LEVEL 0 +BINNAME=pika$(DEBUG_SUFFIX) +endif +BINARY = ${BINNAME} + +.PHONY: distclean clean dbg all + +%.pb.h %.pb.cc: %.proto + $(AM_V_GEN)protoc --proto_path=$(SRC_PATH) --cpp_out=$(SRC_PATH) $< + +%.o: %.cc + $(AM_V_CC)$(CXX) $(CXXFLAGS) -c $< -o $@ + +proto: $(PIKA_PROTO_GENS) + +all: $(BINARY) + +dbg: $(BINARY) + +$(BINARY): $(SLASH) $(PINK) $(ROCKSDB) $(BLACKWIDOW) $(GLOG) $(PROTOOBJECTS) $(LIBOBJECTS) + $(AM_V_at)rm -f $@ + $(AM_V_at)$(AM_LINK) + $(AM_V_at)rm -rf $(OUTPUT) + $(AM_V_at)mkdir -p $(OUTPUT)/bin + $(AM_V_at)mv $@ $(OUTPUT)/bin + $(AM_V_at)cp -r $(CURDIR)/conf $(OUTPUT) + + +$(SLASH): + $(AM_V_at)make -C $(SLASH_PATH)/slash/ DEBUG_LEVEL=$(DEBUG_LEVEL) + +$(PINK): + $(AM_V_at)make -C $(PINK_PATH)/pink/ DEBUG_LEVEL=$(DEBUG_LEVEL) NO_PB=0 SLASH_PATH=$(SLASH_PATH) + +$(ROCKSDB): + $(AM_V_at)make -j $(PROCESSOR_NUMS) -C $(ROCKSDB_PATH)/ static_lib DISABLE_JEMALLOC=1 DEBUG_LEVEL=$(DEBUG_LEVEL) + +$(BLACKWIDOW): + $(AM_V_at)make -C $(BLACKWIDOW_PATH) ROCKSDB_PATH=$(ROCKSDB_PATH) SLASH_PATH=$(SLASH_PATH) DEBUG_LEVEL=$(DEBUG_LEVEL) + +$(GLOG): + cd $(THIRD_PATH)/glog; if [ ! -f ./Makefile ]; then ./configure --disable-shared; fi; make; echo '*' > $(CURDIR)/third/glog/.gitignore; + +clean: + rm -rf $(OUTPUT) + rm -rf $(CLEAN_FILES) + rm -rf $(PIKA_PROTO_GENS) + find $(SRC_PATH) -name "*.[oda]*" -exec rm -f {} \; + find $(SRC_PATH) -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; + +distclean: clean + make -C $(PINK_PATH)/pink/ SLASH_PATH=$(SLASH_PATH) clean + make -C $(SLASH_PATH)/slash/ clean + make -C $(BLACKWIDOW_PATH)/ clean + make -C $(ROCKSDB_PATH)/ clean +# make -C $(GLOG_PATH)/ clean diff --git a/tools/pika_migrate/README.md b/tools/pika_migrate/README.md new file mode 100644 index 0000000000..01afc5bcbb --- /dev/null +++ b/tools/pika_migrate/README.md @@ -0,0 +1,76 @@ + + +## 适用版本 + +适用 PIKA 3.2.0及以上版本,单机模式且只使用了单 DB。若 PIKA 版本低于3.2.0,需将内核版本升级至 3.2.0。具体信息,请参见 升级 PIKA 内核版本至3.2.0。 +### 开发背景: +之前Pika项目官方提供的pika\_to\_redis工具仅支持离线将Pika的DB中的数据迁移到Pika、Redis, 且无法增量同步, 该工具实际上就是一个特殊的Pika, 只不过成为从库之后, 内部会将从主库获取到的数据转发给Redis,同时并支持增量同步, 实现热迁功能. + +## 迁移原理 + +将 PIKA 中的数据在线迁移到 Redis,并支持全量和增量同步。使用 pika-migrate 工具,将工具虚拟为 PIKA 的从库,然后从主库获取到数据转发给 Redis,同时支持增量同步,实现在线热迁的功能。 +1. pika-migrate 通过 dbsync 请求获取主库全量 DB 数据,以及当前 DB 数据所对应的 binlog 点位。 +2. 获取到主库当前全量 DB 数据之后,扫描 DB,将 DB 中的数据打包转发给 Redis。 +3. 通过之前获取的 binlog 的点位向主库进行增量同步, 在增量同步的过程中,将从主库获取到的 binlog 重组成 Redis 命令,转发给 Redis。 + + +## 注意事项 + +PIKA 支持不同数据结构采用同名 Key,但是 Redis 不⽀持,所以在有同 Key 数据的场景下,以第⼀个迁移到 Redis 数据结构为准,其他同名 Key 的数据结构会丢失。 +该工具只支持热迁移单机模式下,并且只采⽤单 DB 版本的 PIKA,如果是集群模式,或者是多 DB 场景,⼯具会报错并且退出。 +为了避免由于主库 binlog 被清理导致该⼯具触发多次全量同步向 Redis 写入脏数据,工具自身做了保护,在第⼆次触发全量同步时会报错退出。 + +## 编译步骤 +```shell +# 若third目录中子仓库为空,需要进入工具根目录更新submodule +git submodule update --init --recursive +# 编译 +make +``` + +### 编译备注 + +1.如果rocksdb编译失败,请先按照[此处](https://github.com/facebook/rocksdb/blob/004237e62790320d8e630456cbeb6f4a1f3579c2/INSTALL.md) 的步骤准备环境 +2.若类似为: +```shell +error: implicitly-declared 'constexpr rocksdb::FileDescriptor::FileDescriptor(const rocksdb::FileDescriptor&)' is deprecated [-Werror=deprecated-copy] +``` +可以修改tools/pika_migrate/third/rocksdb目录下的makefile:
WARNING_FLAGS = -Wno-missing-field-initializers +-Wno-unused-parameter + +## 迁移步骤 + +1. 在 PIKA 主库上执行如下命令,让 PIKA 主库保留10000个 binlog 文件。 + +```shell +config set expire-logs-nums 10000 +``` + +```text +说明: +pika-port 将全量数据写入到 Redis 这段时间可能耗时很长,而导致主库原先 binlog 点位被清理。需要在 PIKA 主库上保留10000个 binlog ⽂件,确保后续该⼯具请求增量同步的时候,对应的 binlog 文件还存在。 +binlog 文件占用磁盘空间,可以根据实际情况确定保留 binlog 的数量。 +``` + +2. 修改迁移工具的配置文件 pika.conf 中的如下参数。 + ![img.png](img.png) + + target-redis-host:指定 Redis 的 IP 地址。 + target-redis-port:指定 Redis 的端口号。 + target-redis-pwd:指定 Redis 默认账号的密码。 + sync-batch-num:指定 pika-migrate 接收到主库的 sync-batch-num 个数据⼀起打包发送给 Redis,提升转发效率。 + redis-sender-num:指定 redis-sender-num 个线程用于转发数据包。转发命令通过 Key 的哈希值将数据分配到不同的线程发送,无需担心多线程发送导致数据错乱的问题。 +3. 在工具包的路径下执行如下命令,启动 pika-migrate 工具,并查看回显信息。 +```shell +pika -c pika.conf +``` + +4. 执行如下命令,将迁移工具伪装成 Slave,向主库请求同步,并观察是否有报错信息。 +```shell +slaveof ip port force +``` + +5. 确认主从关系建立成功之后,pika-migrate 同时向目标 Redis 转发数据。执行如下命令,查看主从同步延迟。可在主库写入⼀个特殊的 Key,然后在 Redis 侧查看是否可立即获取到该 Key,判断数据同步完毕。 +```shell +info Replication +``` diff --git a/tools/pika_migrate/conf/pika.conf b/tools/pika_migrate/conf/pika.conf new file mode 100644 index 0000000000..d1dd3f8831 --- /dev/null +++ b/tools/pika_migrate/conf/pika.conf @@ -0,0 +1,144 @@ +# Pika port +port : 9222 +# Thread Number +thread-num : 1 +# Thread Pool Size +thread-pool-size : 12 +# Sync Thread Number +sync-thread-num : 6 +# Pika log path +log-path : ./log/ +# Pika db path +db-path : ./db/ +# Pika write-buffer-size +write-buffer-size : 268435456 +# Pika timeout +timeout : 60 +# Requirepass +requirepass : +# Masterauth +masterauth : +# Userpass +userpass : +# User Blacklist +userblacklist : +# if this option is set to 'classic', that means pika support multiple DB, in +# this mode, option databases enable +# if this option is set to 'sharding', that means pika support multiple Table, you +# can specify slot num for each table, in this mode, option default-slot-num enable +# Pika instance mode [classic | sharding] +instance-mode : classic +# Set the number of databases. The default database is DB 0, you can select +# a different one on a per-connection basis using SELECT where +# dbid is a number between 0 and 'databases' - 1, limited in [1, 8] +databases : 1 +# default slot number each table in sharding mode +default-slot-num : 1024 +# Dump Prefix +dump-prefix : +# daemonize [yes | no] +#daemonize : yes +# Dump Path +dump-path : ./dump/ +# Expire-dump-days +dump-expire : 0 +# pidfile Path +pidfile : ./pika.pid +# Max Connection +maxclients : 20000 +# the per file size of sst to compact, defalut is 2M +target-file-size-base : 20971520 +# Expire-logs-days +expire-logs-days : 7 +# Expire-logs-nums +expire-logs-nums : 10 +# Root-connection-num +root-connection-num : 2 +# Slowlog-write-errorlog +slowlog-write-errorlog : no +# Slowlog-log-slower-than +slowlog-log-slower-than : 10000 +# Slowlog-max-len +slowlog-max-len : 128 +# Pika db sync path +db-sync-path : ./dbsync/ +# db sync speed(MB) max is set to 1024MB, min is set to 0, and if below 0 or above 1024, the value will be adjust to 1024 +db-sync-speed : -1 +# The slave priority +slave-priority : 100 +# network interface +#network-interface : eth1 +# replication +#slaveof : master-ip:master-port + +# CronTask, format 1: start-end/ratio, like 02-04/60, pika will check to schedule compaction between 2 to 4 o'clock everyday +# if the freesize/disksize > 60%. +# format 2: week/start-end/ratio, like 3/02-04/60, pika will check to schedule compaction between 2 to 4 o'clock +# every wednesday, if the freesize/disksize > 60%. +# NOTICE: if compact-interval is set, compact-cron will be mask and disable. +# +#compact-cron : 3/02-04/60 + +# Compact-interval, format: interval/ratio, like 6/60, pika will check to schedule compaction every 6 hours, +# if the freesize/disksize > 60%. NOTICE:compact-interval is prior than compact-cron; +#compact-interval : + +# server-id for hub +server-id : 1 +# the size of flow control window while sync binlog between master and slave.Default is 9000 and the maximum is 90000. +sync-window-size : 9000 + +################### +## Migrate Settings +################### + +target-redis-host : 127.0.0.1 +target-redis-port : 6379 +target-redis-pwd : + +sync-batch-num : 100 +redis-sender-num : 10 + +################### +## Critical Settings +################### +# write_binlog [yes | no] +write-binlog : yes +# binlog file size: default is 100M, limited in [1K, 2G] +binlog-file-size : 104857600 +# Automatically triggers a small compaction according statistics +# Use the cache to store up to 'max-cache-statistic-keys' keys +# if 'max-cache-statistic-keys' set to '0', that means turn off the statistics function +# it also doesn't automatically trigger a small compact feature +max-cache-statistic-keys : 0 +# When 'delete' or 'overwrite' a specific multi-data structure key 'small-compaction-threshold' times, +# a small compact is triggered automatically, default is 5000, limited in [1, 100000] +small-compaction-threshold : 5000 +# If the total size of all live memtables of all the DBs exceeds +# the limit, a flush will be triggered in the next DB to which the next write +# is issued. +max-write-buffer-size : 10737418240 +# Limit some command response size, like Scan, Keys* +max-client-response-size : 1073741824 +# Compression +compression : snappy +# max-background-flushes: default is 1, limited in [1, 4] +max-background-flushes : 1 +# max-background-compactions: default is 2, limited in [1, 8] +max-background-compactions : 2 +# maximum value of Rocksdb cached open file descriptors +max-cache-files : 5000 +# max_bytes_for_level_multiplier: default is 10, you can change it to 5 +max-bytes-for-level-multiplier : 10 +# BlockBasedTable block_size, default 4k +# block-size: 4096 +# block LRU cache, default 8M, 0 to disable +# block-cache: 8388608 +# whether the block cache is shared among the RocksDB instances, default is per CF +# share-block-cache: no +# whether or not index and filter blocks is stored in block cache +# cache-index-and-filter-blocks: no +# when set to yes, bloomfilter of the last level will not be built +# optimize-filters-for-hits: no +# https://github.com/facebook/rocksdb/wiki/Leveled-Compaction#levels-target-size +# level-compaction-dynamic-level-bytes: no diff --git a/tools/pika_migrate/detect_environment b/tools/pika_migrate/detect_environment new file mode 100755 index 0000000000..a316ec02da --- /dev/null +++ b/tools/pika_migrate/detect_environment @@ -0,0 +1,112 @@ +#!/bin/sh + +OUTPUT=$1 +if test -z "$OUTPUT"; then + echo "usage: $0 " >&2 + exit 1 +fi + +# Delete existing output, if it exists +rm -f "$OUTPUT" +touch "$OUTPUT" + +if test -z "$CXX"; then + CXX=g++ +fi + +# Test whether tcmalloc is available +if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \ + -ltcmalloc 2>/dev/null; then + TCMALLOC_LDFLAGS=" -ltcmalloc" +fi + +# Test whether malloc_extension is available +$CXX $CFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null < + int main() { + MallocExtension::instance()->Initialize();; + return 0; + } +EOF +if [ "$?" = 0 ]; then + TCMALLOC_EXTENSION_FLAGS=" -DTCMALLOC_EXTENSION" +fi + +# Test whether Snappy library is installed +# http://code.google.com/p/snappy/ +$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lsnappy" +fi + +# Test whether gflags library is installed +# http://gflags.github.io/gflags/ +# check if the namespace is gflags +$CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF + #include + using namespace gflags; + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lgflags" +else + # check if namespace is google + $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF + #include + using namespace google; + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lgflags" +fi +fi + +# Test whether zlib library is installed +$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lz" +fi + +# Test whether bzip library is installed +$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lbz2" +fi + +# Test whether lz4 library is installed +$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + #include + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -llz4" +fi + +# Test whether zstd library is installed +$CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null < + int main() {} +EOF +if [ "$?" = 0 ]; then + ROCKSDB_LDFLAGS="$ROCKSDB_LDFLAGS -lzstd" +fi + + + +# Test processor nums +PROCESSOR_NUMS=$(cat /proc/cpuinfo | grep processor | wc -l) + +echo "ROCKSDB_LDFLAGS=$ROCKSDB_LDFLAGS" >> "$OUTPUT" +echo "TCMALLOC_EXTENSION_FLAGS=$TCMALLOC_EXTENSION_FLAGS" >> "$OUTPUT" +echo "TCMALLOC_LDFLAGS=$TCMALLOC_LDFLAGS" >> "$OUTPUT" +echo "PROCESSOR_NUMS=$PROCESSOR_NUMS" >> "$OUTPUT" diff --git a/tools/pika_migrate/img.png b/tools/pika_migrate/img.png new file mode 100644 index 0000000000..756bfa2948 Binary files /dev/null and b/tools/pika_migrate/img.png differ diff --git a/tools/pika_migrate/include/build_version.h b/tools/pika_migrate/include/build_version.h new file mode 100644 index 0000000000..52e583c3a3 --- /dev/null +++ b/tools/pika_migrate/include/build_version.h @@ -0,0 +1,15 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef INCLUDE_BUILD_VERSION_H_ +#define INCLUDE_BUILD_VERSION_H_ + +// this variable tells us about the git revision +extern const char* pika_build_git_sha; + +// Date on which the code was compiled: +extern const char* pika_build_compile_date; + +#endif // INCLUDE_BUILD_VERSION_H_ diff --git a/tools/pika_migrate/include/migrator_thread.h b/tools/pika_migrate/include/migrator_thread.h new file mode 100644 index 0000000000..ed4141f90b --- /dev/null +++ b/tools/pika_migrate/include/migrator_thread.h @@ -0,0 +1,64 @@ +#ifndef MIGRATOR_THREAD_H_ +#define MIGRATOR_THREAD_H_ + +#include + +#include "pink/include/redis_cli.h" + +#include "include/pika_sender.h" + +class MigratorThread : public pink::Thread { + public: + MigratorThread(void *db, std::vector *senders, int type, int thread_num) : + db_(db), + should_exit_(false), + senders_(senders), + type_(type), + thread_num_(thread_num), + thread_index_(0), + num_(0) { + } + + virtual ~ MigratorThread(); + + int64_t num() { + slash::MutexLock l(&num_mutex_); + return num_; + } + + void Stop() { + should_exit_ = true; + } + + private: + void PlusNum() { + slash::MutexLock l(&num_mutex_); + ++num_; + } + + void DispatchKey(const std::string &command, const std::string& key = ""); + + void MigrateDB(); + void MigrateStringsDB(); + void MigrateListsDB(); + void MigrateHashesDB(); + void MigrateSetsDB(); + void MigrateZsetsDB(); + + virtual void *ThreadMain(); + + private: + void* db_; + bool should_exit_; + + std::vector *senders_; + int type_; + int thread_num_; + int thread_index_; + + int64_t num_; + slash::Mutex num_mutex_; +}; + +#endif + diff --git a/tools/pika_migrate/include/pika_admin.h b/tools/pika_migrate/include/pika_admin.h new file mode 100644 index 0000000000..c8484f7e6e --- /dev/null +++ b/tools/pika_migrate/include/pika_admin.h @@ -0,0 +1,459 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_ADMIN_H_ +#define PIKA_ADMIN_H_ + +#include +#include +#include +#include + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" + +/* + * Admin + */ +class SlaveofCmd : public Cmd { + public: + SlaveofCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), is_noone_(false) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlaveofCmd(*this); + } + + private: + std::string master_ip_; + int64_t master_port_; + bool is_noone_; + virtual void DoInitial() override; + virtual void Clear() { + is_noone_ = false; + master_ip_.clear(); + master_port_ = 0; + } +}; + +class DbSlaveofCmd : public Cmd { + public: + DbSlaveofCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DbSlaveofCmd(*this); + } + + private: + std::string db_name_; + bool force_sync_; + bool is_noone_; + bool have_offset_; + int64_t filenum_; + int64_t offset_; + virtual void DoInitial() override; + virtual void Clear() { + db_name_.clear(); + force_sync_ = false; + is_noone_ = false; + have_offset_ = false; + } +}; + +class AuthCmd : public Cmd { + public: + AuthCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new AuthCmd(*this); + } + + private: + std::string pwd_; + virtual void DoInitial() override; +}; + +class BgsaveCmd : public Cmd { + public: + BgsaveCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new BgsaveCmd(*this); + } + + private: + virtual void DoInitial() override; + virtual void Clear() { + bgsave_tables_.clear(); + } + std::set bgsave_tables_; +}; + +class CompactCmd : public Cmd { + public: + CompactCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new CompactCmd(*this); + } + + private: + virtual void DoInitial() override; + virtual void Clear() { + struct_type_.clear(); + compact_tables_.clear(); + } + std::string struct_type_; + std::set compact_tables_; +}; + +class PurgelogstoCmd : public Cmd { + public: + PurgelogstoCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), num_(0) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PurgelogstoCmd(*this); + } + + private: + uint32_t num_; + std::string table_; + virtual void DoInitial() override; +}; + +class PingCmd : public Cmd { + public: + PingCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PingCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class SelectCmd : public Cmd { + public: + SelectCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SelectCmd(*this); + } + + private: + virtual void DoInitial() override; + virtual void Clear() { + table_name_.clear(); + } + std::string table_name_; +}; + +class FlushallCmd : public Cmd { + public: + FlushallCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new FlushallCmd(*this); + } + + private: + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class FlushdbCmd : public Cmd { + public: + FlushdbCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new FlushdbCmd(*this); + } + + private: + std::string db_name_; + virtual void DoInitial() override; + virtual void Clear() { + db_name_.clear(); + } +}; + +class ClientCmd : public Cmd { + public: + ClientCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + const static std::string CLIENT_LIST_S; + const static std::string CLIENT_KILL_S; + virtual Cmd* Clone() override { + return new ClientCmd(*this); + } + + private: + std::string operation_, info_; + virtual void DoInitial() override; +}; + +class InfoCmd : public Cmd { + public: + enum InfoSection { + kInfoErr = 0x0, + kInfoServer, + kInfoClients, + kInfoStats, + kInfoExecCount, + kInfoCPU, + kInfoReplication, + kInfoKeyspace, + kInfoLog, + kInfoData, + kInfo, + kInfoAll, + kInfoDebug + }; + + InfoCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), rescan_(false), off_(false) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new InfoCmd(*this); + } + + private: + InfoSection info_section_; + bool rescan_; //whether to rescan the keyspace + bool off_; + std::set keyspace_scan_tables_; + + const static std::string kInfoSection; + const static std::string kAllSection; + const static std::string kServerSection; + const static std::string kClientsSection; + const static std::string kStatsSection; + const static std::string kExecCountSection; + const static std::string kCPUSection; + const static std::string kReplicationSection; + const static std::string kKeyspaceSection; + const static std::string kDataSection; + const static std::string kDebugSection; + + virtual void DoInitial() override; + virtual void Clear() { + rescan_ = false; + off_ = false; + keyspace_scan_tables_.clear(); + } + + void InfoServer(std::string& info); + void InfoClients(std::string& info); + void InfoStats(std::string& info); + void InfoExecCount(std::string& info); + void InfoCPU(std::string& info); + void InfoShardingReplication(std::string& info); + void InfoReplication(std::string& info); + void InfoKeyspace(std::string& info); + void InfoData(std::string& info); + void InfoDebug(std::string& info); +}; + +class ShutdownCmd : public Cmd { + public: + ShutdownCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ShutdownCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class ConfigCmd : public Cmd { + public: + ConfigCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ConfigCmd(*this); + } + + private: + std::vector config_args_v_; + virtual void DoInitial() override; + void ConfigGet(std::string &ret); + void ConfigSet(std::string &ret); + void ConfigRewrite(std::string &ret); + void ConfigResetstat(std::string &ret); +}; + +class MonitorCmd : public Cmd { + public: + MonitorCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new MonitorCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class DbsizeCmd : public Cmd { + public: + DbsizeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DbsizeCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class TimeCmd : public Cmd { + public: + TimeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new TimeCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class DelbackupCmd : public Cmd { + public: + DelbackupCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DelbackupCmd(*this); + } + + private: + virtual void DoInitial() override; +}; + +class EchoCmd : public Cmd { + public: + EchoCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new EchoCmd(*this); + } + + private: + std::string body_; + virtual void DoInitial() override; +}; + +class ScandbCmd : public Cmd { + public: + ScandbCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), type_(blackwidow::kAll) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ScandbCmd(*this); + } + + private: + blackwidow::DataType type_; + virtual void DoInitial() override; + virtual void Clear() { + type_ = blackwidow::kAll; + } +}; + +class SlowlogCmd : public Cmd { + public: + enum SlowlogCondition{kGET, kLEN, kRESET}; + SlowlogCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), condition_(kGET) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlowlogCmd(*this); + } + private: + int64_t number_; + SlowlogCmd::SlowlogCondition condition_; + virtual void DoInitial() override; + virtual void Clear() { + number_ = 10; + condition_ = kGET; + } +}; + +class PaddingCmd : public Cmd { + public: + PaddingCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PaddingCmd(*this); + } + + private: + virtual void DoInitial(); + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +#ifdef TCMALLOC_EXTENSION +class TcmallocCmd : public Cmd { + public: + TcmallocCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new TcmallocCmd(*this); + } + + private: + int64_t type_; + int64_t rate_; + virtual void DoInitial() override; +}; +#endif + +class PKPatternMatchDelCmd : public Cmd { + public: + PKPatternMatchDelCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKPatternMatchDelCmd(*this); + } + + private: + blackwidow::DataType type_; + std::string pattern_; + virtual void DoInitial() override; +}; +#endif // PIKA_ADMIN_H_ diff --git a/tools/pika_migrate/include/pika_auxiliary_thread.h b/tools/pika_migrate/include/pika_auxiliary_thread.h new file mode 100644 index 0000000000..3e192bf280 --- /dev/null +++ b/tools/pika_migrate/include/pika_auxiliary_thread.h @@ -0,0 +1,25 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_AUXILIARY_THREAD_H_ +#define PIKA_AUXILIARY_THREAD_H_ + +#include "pink/include/pink_thread.h" + +#include "slash/include/slash_mutex.h" + +class PikaAuxiliaryThread : public pink::Thread { + public: + PikaAuxiliaryThread() : + mu_(), + cv_(&mu_) {} + virtual ~PikaAuxiliaryThread(); + slash::Mutex mu_; + slash::CondVar cv_; + private: + virtual void* ThreadMain(); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_binlog.h b/tools/pika_migrate/include/pika_binlog.h new file mode 100644 index 0000000000..48d6b61f40 --- /dev/null +++ b/tools/pika_migrate/include/pika_binlog.h @@ -0,0 +1,115 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_BINLOG_H_ +#define PIKA_BINLOG_H_ + +#include "slash/include/env.h" +#include "slash/include/slash_mutex.h" +#include "slash/include/slash_status.h" + +#include "include/pika_define.h" + +using slash::Status; +using slash::Slice; + +std::string NewFileName(const std::string name, const uint32_t current); + +class Version { + public: + Version(slash::RWFile *save); + ~Version(); + + Status Init(); + + // RWLock should be held when access members. + Status StableSave(); + + uint32_t pro_num_; + uint64_t pro_offset_; + uint64_t logic_id_; + + pthread_rwlock_t rwlock_; + + void debug() { + slash::RWLock(&rwlock_, false); + printf ("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); + } + + private: + + slash::RWFile *save_; + + // No copying allowed; + Version(const Version&); + void operator=(const Version&); +}; + +class Binlog { + public: + Binlog(const std::string& Binlog_path, const int file_size = 100 * 1024 * 1024); + ~Binlog(); + + void Lock() { mutex_.Lock(); } + void Unlock() { mutex_.Unlock(); } + + Status Put(const std::string &item); + Status Put(const char* item, int len); + + Status GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset, uint64_t* logic_id = NULL); + /* + * Set Producer pro_num and pro_offset with lock + */ + Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset); + + static Status AppendPadding(slash::WritableFile* file, uint64_t* len); + + slash::WritableFile *queue() { return queue_; } + + uint64_t file_size() { + return file_size_; + } + + std::string filename; + + private: + + void InitLogFile(); + Status EmitPhysicalRecord(RecordType t, const char *ptr, size_t n, int *temp_pro_offset); + + + /* + * Produce + */ + Status Produce(const Slice &item, int *pro_offset); + + uint32_t consumer_num_; + uint64_t item_num_; + + Version* version_; + slash::WritableFile *queue_; + slash::RWFile *versionfile_; + + slash::Mutex mutex_; + + uint32_t pro_num_; + + int block_offset_; + + char* pool_; + bool exit_all_consume_; + const std::string binlog_path_; + + uint64_t file_size_; + + // Not use + //int32_t retry_; + + // No copying allowed + Binlog(const Binlog&); + void operator=(const Binlog&); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_binlog_reader.h b/tools/pika_migrate/include/pika_binlog_reader.h new file mode 100644 index 0000000000..6b50fa0b4b --- /dev/null +++ b/tools/pika_migrate/include/pika_binlog_reader.h @@ -0,0 +1,48 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_BINLOG_READER_H_ +#define PIKA_BINLOG_READER_H_ + +#include +#include + +#include "slash/include/slash_status.h" +#include "slash/include/env.h" +#include "slash/include/slash_slice.h" + +#include "include/pika_binlog.h" + +using slash::Status; +using slash::Slice; + +class PikaBinlogReader { + public: + PikaBinlogReader(uint32_t cur_filenum, uint64_t cur_offset); + PikaBinlogReader(); + ~PikaBinlogReader(); + Status Get(std::string* scratch, uint32_t* filenum, uint64_t* offset); + int Seek(std::shared_ptr logger, uint32_t filenum, uint64_t offset); + bool ReadToTheEnd(); + void GetReaderStatus(uint32_t* cur_filenum, uint64_t* cur_offset); + private: + bool GetNext(uint64_t* size); + unsigned int ReadPhysicalRecord(slash::Slice *redult, uint32_t* filenum, uint64_t* offset); + // Returns scratch binflog and corresponding offset + Status Consume(std::string* scratch, uint32_t* filenum, uint64_t* offset); + + pthread_rwlock_t rwlock_; + uint32_t cur_filenum_; + uint64_t cur_offset_; + uint64_t last_record_offset_; + + std::shared_ptr logger_; + slash::SequentialFile *queue_; + + char* const backing_store_; + Slice buffer_; +}; + +#endif // PIKA_BINLOG_READER_H_ diff --git a/tools/pika_migrate/include/pika_binlog_transverter.h b/tools/pika_migrate/include/pika_binlog_transverter.h new file mode 100644 index 0000000000..14244f55e0 --- /dev/null +++ b/tools/pika_migrate/include/pika_binlog_transverter.h @@ -0,0 +1,91 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_BINLOG_TRANSVERTER_H_ +#define PIKA_BINLOG_TRANSVERTER_H_ + +#include +#include +#include +#include + + +/* + * ***********************************************Type First Binlog Item Format*********************************************** + * | | | | | | | | | + * 2 Bytes 4 Bytes 4 Bytes 8 Bytes 4 Bytes 8 Bytes 4 Bytes content length Bytes + * + */ +#define BINLOG_ENCODE_LEN 34 + +enum BinlogType { + TypeFirst = 1, +}; + + +const int BINLOG_ITEM_HEADER_SIZE = 34; +const int PADDING_BINLOG_PROTOCOL_SIZE = 22; +const int SPACE_STROE_PARAMETER_LENGTH = 5; + +class BinlogItem { + public: + BinlogItem() : + exec_time_(0), + server_id_(0), + logic_id_(0), + filenum_(0), + offset_(0), + content_("") {} + + friend class PikaBinlogTransverter; + + uint32_t exec_time() const; + uint32_t server_id() const; + uint64_t logic_id() const; + uint32_t filenum() const; + uint64_t offset() const; + std::string content() const; + std::string ToString() const; + + void set_exec_time(uint32_t exec_time); + void set_server_id(uint32_t server_id); + void set_logic_id(uint64_t logic_id); + void set_filenum(uint32_t filenum); + void set_offset(uint64_t offset); + + private: + uint32_t exec_time_; + uint32_t server_id_; + uint64_t logic_id_; + uint32_t filenum_; + uint64_t offset_; + std::string content_; + std::vector extends_; +}; + +class PikaBinlogTransverter{ + public: + PikaBinlogTransverter() {}; + static std::string BinlogEncode(BinlogType type, + uint32_t exec_time, + uint32_t server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset, + const std::string& content, + const std::vector& extends); + + static bool BinlogDecode(BinlogType type, + const std::string& binlog, + BinlogItem* binlog_item); + + static std::string ConstructPaddingBinlog(BinlogType type, uint32_t size); + + static bool BinlogItemWithoutContentDecode(BinlogType type, + const std::string& binlog, + BinlogItem* binlog_item); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_bit.h b/tools/pika_migrate/include/pika_bit.h new file mode 100644 index 0000000000..257e9cb866 --- /dev/null +++ b/tools/pika_migrate/include/pika_bit.h @@ -0,0 +1,142 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_BIT_H_ +#define PIKA_BIT_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * bitoperation + */ +class BitGetCmd : public Cmd { + public: + BitGetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new BitGetCmd(*this); + } + private: + std::string key_; + int64_t bit_offset_; + virtual void Clear() { + key_ = ""; + bit_offset_ = -1; + } + virtual void DoInitial() override; +}; + +class BitSetCmd : public Cmd { + public: + BitSetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new BitSetCmd(*this); + } + private: + std::string key_; + int64_t bit_offset_; + int64_t on_; + virtual void Clear() { + key_ = ""; + bit_offset_ = -1; + on_ = -1; + } + virtual void DoInitial() override; +}; + +class BitCountCmd : public Cmd { + public: + BitCountCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new BitCountCmd(*this); + } + private: + std::string key_; + bool count_all_; + int64_t start_offset_; + int64_t end_offset_; + virtual void Clear() { + key_ = ""; + count_all_ = false; + start_offset_ = -1; + end_offset_ = -1; + } + virtual void DoInitial() override; +}; + +class BitPosCmd : public Cmd { + public: + BitPosCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new BitPosCmd(*this); + } + private: + std::string key_; + bool pos_all_; + bool endoffset_set_; + int64_t bit_val_; + int64_t start_offset_; + int64_t end_offset_; + virtual void Clear() { + key_ = ""; + pos_all_ = false; + endoffset_set_ = false; + bit_val_ = -1; + start_offset_ = -1; + end_offset_ = -1; + } + virtual void DoInitial() override; +}; + +class BitOpCmd : public Cmd { + public: + BitOpCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new BitOpCmd(*this); + } + private: + std::string dest_key_; + std::vector src_keys_; + blackwidow::BitOpType op_; + virtual void Clear() { + dest_key_ = ""; + src_keys_.clear(); + op_ = blackwidow::kBitOpDefault; + } + virtual void DoInitial() override; +}; +#endif diff --git a/tools/pika_migrate/include/pika_client_conn.h b/tools/pika_migrate/include/pika_client_conn.h new file mode 100644 index 0000000000..1bbe82ab9e --- /dev/null +++ b/tools/pika_migrate/include/pika_client_conn.h @@ -0,0 +1,81 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_CLIENT_CONN_H_ +#define PIKA_CLIENT_CONN_H_ + +#include "include/pika_command.h" + +class PikaClientConn: public pink::RedisConn { + public: + struct BgTaskArg { + std::shared_ptr pcc; + std::vector redis_cmds; + std::string* response; + }; + + // Auth related + class AuthStat { + public: + void Init(); + bool IsAuthed(const std::shared_ptr cmd_ptr); + bool ChecknUpdate(const std::string& arg); + private: + enum StatType { + kNoAuthed = 0, + kAdminAuthed, + kLimitAuthed, + }; + StatType stat_; + }; + + PikaClientConn(int fd, std::string ip_port, + pink::Thread *server_thread, + pink::PinkEpoll* pink_epoll, + const pink::HandleType& handle_type); + virtual ~PikaClientConn() {} + + void AsynProcessRedisCmds(const std::vector& argvs, std::string* response) override; + + void BatchExecRedisCmd(const std::vector& argvs, std::string* response); + int DealMessage(const pink::RedisCmdArgsType& argv, std::string* response); + static void DoBackgroundTask(void* arg); + + bool IsPubSub() { return is_pubsub_; } + void SetIsPubSub(bool is_pubsub) { is_pubsub_ = is_pubsub; } + void SetCurrentTable(const std::string& table_name) {current_table_ = table_name;} + + pink::ServerThread* server_thread() { + return server_thread_; + } + + AuthStat& auth_stat() { + return auth_stat_; + } + + private: + pink::ServerThread* const server_thread_; + std::string current_table_; + bool is_pubsub_; + + std::string DoCmd(const PikaCmdArgsType& argv, const std::string& opt); + + void ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t start_us); + void ProcessMonitor(const PikaCmdArgsType& argv); + + AuthStat auth_stat_; +}; + +struct ClientInfo { + int fd; + std::string ip_port; + int64_t last_interaction; + std::shared_ptr conn; +}; + +extern bool AddrCompare(const ClientInfo& lhs, const ClientInfo& rhs); +extern bool IdleCompare(const ClientInfo& lhs, const ClientInfo& rhs); + +#endif diff --git a/tools/pika_migrate/include/pika_cluster.h b/tools/pika_migrate/include/pika_cluster.h new file mode 100644 index 0000000000..bb34c37c31 --- /dev/null +++ b/tools/pika_migrate/include/pika_cluster.h @@ -0,0 +1,114 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_CLUSTER_H_ +#define PIKA_CLUSTER_H_ + +#include "include/pika_command.h" + +class PkClusterInfoCmd : public Cmd { + public: + enum InfoSection { + kInfoErr = 0x0, + kInfoSlot + }; + enum InfoRange { + kSingle = 0x0, + kAll + }; + PkClusterInfoCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), + info_section_(kInfoErr), info_range_(kAll), partition_id_(0) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PkClusterInfoCmd(*this); + } + + private: + InfoSection info_section_; + InfoRange info_range_; + + std::string table_name_; + uint32_t partition_id_; + + virtual void DoInitial() override; + virtual void Clear() { + info_section_ = kInfoErr; + info_range_ = kAll; + table_name_.clear(); + partition_id_ = 0; + } + const static std::string kSlotSection; + void ClusterInfoSlotAll(std::string* info); + Status GetSlotInfo(const std::string table_name, uint32_t partition_id, std::string* info); + bool ParseInfoSlotSubCmd(); +}; + +class SlotParentCmd : public Cmd { + public: + SlotParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + + protected: + std::set slots_; + std::set p_infos_; + virtual void DoInitial(); + virtual void Clear() { + slots_.clear(); + p_infos_.clear(); + } +}; + +class PkClusterAddSlotsCmd : public SlotParentCmd { + public: + PkClusterAddSlotsCmd(const std::string& name, int arity, uint16_t flag) + : SlotParentCmd(name, arity, flag) {} + virtual Cmd* Clone() override { + return new PkClusterAddSlotsCmd(*this); + } + virtual void Do(std::shared_ptr partition = nullptr); + private: + virtual void DoInitial() override; + Status AddSlotsSanityCheck(const std::string& table_name); +}; + +class PkClusterDelSlotsCmd : public SlotParentCmd { + public: + PkClusterDelSlotsCmd(const std::string& name, int32_t arity, uint16_t flag) + : SlotParentCmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PkClusterDelSlotsCmd(*this); + } + private: + virtual void DoInitial() override; + Status RemoveSlotsSanityCheck(const std::string& table_name); +}; + +class PkClusterSlotsSlaveofCmd : public Cmd { + public: + PkClusterSlotsSlaveofCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PkClusterSlotsSlaveofCmd(*this); + } + private: + std::string ip_; + int64_t port_; + std::set slots_; + bool force_sync_; + bool is_noone_; + virtual void DoInitial() override; + virtual void Clear() { + ip_.clear(); + port_ = 0; + slots_.clear(); + force_sync_ = false; + is_noone_ = false; + } +}; + +#endif // PIKA_CLUSTER_H_ diff --git a/tools/pika_migrate/include/pika_cmd_table_manager.h b/tools/pika_migrate/include/pika_cmd_table_manager.h new file mode 100644 index 0000000000..bd87296698 --- /dev/null +++ b/tools/pika_migrate/include/pika_cmd_table_manager.h @@ -0,0 +1,32 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_CMD_TABLE_MANAGER_H_ +#define PIKA_CMD_TABLE_MANAGER_H_ + +#include "include/pika_command.h" +#include "include/pika_data_distribution.h" + + +class PikaCmdTableManager { + public: + PikaCmdTableManager(); + virtual ~PikaCmdTableManager(); + std::shared_ptr GetCmd(const std::string& opt); + uint32_t DistributeKey(const std::string& key, uint32_t partition_num); + private: + std::shared_ptr NewCommand(const std::string& opt); + + void InsertCurrentThreadDistributionMap(); + bool CheckCurrentThreadDistributionMapExist(const pid_t& tid); + + void TryChangeToAlias(std::string *internal_opt); + + CmdTable* cmds_; + + pthread_rwlock_t map_protector_; + std::unordered_map thread_distribution_map_; +}; +#endif diff --git a/tools/pika_migrate/include/pika_command.h b/tools/pika_migrate/include/pika_command.h new file mode 100644 index 0000000000..dec0b50924 --- /dev/null +++ b/tools/pika_migrate/include/pika_command.h @@ -0,0 +1,486 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_COMMAND_H_ +#define PIKA_COMMAND_H_ + +#include + +#include "pink/include/redis_conn.h" +#include "pink/include/pink_conn.h" +#include "slash/include/slash_string.h" + +#include "include/pika_partition.h" + +//Constant for command name +//Admin +const std::string kCmdNameSlaveof = "slaveof"; +const std::string kCmdNameDbSlaveof = "dbslaveof"; +const std::string kCmdNameAuth = "auth"; +const std::string kCmdNameBgsave = "bgsave"; +const std::string kCmdNameCompact = "compact"; +const std::string kCmdNamePurgelogsto = "purgelogsto"; +const std::string kCmdNamePing = "ping"; +const std::string kCmdNameSelect = "select"; +const std::string kCmdNameFlushall = "flushall"; +const std::string kCmdNameFlushdb = "flushdb"; +const std::string kCmdNameClient = "client"; +const std::string kCmdNameShutdown = "shutdown"; +const std::string kCmdNameInfo = "info"; +const std::string kCmdNameConfig = "config"; +const std::string kCmdNameMonitor = "monitor"; +const std::string kCmdNameDbsize = "dbsize"; +const std::string kCmdNameTime = "time"; +const std::string kCmdNameDelbackup = "delbackup"; +const std::string kCmdNameEcho = "echo"; +const std::string kCmdNameScandb = "scandb"; +const std::string kCmdNameSlowlog = "slowlog"; +const std::string kCmdNamePadding = "padding"; +#ifdef TCMALLOC_EXTENSION +const std::string kCmdNameTcmalloc = "tcmalloc"; +#endif +const std::string kCmdNamePKPatternMatchDel = "pkpatternmatchdel"; + +//Kv +const std::string kCmdNameSet = "set"; +const std::string kCmdNameGet = "get"; +const std::string kCmdNameDel = "del"; +const std::string kCmdNameIncr = "incr"; +const std::string kCmdNameIncrby = "incrby"; +const std::string kCmdNameIncrbyfloat = "incrbyfloat"; +const std::string kCmdNameDecr = "decr"; +const std::string kCmdNameDecrby = "decrby"; +const std::string kCmdNameGetset = "getset"; +const std::string kCmdNameAppend = "append"; +const std::string kCmdNameMget = "mget"; +const std::string kCmdNameKeys = "keys"; +const std::string kCmdNameSetnx = "setnx"; +const std::string kCmdNameSetex = "setex"; +const std::string kCmdNamePsetex = "psetex"; +const std::string kCmdNameDelvx = "delvx"; +const std::string kCmdNameMset = "mset"; +const std::string kCmdNameMsetnx = "msetnx"; +const std::string kCmdNameGetrange = "getrange"; +const std::string kCmdNameSetrange = "setrange"; +const std::string kCmdNameStrlen = "strlen"; +const std::string kCmdNameExists = "exists"; +const std::string kCmdNameExpire = "expire"; +const std::string kCmdNamePexpire = "pexpire"; +const std::string kCmdNameExpireat = "expireat"; +const std::string kCmdNamePexpireat = "pexpireat"; +const std::string kCmdNameTtl = "ttl"; +const std::string kCmdNamePttl = "pttl"; +const std::string kCmdNamePersist = "persist"; +const std::string kCmdNameType = "type"; +const std::string kCmdNameScan = "scan"; +const std::string kCmdNameScanx = "scanx"; +const std::string kCmdNamePKSetexAt = "pksetexat"; +const std::string kCmdNamePKScanRange = "pkscanrange"; +const std::string kCmdNamePKRScanRange = "pkrscanrange"; + +//Hash +const std::string kCmdNameHDel = "hdel"; +const std::string kCmdNameHSet = "hset"; +const std::string kCmdNameHGet = "hget"; +const std::string kCmdNameHGetall = "hgetall"; +const std::string kCmdNameHExists = "hexists"; +const std::string kCmdNameHIncrby = "hincrby"; +const std::string kCmdNameHIncrbyfloat = "hincrbyfloat"; +const std::string kCmdNameHKeys = "hkeys"; +const std::string kCmdNameHLen = "hlen"; +const std::string kCmdNameHMget = "hmget"; +const std::string kCmdNameHMset = "hmset"; +const std::string kCmdNameHSetnx = "hsetnx"; +const std::string kCmdNameHStrlen = "hstrlen"; +const std::string kCmdNameHVals = "hvals"; +const std::string kCmdNameHScan = "hscan"; +const std::string kCmdNameHScanx = "hscanx"; +const std::string kCmdNamePKHScanRange = "pkhscanrange"; +const std::string kCmdNamePKHRScanRange = "pkhrscanrange"; + +//List +const std::string kCmdNameLIndex = "lindex"; +const std::string kCmdNameLInsert = "linsert"; +const std::string kCmdNameLLen = "llen"; +const std::string kCmdNameLPop = "lpop"; +const std::string kCmdNameLPush = "lpush"; +const std::string kCmdNameLPushx = "lpushx"; +const std::string kCmdNameLRange = "lrange"; +const std::string kCmdNameLRem = "lrem"; +const std::string kCmdNameLSet = "lset"; +const std::string kCmdNameLTrim = "ltrim"; +const std::string kCmdNameRPop = "rpop"; +const std::string kCmdNameRPopLPush = "rpoplpush"; +const std::string kCmdNameRPush = "rpush"; +const std::string kCmdNameRPushx = "rpushx"; + +//BitMap +const std::string kCmdNameBitSet = "setbit"; +const std::string kCmdNameBitGet = "getbit"; +const std::string kCmdNameBitPos = "bitpos"; +const std::string kCmdNameBitOp = "bitop"; +const std::string kCmdNameBitCount = "bitcount"; + +//Zset +const std::string kCmdNameZAdd = "zadd"; +const std::string kCmdNameZCard = "zcard"; +const std::string kCmdNameZScan = "zscan"; +const std::string kCmdNameZIncrby = "zincrby"; +const std::string kCmdNameZRange = "zrange"; +const std::string kCmdNameZRangebyscore = "zrangebyscore"; +const std::string kCmdNameZCount = "zcount"; +const std::string kCmdNameZRem = "zrem"; +const std::string kCmdNameZUnionstore = "zunionstore"; +const std::string kCmdNameZInterstore = "zinterstore"; +const std::string kCmdNameZRank = "zrank"; +const std::string kCmdNameZRevrank = "zrevrank"; +const std::string kCmdNameZScore = "zscore"; +const std::string kCmdNameZRevrange = "zrevrange"; +const std::string kCmdNameZRevrangebyscore = "zrevrangebyscore"; +const std::string kCmdNameZRangebylex = "zrangebylex"; +const std::string kCmdNameZRevrangebylex = "zrevrangebylex"; +const std::string kCmdNameZLexcount = "zlexcount"; +const std::string kCmdNameZRemrangebyrank = "zremrangebyrank"; +const std::string kCmdNameZRemrangebylex = "zremrangebylex"; +const std::string kCmdNameZRemrangebyscore = "zremrangebyscore"; +const std::string kCmdNameZPopmax = "zpopmax"; +const std::string kCmdNameZPopmin = "zpopmin"; + +//Set +const std::string kCmdNameSAdd = "sadd"; +const std::string kCmdNameSPop = "spop"; +const std::string kCmdNameSCard = "scard"; +const std::string kCmdNameSMembers = "smembers"; +const std::string kCmdNameSScan = "sscan"; +const std::string kCmdNameSRem = "srem"; +const std::string kCmdNameSUnion = "sunion"; +const std::string kCmdNameSUnionstore = "sunionstore"; +const std::string kCmdNameSInter = "sinter"; +const std::string kCmdNameSInterstore = "sinterstore"; +const std::string kCmdNameSIsmember = "sismember"; +const std::string kCmdNameSDiff = "sdiff"; +const std::string kCmdNameSDiffstore = "sdiffstore"; +const std::string kCmdNameSMove = "smove"; +const std::string kCmdNameSRandmember = "srandmember"; + +//HyperLogLog +const std::string kCmdNamePfAdd = "pfadd"; +const std::string kCmdNamePfCount = "pfcount"; +const std::string kCmdNamePfMerge = "pfmerge"; + +//GEO +const std::string kCmdNameGeoAdd = "geoadd"; +const std::string kCmdNameGeoPos = "geopos"; +const std::string kCmdNameGeoDist = "geodist"; +const std::string kCmdNameGeoHash = "geohash"; +const std::string kCmdNameGeoRadius = "georadius"; +const std::string kCmdNameGeoRadiusByMember = "georadiusbymember"; + +//Pub/Sub +const std::string kCmdNamePublish = "publish"; +const std::string kCmdNameSubscribe = "subscribe"; +const std::string kCmdNameUnSubscribe = "unsubscribe"; +const std::string kCmdNamePubSub = "pubsub"; +const std::string kCmdNamePSubscribe = "psubscribe"; +const std::string kCmdNamePUnSubscribe = "punsubscribe"; + +//Codis Slots +const std::string kCmdNameSlotsInfo = "slotsinfo"; +const std::string kCmdNameSlotsHashKey = "slotshashkey"; +const std::string kCmdNameSlotsMgrtTagSlotAsync = "slotsmgrttagslot-async"; +const std::string kCmdNameSlotsMgrtSlotAsync = "slotsmgrtslot-async"; +const std::string kCmdNameSlotsDel = "slotsdel"; +const std::string kCmdNameSlotsScan = "slotsscan"; +const std::string kCmdNameSlotsMgrtExecWrapper = "slotsmgrt-exec-wrapper"; +const std::string kCmdNameSlotsMgrtAsyncStatus = "slotsmgrt-async-status"; +const std::string kCmdNameSlotsMgrtAsyncCancel = "slotsmgrt-async-cancel"; +const std::string kCmdNameSlotsMgrtSlot = "slotsmgrtslot"; +const std::string kCmdNameSlotsMgrtTagSlot = "slotsmgrttagslot"; +const std::string kCmdNameSlotsMgrtOne = "slotsmgrtone"; +const std::string kCmdNameSlotsMgrtTagOne = "slotsmgrttagone"; + + +//Cluster +const std::string kCmdNamePkClusterInfo = "pkclusterinfo"; +const std::string kCmdNamePkClusterAddSlots = "pkclusteraddslots"; +const std::string kCmdNamePkClusterDelSlots = "pkclusterdelslots"; +const std::string kCmdNamePkClusterSlotsSlaveof = "pkclusterslotsslaveof"; + +const std::string kClusterPrefix = "pkcluster"; +typedef pink::RedisCmdArgsType PikaCmdArgsType; +static const int RAW_ARGS_LEN = 1024 * 1024; + +enum CmdFlagsMask { + kCmdFlagsMaskRW = 1, + kCmdFlagsMaskType = 30, + kCmdFlagsMaskLocal = 32, + kCmdFlagsMaskSuspend = 64, + kCmdFlagsMaskPrior = 128, + kCmdFlagsMaskAdminRequire = 256, + kCmdFlagsMaskPartition = 1536 +}; + +enum CmdFlags { + kCmdFlagsRead = 0, //default rw + kCmdFlagsWrite = 1, + kCmdFlagsAdmin = 0, //default type + kCmdFlagsKv = 2, + kCmdFlagsHash = 4, + kCmdFlagsList = 6, + kCmdFlagsSet = 8, + kCmdFlagsZset = 10, + kCmdFlagsBit = 12, + kCmdFlagsHyperLogLog = 14, + kCmdFlagsGeo = 16, + kCmdFlagsPubSub = 18, + kCmdFlagsNoLocal = 0, //default nolocal + kCmdFlagsLocal = 32, + kCmdFlagsNoSuspend = 0, //default nosuspend + kCmdFlagsSuspend = 64, + kCmdFlagsNoPrior = 0, //default noprior + kCmdFlagsPrior = 128, + kCmdFlagsNoAdminRequire = 0, //default no need admin + kCmdFlagsAdminRequire = 256, + kCmdFlagsDoNotSpecifyPartition = 0, //default do not specify partition + kCmdFlagsSinglePartition = 512, + kCmdFlagsMultiPartition = 1024 +}; + + +void inline RedisAppendContent(std::string& str, const std::string& value); +void inline RedisAppendLen(std::string& str, int64_t ori, const std::string &prefix); + +const std::string kNewLine = "\r\n"; + +class CmdRes { +public: + enum CmdRet { + kNone = 0, + kOk, + kPong, + kSyntaxErr, + kInvalidInt, + kInvalidBitInt, + kInvalidBitOffsetInt, + kInvalidBitPosArgument, + kWrongBitOpNotNum, + kInvalidFloat, + kOverFlow, + kNotFound, + kOutOfRange, + kInvalidPwd, + kNoneBgsave, + kPurgeExist, + kInvalidParameter, + kWrongNum, + kInvalidIndex, + kInvalidDbType, + kInvalidTable, + kErrOther, + }; + + CmdRes():ret_(kNone) {} + + bool none() const { + return ret_ == kNone && message_.empty(); + } + bool ok() const { + return ret_ == kOk || ret_ == kNone; + } + void clear() { + message_.clear(); + ret_ = kNone; + } + std::string raw_message() const { + return message_; + } + std::string message() const { + std::string result; + switch (ret_) { + case kNone: + return message_; + case kOk: + return "+OK\r\n"; + case kPong: + return "+PONG\r\n"; + case kSyntaxErr: + return "-ERR syntax error\r\n"; + case kInvalidInt: + return "-ERR value is not an integer or out of range\r\n"; + case kInvalidBitInt: + return "-ERR bit is not an integer or out of range\r\n"; + case kInvalidBitOffsetInt: + return "-ERR bit offset is not an integer or out of range\r\n"; + case kWrongBitOpNotNum: + return "-ERR BITOP NOT must be called with a single source key.\r\n"; + + case kInvalidBitPosArgument: + return "-ERR The bit argument must be 1 or 0.\r\n"; + case kInvalidFloat: + return "-ERR value is not a valid float\r\n"; + case kOverFlow: + return "-ERR increment or decrement would overflow\r\n"; + case kNotFound: + return "-ERR no such key\r\n"; + case kOutOfRange: + return "-ERR index out of range\r\n"; + case kInvalidPwd: + return "-ERR invalid password\r\n"; + case kNoneBgsave: + return "-ERR No BGSave Works now\r\n"; + case kPurgeExist: + return "-ERR binlog already in purging...\r\n"; + case kInvalidParameter: + return "-ERR Invalid Argument\r\n"; + case kWrongNum: + result = "-ERR wrong number of arguments for '"; + result.append(message_); + result.append("' command\r\n"); + break; + case kInvalidIndex: + result = "-ERR invalid DB index for '"; + result.append(message_); + result.append("'\r\n"); + break; + case kInvalidDbType: + result = "-ERR invalid DB for '"; + result.append(message_); + result.append("'\r\n"); + break; + case kInvalidTable: + result = "-ERR invalid Table for '"; + result.append(message_); + result.append("'\r\n"); + break; + case kErrOther: + result = "-ERR "; + result.append(message_); + result.append(kNewLine); + break; + default: + break; + } + return result; + } + + // Inline functions for Create Redis protocol + void AppendStringLen(int64_t ori) { + RedisAppendLen(message_, ori, "$"); + } + void AppendArrayLen(int64_t ori) { + RedisAppendLen(message_, ori, "*"); + } + void AppendInteger(int64_t ori) { + RedisAppendLen(message_, ori, ":"); + } + void AppendContent(const std::string& value) { + RedisAppendContent(message_, value); + } + void AppendString(const std::string& value) { + AppendStringLen(value.size()); + AppendContent(value); + } + void AppendStringRaw(const std::string& value) { + message_.append(value); + } + void SetRes(CmdRet _ret, const std::string content = "") { + ret_ = _ret; + if (!content.empty()) { + message_ = content; + } + } + +private: + std::string message_; + CmdRet ret_; +}; + +class Cmd { + public: + Cmd(const std::string& name, int arity, uint16_t flag) + : name_(name), arity_(arity), flag_(flag) {} + virtual ~Cmd() {} + + virtual std::vector current_key() const; + virtual void Execute(); + virtual void ProcessFlushDBCmd(); + virtual void ProcessFlushAllCmd(); + virtual void ProcessSinglePartitionCmd(); + virtual void ProcessMultiPartitionCmd(); + virtual void ProcessDoNotSpecifyPartitionCmd(); + virtual void Do(std::shared_ptr partition = nullptr) = 0; + virtual Cmd* Clone() = 0; + + void Initial(const PikaCmdArgsType& argv, + const std::string& table_name); + + bool is_write() const; + bool is_local() const; + bool is_suspend() const; + bool is_admin_require() const; + bool is_single_partition() const; + bool is_multi_partition() const; + + std::string name() const; + CmdRes& res(); + + virtual std::string ToBinlog(uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset); + + void SetConn(const std::shared_ptr conn); + std::shared_ptr GetConn(); + + protected: + // enable copy, used default copy + //Cmd(const Cmd&); + void ProcessCommand(std::shared_ptr partition); + void DoCommand(std::shared_ptr partition); + void DoBinlog(std::shared_ptr partition); + bool CheckArg(int num) const; + void LogCommand() const; + + std::string name_; + int arity_; + uint16_t flag_; + + CmdRes res_; + PikaCmdArgsType argv_; + std::string table_name_; + + std::weak_ptr conn_; + + private: + virtual void DoInitial() = 0; + virtual void Clear() {}; + + Cmd& operator=(const Cmd&); +}; + +typedef std::unordered_map CmdTable; + +// Method for Cmd Table +void InitCmdTable(CmdTable* cmd_table); +Cmd* GetCmdFromTable(const std::string& opt, const CmdTable& cmd_table); +void DestoryCmdTable(CmdTable* cmd_table); + +void RedisAppendContent(std::string& str, const std::string& value) { + str.append(value.data(), value.size()); + str.append(kNewLine); +} + +void RedisAppendLen(std::string& str, int64_t ori, const std::string &prefix) { + char buf[32]; + slash::ll2string(buf, 32, static_cast(ori)); + str.append(prefix); + str.append(buf); + str.append(kNewLine); +} + +void TryAliasChange(std::vector* argv); + +#endif diff --git a/tools/pika_migrate/include/pika_conf.h b/tools/pika_migrate/include/pika_conf.h new file mode 100644 index 0000000000..83149be514 --- /dev/null +++ b/tools/pika_migrate/include/pika_conf.h @@ -0,0 +1,340 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_CONF_H_ +#define PIKA_CONF_H_ + +#include +#include +#include +#include + +#include "slash/include/base_conf.h" +#include "slash/include/slash_mutex.h" +#include "slash/include/slash_string.h" + +#include "include/pika_define.h" +#include "include/pika_meta.h" + +#define kBinlogReadWinDefaultSize 9000 +#define kBinlogReadWinMaxSize 90000 + +typedef slash::RWLock RWLock; + +// global class, class members well initialized +class PikaConf : public slash::BaseConf { + public: + PikaConf(const std::string& path); + ~PikaConf(); + + // Getter + int port() { RWLock l(&rwlock_, false); return port_; } + std::string slaveof() { RWLock l(&rwlock_, false); return slaveof_;} + int slave_priority() { RWLock l(&rwlock_, false); return slave_priority_;} + bool write_binlog() { RWLock l(&rwlock_, false); return write_binlog_;} + int thread_num() { RWLock l(&rwlock_, false); return thread_num_; } + int thread_pool_size() { RWLock l(&rwlock_, false); return thread_pool_size_; } + int sync_thread_num() { RWLock l(&rwlock_, false); return sync_thread_num_; } + std::string log_path() { RWLock l(&rwlock_, false); return log_path_; } + std::string db_path() { RWLock l(&rwlock_, false); return db_path_; } + std::string db_sync_path() { RWLock l(&rwlock_, false); return db_sync_path_; } + int db_sync_speed() { RWLock l(&rwlock_, false); return db_sync_speed_; } + std::string compact_cron() { RWLock l(&rwlock_, false); return compact_cron_; } + std::string compact_interval() { RWLock l(&rwlock_, false); return compact_interval_; } + int64_t write_buffer_size() { RWLock l(&rwlock_, false); return write_buffer_size_; } + int64_t max_write_buffer_size() { RWLock l(&rwlock_, false); return max_write_buffer_size_; } + int64_t max_client_response_size() { RWLock L(&rwlock_, false); return max_client_response_size_;} + int timeout() { RWLock l(&rwlock_, false); return timeout_; } + std::string server_id() { RWLock l(&rwlock_, false); return server_id_; } + std::string requirepass() { RWLock l(&rwlock_, false); return requirepass_; } + std::string masterauth() { RWLock l(&rwlock_, false); return masterauth_; } + std::string bgsave_path() { RWLock l(&rwlock_, false); return bgsave_path_; } + int expire_dump_days() { RWLock l(&rwlock_, false); return expire_dump_days_; } + std::string bgsave_prefix() { RWLock l(&rwlock_, false); return bgsave_prefix_; } + std::string userpass() { RWLock l(&rwlock_, false); return userpass_; } + const std::string suser_blacklist() { RWLock l(&rwlock_, false); return slash::StringConcat(user_blacklist_, COMMA); } + const std::vector& vuser_blacklist() { RWLock l(&rwlock_, false); return user_blacklist_;} + bool classic_mode() { return classic_mode_.load();} + int databases() { RWLock l(&rwlock_, false); return databases_;} + int default_slot_num() { RWLock l(&rwlock_, false); return default_slot_num_;} + const std::vector& table_structs() { RWLock l(&rwlock_, false); return table_structs_; } + std::string default_table() { RWLock l(&rwlock_, false); return default_table_;} + std::string compression() { RWLock l(&rwlock_, false); return compression_; } + int target_file_size_base() { RWLock l(&rwlock_, false); return target_file_size_base_; } + int max_cache_statistic_keys() { RWLock l(&rwlock_, false); return max_cache_statistic_keys_;} + int small_compaction_threshold() { RWLock l(&rwlock_, false); return small_compaction_threshold_;} + int max_background_flushes() { RWLock l(&rwlock_, false); return max_background_flushes_; } + int max_background_compactions() { RWLock l(&rwlock_, false); return max_background_compactions_; } + int max_cache_files() { RWLock l(&rwlock_, false); return max_cache_files_; } + int max_bytes_for_level_multiplier() { RWLock l(&rwlock_, false); return max_bytes_for_level_multiplier_; } + int64_t block_size() { RWLock l(&rwlock_, false); return block_size_; } + int64_t block_cache() { RWLock l(&rwlock_, false); return block_cache_; } + bool share_block_cache() { RWLock l(&rwlock_, false); return share_block_cache_; } + bool cache_index_and_filter_blocks() { RWLock l(&rwlock_, false); return cache_index_and_filter_blocks_; } + bool optimize_filters_for_hits() { RWLock l(&rwlock_, false); return optimize_filters_for_hits_; } + bool level_compaction_dynamic_level_bytes() { RWLock l(&rwlock_, false); return level_compaction_dynamic_level_bytes_; } + int expire_logs_nums() { RWLock l(&rwlock_, false); return expire_logs_nums_; } + int expire_logs_days() { RWLock l(&rwlock_, false); return expire_logs_days_; } + std::string conf_path() { RWLock l(&rwlock_, false); return conf_path_; } + bool slave_read_only() { RWLock l(&rwlock_, false); return slave_read_only_; } + int maxclients() { RWLock l(&rwlock_, false); return maxclients_; } + int root_connection_num() { RWLock l(&rwlock_, false); return root_connection_num_; } + bool slowlog_write_errorlog() { return slowlog_write_errorlog_.load();} + int slowlog_slower_than() { return slowlog_log_slower_than_.load(); } + int slowlog_max_len() { RWLock L(&rwlock_, false); return slowlog_max_len_; } + std::string network_interface() { RWLock l(&rwlock_, false); return network_interface_; } + int sync_window_size() { return sync_window_size_.load(); } + + std::string target_redis_host() { return target_redis_host_; } + int target_redis_port() { return target_redis_port_; } + std::string target_redis_pwd() { return target_redis_pwd_; } + int sync_batch_num() { return sync_batch_num_; } + int redis_sender_num() { return redis_sender_num_; } + + // Immutable config items, we don't use lock. + bool daemonize() { return daemonize_; } + std::string pidfile() { return pidfile_; } + int binlog_file_size() { return binlog_file_size_; } + + // Setter + void SetPort(const int value) { + RWLock l(&rwlock_, true); + port_ = value; + } + void SetThreadNum(const int value) { + RWLock l(&rwlock_, true); + thread_num_ = value; + } + void SetTimeout(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("timeout", std::to_string(value)); + timeout_ = value; + } + void SetThreadPoolSize(const int value) { + RWLock l(&rwlock_, true); + thread_pool_size_ = value; + } + void SetSlaveof(const std::string value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("slaveof", value); + slaveof_ = value; + } + void SetSlavePriority(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("slave-priority", std::to_string(value)); + slave_priority_ = value; + } + void SetWriteBinlog(const std::string& value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("write-binlog", value); + write_binlog_ = (value == "yes") ? true : false; + } + void SetMaxCacheStatisticKeys(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("max-cache-statistic-keys", std::to_string(value)); + max_cache_statistic_keys_ = value; + } + void SetSmallCompactionThreshold(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("small-compaction-threshold", std::to_string(value)); + small_compaction_threshold_ = value; + } + void SetMaxClientResponseSize(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("max-client-response-size", std::to_string(value)); + max_client_response_size_ = value; + } + void SetBgsavePath(const std::string &value) { + RWLock l(&rwlock_, true); + bgsave_path_ = value; + if (value[value.length() - 1] != '/') { + bgsave_path_ += "/"; + } + } + void SetExpireDumpDays(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("dump-expire", std::to_string(value)); + expire_dump_days_ = value; + } + void SetBgsavePrefix(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("dump-prefix", value); + bgsave_prefix_ = value; + } + void SetRequirePass(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("requirepass", value); + requirepass_ = value; + } + void SetMasterAuth(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("masterauth", value); + masterauth_ = value; + } + void SetUserPass(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("userpass", value); + userpass_ = value; + } + void SetUserBlackList(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("userblacklist", value); + slash::StringSplit(value, COMMA, user_blacklist_); + for (auto& item : user_blacklist_) { + slash::StringToLower(item); + } + } + void SetExpireLogsNums(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("expire-logs-nums", std::to_string(value)); + expire_logs_nums_ = value; + } + void SetExpireLogsDays(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("expire-logs-days", std::to_string(value)); + expire_logs_days_ = value; + } + void SetMaxConnection(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("maxclients", std::to_string(value)); + maxclients_ = value; + } + void SetRootConnectionNum(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("root-connection-num", std::to_string(value)); + root_connection_num_ = value; + } + void SetSlowlogWriteErrorlog(const bool value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("slowlog-write-errorlog", value == true ? "yes" : "no"); + slowlog_write_errorlog_.store(value); + } + void SetSlowlogSlowerThan(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("slowlog-log-slower-than", std::to_string(value)); + slowlog_log_slower_than_.store(value); + } + void SetSlowlogMaxLen(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("slowlog-max-len", std::to_string(value)); + slowlog_max_len_ = value; + } + void SetDbSyncSpeed(const int value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("db-sync-speed", std::to_string(value)); + db_sync_speed_ = value; + } + void SetCompactCron(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("compact-cron", value); + compact_cron_ = value; + } + void SetCompactInterval(const std::string &value) { + RWLock l(&rwlock_, true); + TryPushDiffCommands("compact-interval", value); + compact_interval_ = value; + } + void SetSyncWindowSize(const int &value) { + TryPushDiffCommands("sync-window-size", std::to_string(value)); + sync_window_size_.store(value); + } + + Status TablePartitionsSanityCheck(const std::string& table_name, + const std::set& partition_ids, + bool is_add); + Status AddTablePartitions(const std::string& table_name, + const std::set& partition_ids); + Status RemoveTablePartitions(const std::string& table_name, + const std::set& partition_ids); + + int Load(); + int ConfigRewrite(); + + private: + Status InternalGetTargetTable(const std::string& table_name, + uint32_t* const target); + + int port_; + std::string slaveof_; + int slave_priority_; + int thread_num_; + int thread_pool_size_; + int sync_thread_num_; + std::string log_path_; + std::string db_path_; + std::string db_sync_path_; + int expire_dump_days_; + int db_sync_speed_; + std::string compact_cron_; + std::string compact_interval_; + int64_t write_buffer_size_; + int64_t max_write_buffer_size_; + int64_t max_client_response_size_; + bool daemonize_; + int timeout_; + std::string server_id_; + std::string requirepass_; + std::string masterauth_; + std::string userpass_; + std::vector user_blacklist_; + std::atomic classic_mode_; + int databases_; + int default_slot_num_; + std::vector table_structs_; + std::string default_table_; + std::string bgsave_path_; + std::string bgsave_prefix_; + std::string pidfile_; + + std::string compression_; + int maxclients_; + int root_connection_num_; + std::atomic slowlog_write_errorlog_; + std::atomic slowlog_log_slower_than_; + int slowlog_max_len_; + int expire_logs_days_; + int expire_logs_nums_; + bool slave_read_only_; + std::string conf_path_; + int max_cache_statistic_keys_; + int small_compaction_threshold_; + int max_background_flushes_; + int max_background_compactions_; + int max_cache_files_; + int max_bytes_for_level_multiplier_; + int64_t block_size_; + int64_t block_cache_; + bool share_block_cache_; + bool cache_index_and_filter_blocks_; + bool optimize_filters_for_hits_; + bool level_compaction_dynamic_level_bytes_; + std::atomic sync_window_size_; + + std::string network_interface_; + + // diff commands between cached commands and config file commands + std::map diff_commands_; + void TryPushDiffCommands(const std::string& command, const std::string& value); + + // migrate configure items + std::string target_redis_host_; + int target_redis_port_; + std::string target_redis_pwd_; + int sync_batch_num_; + int redis_sender_num_; + + // + // Critical configure items + // + bool write_binlog_; + int target_file_size_base_; + int binlog_file_size_; + + PikaMeta* local_meta_; + + pthread_rwlock_t rwlock_; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_data_distribution.h b/tools/pika_migrate/include/pika_data_distribution.h new file mode 100644 index 0000000000..19128e3704 --- /dev/null +++ b/tools/pika_migrate/include/pika_data_distribution.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_DATA_DISTRIBUTION_H_ +#define PIKA_DATA_DISTRIBUTION_H_ + +#include "slash/include/slash_status.h" + +// polynomial reserved Crc32 magic num +const uint32_t IEEE_POLY = 0xedb88320; + +class PikaDataDistribution { + public: + virtual ~PikaDataDistribution() = default; + // Initialization + virtual void Init() = 0; + // key map to partition id + virtual uint32_t Distribute(const std::string& str, uint32_t partition_num) = 0; +}; + +class HashModulo : public PikaDataDistribution { + public: + virtual ~HashModulo() = default; + virtual void Init(); + virtual uint32_t Distribute(const std::string& str, uint32_t partition_num); +}; + +class Crc32 : public PikaDataDistribution { + public: + virtual void Init(); + virtual uint32_t Distribute(const std::string& str, uint32_t partition_num); + private: + void Crc32TableInit(uint32_t poly); + uint32_t Crc32Update(uint32_t crc, const char* buf, int len); + uint32_t crc32tab[256]; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_define.h b/tools/pika_migrate/include/pika_define.h new file mode 100644 index 0000000000..4610a84df0 --- /dev/null +++ b/tools/pika_migrate/include/pika_define.h @@ -0,0 +1,424 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_DEFINE_H_ +#define PIKA_DEFINE_H_ + +#include +#include + +#include "pink/include/redis_cli.h" + +#define PIKA_SYNC_BUFFER_SIZE 10 +#define PIKA_MAX_WORKER_THREAD_NUM 24 +#define PIKA_REPL_SERVER_TP_SIZE 3 +#define PIKA_META_SYNC_MAX_WAIT_TIME 10 +#define PIKA_SCAN_STEP_LENGTH 1000 + +class PikaServer; + +/* Port shift */ +const int kPortShiftRSync = 1000; +const int kPortShiftReplServer = 2000; + +const std::string kPikaPidFile = "pika.pid"; +const std::string kPikaSecretFile = "rsync.secret"; +const std::string kDefaultRsyncAuth = "default"; + +struct TableStruct { + TableStruct(const std::string& tn, + const uint32_t pn, + const std::set& pi) + : table_name(tn), partition_num(pn), partition_ids(pi) {} + + bool operator == (const TableStruct& table_struct) const { + return table_name == table_struct.table_name + && partition_num == table_struct.partition_num + && partition_ids == table_struct.partition_ids; + } + std::string table_name; + uint32_t partition_num; + std::set partition_ids; +}; + +struct WorkerCronTask { + int task; + std::string ip_port; +}; +typedef WorkerCronTask MonitorCronTask; +//task define +#define TASK_KILL 0 +#define TASK_KILLALL 1 + +//slave item +struct SlaveItem { + std::string ip_port; + std::string ip; + int port; + int conn_fd; + int stage; + std::vector table_structs; + struct timeval create_time; +}; + +enum ReplState { + kNoConnect = 0, + kTryConnect = 1, + kTryDBSync = 2, + kWaitDBSync = 3, + kWaitReply = 4, + kConnected = 5, + kError = 6 +}; + +// debug only +const std::string ReplStateMsg[] = { + "kNoConnect", + "kTryConnect", + "kTryDBSync", + "kWaitDBSync", + "kWaitReply", + "kConnected", + "kError" +}; + +enum SlotState { + INFREE = 0, + INBUSY = 1, +}; + +struct BinlogOffset { + uint32_t filenum; + uint64_t offset; + BinlogOffset() + : filenum(0), offset(0) {} + BinlogOffset(uint32_t num, uint64_t off) + : filenum(num), offset(off) {} + BinlogOffset(const BinlogOffset& other) { + filenum = other.filenum; + offset = other.offset; + } + std::string ToString() const { + return "filenum: " + std::to_string(filenum) + " offset: " + std::to_string(offset); + } + bool operator==(const BinlogOffset& other) const { + if (filenum == other.filenum && offset == other.offset) { + return true; + } + return false; + } +}; + +//dbsync arg +struct DBSyncArg { + PikaServer* p; + std::string ip; + int port; + std::string table_name; + uint32_t partition_id; + DBSyncArg(PikaServer* const _p, + const std::string& _ip, + int _port, + const std::string& _table_name, + uint32_t _partition_id) + : p(_p), ip(_ip), port(_port), + table_name(_table_name), partition_id(_partition_id) {} +}; + +// rm define +enum SlaveState { + kSlaveNotSync = 0, + kSlaveDbSync = 1, + kSlaveBinlogSync = 2, +}; + +// debug only +const std::string SlaveStateMsg[] = { + "SlaveNotSync", + "SlaveDbSync", + "SlaveBinlogSync" +}; + +enum BinlogSyncState { + kNotSync = 0, + kReadFromCache = 1, + kReadFromFile = 2, +}; + +// debug only +const std::string BinlogSyncStateMsg[] = { + "NotSync", + "ReadFromCache", + "ReadFromFile" +}; + +struct BinlogChip { + BinlogOffset offset_; + std::string binlog_; + BinlogChip(BinlogOffset offset, std::string binlog) : offset_(offset), binlog_(binlog) { + } + BinlogChip(const BinlogChip& binlog_chip) { + offset_ = binlog_chip.offset_; + binlog_ = binlog_chip.binlog_; + } +}; + +struct PartitionInfo { + PartitionInfo(const std::string& table_name, uint32_t partition_id) + : table_name_(table_name), partition_id_(partition_id) { + } + PartitionInfo() : partition_id_(0) { + } + bool operator==(const PartitionInfo& other) const { + if (table_name_ == other.table_name_ + && partition_id_ == other.partition_id_) { + return true; + } + return false; + } + int operator<(const PartitionInfo& other) const { + int ret = strcmp(table_name_.data(), other.table_name_.data()); + if (!ret) { + if (partition_id_ < other.partition_id_) { + ret = -1; + } else if (partition_id_ > other.partition_id_) { + ret = 1; + } else { + ret = 0; + } + } + return ret; + } + std::string ToString() const { + return "(" + table_name_ + ":" + std::to_string(partition_id_) + ")"; + } + std::string table_name_; + uint32_t partition_id_; +}; + +struct hash_partition_info { + size_t operator()(const PartitionInfo& n) const { + return std::hash()(n.table_name_) ^ std::hash()(n.partition_id_); + } +}; + +class Node { + public: + Node(const std::string& ip, int port) : ip_(ip), port_(port) { + } + virtual ~Node() = default; + Node() : port_(0) { + } + const std::string& Ip() const { + return ip_; + } + int Port() const { + return port_; + } + std::string ToString() const { + return ip_ + ":" + std::to_string(port_); + } + private: + std::string ip_; + int port_; +}; + +class RmNode : public Node { + public: + RmNode(const std::string& ip, int port, + const PartitionInfo& partition_info) + : Node(ip, port), + partition_info_(partition_info), + session_id_(0), + last_send_time_(0), + last_recv_time_(0) {} + + RmNode(const std::string& ip, + int port, + const std::string& table_name, + uint32_t partition_id) + : Node(ip, port), + partition_info_(table_name, partition_id), + session_id_(0), + last_send_time_(0), + last_recv_time_(0) {} + + RmNode(const std::string& ip, + int port, + const std::string& table_name, + uint32_t partition_id, + int32_t session_id) + : Node(ip, port), + partition_info_(table_name, partition_id), + session_id_(session_id), + last_send_time_(0), + last_recv_time_(0) {} + + RmNode(const std::string& table_name, + uint32_t partition_id) + : Node(), + partition_info_(table_name, partition_id), + session_id_(0), + last_send_time_(0), + last_recv_time_(0) {} + RmNode() + : Node(), + partition_info_(), + session_id_(0), + last_send_time_(0), + last_recv_time_(0) {} + + virtual ~RmNode() = default; + bool operator==(const RmNode& other) const { + if (partition_info_.table_name_ == other.TableName() + && partition_info_.partition_id_ == other.PartitionId() + && Ip() == other.Ip() && Port() == other.Port()) { + return true; + } + return false; + } + + const std::string& TableName() const { + return partition_info_.table_name_; + } + uint32_t PartitionId() const { + return partition_info_.partition_id_; + } + const PartitionInfo& NodePartitionInfo() const { + return partition_info_; + } + void SetSessionId(uint32_t session_id) { + session_id_ = session_id; + } + int32_t SessionId() const { + return session_id_; + } + std::string ToString() const { + return "partition=" + TableName() + "_" + std::to_string(PartitionId()) + ",ip_port=" + + Ip() + ":" + std::to_string(Port()) + ",session id=" + std::to_string(SessionId()); + } + void SetLastSendTime(uint64_t last_send_time) { + last_send_time_ = last_send_time; + } + uint64_t LastSendTime() const { + return last_send_time_; + } + void SetLastRecvTime(uint64_t last_recv_time) { + last_recv_time_ = last_recv_time; + } + uint64_t LastRecvTime() const { + return last_recv_time_; + } + private: + PartitionInfo partition_info_; + int32_t session_id_; + uint64_t last_send_time_; + uint64_t last_recv_time_; +}; + +struct hash_rm_node { + size_t operator()(const RmNode& n) const { + return std::hash()(n.TableName()) ^ std::hash()(n.PartitionId()) ^ std::hash()(n.Ip()) ^ std::hash()(n.Port()); + } +}; + +struct WriteTask { + struct RmNode rm_node_; + struct BinlogChip binlog_chip_; + WriteTask(RmNode rm_node, BinlogChip binlog_chip) : rm_node_(rm_node), binlog_chip_(binlog_chip) { + } +}; + +//slowlog define +#define SLOWLOG_ENTRY_MAX_ARGC 32 +#define SLOWLOG_ENTRY_MAX_STRING 128 + +//slowlog entry +struct SlowlogEntry { + int64_t id; + int64_t start_time; + int64_t duration; + pink::RedisCmdArgsType argv; +}; + +#define PIKA_MIN_RESERVED_FDS 5000 + +const int SLAVE_ITEM_STAGE_ONE = 1; +const int SLAVE_ITEM_STAGE_TWO = 2; + +//repl_state_ +const int PIKA_REPL_NO_CONNECT = 0; +const int PIKA_REPL_SHOULD_META_SYNC = 1; +const int PIKA_REPL_META_SYNC_DONE = 2; +const int PIKA_REPL_ERROR = 3; + +//role +const int PIKA_ROLE_SINGLE = 0; +const int PIKA_ROLE_SLAVE = 1; +const int PIKA_ROLE_MASTER = 2; + +/* + * The size of Binlogfile + */ +//static uint64_t kBinlogSize = 128; +//static const uint64_t kBinlogSize = 1024 * 1024 * 100; + +enum RecordType { + kZeroType = 0, + kFullType = 1, + kFirstType = 2, + kMiddleType = 3, + kLastType = 4, + kEof = 5, + kBadRecord = 6, + kOldRecord = 7 +}; + +/* + * the block size that we read and write from write2file + * the default size is 64KB + */ +static const size_t kBlockSize = 64 * 1024; + +/* + * Header is Type(1 byte), length (3 bytes), time (4 bytes) + */ +static const size_t kHeaderSize = 1 + 3 + 4; + +/* + * the size of memory when we use memory mode + * the default memory size is 2GB + */ +const int64_t kPoolSize = 1073741824; + +const std::string kBinlogPrefix = "write2file"; +const size_t kBinlogPrefixLen = 10; + +const std::string kPikaMeta = "meta"; +const std::string kManifest = "manifest"; + +/* + * define common character + * + */ +#define COMMA ',' + +/* + * define reply between master and slave + * + */ +const std::string kInnerReplOk = "ok"; +const std::string kInnerReplWait = "wait"; + +const unsigned int kMaxBitOpInputKey = 12800; +const int kMaxBitOpInputBit = 21; +/* + * db sync + */ +const uint32_t kDBSyncMaxGap = 50; +const std::string kDBSyncModule = "document"; + +const std::string kBgsaveInfoFile = "info"; +#endif diff --git a/tools/pika_migrate/include/pika_dispatch_thread.h b/tools/pika_migrate/include/pika_dispatch_thread.h new file mode 100644 index 0000000000..8c52053013 --- /dev/null +++ b/tools/pika_migrate/include/pika_dispatch_thread.h @@ -0,0 +1,57 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_DISPATCH_THREAD_H_ +#define PIKA_DISPATCH_THREAD_H_ + +#include "include/pika_client_conn.h" + +class PikaDispatchThread { + public: + PikaDispatchThread(std::set &ips, int port, int work_num, + int cron_interval, int queue_limit); + ~PikaDispatchThread(); + int StartThread(); + + int64_t ThreadClientList(std::vector *clients); + + bool ClientKill(const std::string& ip_port); + void ClientKillAll(); + + void SetQueueLimit(int queue_limit) { + thread_rep_->SetQueueLimit(queue_limit); + } + + private: + class ClientConnFactory : public pink::ConnFactory { + public: + virtual std::shared_ptr NewPinkConn( + int connfd, + const std::string &ip_port, + pink::Thread* server_thread, + void* worker_specific_data, + pink::PinkEpoll* pink_epoll) const { + return std::make_shared(connfd, ip_port, server_thread, pink_epoll, pink::HandleType::kAsynchronous); + } + }; + + class Handles : public pink::ServerHandle { + public: + explicit Handles(PikaDispatchThread* pika_disptcher) + : pika_disptcher_(pika_disptcher) { + } + using pink::ServerHandle::AccessHandle; + bool AccessHandle(std::string& ip) const override; + void CronHandle() const override; + + private: + PikaDispatchThread* pika_disptcher_; + }; + + ClientConnFactory conn_factory_; + Handles handles_; + pink::ServerThread* thread_rep_; +}; +#endif diff --git a/tools/pika_migrate/include/pika_geo.h b/tools/pika_migrate/include/pika_geo.h new file mode 100644 index 0000000000..6d8ac4495c --- /dev/null +++ b/tools/pika_migrate/include/pika_geo.h @@ -0,0 +1,176 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_GEO_H_ +#define PIKA_GEO_H_ + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * zset + */ +enum Sort { + Unsort, //default + Asc, + Desc +}; + +struct GeoPoint { + std::string member; + double longitude; + double latitude; +}; + +struct NeighborPoint { + std::string member; + double score; + double distance; +}; + +struct GeoRange { + std::string member; + double longitude; + double latitude; + double distance; + std::string unit; + bool withdist; + bool withhash; + bool withcoord; + int option_num; + bool count; + int count_limit; + bool store; + bool storedist; + std::string storekey; + Sort sort; +}; + +class GeoAddCmd : public Cmd { + public: + GeoAddCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoAddCmd(*this); + } + private: + std::string key_; + std::vector pos_; + virtual void DoInitial(); +}; + +class GeoPosCmd : public Cmd { + public: + GeoPosCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoPosCmd(*this); + } + private: + std::string key_; + std::vector members_; + virtual void DoInitial(); +}; + +class GeoDistCmd : public Cmd { + public: + GeoDistCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoDistCmd(*this); + } + private: + std::string key_, first_pos_, second_pos_, unit_; + virtual void DoInitial(); +}; + +class GeoHashCmd : public Cmd { + public: + GeoHashCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoHashCmd(*this); + } + private: + std::string key_; + std::vector members_; + virtual void DoInitial(); +}; + +class GeoRadiusCmd : public Cmd { + public: + GeoRadiusCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoRadiusCmd(*this); + } + private: + std::string key_; + GeoRange range_; + virtual void DoInitial(); + virtual void Clear() { + range_.withdist = false; + range_.withcoord = false; + range_.withhash = false; + range_.count = false; + range_.store = false; + range_.storedist = false; + range_.option_num = 0; + range_.count_limit = 0; + range_.sort = Unsort; + } +}; + +class GeoRadiusByMemberCmd : public Cmd { + public: + GeoRadiusByMemberCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GeoRadiusByMemberCmd(*this); + } + private: + std::string key_; + GeoRange range_; + virtual void DoInitial(); + virtual void Clear() { + range_.withdist = false; + range_.withcoord = false; + range_.withhash = false; + range_.count = false; + range_.store = false; + range_.storedist = false; + range_.option_num = 0; + range_.count_limit = 0; + range_.sort = Unsort; + } +}; + +#endif diff --git a/tools/pika_migrate/include/pika_geohash.h b/tools/pika_migrate/include/pika_geohash.h new file mode 100644 index 0000000000..e963839a4a --- /dev/null +++ b/tools/pika_migrate/include/pika_geohash.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2013-2014, yinqiwen + * Copyright (c) 2014, Matt Stancliff . + * Copyright (c) 2015, Salvatore Sanfilippo . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PIKA_GEOHASH_H_ +#define PIKA_GEOHASH_H_ + +#include +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +#define HASHISZERO(r) (!(r).bits && !(r).step) +#define RANGEISZERO(r) (!(r).max && !(r).min) +#define RANGEPISZERO(r) (r == NULL || RANGEISZERO(*r)) + +#define GEO_STEP_MAX 26 /* 26*2 = 52 bits. */ + +/* Limits from EPSG:900913 / EPSG:3785 / OSGEO:41001 */ +#define GEO_LAT_MIN -85.05112878 +#define GEO_LAT_MAX 85.05112878 +#define GEO_LONG_MIN -180 +#define GEO_LONG_MAX 180 + +typedef enum { + GEOHASH_NORTH = 0, + GEOHASH_EAST, + GEOHASH_WEST, + GEOHASH_SOUTH, + GEOHASH_SOUTH_WEST, + GEOHASH_SOUTH_EAST, + GEOHASH_NORT_WEST, + GEOHASH_NORT_EAST +} GeoDirection; + +typedef struct { + uint64_t bits; + uint8_t step; +} GeoHashBits; + +typedef struct { + double min; + double max; +} GeoHashRange; + +typedef struct { + GeoHashBits hash; + GeoHashRange longitude; + GeoHashRange latitude; +} GeoHashArea; + +typedef struct { + GeoHashBits north; + GeoHashBits east; + GeoHashBits west; + GeoHashBits south; + GeoHashBits north_east; + GeoHashBits south_east; + GeoHashBits north_west; + GeoHashBits south_west; +} GeoHashNeighbors; + +/* + * 0:success + * -1:failed + */ +void geohashGetCoordRange(GeoHashRange *long_range, GeoHashRange *lat_range); +int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range, + double longitude, double latitude, uint8_t step, + GeoHashBits *hash); +int geohashEncodeType(double longitude, double latitude, + uint8_t step, GeoHashBits *hash); +int geohashEncodeWGS84(double longitude, double latitude, uint8_t step, + GeoHashBits *hash); +int geohashDecode(const GeoHashRange long_range, const GeoHashRange lat_range, + const GeoHashBits hash, GeoHashArea *area); +int geohashDecodeType(const GeoHashBits hash, GeoHashArea *area); +int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area); +int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy); +int geohashDecodeToLongLatType(const GeoHashBits hash, double *xy); +int geohashDecodeToLongLatWGS84(const GeoHashBits hash, double *xy); +int geohashDecodeToLongLatMercator(const GeoHashBits hash, double *xy); +void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors); + +#if defined(__cplusplus) +} +#endif +#endif /* PIKA_GEOHASH_H_ */ diff --git a/tools/pika_migrate/include/pika_geohash_helper.h b/tools/pika_migrate/include/pika_geohash_helper.h new file mode 100644 index 0000000000..0642455fa4 --- /dev/null +++ b/tools/pika_migrate/include/pika_geohash_helper.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2013-2014, yinqiwen + * Copyright (c) 2014, Matt Stancliff . + * Copyright (c) 2015, Salvatore Sanfilippo . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef PIKA_GEOHASH_HELPER_HPP_ +#define PIKA_GEOHASH_HELPER_HPP_ + +#include "include/pika_geohash.h" + +#define GZERO(s) s.bits = s.step = 0; +#define GISZERO(s) (!s.bits && !s.step) +#define GISNOTZERO(s) (s.bits || s.step) + +typedef uint64_t GeoHashFix52Bits; +typedef uint64_t GeoHashVarBits; + +typedef struct { + GeoHashBits hash; + GeoHashArea area; + GeoHashNeighbors neighbors; +} GeoHashRadius; + +int GeoHashBitsComparator(const GeoHashBits *a, const GeoHashBits *b); +uint8_t geohashEstimateStepsByRadius(double range_meters, double lat); +int geohashBoundingBox(double longitude, double latitude, double radius_meters, + double *bounds); +GeoHashRadius geohashGetAreasByRadius(double longitude, + double latitude, double radius_meters); +GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude, + double radius_meters); +GeoHashRadius geohashGetAreasByRadiusMercator(double longitude, double latitude, + double radius_meters); +GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash); +double geohashGetDistance(double lon1d, double lat1d, + double lon2d, double lat2d); +int geohashGetDistanceIfInRadius(double x1, double y1, + double x2, double y2, double radius, + double *distance); +int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2, + double y2, double radius, + double *distance); + +#endif /* PIKA_GEOHASH_HELPER_HPP_ */ diff --git a/tools/pika_migrate/include/pika_hash.h b/tools/pika_migrate/include/pika_hash.h new file mode 100644 index 0000000000..0658f0e73e --- /dev/null +++ b/tools/pika_migrate/include/pika_hash.h @@ -0,0 +1,370 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_HASH_H_ +#define PIKA_HASH_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * hash + */ +class HDelCmd : public Cmd { + public: + HDelCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HDelCmd(*this); + } + private: + std::string key_; + std::vector fields_; + virtual void DoInitial() override; +}; + +class HGetCmd : public Cmd { + public: + HGetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HGetCmd(*this); + } + private: + std::string key_, field_; + virtual void DoInitial() override; +}; + +class HGetallCmd : public Cmd { + public: + HGetallCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HGetallCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class HSetCmd : public Cmd { + public: + HSetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HSetCmd(*this); + } + private: + std::string key_, field_, value_; + virtual void DoInitial() override; +}; + +class HExistsCmd : public Cmd { + public: + HExistsCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HExistsCmd(*this); + } + private: + std::string key_, field_; + virtual void DoInitial() override; +}; + +class HIncrbyCmd : public Cmd { + public: + HIncrbyCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HIncrbyCmd(*this); + } + private: + std::string key_, field_; + int64_t by_; + virtual void DoInitial() override; +}; + +class HIncrbyfloatCmd : public Cmd { + public: + HIncrbyfloatCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HIncrbyfloatCmd(*this); + } + private: + std::string key_, field_, by_; + virtual void DoInitial() override; +}; + +class HKeysCmd : public Cmd { + public: + HKeysCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HKeysCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class HLenCmd : public Cmd { + public: + HLenCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HLenCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class HMgetCmd : public Cmd { + public: + HMgetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HMgetCmd(*this); + } + private: + std::string key_; + std::vector fields_; + virtual void DoInitial() override; +}; + +class HMsetCmd : public Cmd { + public: + HMsetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HMsetCmd(*this); + } + private: + std::string key_; + std::vector fvs_; + virtual void DoInitial() override; +}; + +class HSetnxCmd : public Cmd { + public: + HSetnxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HSetnxCmd(*this); + } + private: + std::string key_, field_, value_; + virtual void DoInitial() override; +}; + +class HStrlenCmd : public Cmd { + public: + HStrlenCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HStrlenCmd(*this); + } + private: + std::string key_, field_; + virtual void DoInitial() override; +}; + +class HValsCmd : public Cmd { + public: + HValsCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HValsCmd(*this); + } + private: + std::string key_, field_; + virtual void DoInitial() override; +}; + +class HScanCmd : public Cmd { + public: + HScanCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HScanCmd(*this); + } + private: + std::string key_, pattern_; + int64_t cursor_, count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class HScanxCmd : public Cmd { + public: + HScanxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new HScanxCmd(*this); + } + private: + std::string key_, start_field_, pattern_; + int64_t count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class PKHScanRangeCmd : public Cmd { + public: + PKHScanRangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), limit_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKHScanRangeCmd(*this); + } + private: + std::string key_; + std::string field_start_; + std::string field_end_; + std::string pattern_; + int64_t limit_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + limit_ = 10; + } +}; + +class PKHRScanRangeCmd : public Cmd { + public: + PKHRScanRangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), limit_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKHRScanRangeCmd(*this); + } + private: + std::string key_; + std::string field_start_; + std::string field_end_; + std::string pattern_; + int64_t limit_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + limit_ = 10; + } +}; +#endif diff --git a/tools/pika_migrate/include/pika_hyperloglog.h b/tools/pika_migrate/include/pika_hyperloglog.h new file mode 100644 index 0000000000..ecf3b9036f --- /dev/null +++ b/tools/pika_migrate/include/pika_hyperloglog.h @@ -0,0 +1,69 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_HYPERLOGLOG_H_ +#define PIKA_HYPERLOGLOG_H_ + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * hyperloglog + */ +class PfAddCmd : public Cmd { + public: + PfAddCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PfAddCmd(*this); + } + private: + std::string key_; + std::vector values_; + virtual void DoInitial() override; + virtual void Clear() { + values_.clear(); + } +}; + +class PfCountCmd : public Cmd { + public: + PfCountCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PfCountCmd(*this); + } + private: + std::vector keys_; + virtual void DoInitial() override; + virtual void Clear() { + keys_.clear(); + } +}; + +class PfMergeCmd : public Cmd { + public: + PfMergeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PfMergeCmd(*this); + } + private: + std::vector keys_; + virtual void DoInitial() override; + virtual void Clear() { + keys_.clear(); + } +}; + +#endif diff --git a/tools/pika_migrate/include/pika_kv.h b/tools/pika_migrate/include/pika_kv.h new file mode 100644 index 0000000000..f23c8c07ca --- /dev/null +++ b/tools/pika_migrate/include/pika_kv.h @@ -0,0 +1,736 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_KV_H_ +#define PIKA_KV_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + + +/* + * kv + */ +class SetCmd : public Cmd { + public: + enum SetCondition {kNONE, kNX, kXX, kVX, kEXORPX}; + SetCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag), sec_(0), condition_(kNONE) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SetCmd(*this); + } + + private: + std::string key_; + std::string value_; + std::string target_; + int32_t success_; + int64_t sec_; + SetCmd::SetCondition condition_; + virtual void DoInitial() override; + virtual void Clear() override { + sec_ = 0; + success_ = 0; + condition_ = kNONE; + } + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class GetCmd : public Cmd { + public: + GetCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GetCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class DelCmd : public Cmd { + public: + DelCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual void Do(std::shared_ptr partition = nullptr); + virtual std::vector current_key() const { + return keys_; + } + virtual Cmd* Clone() override { + return new DelCmd(*this); + } + + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class IncrCmd : public Cmd { + public: + IncrCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new IncrCmd(*this); + } + private: + std::string key_; + int64_t new_value_; + virtual void DoInitial() override; +}; + +class IncrbyCmd : public Cmd { + public: + IncrbyCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new IncrbyCmd(*this); + } + private: + std::string key_; + int64_t by_, new_value_; + virtual void DoInitial() override; +}; + +class IncrbyfloatCmd : public Cmd { + public: + IncrbyfloatCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new IncrbyfloatCmd(*this); + } + private: + std::string key_, value_, new_value_; + double by_; + virtual void DoInitial() override; +}; + +class DecrCmd : public Cmd { + public: + DecrCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DecrCmd(*this); + } + private: + std::string key_; + int64_t new_value_; + virtual void DoInitial() override; +}; + +class DecrbyCmd : public Cmd { + public: + DecrbyCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DecrbyCmd(*this); + } + private: + std::string key_; + int64_t by_, new_value_; + virtual void DoInitial() override; +}; + +class GetsetCmd : public Cmd { + public: + GetsetCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GetsetCmd(*this); + } + private: + std::string key_; + std::string new_value_; + virtual void DoInitial() override; +}; + +class AppendCmd : public Cmd { + public: + AppendCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new AppendCmd(*this); + } + private: + std::string key_; + std::string value_; + virtual void DoInitial() override; +}; + +class MgetCmd : public Cmd { + public: + MgetCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual void Do(std::shared_ptr partition = nullptr); + virtual std::vector current_key() const { + return keys_; + } + virtual Cmd* Clone() override { + return new MgetCmd(*this); + } + + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class KeysCmd : public Cmd { + public: + KeysCmd(const std::string& name , int arity, uint16_t flag) + : Cmd(name, arity, flag), type_(blackwidow::DataType::kAll) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new KeysCmd(*this); + } + private: + std::string pattern_; + blackwidow::DataType type_; + virtual void DoInitial() override; + virtual void Clear() { + type_ = blackwidow::DataType::kAll; + } +}; + +class SetnxCmd : public Cmd { + public: + SetnxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SetnxCmd(*this); + } + private: + std::string key_; + std::string value_; + int32_t success_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class SetexCmd : public Cmd { + public: + SetexCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SetexCmd(*this); + } + private: + std::string key_; + int64_t sec_; + std::string value_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class PsetexCmd : public Cmd { + public: + PsetexCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PsetexCmd(*this); + } + private: + std::string key_; + int64_t usec_; + std::string value_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class DelvxCmd : public Cmd { + public: + DelvxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new DelvxCmd(*this); + } + private: + std::string key_; + std::string value_; + int32_t success_; + virtual void DoInitial() override; +}; + +class MsetCmd : public Cmd { + public: + MsetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual std::vector current_key() const { + std::vector res; + for (auto& kv : kvs_) { + res.push_back(kv.key); + } + return res; + } + virtual Cmd* Clone() override { + return new MsetCmd(*this); + } + private: + std::vector kvs_; + virtual void DoInitial() override; +}; + +class MsetnxCmd : public Cmd { + public: + MsetnxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new MsetnxCmd(*this); + } + private: + std::vector kvs_; + int32_t success_; + virtual void DoInitial() override; +}; + +class GetrangeCmd : public Cmd { + public: + GetrangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new GetrangeCmd(*this); + } + private: + std::string key_; + int64_t start_; + int64_t end_; + virtual void DoInitial() override; +}; + +class SetrangeCmd : public Cmd { + public: + SetrangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SetrangeCmd(*this); + } + private: + std::string key_; + int64_t offset_; + std::string value_; + virtual void DoInitial() override; +}; + +class StrlenCmd : public Cmd { + public: + StrlenCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new StrlenCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class ExistsCmd : public Cmd { + public: + ExistsCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual std::vector current_key() const { + return keys_; + } + virtual Cmd* Clone() override { + return new ExistsCmd(*this); + } + + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class ExpireCmd : public Cmd { + public: + ExpireCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ExpireCmd(*this); + } + private: + std::string key_; + int64_t sec_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class PexpireCmd : public Cmd { + public: + PexpireCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PexpireCmd(*this); + } + private: + std::string key_; + int64_t msec_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class ExpireatCmd : public Cmd { + public: + ExpireatCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ExpireatCmd(*this); + } + private: + std::string key_; + int64_t time_stamp_; + virtual void DoInitial() override; +}; + +class PexpireatCmd : public Cmd { + public: + PexpireatCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PexpireatCmd(*this); + } + private: + std::string key_; + int64_t time_stamp_ms_; + virtual void DoInitial() override; + virtual std::string ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) override; +}; + +class TtlCmd : public Cmd { + public: + TtlCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new TtlCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class PttlCmd : public Cmd { + public: + PttlCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PttlCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class PersistCmd : public Cmd { + public: + PersistCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PersistCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class TypeCmd : public Cmd { + public: + TypeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new TypeCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class ScanCmd : public Cmd { + public: + ScanCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ScanCmd(*this); + } + private: + int64_t cursor_; + std::string pattern_; + int64_t count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class ScanxCmd : public Cmd { + public: + ScanxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ScanxCmd(*this); + } + private: + blackwidow::DataType type_; + std::string start_key_; + std::string pattern_; + int64_t count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class PKSetexAtCmd : public Cmd { +public: + PKSetexAtCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), time_stamp_(0) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKSetexAtCmd(*this); + } +private: + std::string key_; + std::string value_; + int64_t time_stamp_; + virtual void DoInitial() override; + virtual void Clear() { + time_stamp_ = 0; + } +}; + +class PKScanRangeCmd : public Cmd { + public: + PKScanRangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), limit_(10), string_with_value(false) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKScanRangeCmd(*this); + } + private: + blackwidow::DataType type_; + std::string key_start_; + std::string key_end_; + std::string pattern_; + int64_t limit_; + bool string_with_value; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + limit_ = 10; + string_with_value = false; + } +}; + +class PKRScanRangeCmd : public Cmd { + public: + PKRScanRangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), limit_(10), string_with_value(false) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new PKRScanRangeCmd(*this); + } + private: + blackwidow::DataType type_; + std::string key_start_; + std::string key_end_; + std::string pattern_; + int64_t limit_; + bool string_with_value; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + limit_ = 10; + string_with_value = false; + } +}; +#endif diff --git a/tools/pika_migrate/include/pika_list.h b/tools/pika_migrate/include/pika_list.h new file mode 100644 index 0000000000..3f18129554 --- /dev/null +++ b/tools/pika_migrate/include/pika_list.h @@ -0,0 +1,289 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_LIST_H_ +#define PIKA_LIST_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * list + */ +class LIndexCmd : public Cmd { + public: + LIndexCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), index_(0) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LIndexCmd(*this); + } + private: + std::string key_; + int64_t index_; + virtual void DoInitial() override; + virtual void Clear() { + index_ = 0; + } +}; + +class LInsertCmd : public Cmd { + public: + LInsertCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), dir_(blackwidow::After) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LInsertCmd(*this); + } + private: + std::string key_; + blackwidow::BeforeOrAfter dir_; + std::string pivot_; + std::string value_; + virtual void DoInitial() override; +}; + +class LLenCmd : public Cmd { + public: + LLenCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LLenCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class LPopCmd : public Cmd { + public: + LPopCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LPopCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class LPushCmd : public Cmd { + public: + LPushCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LPushCmd(*this); + } + private: + std::string key_; + std::vector values_; + virtual void DoInitial() override; + virtual void Clear() { + values_.clear(); + } +}; + +class LPushxCmd : public Cmd { + public: + LPushxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LPushxCmd(*this); + } + private: + std::string key_; + std::string value_; + virtual void DoInitial() override; +}; + +class LRangeCmd : public Cmd { + public: + LRangeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_(0), right_(0) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LRangeCmd(*this); + } + private: + std::string key_; + int64_t left_; + int64_t right_; + virtual void DoInitial() override; +}; + +class LRemCmd : public Cmd { + public: + LRemCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), count_(0) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LRemCmd(*this); + } + private: + std::string key_; + int64_t count_; + std::string value_; + virtual void DoInitial() override; +}; + +class LSetCmd : public Cmd { + public: + LSetCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), index_(0) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LSetCmd(*this); + } + private: + std::string key_; + int64_t index_; + std::string value_; + virtual void DoInitial() override; +}; + +class LTrimCmd : public Cmd { + public: + LTrimCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), start_(0), stop_(0) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new LTrimCmd(*this); + } + private: + std::string key_; + int64_t start_; + int64_t stop_; + virtual void DoInitial() override; +}; + +class RPopCmd : public Cmd { + public: + RPopCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new RPopCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class RPopLPushCmd : public Cmd { + public: + RPopLPushCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new RPopLPushCmd(*this); + } + private: + std::string source_; + std::string receiver_; + virtual void DoInitial() override; +}; + +class RPushCmd : public Cmd { + public: + RPushCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new RPushCmd(*this); + } + private: + std::string key_; + std::vector values_; + virtual void DoInitial() override; + virtual void Clear() { + values_.clear(); + } +}; + +class RPushxCmd : public Cmd { + public: + RPushxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {}; + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new RPushxCmd(*this); + } + private: + std::string key_; + std::string value_; + virtual void DoInitial() override; +}; +#endif diff --git a/tools/pika_migrate/include/pika_meta.h b/tools/pika_migrate/include/pika_meta.h new file mode 100644 index 0000000000..de576bfa63 --- /dev/null +++ b/tools/pika_migrate/include/pika_meta.h @@ -0,0 +1,35 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_META +#define PIKA_META + +#include "slash/include/env.h" +#include "slash/include/slash_mutex.h" + +#include "include/pika_define.h" + +using slash::Status; + +class PikaMeta { + public: + PikaMeta(); + ~PikaMeta(); + + void SetPath(const std::string& path); + + Status StableSave(const std::vector& table_structs); + Status ParseMeta(std::vector* const table_structs); + + private: + pthread_rwlock_t rwlock_; + std::string local_meta_path_; + + // No copying allowed; + PikaMeta(const PikaMeta&); + void operator=(const PikaMeta&); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_monitor_thread.h b/tools/pika_migrate/include/pika_monitor_thread.h new file mode 100644 index 0000000000..f7900e2af7 --- /dev/null +++ b/tools/pika_migrate/include/pika_monitor_thread.h @@ -0,0 +1,48 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_MONITOR_THREAD_H_ +#define PIKA_MONITOR_THREAD_H_ + +#include +#include +#include +#include + +#include "pink/include/pink_thread.h" +#include "slash/include/slash_mutex.h" + +#include "include/pika_define.h" +#include "include/pika_client_conn.h" + +class PikaMonitorThread : public pink::Thread { + public: + PikaMonitorThread(); + virtual ~PikaMonitorThread(); + + void AddMonitorClient(std::shared_ptr client_ptr); + void AddMonitorMessage(const std::string &monitor_message); + int32_t ThreadClientList(std::vector* client = NULL); + bool ThreadClientKill(const std::string& ip_port = "all"); + bool HasMonitorClients(); + + private: + void AddCronTask(MonitorCronTask task); + bool FindClient(const std::string& ip_port); + pink::WriteStatus SendMessage(int32_t fd, std::string& message); + void RemoveMonitorClient(const std::string& ip_port); + + std::atomic has_monitor_clients_; + slash::Mutex monitor_mutex_protector_; + slash::CondVar monitor_cond_; + + std::list monitor_clients_; + std::deque monitor_messages_; + std::queue cron_tasks_; + + virtual void* ThreadMain(); + void RemoveMonitorClient(int32_t client_fd); +}; +#endif diff --git a/tools/pika_migrate/include/pika_partition.h b/tools/pika_migrate/include/pika_partition.h new file mode 100644 index 0000000000..461955b85a --- /dev/null +++ b/tools/pika_migrate/include/pika_partition.h @@ -0,0 +1,169 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_PARTITION_H_ +#define PIKA_PARTITION_H_ + +#include "blackwidow/blackwidow.h" +#include "blackwidow/backupable.h" +#include "slash/include/scope_record_lock.h" + +#include "include/pika_binlog.h" + +class Cmd; + +/* + *Keyscan used + */ +struct KeyScanInfo { + time_t start_time; + std::string s_start_time; + int32_t duration; + std::vector key_infos; //the order is strings, hashes, lists, zsets, sets + bool key_scaning_; + KeyScanInfo() : + start_time(0), + s_start_time("1970-01-01 08:00:00"), + duration(-3), + key_infos({{0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}, {0, 0, 0, 0}}), + key_scaning_(false) { + } +}; + + +struct BgSaveInfo { + bool bgsaving; + time_t start_time; + std::string s_start_time; + std::string path; + uint32_t filenum; + uint64_t offset; + BgSaveInfo() : bgsaving(false), filenum(0), offset(0) {} + void Clear() { + bgsaving = false; + path.clear(); + filenum = 0; + offset = 0; + } +}; + +class Partition : public std::enable_shared_from_this { + public: + Partition(const std::string& table_name, + uint32_t partition_id, + const std::string& table_db_path, + const std::string& table_log_path); + virtual ~Partition(); + + std::string GetTableName() const; + uint32_t GetPartitionId() const; + std::string GetPartitionName() const; + std::shared_ptr logger() const; + std::shared_ptr db() const; + + void Compact(const blackwidow::DataType& type); + // needd to hold logger_->Lock() + Status WriteBinlog(const std::string& binlog); + + void DbRWLockWriter(); + void DbRWLockReader(); + void DbRWUnLock(); + + slash::lock::LockMgr* LockMgr(); + + void SetBinlogIoError(bool error); + bool IsBinlogIoError(); + bool GetBinlogOffset(BinlogOffset* const boffset); + bool SetBinlogOffset(const BinlogOffset& boffset); + + void PrepareRsync(); + bool TryUpdateMasterOffset(); + bool ChangeDb(const std::string& new_path); + + void Leave(); + void Close(); + void MoveToTrash(); + + // BgSave use; + bool IsBgSaving(); + void BgSavePartition(); + BgSaveInfo bgsave_info(); + + // FlushDB & FlushSubDB use + bool FlushDB(); + bool FlushSubDB(const std::string& db_name); + + // Purgelogs use + bool PurgeLogs(uint32_t to = 0, bool manual = false); + void ClearPurge(); + + // key scan info use + Status GetKeyNum(std::vector* key_info); + KeyScanInfo GetKeyScanInfo(); + + private: + std::string table_name_; + uint32_t partition_id_; + + std::string db_path_; + std::string log_path_; + std::string bgsave_sub_path_; + std::string dbsync_path_; + std::string partition_name_; + + bool opened_; + std::shared_ptr logger_; + std::atomic binlog_io_error_; + + pthread_rwlock_t db_rwlock_; + slash::lock::LockMgr* lock_mgr_; + std::shared_ptr db_; + + bool full_sync_; + + slash::Mutex key_info_protector_; + KeyScanInfo key_scan_info_; + + /* + * BgSave use + */ + static void DoBgSave(void* arg); + bool RunBgsaveEngine(); + bool InitBgsaveEnv(); + bool InitBgsaveEngine(); + void ClearBgsave(); + void FinishBgsave(); + BgSaveInfo bgsave_info_; + slash::Mutex bgsave_protector_; + blackwidow::BackupEngine* bgsave_engine_; + + /* + * Purgelogs use + */ + static void DoPurgeLogs(void* arg); + bool PurgeFiles(uint32_t to, bool manual); + bool GetBinlogFiles(std::map& binlogs); + std::atomic purging_; + + // key scan info use + void InitKeyScan(); + + /* + * No allowed copy and copy assign + */ + Partition(const Partition&); + void operator=(const Partition&); + +}; + +struct PurgeArg { + std::shared_ptr partition; + uint32_t to; + bool manual; + bool force; // Ignore the delete window +}; + + +#endif diff --git a/tools/pika_migrate/include/pika_pubsub.h b/tools/pika_migrate/include/pika_pubsub.h new file mode 100644 index 0000000000..737dc752c0 --- /dev/null +++ b/tools/pika_migrate/include/pika_pubsub.h @@ -0,0 +1,93 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_PUBSUB_H_ +#define PIKA_PUBSUB_H_ + +#include "pika_command.h" + +/* + * pubsub + */ +class PublishCmd : public Cmd { + public: + PublishCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new PublishCmd(*this); + } + private: + std::string channel_; + std::string msg_; + virtual void DoInitial() override; +}; + +class SubscribeCmd : public Cmd { + public: + SubscribeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new SubscribeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class UnSubscribeCmd : public Cmd { + public: + UnSubscribeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new UnSubscribeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class PUnSubscribeCmd : public Cmd { + public: + PUnSubscribeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new PUnSubscribeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class PSubscribeCmd : public Cmd { + public: + PSubscribeCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new PSubscribeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class PubSubCmd : public Cmd { + public: + PubSubCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr) override; + virtual Cmd* Clone() override { + return new PubSubCmd(*this); + } + private: + std::string subcommand_; + std::vector arguments_; + virtual void DoInitial() override; + virtual void Clear() { + arguments_.clear(); + } +}; + +#endif // INCLUDE_PIKA_PUBSUB_H_ diff --git a/tools/pika_migrate/include/pika_repl_bgworker.h b/tools/pika_migrate/include/pika_repl_bgworker.h new file mode 100644 index 0000000000..e74f41e3a9 --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_bgworker.h @@ -0,0 +1,43 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_BGWROKER_H_ +#define PIKA_REPL_BGWROKER_H_ + +#include +#include + +#include "pink/include/pb_conn.h" +#include "pink/include/bg_thread.h" +#include "pink/include/thread_pool.h" + +#include "src/pika_inner_message.pb.h" + +#include "include/pika_command.h" +#include "include/pika_binlog_transverter.h" + +class PikaReplBgWorker { + public: + explicit PikaReplBgWorker(int queue_size); + ~PikaReplBgWorker(); + int StartThread(); + int StopThread(); + void Schedule(pink::TaskFunc func, void* arg); + void QueueClear(); + static void HandleBGWorkerWriteBinlog(void* arg); + static void HandleBGWorkerWriteDB(void* arg); + + BinlogItem binlog_item_; + pink::RedisParser redis_parser_; + std::string ip_port_; + std::string table_name_; + uint32_t partition_id_; + + private: + pink::BGThread bg_thread_; + static int HandleWriteBinlog(pink::RedisParser* parser, const pink::RedisCmdArgsType& argv); +}; + +#endif // PIKA_REPL_BGWROKER_H_ diff --git a/tools/pika_migrate/include/pika_repl_client.h b/tools/pika_migrate/include/pika_repl_client.h new file mode 100644 index 0000000000..d786af489f --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_client.h @@ -0,0 +1,126 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_CLIENT_H_ +#define PIKA_REPL_CLIENT_H_ + +#include +#include + +#include "pink/include/pink_conn.h" +#include "pink/include/client_thread.h" +#include "pink/include/thread_pool.h" +#include "slash/include/slash_status.h" + +#include "include/pika_define.h" +#include "include/pika_partition.h" +#include "include/pika_binlog_reader.h" +#include "include/pika_repl_bgworker.h" +#include "include/pika_repl_client_thread.h" + +#include "pink/include/thread_pool.h" +#include "src/pika_inner_message.pb.h" + +using slash::Status; + +struct ReplClientTaskArg { + std::shared_ptr res; + std::shared_ptr conn; + ReplClientTaskArg(std::shared_ptr _res, + std::shared_ptr _conn) + : res(_res), conn(_conn) {} +}; + +struct ReplClientWriteBinlogTaskArg { + std::shared_ptr res; + std::shared_ptr conn; + void* res_private_data; + PikaReplBgWorker* worker; + ReplClientWriteBinlogTaskArg( + const std::shared_ptr _res, + std::shared_ptr _conn, + void* _res_private_data, + PikaReplBgWorker* _worker) : + res(_res), conn(_conn), + res_private_data(_res_private_data), worker(_worker) {} +}; + +struct ReplClientWriteDBTaskArg { + PikaCmdArgsType* argv; + BinlogItem* binlog_item; + std::string table_name; + uint32_t partition_id; + ReplClientWriteDBTaskArg(PikaCmdArgsType* _argv, + BinlogItem* _binlog_item, + const std::string _table_name, + uint32_t _partition_id) + : argv(_argv), binlog_item(_binlog_item), + table_name(_table_name), partition_id(_partition_id) {} + ~ReplClientWriteDBTaskArg() { + delete argv; + delete binlog_item; + } +}; + + +class PikaReplClient { + public: + PikaReplClient(int cron_interval, int keepalive_timeout); + ~PikaReplClient(); + + int Start(); + int Stop(); + + slash::Status Write(const std::string& ip, const int port, const std::string& msg); + slash::Status Close(const std::string& ip, const int port); + + void Schedule(pink::TaskFunc func, void* arg); + void ScheduleWriteBinlogTask(std::string table_partition, + const std::shared_ptr res, + std::shared_ptr conn, + void* req_private_data); + void ScheduleWriteDBTask(const std::string& dispatch_key, + PikaCmdArgsType* argv, BinlogItem* binlog_item, + const std::string& table_name, uint32_t partition_id); + + Status SendMetaSync(); + Status SendPartitionDBSync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& boffset, + const std::string& local_ip); + Status SendPartitionTrySync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& boffset, + const std::string& local_ip); + Status SendPartitionBinlogSync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& ack_start, + const BinlogOffset& ack_end, + const std::string& local_ip, + bool is_frist_send); + Status SendRemoveSlaveNode(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const std::string& local_ip); + private: + size_t GetHashIndex(std::string key, bool upper_half); + void UpdateNextAvail() { + next_avail_ = (next_avail_ + 1) % bg_workers_.size(); + } + + PikaReplClientThread* client_thread_; + int next_avail_; + std::hash str_hash; + std::vector bg_workers_; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_repl_client_conn.h b/tools/pika_migrate/include/pika_repl_client_conn.h new file mode 100644 index 0000000000..516507f2d5 --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_client_conn.h @@ -0,0 +1,40 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_CLIENT_CONN_H_ +#define PIKA_REPL_CLIENT_CONN_H_ + +#include "pink/include/pb_conn.h" + +#include + +#include "include/pika_conf.h" +#include "src/pika_inner_message.pb.h" + +class PikaReplClientConn: public pink::PbConn { + public: + PikaReplClientConn(int fd, const std::string& ip_port, pink::Thread *thread, void* worker_specific_data, pink::PinkEpoll* epoll); + virtual ~PikaReplClientConn() = default; + + static void HandleMetaSyncResponse(void* arg); + static void HandleDBSyncResponse(void* arg); + static void HandleTrySyncResponse(void* arg); + static void HandleRemoveSlaveNodeResponse(void* arg); + static bool IsTableStructConsistent(const std::vector& current_tables, + const std::vector& expect_tables); + int DealMessage() override; + private: + // dispatch binlog by its table_name + partition + void DispatchBinlogRes(const std::shared_ptr response); + + struct ReplRespArg { + std::shared_ptr resp; + std::shared_ptr conn; + ReplRespArg(std::shared_ptr _resp, std::shared_ptr _conn) : resp(_resp), conn(_conn) { + } + }; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_repl_client_thread.h b/tools/pika_migrate/include/pika_repl_client_thread.h new file mode 100644 index 0000000000..c0ed6ab48b --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_client_thread.h @@ -0,0 +1,63 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_CLIENT_THREAD_H_ +#define PIKA_REPL_CLIENT_THREAD_H_ + +#include +#include + +#include "include/pika_repl_client_conn.h" + +#include "pink/include/pink_conn.h" +#include "pink/include/client_thread.h" + +class PikaReplClientThread : public pink::ClientThread { + public: + PikaReplClientThread(int cron_interval, int keepalive_timeout); + virtual ~PikaReplClientThread() = default; + int Start(); + + private: + class ReplClientConnFactory : public pink::ConnFactory { + public: + virtual std::shared_ptr NewPinkConn( + int connfd, + const std::string &ip_port, + pink::Thread *thread, + void* worker_specific_data, + pink::PinkEpoll* pink_epoll) const override { + return std::make_shared(connfd, ip_port, thread, worker_specific_data, pink_epoll); + } + }; + class ReplClientHandle : public pink::ClientHandle { + public: + void CronHandle() const override { + } + void FdTimeoutHandle(int fd, const std::string& ip_port) const override; + void FdClosedHandle(int fd, const std::string& ip_port) const override; + bool AccessHandle(std::string& ip) const override { + // ban 127.0.0.1 if you want to test this routine + // if (ip.find("127.0.0.2") != std::string::npos) { + // std::cout << "AccessHandle " << ip << std::endl; + // return false; + // } + return true; + } + int CreateWorkerSpecificData(void** data) const override { + return 0; + } + int DeleteWorkerSpecificData(void* data) const override { + return 0; + } + void DestConnectFailedHandle(std::string ip_port, std::string reason) const override { + } + }; + + ReplClientConnFactory conn_factory_; + ReplClientHandle handle_; +}; + +#endif // PIKA_REPL_CLIENT_THREAD_H_ diff --git a/tools/pika_migrate/include/pika_repl_server.h b/tools/pika_migrate/include/pika_repl_server.h new file mode 100644 index 0000000000..592052d92a --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_server.h @@ -0,0 +1,48 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_SERVER_H_ +#define PIKA_REPL_SERVER_H_ + +#include "pink/include/thread_pool.h" + +#include + +#include "include/pika_command.h" +#include "include/pika_repl_bgworker.h" +#include "include/pika_repl_server_thread.h" + +struct ReplServerTaskArg { + std::shared_ptr req; + std::shared_ptr conn; + ReplServerTaskArg(std::shared_ptr _req, std::shared_ptr _conn) + : req(_req), conn(_conn) {} +}; + +class PikaReplServer { + public: + PikaReplServer(const std::set& ips, int port, int cron_interval); + ~PikaReplServer(); + + int Start(); + int Stop(); + + slash::Status SendSlaveBinlogChips(const std::string& ip, int port, const std::vector& tasks); + slash::Status Write(const std::string& ip, const int port, const std::string& msg); + + void Schedule(pink::TaskFunc func, void* arg); + void UpdateClientConnMap(const std::string& ip_port, int fd); + void RemoveClientConn(int fd); + void KillAllConns(); + + private: + pink::ThreadPool* server_tp_; + PikaReplServerThread* pika_repl_server_thread_; + + pthread_rwlock_t client_conn_rwlock_; + std::map client_conn_map_; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_repl_server_conn.h b/tools/pika_migrate/include/pika_repl_server_conn.h new file mode 100644 index 0000000000..d5757f0373 --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_server_conn.h @@ -0,0 +1,30 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_SERVER_CONN_H_ +#define PIKA_REPL_SERVER_CONN_H_ + +#include + +#include "pink/include/pb_conn.h" +#include "pink/include/pink_thread.h" + +#include "src/pika_inner_message.pb.h" + +class PikaReplServerConn: public pink::PbConn { + public: + PikaReplServerConn(int fd, std::string ip_port, pink::Thread* thread, void* worker_specific_data, pink::PinkEpoll* epoll); + virtual ~PikaReplServerConn(); + + static void HandleMetaSyncRequest(void* arg); + static void HandleTrySyncRequest(void* arg); + static void HandleDBSyncRequest(void* arg); + static void HandleBinlogSyncRequest(void* arg); + static void HandleRemoveSlaveNodeRequest(void* arg); + + int DealMessage(); +}; + +#endif // INCLUDE_PIKA_REPL_SERVER_CONN_H_ diff --git a/tools/pika_migrate/include/pika_repl_server_thread.h b/tools/pika_migrate/include/pika_repl_server_thread.h new file mode 100644 index 0000000000..f322a1df7c --- /dev/null +++ b/tools/pika_migrate/include/pika_repl_server_thread.h @@ -0,0 +1,55 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_REPL_SERVER_THREAD_H_ +#define PIKA_REPL_SERVER_THREAD_H_ + +#include "pink/src/holy_thread.h" + +#include "include/pika_repl_server_conn.h" + +class PikaReplServerThread : public pink::HolyThread { + public: + PikaReplServerThread(const std::set& ips, int port, int cron_interval); + virtual ~PikaReplServerThread() = default; + + int ListenPort(); + + // for ProcessBinlogData use + uint64_t GetnPlusSerial() { + return serial_++; + } + + private: + class ReplServerConnFactory : public pink::ConnFactory { + public: + explicit ReplServerConnFactory(PikaReplServerThread* binlog_receiver) + : binlog_receiver_(binlog_receiver) { + } + + virtual std::shared_ptr NewPinkConn( + int connfd, + const std::string& ip_port, + pink::Thread* thread, + void* worker_specific_data, + pink::PinkEpoll* pink_epoll) const override { + return std::make_shared(connfd, ip_port, thread, binlog_receiver_, pink_epoll); + } + private: + PikaReplServerThread* binlog_receiver_; + }; + + class ReplServerHandle : public pink::ServerHandle { + public: + virtual void FdClosedHandle(int fd, const std::string& ip_port) const override; + }; + + ReplServerConnFactory conn_factory_; + ReplServerHandle handle_; + int port_; + uint64_t serial_; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_rm.h b/tools/pika_migrate/include/pika_rm.h new file mode 100644 index 0000000000..cb20a8b250 --- /dev/null +++ b/tools/pika_migrate/include/pika_rm.h @@ -0,0 +1,359 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_RM_H_ +#define PIKA_RM_H_ + +#include +#include +#include +#include +#include + +#include "slash/include/slash_status.h" + +#include "include/pika_binlog_reader.h" +#include "include/pika_repl_client.h" +#include "include/pika_repl_server.h" + +#define kBinlogSendPacketNum 40 +#define kBinlogSendBatchNum 100 + +// unit seconds +#define kSendKeepAliveTimeout (10 * 1000000) +#define kRecvKeepAliveTimeout (20 * 1000000) + +using slash::Status; + +struct SyncWinItem { + BinlogOffset offset_; + bool acked_; + bool operator==(const SyncWinItem& other) const { + if (offset_.filenum == other.offset_.filenum && offset_.offset == other.offset_.offset) { + return true; + } + return false; + } + explicit SyncWinItem(const BinlogOffset& offset) : offset_(offset), acked_(false) { + } + SyncWinItem(uint32_t filenum, uint64_t offset) : offset_(filenum, offset), acked_(false) { + } + std::string ToString() const { + return offset_.ToString() + " acked: " + std::to_string(acked_); + } +}; + +class SyncWindow { + public: + SyncWindow() { + } + void Push(const SyncWinItem& item); + bool Update(const SyncWinItem& start_item, const SyncWinItem& end_item, BinlogOffset* acked_offset); + int Remainings(); + std::string ToStringStatus() const { + if (win_.empty()) { + return " Size: " + std::to_string(win_.size()) + "\r\n"; + } else { + std::string res; + res += " Size: " + std::to_string(win_.size()) + "\r\n"; + res += (" Begin_item: " + win_.begin()->ToString() + "\r\n"); + res += (" End_item: " + win_.rbegin()->ToString() + "\r\n"); + return res; + } + } + private: + // TODO(whoiami) ring buffer maybe + std::deque win_; +}; + +// role master use +class SlaveNode : public RmNode { + public: + SlaveNode(const std::string& ip, int port, const std::string& table_name, uint32_t partition_id, int session_id); + ~SlaveNode(); + void Lock() { + slave_mu.Lock(); + } + void Unlock() { + slave_mu.Unlock(); + } + SlaveState slave_state; + + BinlogSyncState b_state; + SyncWindow sync_win; + BinlogOffset sent_offset; + BinlogOffset acked_offset; + + std::string ToStringStatus(); + + std::shared_ptr binlog_reader; + Status InitBinlogFileReader(const std::shared_ptr& binlog, const BinlogOffset& offset); + void ReleaseBinlogFileReader(); + + slash::Mutex slave_mu; +}; + +class SyncPartition { + public: + SyncPartition(const std::string& table_name, uint32_t partition_id); + virtual ~SyncPartition() = default; + + PartitionInfo& SyncPartitionInfo() { + return partition_info_; + } + protected: + // std::shared_ptr binlog_; + PartitionInfo partition_info_; +}; + +class SyncMasterPartition : public SyncPartition { + public: + SyncMasterPartition(const std::string& table_name, uint32_t partition_id); + Status AddSlaveNode(const std::string& ip, int port, int session_id); + Status RemoveSlaveNode(const std::string& ip, int port); + + Status ActivateSlaveBinlogSync(const std::string& ip, int port, const std::shared_ptr binlog, const BinlogOffset& offset); + Status ActivateSlaveDbSync(const std::string& ip, int port); + + Status SyncBinlogToWq(const std::string& ip, int port); + Status UpdateSlaveBinlogAckInfo(const std::string& ip, int port, const BinlogOffset& start, const BinlogOffset& end); + Status GetSlaveSyncBinlogInfo(const std::string& ip, int port, BinlogOffset* sent_offset, BinlogOffset* acked_offset); + Status GetSlaveState(const std::string& ip, int port, SlaveState* const slave_state); + + Status SetLastSendTime(const std::string& ip, int port, uint64_t time); + Status GetLastSendTime(const std::string& ip, int port, uint64_t* time); + + Status SetLastRecvTime(const std::string& ip, int port, uint64_t time); + Status GetLastRecvTime(const std::string& ip, int port, uint64_t* time); + + Status GetSafetyPurgeBinlog(std::string* safety_purge); + bool BinlogCloudPurge(uint32_t index); + + Status WakeUpSlaveBinlogSync(); + Status CheckSyncTimeout(uint64_t now); + + int GetNumberOfSlaveNode(); + bool CheckSlaveNodeExist(const std::string& ip, int port); + Status GetSlaveNodeSession(const std::string& ip, int port, int32_t* session); + + void GetValidSlaveNames(std::vector* slavenames); + // display use + Status GetInfo(std::string* info); + // debug use + std::string ToStringStatus(); + + int32_t GenSessionId(); + bool CheckSessionId(const std::string& ip, int port, + const std::string& table_name, + uint64_t partition_id, int session_id); + + private: + bool CheckReadBinlogFromCache(); + // inovker need to hold partition_mu_ + void CleanMasterNode(); + void CleanSlaveNode(); + // invoker need to hold slave_mu_ + Status ReadCachedBinlogToWq(const std::shared_ptr& slave_ptr); + Status ReadBinlogFileToWq(const std::shared_ptr& slave_ptr); + // inovker need to hold partition_mu_ + Status GetSlaveNode(const std::string& ip, int port, std::shared_ptr* slave_node); + + slash::Mutex partition_mu_; + std::vector> slaves_; + + slash::Mutex session_mu_; + int32_t session_id_; + + // BinlogCacheWindow win_; +}; + +class SyncSlavePartition : public SyncPartition { + public: + SyncSlavePartition(const std::string& table_name, uint32_t partition_id); + + void Activate(const RmNode& master, const ReplState& repl_state); + void Deactivate(); + + void SetLastRecvTime(uint64_t time); + uint64_t LastRecvTime(); + + void SetReplState(const ReplState& repl_state); + ReplState State(); + + Status CheckSyncTimeout(uint64_t now); + + // For display + Status GetInfo(std::string* info); + // For debug + std::string ToStringStatus(); + + const std::string& MasterIp() { + return m_info_.Ip(); + } + int MasterPort() { + return m_info_.Port(); + } + void SetMasterSessionId(int32_t session_id) { + m_info_.SetSessionId(session_id); + } + int32_t MasterSessionId() { + return m_info_.SessionId(); + } + void SetLocalIp(const std::string& local_ip) { + local_ip_ = local_ip; + } + std::string LocalIp() { + return local_ip_; + } + + private: + slash::Mutex partition_mu_; + RmNode m_info_; + ReplState repl_state_; + std::string local_ip_; +}; + +class BinlogReaderManager { + public: + ~BinlogReaderManager(); + Status FetchBinlogReader(const RmNode& rm_node, std::shared_ptr* reader); + Status ReleaseBinlogReader(const RmNode& rm_node); + private: + slash::Mutex reader_mu_; + std::unordered_map, hash_rm_node> occupied_; + std::vector> vacant_; +}; + +class PikaReplicaManager { + public: + PikaReplicaManager(); + ~PikaReplicaManager(); + + void Start(); + void Stop(); + + Status AddSyncPartitionSanityCheck(const std::set& p_infos); + Status AddSyncPartition(const std::set& p_infos); + Status RemoveSyncPartitionSanityCheck(const std::set& p_infos); + Status RemoveSyncPartition(const std::set& p_infos); + Status SelectLocalIp(const std::string& remote_ip, + const int remote_port, + std::string* const local_ip); + Status ActivateSyncSlavePartition(const RmNode& node, const ReplState& repl_state); + Status UpdateSyncSlavePartitionSessionId(const PartitionInfo& p_info, int32_t session_id); + Status DeactivateSyncSlavePartition(const PartitionInfo& p_info); + Status SetSlaveReplState(const PartitionInfo& p_info, const ReplState& repl_state); + Status GetSlaveReplState(const PartitionInfo& p_info, ReplState* repl_state); + + // For Pika Repl Client Thread + Status SendMetaSyncRequest(); + Status SendRemoveSlaveNodeRequest(const std::string& table, uint32_t partition_id); + Status SendPartitionTrySyncRequest(const std::string& table_name, size_t partition_id); + Status SendPartitionDBSyncRequest(const std::string& table_name, size_t partition_id); + Status SendPartitionBinlogSyncAckRequest(const std::string& table, uint32_t partition_id, + const BinlogOffset& ack_start, const BinlogOffset& ack_end, + bool is_first_send = false); + Status CloseReplClientConn(const std::string& ip, int32_t port); + + // For Pika Repl Server Thread + Status SendSlaveBinlogChipsRequest(const std::string& ip, int port, const std::vector& tasks); + + // For SyncMasterPartition + std::shared_ptr GetSyncMasterPartitionByName(const PartitionInfo& p_info); + Status GetSafetyPurgeBinlogFromSMP(const std::string& table_name, + uint32_t partition_id, std::string* safety_purge); + bool BinlogCloudPurgeFromSMP(const std::string& table_name, + uint32_t partition_id, uint32_t index); + + // For SyncSlavePartition + std::shared_ptr GetSyncSlavePartitionByName(const PartitionInfo& p_info); + + + + Status RunSyncSlavePartitionStateMachine(); + + Status SetMasterLastRecvTime(const RmNode& slave, uint64_t time); + Status SetSlaveLastRecvTime(const RmNode& slave, uint64_t time); + + Status CheckSyncTimeout(uint64_t now); + + // To check partition info + // For pkcluster info command + Status GetPartitionInfo( + const std::string& table, uint32_t partition_id, std::string* info); + + void FindCompleteReplica(std::vector* replica); + void FindCommonMaster(std::string* master); + + Status CheckPartitionRole( + const std::string& table, uint32_t partition_id, int* role); + + void RmStatus(std::string* debug_info); + + // following funcs invoked by master partition only + + Status AddPartitionSlave(const RmNode& slave); + Status RemovePartitionSlave(const RmNode& slave); + bool CheckPartitionSlaveExist(const RmNode& slave); + Status GetPartitionSlaveSession(const RmNode& slave, int32_t* session); + + Status LostConnection(const std::string& ip, int port); + + Status ActivateBinlogSync(const RmNode& slave, const BinlogOffset& offset); + Status ActivateDbSync(const RmNode& slave); + + // Update binlog win and try to send next binlog + Status UpdateSyncBinlogStatus(const RmNode& slave, const BinlogOffset& offset_start, const BinlogOffset& offset_end); + Status GetSyncBinlogStatus(const RmNode& slave, BinlogOffset* sent_boffset, BinlogOffset* acked_boffset); + Status GetSyncMasterPartitionSlaveState(const RmNode& slave, SlaveState* const slave_state); + + Status WakeUpBinlogSync(); + + // Session Id + int32_t GenPartitionSessionId(const std::string& table_name, uint32_t partition_id); + int32_t GetSlavePartitionSessionId(const std::string& table_name, uint32_t partition_id); + bool CheckSlavePartitionSessionId(const std::string& table_name, uint32_t partition_id, + int session_id); + bool CheckMasterPartitionSessionId(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id, int session_id); + + // write_queue related + void ProduceWriteQueue(const std::string& ip, int port, const std::vector& tasks); + int ConsumeWriteQueue(); + void DropItemInWriteQueue(const std::string& ip, int port); + + // Schedule Task + void ScheduleReplServerBGTask(pink::TaskFunc func, void* arg); + void ScheduleReplClientBGTask(pink::TaskFunc func, void* arg); + void ScheduleWriteBinlogTask(const std::string& table_partition, + const std::shared_ptr res, + std::shared_ptr conn, void* res_private_data); + void ScheduleWriteDBTask(const std::string& dispatch_key, + PikaCmdArgsType* argv, BinlogItem* binlog_item, + const std::string& table_name, uint32_t partition_id); + + void ReplServerRemoveClientConn(int fd); + void ReplServerUpdateClientConnMap(const std::string& ip_port, int fd); + + BinlogReaderManager binlog_reader_mgr; + + private: + void InitPartition(); + + pthread_rwlock_t partitions_rw_; + std::unordered_map, hash_partition_info> sync_master_partitions_; + std::unordered_map, hash_partition_info> sync_slave_partitions_; + + slash::Mutex write_queue_mu_; + // every host owns a queue + std::unordered_map> write_queues_; // ip+port, queue + + PikaReplClient* pika_repl_client_; + PikaReplServer* pika_repl_server_; + int last_meta_sync_timestamp_; +}; + +#endif // PIKA_RM_H diff --git a/tools/pika_migrate/include/pika_rsync_service.h b/tools/pika_migrate/include/pika_rsync_service.h new file mode 100644 index 0000000000..f728f52b57 --- /dev/null +++ b/tools/pika_migrate/include/pika_rsync_service.h @@ -0,0 +1,28 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_RSYNC_SERVICE_H_ +#define PIKA_RSYNC_SERVICE_H_ + +#include "iostream" + +class PikaRsyncService { + public: + PikaRsyncService(const std::string& raw_path, + const int port); + ~PikaRsyncService(); + int StartRsync(); + bool CheckRsyncAlive(); + int ListenPort(); + + private: + int CreateSecretFile(); + std::string raw_path_; + std::string rsync_path_; + std::string pid_path_; + int port_; +}; + +#endif diff --git a/tools/pika_migrate/include/pika_sender.h b/tools/pika_migrate/include/pika_sender.h new file mode 100644 index 0000000000..1cdb38f34b --- /dev/null +++ b/tools/pika_migrate/include/pika_sender.h @@ -0,0 +1,41 @@ +#ifndef PIKA_SENDER_H_ +#define PIKA_SENDER_H_ + +#include +#include +#include +#include +#include + +#include "pink/include/bg_thread.h" +#include "pink/include/pink_cli.h" +#include "pink/include/redis_cli.h" + +class PikaSender : public pink::Thread { +public: + PikaSender(std::string ip, int64_t port, std::string password); + virtual ~PikaSender(); + void LoadKey(const std::string &cmd); + void Stop(); + + int64_t elements() { return elements_; } + + void SendCommand(std::string &command, const std::string &key); + int QueueSize(); + void ConnectRedis(); + +private: + pink::PinkCli *cli_; + slash::CondVar signal_; + slash::Mutex keys_mutex_; + std::queue keys_queue_; + std::string ip_; + int port_; + std::string password_; + std::atomic should_exit_; + int64_t elements_; + + virtual void *ThreadMain(); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_server.h b/tools/pika_migrate/include/pika_server.h new file mode 100644 index 0000000000..49085088b3 --- /dev/null +++ b/tools/pika_migrate/include/pika_server.h @@ -0,0 +1,425 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_SERVER_H_ +#define PIKA_SERVER_H_ + +#include +#include + +#include "slash/include/slash_mutex.h" +#include "slash/include/slash_status.h" +#include "slash/include/slash_string.h" +#include "pink/include/bg_thread.h" +#include "pink/include/thread_pool.h" +#include "pink/include/pink_pubsub.h" +#include "blackwidow/blackwidow.h" +#include "blackwidow/backupable.h" + +#include "include/redis_sender.h" +#include "include/pika_conf.h" +#include "include/pika_table.h" +#include "include/pika_binlog.h" +#include "include/pika_define.h" +#include "include/pika_monitor_thread.h" +#include "include/pika_rsync_service.h" +#include "include/pika_dispatch_thread.h" +#include "include/pika_repl_client.h" +#include "include/pika_repl_server.h" +#include "include/pika_auxiliary_thread.h" + +using slash::Status; +using slash::Slice; + +struct StatisticData { + StatisticData() + : accumulative_connections(0), + thread_querynum(0), + last_thread_querynum(0), + last_sec_thread_querynum(0), + last_time_us(0) { + CmdTable* cmds = new CmdTable(); + cmds->reserve(300); + InitCmdTable(cmds); + CmdTable::const_iterator it = cmds->begin(); + for (; it != cmds->end(); ++it) { + std::string tmp = it->first; + exec_count_table[slash::StringToUpper(tmp)].store(0); + } + DestoryCmdTable(cmds); + delete cmds; + } + + std::atomic accumulative_connections; + std::unordered_map> exec_count_table; + std::atomic thread_querynum; + std::atomic last_thread_querynum; + std::atomic last_sec_thread_querynum; + std::atomic last_time_us; +}; +/* +static std::set MultiKvCommands {kCmdNameDel, + kCmdNameMget, kCmdNameKeys, kCmdNameMset, + kCmdNameMsetnx, kCmdNameExists, kCmdNameScan, + kCmdNameScanx, kCmdNamePKScanRange, kCmdNamePKRScanRange, + kCmdNameRPopLPush, kCmdNameZUnionstore, kCmdNameZInterstore, + kCmdNameSUnion, kCmdNameSUnionstore, kCmdNameSInter, + kCmdNameSInterstore, kCmdNameSDiff, kCmdNameSDiffstore, + kCmdNameSMove, kCmdNameBitOp, kCmdNamePfAdd, + kCmdNamePfCount, kCmdNamePfMerge, kCmdNameGeoAdd, + kCmdNameGeoPos, kCmdNameGeoDist, kCmdNameGeoHash, + kCmdNameGeoRadius, kCmdNameGeoRadiusByMember}; +*/ + +static std::set ShardingModeNotSupportCommands { + kCmdNameMsetnx, kCmdNameScan, kCmdNameKeys, + kCmdNameScanx, kCmdNamePKScanRange, kCmdNamePKRScanRange, + kCmdNameRPopLPush, kCmdNameZUnionstore, kCmdNameZInterstore, + kCmdNameSUnion, kCmdNameSUnionstore, kCmdNameSInter, + kCmdNameSInterstore, kCmdNameSDiff, kCmdNameSDiffstore, + kCmdNameSMove, kCmdNameBitOp, kCmdNamePfAdd, + kCmdNamePfCount, kCmdNamePfMerge, kCmdNameGeoAdd, + kCmdNameGeoPos, kCmdNameGeoDist, kCmdNameGeoHash, + kCmdNameGeoRadius, kCmdNameGeoRadiusByMember, kCmdNamePKPatternMatchDel}; + + +extern PikaConf *g_pika_conf; + +enum TaskType { + kCompactAll, + kCompactStrings, + kCompactHashes, + kCompactSets, + kCompactZSets, + kCompactList, + kResetReplState, + kPurgeLog, + kStartKeyScan, + kStopKeyScan, + kBgSave, +}; + +class PikaServer { + public: + PikaServer(); + ~PikaServer(); + + /* + * Server init info + */ + bool ServerInit(); + + void Start(); + void Exit(); + + std::string host(); + int port(); + time_t start_time_s(); + std::string master_ip(); + int master_port(); + int role(); + bool readonly(const std::string& table, const std::string& key); + int repl_state(); + std::string repl_state_str(); + bool force_full_sync(); + void SetForceFullSync(bool v); + void SetDispatchQueueLimit(int queue_limit); + blackwidow::BlackwidowOptions bw_options(); + + /* + * Table use + */ + void InitTableStruct(); + std::shared_ptr GetTable(const std::string& table_name); + std::set GetTablePartitionIds(const std::string& table_name); + bool IsBgSaving(); + bool IsKeyScaning(); + bool IsCompacting(); + bool IsTableExist(const std::string& table_name); + bool IsTablePartitionExist(const std::string& table_name, uint32_t partition_id); + bool IsCommandSupport(const std::string& command); + bool IsTableBinlogIoError(const std::string& table_name); + Status DoSameThingSpecificTable(const TaskType& type, const std::set& tables = {}); + + /* + * Partition use + */ + void PreparePartitionTrySync(); + void PartitionSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys); + void PartitionSetSmallCompactionThreshold(uint32_t small_compaction_threshold); + bool GetTablePartitionBinlogOffset(const std::string& table_name, + uint32_t partition_id, + BinlogOffset* const boffset); + std::shared_ptr GetPartitionByDbName(const std::string& db_name); + std::shared_ptr GetTablePartitionById( + const std::string& table_name, + uint32_t partition_id); + std::shared_ptr GetTablePartitionByKey( + const std::string& table_name, + const std::string& key); + Status DoSameThingEveryPartition(const TaskType& type); + + /* + * Master use + */ + void BecomeMaster(); + void DeleteSlave(int fd); //conn fd + int32_t CountSyncSlaves(); + int32_t GetSlaveListString(std::string& slave_list_str); + int32_t GetShardingSlaveListString(std::string& slave_list_str); + bool TryAddSlave(const std::string& ip, int64_t port, int fd, + const std::vector& table_structs); + slash::Mutex slave_mutex_; // protect slaves_; + std::vector slaves_; + + + /* + * Slave use + */ + void SyncError(); + void RemoveMaster(); + bool SetMaster(std::string& master_ip, int master_port); + + /* + * Slave State Machine + */ + bool ShouldMetaSync(); + void FinishMetaSync(); + bool MetaSyncDone(); + void ResetMetaSyncStatus(); + bool AllPartitionConnectSuccess(); + bool LoopPartitionStateMachine(); + void SetLoopPartitionStateMachine(bool need_loop); + + /* + * ThreadPool Process Task + */ + void Schedule(pink::TaskFunc func, void* arg); + + /* + * BGSave used + */ + void BGSaveTaskSchedule(pink::TaskFunc func, void* arg); + + /* + * PurgeLog used + */ + void PurgelogsTaskSchedule(pink::TaskFunc func, void* arg); + + /* + * Flushall & Flushdb used + */ + void PurgeDir(const std::string& path); + void PurgeDirTaskSchedule(void (*function)(void*), void* arg); + + /* + * DBSync used + */ + void DBSync(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id); + void TryDBSync(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id, int32_t top); + void DbSyncSendFile(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id); + std::string DbSyncTaskIndex(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id); + + /* + * Keyscan used + */ + void KeyScanTaskSchedule(pink::TaskFunc func, void* arg); + + /* + * Client used + */ + void ClientKillAll(); + int ClientKill(const std::string &ip_port); + int64_t ClientList(std::vector *clients = nullptr); + + /* + * Monitor used + */ + bool HasMonitorClients(); + void AddMonitorMessage(const std::string &monitor_message); + void AddMonitorClient(std::shared_ptr client_ptr); + + /* + * Slowlog used + */ + void SlowlogTrim(); + void SlowlogReset(); + uint32_t SlowlogLen(); + void SlowlogObtain(int64_t number, std::vector* slowlogs); + void SlowlogPushEntry(const PikaCmdArgsType& argv, int32_t time, int64_t duration); + + /* + * Statistic used + */ + void ResetStat(); + uint64_t ServerQueryNum(); + uint64_t ServerCurrentQps(); + uint64_t accumulative_connections(); + void incr_accumulative_connections(); + void ResetLastSecQuerynum(); + void UpdateQueryNumAndExecCountTable(const std::string& command); + std::unordered_map ServerExecCountTable(); + + /* + * Slave to Master communication used + */ + int SendToPeer(); + void SignalAuxiliary(); + Status TriggerSendBinlogSync(); + + /* + * PubSub used + */ + int PubSubNumPat(); + int Publish(const std::string& channel, const std::string& msg); + int UnSubscribe(std::shared_ptr conn, + const std::vector& channels, + const bool pattern, + std::vector>* result); + void Subscribe(std::shared_ptr conn, + const std::vector& channels, + const bool pattern, + std::vector>* result); + void PubSubChannels(const std::string& pattern, + std::vector* result); + void PubSubNumSub(const std::vector& channels, + std::vector>* result); + + /* + * migrate used + */ + int SendRedisCommand(const std::string& command, const std::string& key); + void RetransmitData(const std::string& path); + + + friend class Cmd; + friend class InfoCmd; + friend class PkClusterAddSlotsCmd; + friend class PkClusterDelSlotsCmd; + friend class PikaReplClientConn; + friend class PkClusterInfoCmd; + + private: + /* + * TimingTask use + */ + void DoTimingTask(); + void AutoCompactRange(); + void AutoPurge(); + void AutoDeleteExpiredDump(); + void AutoKeepAliveRSync(); + + std::string host_; + int port_; + time_t start_time_s_; + + blackwidow::BlackwidowOptions bw_options_; + void InitBlackwidowOptions(); + + std::atomic exit_; + + /* + * Table used + */ + std::atomic slot_state_; + pthread_rwlock_t tables_rw_; + std::map> tables_; + + /* + * CronTask used + */ + bool have_scheduled_crontask_; + struct timeval last_check_compact_time_; + + /* + * Communicate with the client used + */ + int worker_num_; + pink::ThreadPool* pika_thread_pool_; + PikaDispatchThread* pika_dispatch_thread_; + + + /* + * Slave used + */ + std::string master_ip_; + int master_port_; + int repl_state_; + int role_; + bool loop_partition_state_machine_; + bool force_full_sync_; + pthread_rwlock_t state_protector_; //protect below, use for master-slave mode + + /* + * Bgsave used + */ + pink::BGThread bgsave_thread_; + + /* + * Purgelogs use + */ + pink::BGThread purge_thread_; + + /* + * DBSync used + */ + slash::Mutex db_sync_protector_; + std::unordered_set db_sync_slaves_; + + /* + * Keyscan used + */ + pink::BGThread key_scan_thread_; + + /* + * Monitor used + */ + PikaMonitorThread* pika_monitor_thread_; + + /* + * Rsync used + */ + PikaRsyncService* pika_rsync_service_; + + /* + * Pubsub used + */ + pink::PubSubThread* pika_pubsub_thread_; + + /* + * Communication used + */ + PikaAuxiliaryThread* pika_auxiliary_thread_; + + /* + * + */ + std::vector redis_senders_; + + /* + * Slowlog used + */ + uint64_t slowlog_entry_id_; + pthread_rwlock_t slowlog_protector_; + std::list slowlog_list_; + + /* + * Statistic used + */ + StatisticData statistic_data_; + + PikaServer(PikaServer &ps); + void operator =(const PikaServer &ps); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_set.h b/tools/pika_migrate/include/pika_set.h new file mode 100644 index 0000000000..fe00850751 --- /dev/null +++ b/tools/pika_migrate/include/pika_set.h @@ -0,0 +1,266 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_SET_H_ +#define PIKA_SET_H_ + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * set + */ +class SAddCmd : public Cmd { + public: + SAddCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SAddCmd(*this); + } + private: + std::string key_; + std::vector members_; + virtual void DoInitial() override; +}; + +class SPopCmd : public Cmd { + public: + SPopCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SPopCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class SCardCmd : public Cmd { + public: + SCardCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SCardCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class SMembersCmd : public Cmd { + public: + SMembersCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SMembersCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class SScanCmd : public Cmd { + public: + SScanCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SScanCmd(*this); + } + private: + std::string key_, pattern_; + int64_t cursor_, count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class SRemCmd : public Cmd { + public: + SRemCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SRemCmd(*this); + } + private: + std::string key_; + std::vector members_; + virtual void DoInitial() override; +}; + +class SUnionCmd : public Cmd { + public: + SUnionCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SUnionCmd(*this); + } + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class SUnionstoreCmd : public Cmd { + public: + SUnionstoreCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SUnionstoreCmd(*this); + } + private: + std::string dest_key_; + std::vector keys_; + virtual void DoInitial() override; +}; + +class SInterCmd : public Cmd { + public: + SInterCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SInterCmd(*this); + } + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class SInterstoreCmd : public Cmd { + public: + SInterstoreCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SInterstoreCmd(*this); + } + private: + std::string dest_key_; + std::vector keys_; + virtual void DoInitial() override; +}; + +class SIsmemberCmd : public Cmd { + public: + SIsmemberCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SIsmemberCmd(*this); + } + private: + std::string key_, member_; + virtual void DoInitial() override; +}; + +class SDiffCmd : public Cmd { + public: + SDiffCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SDiffCmd(*this); + } + private: + std::vector keys_; + virtual void DoInitial() override; +}; + +class SDiffstoreCmd : public Cmd { + public: + SDiffstoreCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SDiffstoreCmd(*this); + } + private: + std::string dest_key_; + std::vector keys_; + virtual void DoInitial() override; +}; + +class SMoveCmd : public Cmd { + public: + SMoveCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SMoveCmd(*this); + } + private: + std::string src_key_, dest_key_, member_; + virtual void DoInitial() override; +}; + +class SRandmemberCmd : public Cmd { + public: + SRandmemberCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), count_(1) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SRandmemberCmd(*this); + } + private: + std::string key_; + int64_t count_; + bool reply_arr; + virtual void DoInitial() override; + virtual void Clear() { + count_ = 1; + reply_arr = false; + } +}; + +#endif diff --git a/tools/pika_migrate/include/pika_slaveping_thread.h b/tools/pika_migrate/include/pika_slaveping_thread.h new file mode 100644 index 0000000000..bc8e6a7ef9 --- /dev/null +++ b/tools/pika_migrate/include/pika_slaveping_thread.h @@ -0,0 +1,44 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_SLAVEPING_THREAD_H_ +#define PIKA_SLAVEPING_THREAD_H_ + +#include + +#include "slash/include/slash_status.h" +#include "pink/include/pink_cli.h" +#include "pink/include/pink_thread.h" + +using slash::Status; + +class PikaSlavepingThread : public pink::Thread { + public: + PikaSlavepingThread(int64_t sid) + : sid_(sid), is_first_send_(true) { + cli_ = pink::NewPbCli(); + cli_->set_connect_timeout(1500); + set_thread_name("SlavePingThread"); + }; + virtual ~PikaSlavepingThread() { + StopThread(); + delete cli_; + LOG(INFO) << "SlavepingThread " << thread_id() << " exit!!!"; + }; + + Status Send(); + Status RecvProc(); + + private: + int64_t sid_; + bool is_first_send_; + + int sockfd_; + pink::PinkCli *cli_; + + virtual void* ThreadMain(); +}; + +#endif diff --git a/tools/pika_migrate/include/pika_slot.h b/tools/pika_migrate/include/pika_slot.h new file mode 100644 index 0000000000..052f87269b --- /dev/null +++ b/tools/pika_migrate/include/pika_slot.h @@ -0,0 +1,191 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_SLOT_H_ +#define PIKA_SLOT_H_ + +#include "include/pika_command.h" + +class SlotsInfoCmd : public Cmd { + public: + SlotsInfoCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsInfoCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsHashKeyCmd : public Cmd { + public: + SlotsHashKeyCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsHashKeyCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtSlotAsyncCmd : public Cmd { + public: + SlotsMgrtSlotAsyncCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtSlotAsyncCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtTagSlotAsyncCmd : public Cmd { + public: + SlotsMgrtTagSlotAsyncCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), dest_port_(0), slot_num_(-1) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtTagSlotAsyncCmd(*this); + } + private: + virtual void DoInitial() override; + std::string dest_ip_; + int64_t dest_port_; + int64_t slot_num_; + virtual void Clear() { + dest_ip_.clear(); + dest_port_ = 0; + slot_num_ = -1; + } +}; + +class SlotsScanCmd : public Cmd { + public: + SlotsScanCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsScanCmd(*this); + } + private: + int64_t cursor_; + uint32_t slotnum_; + std::string pattern_; + int64_t count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class SlotsDelCmd : public Cmd { + public: + SlotsDelCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsDelCmd(*this); + } + private: + std::vector slots_; + virtual void DoInitial() override; + virtual void Clear() { + slots_.clear(); + } +}; + +class SlotsMgrtExecWrapperCmd : public Cmd { + public: + SlotsMgrtExecWrapperCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtExecWrapperCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; + virtual void Clear() { + key_.clear(); + } +}; + +class SlotsMgrtAsyncStatusCmd : public Cmd { + public: + SlotsMgrtAsyncStatusCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtAsyncStatusCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtAsyncCancelCmd : public Cmd { + public: + SlotsMgrtAsyncCancelCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtAsyncCancelCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtSlotCmd : public Cmd { + public: + SlotsMgrtSlotCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtSlotCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtTagSlotCmd : public Cmd { + public: + SlotsMgrtTagSlotCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtTagSlotCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtOneCmd : public Cmd { + public: + SlotsMgrtOneCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtOneCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class SlotsMgrtTagOneCmd : public Cmd { + public: + SlotsMgrtTagOneCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new SlotsMgrtTagOneCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +#endif // PIKA_SLOT_H_ diff --git a/tools/pika_migrate/include/pika_table.h b/tools/pika_migrate/include/pika_table.h new file mode 100644 index 0000000000..adf6b62b6c --- /dev/null +++ b/tools/pika_migrate/include/pika_table.h @@ -0,0 +1,89 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_TABLE_H_ +#define PIKA_TABLE_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +class Table : public std::enable_shared_from_this
{ + public: + Table(const std::string& table_name, + uint32_t partition_num, + const std::string& db_path, + const std::string& log_path); + virtual ~Table(); + + friend class Cmd; + friend class InfoCmd; + friend class PkClusterInfoCmd; + friend class PikaServer; + + std::string GetTableName(); + void BgSaveTable(); + void CompactTable(const blackwidow::DataType& type); + bool FlushPartitionDB(); + bool FlushPartitionSubDB(const std::string& db_name); + bool IsBinlogIoError(); + uint32_t PartitionNum(); + + // Dynamic change partition + Status AddPartitions(const std::set& partition_ids); + Status RemovePartitions(const std::set& partition_ids); + + // KeyScan use; + void KeyScan(); + bool IsKeyScaning(); + void RunKeyScan(); + void StopKeyScan(); + void ScanDatabase(const blackwidow::DataType& type); + KeyScanInfo GetKeyScanInfo(); + Status GetPartitionsKeyScanInfo(std::map* infos); + + // Compact use; + void Compact(const blackwidow::DataType& type); + + void LeaveAllPartition(); + std::set GetPartitionIds(); + std::shared_ptr GetPartitionById(uint32_t partition_id); + std::shared_ptr GetPartitionByKey(const std::string& key); + + private: + std::string table_name_; + uint32_t partition_num_; + std::string db_path_; + std::string log_path_; + + // lock order + // partitions_rw_ > key_scan_protector_ + + pthread_rwlock_t partitions_rw_; + std::map> partitions_; + + /* + * KeyScan use + */ + static void DoKeyScan(void *arg); + void InitKeyScan(); + slash::Mutex key_scan_protector_; + KeyScanInfo key_scan_info_; + + /* + * No allowed copy and copy assign + */ + Table(const Table&); + void operator=(const Table&); +}; + +struct BgTaskArg { + std::shared_ptr
table; + std::shared_ptr partition; +}; + + +#endif diff --git a/tools/pika_migrate/include/pika_version.h b/tools/pika_migrate/include/pika_version.h new file mode 100644 index 0000000000..c0c6a2b617 --- /dev/null +++ b/tools/pika_migrate/include/pika_version.h @@ -0,0 +1,13 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef INCLUDE_PIKA_VERSION_H_ +#define INCLUDE_PIKA_VERSION_H_ + +#define PIKA_MAJOR 3 +#define PIKA_MINOR 2 +#define PIKA_PATCH 7 + +#endif // INCLUDE_PIKA_VERSION_H_ diff --git a/tools/pika_migrate/include/pika_zset.h b/tools/pika_migrate/include/pika_zset.h new file mode 100644 index 0000000000..d4ab4ca6ea --- /dev/null +++ b/tools/pika_migrate/include/pika_zset.h @@ -0,0 +1,516 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef PIKA_ZSET_H_ +#define PIKA_ZSET_H_ + +#include "blackwidow/blackwidow.h" + +#include "include/pika_command.h" +#include "include/pika_partition.h" + +/* + * zset + */ +class ZAddCmd : public Cmd { + public: + ZAddCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZAddCmd(*this); + } + private: + std::string key_; + std::vector score_members; + virtual void DoInitial() override; +}; + +class ZCardCmd : public Cmd { + public: + ZCardCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZCardCmd(*this); + } + private: + std::string key_; + virtual void DoInitial() override; +}; + +class ZScanCmd : public Cmd { + public: + ZScanCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), pattern_("*"), count_(10) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZScanCmd(*this); + } + private: + std::string key_, pattern_; + int64_t cursor_, count_; + virtual void DoInitial() override; + virtual void Clear() { + pattern_ = "*"; + count_ = 10; + } +}; + +class ZIncrbyCmd : public Cmd { + public: + ZIncrbyCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZIncrbyCmd(*this); + } + private: + std::string key_, member_; + double by_; + virtual void DoInitial() override; +}; + +class ZsetRangeParentCmd : public Cmd { + public: + ZsetRangeParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), is_ws_(false) {} + protected: + std::string key_; + int64_t start_, stop_; + bool is_ws_; + virtual void DoInitial() override; + virtual void Clear() { + is_ws_ = false; + } +}; + +class ZRangeCmd : public ZsetRangeParentCmd { + public: + ZRangeCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangeParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRangeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZRevrangeCmd : public ZsetRangeParentCmd { + public: + ZRevrangeCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangeParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRevrangeCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZsetRangebyscoreParentCmd : public Cmd { + public: + ZsetRangebyscoreParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true), with_scores_(false), offset_(0), count_(-1) {} + protected: + std::string key_; + double min_score_, max_score_; + bool left_close_, right_close_, with_scores_; + int64_t offset_, count_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = right_close_ = true; + with_scores_ = false; + offset_ = 0; + count_ = -1; + } +}; + +class ZRangebyscoreCmd : public ZsetRangebyscoreParentCmd { + public: + ZRangebyscoreCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangebyscoreParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRangebyscoreCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZRevrangebyscoreCmd : public ZsetRangebyscoreParentCmd { + public: + ZRevrangebyscoreCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangebyscoreParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRevrangebyscoreCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZCountCmd : public Cmd { + public: + ZCountCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZCountCmd(*this); + } + private: + std::string key_; + double min_score_, max_score_; + bool left_close_, right_close_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = true; + right_close_ = true; + } +}; + +class ZRemCmd : public Cmd { + public: + ZRemCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRemCmd(*this); + } + private: + std::string key_; + std::vector members_; + virtual void DoInitial() override; +}; + +class ZsetUIstoreParentCmd : public Cmd { + public: + ZsetUIstoreParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), aggregate_(blackwidow::SUM) {} + protected: + std::string dest_key_; + int64_t num_keys_; + blackwidow::AGGREGATE aggregate_; + std::vector keys_; + std::vector weights_; + virtual void DoInitial() override; + virtual void Clear() { + aggregate_ = blackwidow::SUM; + } +}; + +class ZUnionstoreCmd : public ZsetUIstoreParentCmd { + public: + ZUnionstoreCmd(const std::string& name, int arity, uint16_t flag) + : ZsetUIstoreParentCmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZUnionstoreCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZInterstoreCmd : public ZsetUIstoreParentCmd { + public: + ZInterstoreCmd(const std::string& name, int arity, uint16_t flag) + : ZsetUIstoreParentCmd(name, arity, flag) {} + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZInterstoreCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZsetRankParentCmd : public Cmd { + public: + ZsetRankParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + protected: + std::string key_, member_; + virtual void DoInitial() override; +}; + +class ZRankCmd : public ZsetRankParentCmd { + public: + ZRankCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRankParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRankCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZRevrankCmd : public ZsetRankParentCmd { + public: + ZRevrankCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRankParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRevrankCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZScoreCmd : public ZsetRankParentCmd { + public: + ZScoreCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRankParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZScoreCmd(*this); + } + private: + std::string key_, member_; + virtual void DoInitial() override; +}; + + +class ZsetRangebylexParentCmd : public Cmd { + public: + ZsetRangebylexParentCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true), offset_(0), count_(-1) {} + protected: + std::string key_, min_member_, max_member_; + bool left_close_, right_close_; + int64_t offset_, count_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = right_close_ = true; + offset_ = 0; + count_ = -1; + } +}; + +class ZRangebylexCmd : public ZsetRangebylexParentCmd { + public: + ZRangebylexCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangebylexParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRangebylexCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZRevrangebylexCmd : public ZsetRangebylexParentCmd { + public: + ZRevrangebylexCmd(const std::string& name, int arity, uint16_t flag) + : ZsetRangebylexParentCmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRevrangebylexCmd(*this); + } + private: + virtual void DoInitial() override; +}; + +class ZLexcountCmd : public Cmd { + public: + ZLexcountCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZLexcountCmd(*this); + } + private: + std::string key_, min_member_, max_member_; + bool left_close_, right_close_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = right_close_ = true; + } +}; + +class ZRemrangebyrankCmd : public Cmd { + public: + ZRemrangebyrankCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRemrangebyrankCmd(*this); + } + private: + std::string key_; + int64_t start_rank_, stop_rank_; + virtual void DoInitial() override; +}; + +class ZRemrangebyscoreCmd : public Cmd { + public: + ZRemrangebyscoreCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRemrangebyscoreCmd(*this); + } + private: + std::string key_; + double min_score_, max_score_; + bool left_close_, right_close_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = right_close_ = true; + } +}; + +class ZRemrangebylexCmd : public Cmd { + public: + ZRemrangebylexCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag), left_close_(true), right_close_(true) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZRemrangebylexCmd(*this); + } + private: + std::string key_; + std::string min_member_, max_member_; + bool left_close_, right_close_; + virtual void DoInitial() override; + virtual void Clear() { + left_close_ = right_close_ = true; + } +}; + +class ZPopmaxCmd : public Cmd { + public: + ZPopmaxCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.emplace_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZPopmaxCmd(*this); + } + private: + virtual void DoInitial() override; + std::string key_; + int64_t count_; +}; + +class ZPopminCmd : public Cmd { + public: + ZPopminCmd(const std::string& name, int arity, uint16_t flag) + : Cmd(name, arity, flag) {} + virtual std::vector current_key() const { + std::vector res; + res.push_back(key_); + return res; + } + virtual void Do(std::shared_ptr partition = nullptr); + virtual Cmd* Clone() override { + return new ZPopminCmd(*this); + } + private: + virtual void DoInitial() override; + std::string key_; + int64_t count_; +}; + +#endif diff --git a/tools/pika_migrate/include/redis_sender.h b/tools/pika_migrate/include/redis_sender.h new file mode 100644 index 0000000000..aa905e3d68 --- /dev/null +++ b/tools/pika_migrate/include/redis_sender.h @@ -0,0 +1,47 @@ +#ifndef REDIS_SENDER_H_ +#define REDIS_SENDER_H_ + +#include +#include +#include +#include +#include + +#include "pink/include/bg_thread.h" +#include "pink/include/pink_cli.h" +#include "pink/include/redis_cli.h" + +class RedisSender : public pink::Thread { + public: + RedisSender(int id, std::string ip, int64_t port, std::string password); + virtual ~RedisSender(); + void Stop(void); + int64_t elements() { + return elements_; + } + + void SendRedisCommand(const std::string &command); + + private: + int SendCommand(std::string &command); + void ConnectRedis(); + + private: + int id_; + pink::PinkCli *cli_; + slash::CondVar rsignal_; + slash::CondVar wsignal_; + slash::Mutex commands_mutex_; + std::queue commands_queue_; + std::string ip_; + int port_; + std::string password_; + bool should_exit_; + int32_t cnt_; + int64_t elements_; + std::atomic last_write_time_; + + virtual void *ThreadMain(); +}; + +#endif diff --git a/tools/pika_migrate/pikatests.sh b/tools/pika_migrate/pikatests.sh new file mode 100755 index 0000000000..bf17cf73e5 --- /dev/null +++ b/tools/pika_migrate/pikatests.sh @@ -0,0 +1,10 @@ +#!/bin/bash +rm -rf ./log +rm -rf .db +cp output/bin/pika src/redis-server +cp output/conf/pika.conf tests/assets/default.conf + +tclsh tests/test_helper.tcl --clients 1 --single unit/$1 +rm src/redis-server +rm -rf ./log +rm -rf ./db diff --git a/tools/pika_migrate/src/build_version.cc.in b/tools/pika_migrate/src/build_version.cc.in new file mode 100644 index 0000000000..de52eeaeba --- /dev/null +++ b/tools/pika_migrate/src/build_version.cc.in @@ -0,0 +1,10 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/build_version.h" +const char* pika_build_git_sha = + "pika_git_sha:@@GIT_SHA@@"; +const char* pika_build_git_date = "pika_build_git_date:@@GIT_DATE_TIME@@"; +const char* pika_build_compile_date = __DATE__; diff --git a/tools/pika_migrate/src/migrator_thread.cc b/tools/pika_migrate/src/migrator_thread.cc new file mode 100644 index 0000000000..a7b7122c51 --- /dev/null +++ b/tools/pika_migrate/src/migrator_thread.cc @@ -0,0 +1,475 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/migrator_thread.h" + +#include + +#include +#include +#include + +#include "blackwidow/blackwidow.h" +#include "src/redis_strings.h" +#include "src/redis_lists.h" +#include "src/redis_hashes.h" +#include "src/redis_sets.h" +#include "src/redis_zsets.h" +#include "src/scope_snapshot.h" +#include "src/strings_value_format.h" + +#include "include/pika_conf.h" + +const int64_t MAX_BATCH_NUM = 30000; + +extern PikaConf* g_pika_conf; + +MigratorThread::~MigratorThread() { +} + +void MigratorThread::MigrateStringsDB() { + blackwidow::BlackWidow *bw = (blackwidow::BlackWidow*)(db_); + + int64_t scan_batch_num = g_pika_conf->sync_batch_num() * 10; + if (MAX_BATCH_NUM < scan_batch_num) { + if (g_pika_conf->sync_batch_num() < MAX_BATCH_NUM) { + scan_batch_num = MAX_BATCH_NUM; + } else { + scan_batch_num = g_pika_conf->sync_batch_num() * 2; + } + } + + int64_t ttl = -1; + int64_t cursor = 0; + blackwidow::Status s; + std::string value; + std::vector keys; + std::map type_timestamp; + std::map type_status; + while (true) { + cursor = bw->Scan(blackwidow::DataType::kStrings, cursor, "*", scan_batch_num, &keys); + + for (const auto& key : keys) { + s = bw->Get(key, &value); + if (!s.ok()) { + LOG(WARNING) << "get " << key << " error: " << s.ToString(); + continue; + } + + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("SET"); + argv.push_back(key); + argv.push_back(value); + + ttl = -1; + type_status.clear(); + type_timestamp = bw->TTL(key, &type_status); + if (type_timestamp[blackwidow::kStrings] != -2) { + ttl = type_timestamp[blackwidow::kStrings]; + } + + if (ttl > 0) { + argv.push_back("EX"); + argv.push_back(std::to_string(ttl)); + } + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + + if (!cursor) { + break; + } + } +} + +void MigratorThread::MigrateListsDB() { + blackwidow::BlackWidow *bw = (blackwidow::BlackWidow*)(db_); + + int64_t scan_batch_num = g_pika_conf->sync_batch_num() * 10; + if (MAX_BATCH_NUM < scan_batch_num) { + if (g_pika_conf->sync_batch_num() < MAX_BATCH_NUM) { + scan_batch_num = MAX_BATCH_NUM; + } else { + scan_batch_num = g_pika_conf->sync_batch_num() * 2; + } + } + + int64_t ttl = -1; + int64_t cursor = 0; + blackwidow::Status s; + std::vector keys; + std::map type_timestamp; + std::map type_status; + + while (true) { + cursor = bw->Scan(blackwidow::DataType::kLists, cursor, "*", scan_batch_num, &keys); + + for (const auto& key : keys) { + int64_t pos = 0; + std::vector nodes; + blackwidow::Status s = bw->LRange(key, pos, pos + g_pika_conf->sync_batch_num() - 1, &nodes); + if (!s.ok()) { + LOG(WARNING) << "db->LRange(key:" << key << ", pos:" << pos + << ", batch size: " << g_pika_conf->sync_batch_num() << ") = " << s.ToString(); + continue; + } + + while (s.ok() && !should_exit_ && !nodes.empty()) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("RPUSH"); + argv.push_back(key); + for (const auto& node : nodes) { + argv.push_back(node); + } + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + + pos += g_pika_conf->sync_batch_num(); + nodes.clear(); + s = bw->LRange(key, pos, pos + g_pika_conf->sync_batch_num() - 1, &nodes); + if (!s.ok()) { + LOG(WARNING) << "db->LRange(key:" << key << ", pos:" << pos + << ", batch size:" << g_pika_conf->sync_batch_num() << ") = " << s.ToString(); + } + } + + ttl = -1; + type_status.clear(); + type_timestamp = bw->TTL(key, &type_status); + if (type_timestamp[blackwidow::kLists] != -2) { + ttl = type_timestamp[blackwidow::kLists]; + } + + if (s.ok() && ttl > 0) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("EXPIRE"); + argv.push_back(key); + argv.push_back(std::to_string(ttl)); + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + } + + if (!cursor) { + break; + } + } +} + +void MigratorThread::MigrateHashesDB() { + blackwidow::BlackWidow *bw = (blackwidow::BlackWidow*)(db_); + + int64_t scan_batch_num = g_pika_conf->sync_batch_num() * 10; + if (MAX_BATCH_NUM < scan_batch_num) { + if (g_pika_conf->sync_batch_num() < MAX_BATCH_NUM) { + scan_batch_num = MAX_BATCH_NUM; + } else { + scan_batch_num = g_pika_conf->sync_batch_num() * 2; + } + } + + int64_t ttl = -1; + int64_t cursor = 0; + blackwidow::Status s; + std::vector keys; + std::map type_timestamp; + std::map type_status; + + while (true) { + cursor = bw->Scan(blackwidow::DataType::kHashes, cursor, "*", scan_batch_num, &keys); + + for (const auto& key : keys) { + std::vector fvs; + blackwidow::Status s = bw->HGetall(key, &fvs); + if (!s.ok()) { + LOG(WARNING) << "db->HGetall(key:" << key << ") = " << s.ToString(); + continue; + } + + auto it = fvs.begin(); + while (!should_exit_ && it != fvs.end()) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("HMSET"); + argv.push_back(key); + for (int idx = 0; + idx < g_pika_conf->sync_batch_num() && !should_exit_ && it != fvs.end(); + idx++, it++) { + argv.push_back(it->field); + argv.push_back(it->value); + } + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + + ttl = -1; + type_status.clear(); + type_timestamp = bw->TTL(key, &type_status); + if (type_timestamp[blackwidow::kHashes] != -2) { + ttl = type_timestamp[blackwidow::kHashes]; + } + + if (s.ok() && ttl > 0) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("EXPIRE"); + argv.push_back(key); + argv.push_back(std::to_string(ttl)); + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + } + + if (!cursor) { + break; + } + } +} + +void MigratorThread::MigrateSetsDB() { + blackwidow::BlackWidow *bw = (blackwidow::BlackWidow*)(db_); + + int64_t scan_batch_num = g_pika_conf->sync_batch_num() * 10; + if (MAX_BATCH_NUM < scan_batch_num) { + if (g_pika_conf->sync_batch_num() < MAX_BATCH_NUM) { + scan_batch_num = MAX_BATCH_NUM; + } else { + scan_batch_num = g_pika_conf->sync_batch_num() * 2; + } + } + + int64_t ttl = -1; + int64_t cursor = 0; + blackwidow::Status s; + std::vector keys; + std::map type_timestamp; + std::map type_status; + + while (true) { + cursor = bw->Scan(blackwidow::DataType::kSets, cursor, "*", scan_batch_num, &keys); + + for (const auto& key : keys) { + std::vector members; + blackwidow::Status s = bw->SMembers(key, &members); + if (!s.ok()) { + LOG(WARNING) << "db->SMembers(key:" << key << ") = " << s.ToString(); + continue; + } + auto it = members.begin(); + while (!should_exit_ && it != members.end()) { + std::string cmd; + pink::RedisCmdArgsType argv; + + argv.push_back("SADD"); + argv.push_back(key); + for (int idx = 0; + idx < g_pika_conf->sync_batch_num() && !should_exit_ && it != members.end(); + idx++, it++) { + argv.push_back(*it); + } + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + + ttl = -1; + type_status.clear(); + type_timestamp = bw->TTL(key, &type_status); + if (type_timestamp[blackwidow::kSets] != -2) { + ttl = type_timestamp[blackwidow::kSets]; + } + + if (s.ok() && ttl > 0) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("EXPIRE"); + argv.push_back(key); + argv.push_back(std::to_string(ttl)); + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + } + + if (!cursor) { + break; + } + } +} + +void MigratorThread::MigrateZsetsDB() { + blackwidow::BlackWidow *bw = (blackwidow::BlackWidow*)(db_); + + int64_t scan_batch_num = g_pika_conf->sync_batch_num() * 10; + if (MAX_BATCH_NUM < scan_batch_num) { + if (g_pika_conf->sync_batch_num() < MAX_BATCH_NUM) { + scan_batch_num = MAX_BATCH_NUM; + } else { + scan_batch_num = g_pika_conf->sync_batch_num() * 2; + } + } + + int64_t ttl = -1; + int64_t cursor = 0; + blackwidow::Status s; + std::vector keys; + std::map type_timestamp; + std::map type_status; + + while (true) { + cursor = bw->Scan(blackwidow::DataType::kZSets, cursor, "*", scan_batch_num, &keys); + + for (const auto& key : keys) { + std::vector score_members; + blackwidow::Status s = bw->ZRange(key, 0, -1, &score_members); + if (!s.ok()) { + LOG(WARNING) << "db->ZRange(key:" << key << ") = " << s.ToString(); + continue; + } + auto it = score_members.begin(); + while (!should_exit_ && it != score_members.end()) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("ZADD"); + argv.push_back(key); + for (int idx = 0; + idx < g_pika_conf->sync_batch_num() && !should_exit_ && it != score_members.end(); + idx++, it++) { + argv.push_back(std::to_string(it->score)); + argv.push_back(it->member); + } + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + + ttl = -1; + type_status.clear(); + type_timestamp = bw->TTL(key, &type_status); + if (type_timestamp[blackwidow::kZSets] != -2) { + ttl = type_timestamp[blackwidow::kZSets]; + } + + if (s.ok() && ttl > 0) { + pink::RedisCmdArgsType argv; + std::string cmd; + + argv.push_back("EXPIRE"); + argv.push_back(key); + argv.push_back(std::to_string(ttl)); + + pink::SerializeRedisCommand(argv, &cmd); + PlusNum(); + DispatchKey(cmd, key); + } + } + + if (!cursor) { + break; + } + } +} + +void MigratorThread::MigrateDB() { + switch (int(type_)) { + case int(blackwidow::kStrings) : { + MigrateStringsDB(); + break; + } + + case int(blackwidow::kLists) : { + MigrateListsDB(); + break; + } + + case int(blackwidow::kHashes) : { + MigrateHashesDB(); + break; + } + + case int(blackwidow::kSets) : { + MigrateSetsDB(); + break; + } + + case int(blackwidow::kZSets) : { + MigrateZsetsDB(); + break; + } + + default: { + LOG(WARNING) << "illegal db type " << type_; + break; + } + } +} + +void MigratorThread::DispatchKey(const std::string &command, const std::string& key) { + thread_index_ = (thread_index_ + 1) % thread_num_; + size_t idx = thread_index_; + if (key.size()) { // no empty + idx = std::hash()(key) % thread_num_; + } + (*senders_)[idx]->LoadKey(command); +} + +const char* GetDBTypeString(int type) { + switch (type) { + case int(blackwidow::kStrings) : { + return "blackwidow::kStrings"; + } + + case int(blackwidow::kLists) : { + return "blackwidow::kLists"; + } + + case int(blackwidow::kHashes) : { + return "blackwidow::kHashes"; + } + + case int(blackwidow::kSets) : { + return "blackwidow::kSets"; + } + + case int(blackwidow::kZSets) : { + return "blackwidow::kZSets"; + } + + default: { + return "blackwidow::Unknown"; + } + } +} + +void *MigratorThread::ThreadMain() { + MigrateDB(); + should_exit_ = true; + LOG(INFO) << GetDBTypeString(type_) << " keys have been dispatched completly"; + return NULL; +} + diff --git a/tools/pika_migrate/src/pika.cc b/tools/pika_migrate/src/pika.cc new file mode 100644 index 0000000000..0408752dd9 --- /dev/null +++ b/tools/pika_migrate/src/pika.cc @@ -0,0 +1,214 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include +#include +#include + +#include "slash/include/env.h" +#include "include/pika_rm.h" +#include "include/pika_server.h" +#include "include/pika_command.h" +#include "include/pika_conf.h" +#include "include/pika_define.h" +#include "include/pika_version.h" +#include "include/pika_cmd_table_manager.h" + +#ifdef TCMALLOC_EXTENSION +#include +#endif + +PikaConf* g_pika_conf; +PikaServer* g_pika_server; +PikaReplicaManager* g_pika_rm; + +PikaCmdTableManager* g_pika_cmd_table_manager; + +static void version() { + char version[32]; + snprintf(version, sizeof(version), "%d.%d.%d", PIKA_MAJOR, + PIKA_MINOR, PIKA_PATCH); + printf("-----------Pika server %s ----------\n", version); +} + +static void PikaConfInit(const std::string& path) { + printf("path : %s\n", path.c_str()); + g_pika_conf = new PikaConf(path); + if (g_pika_conf->Load() != 0) { + LOG(FATAL) << "pika load conf error"; + } + version(); + printf("-----------Pika config list----------\n"); + g_pika_conf->DumpConf(); + printf("-----------Pika config end----------\n"); +} + +static void PikaGlogInit() { + if (!slash::FileExists(g_pika_conf->log_path())) { + slash::CreatePath(g_pika_conf->log_path()); + } + + if (!g_pika_conf->daemonize()) { + FLAGS_alsologtostderr = true; + } + FLAGS_log_dir = g_pika_conf->log_path(); + FLAGS_minloglevel = 0; + FLAGS_max_log_size = 1800; + FLAGS_logbufsecs = 0; + ::google::InitGoogleLogging("pika"); +} + +static void daemonize() { + if (fork() != 0) exit(0); /* parent exits */ + setsid(); /* create a new session */ +} + +static void close_std() { + int fd; + if ((fd = open("/dev/null", O_RDWR, 0)) != -1) { + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + close(fd); + } +} + +static void create_pid_file(void) { + /* Try to write the pid file in a best-effort way. */ + std::string path(g_pika_conf->pidfile()); + + size_t pos = path.find_last_of('/'); + if (pos != std::string::npos) { + // mkpath(path.substr(0, pos).c_str(), 0755); + slash::CreateDir(path.substr(0, pos)); + } else { + path = kPikaPidFile; + } + + FILE *fp = fopen(path.c_str(), "w"); + if (fp) { + fprintf(fp,"%d\n",(int)getpid()); + fclose(fp); + } +} + +static void IntSigHandle(const int sig) { + LOG(INFO) << "Catch Signal " << sig << ", cleanup..."; + g_pika_server->Exit(); +} + +static void PikaSignalSetup() { + signal(SIGHUP, SIG_IGN); + signal(SIGPIPE, SIG_IGN); + signal(SIGINT, &IntSigHandle); + signal(SIGQUIT, &IntSigHandle); + signal(SIGTERM, &IntSigHandle); +} + +static void usage() +{ + char version[32]; + snprintf(version, sizeof(version), "%d.%d.%d", PIKA_MAJOR, + PIKA_MINOR, PIKA_PATCH); + fprintf(stderr, + "Pika module %s\n" + "usage: pika [-hv] [-c conf/file]\n" + "\t-h -- show this help\n" + "\t-c conf/file -- config file \n" + " example: ./output/bin/pika -c ./conf/pika.conf\n", + version + ); +} + +int main(int argc, char *argv[]) { + if (argc != 2 && argc != 3) { + usage(); + exit(-1); + } + + bool path_opt = false; + char c; + char path[1024]; + while (-1 != (c = getopt(argc, argv, "c:hv"))) { + switch (c) { + case 'c': + snprintf(path, 1024, "%s", optarg); + path_opt = true; + break; + case 'h': + usage(); + return 0; + case 'v': + version(); + return 0; + default: + usage(); + return 0; + } + } + + if (path_opt == false) { + fprintf (stderr, "Please specify the conf file path\n" ); + usage(); + exit(-1); + } +#ifdef TCMALLOC_EXTENSION + MallocExtension::instance()->Initialize(); +#endif + PikaConfInit(path); + + rlimit limit; + rlim_t maxfiles = g_pika_conf->maxclients() + PIKA_MIN_RESERVED_FDS; + if (getrlimit(RLIMIT_NOFILE, &limit) == -1) { + LOG(WARNING) << "getrlimit error: " << strerror(errno); + } else if (limit.rlim_cur < maxfiles) { + rlim_t old_limit = limit.rlim_cur; + limit.rlim_cur = maxfiles; + limit.rlim_max = maxfiles; + if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { + LOG(WARNING) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika have successfully reconfig it to " << limit.rlim_cur; + } else { + LOG(FATAL) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika can not reconfig it(" << strerror(errno) << "), do it by yourself"; + } + } + + // daemonize if needed + if (g_pika_conf->daemonize()) { + daemonize(); + create_pid_file(); + } + + + PikaGlogInit(); + PikaSignalSetup(); + + LOG(INFO) << "Server at: " << path; + g_pika_server = new PikaServer(); + g_pika_rm = new PikaReplicaManager(); + g_pika_cmd_table_manager = new PikaCmdTableManager(); + + if (g_pika_conf->daemonize()) { + close_std(); + } + + g_pika_rm->Start(); + g_pika_server->Start(); + + if (g_pika_conf->daemonize()) { + unlink(g_pika_conf->pidfile().c_str()); + } + + // stop PikaReplicaManager first,avoid internal threads + // may references to dead PikaServer + g_pika_rm->Stop(); + + delete g_pika_server; + delete g_pika_rm; + delete g_pika_cmd_table_manager; + ::google::ShutdownGoogleLogging(); + delete g_pika_conf; + + return 0; +} diff --git a/tools/pika_migrate/src/pika_admin.cc b/tools/pika_migrate/src/pika_admin.cc new file mode 100644 index 0000000000..8424a6e8c0 --- /dev/null +++ b/tools/pika_migrate/src/pika_admin.cc @@ -0,0 +1,2106 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_admin.h" + +#include +#include +#include + +#include "slash/include/rsync.h" + +#include "include/pika_conf.h" +#include "include/pika_server.h" +#include "include/pika_rm.h" +#include "include/pika_version.h" +#include "include/build_version.h" + +#ifdef TCMALLOC_EXTENSION +#include +#endif + +extern PikaServer *g_pika_server; +extern PikaConf *g_pika_conf; +extern PikaReplicaManager *g_pika_rm; + +static std::string ConstructPinginPubSubResp(const PikaCmdArgsType &argv) { + if (argv.size() > 2) { + return "-ERR wrong number of arguments for " + kCmdNamePing + + " command\r\n"; + } + std::stringstream resp; + + resp << "*2\r\n" + << "$4\r\n" + << "pong\r\n"; + if (argv.size() == 2) { + resp << "$" << argv[1].size() << "\r\n" << argv[1] << "\r\n"; + } else { + resp << "$0\r\n\r\n"; + } + return resp.str(); +} + +/* + * slaveof no one + * slaveof ip port + * slaveof ip port force + */ +void SlaveofCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlaveof); + return; + } + + if (argv_.size() > 4) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlaveof); + return; + } + + if (argv_.size() == 3 + && !strcasecmp(argv_[1].data(), "no") + && !strcasecmp(argv_[2].data(), "one")) { + is_noone_ = true; + return; + } + + // self is master of A , want to slavof B + if (g_pika_server->role() & PIKA_ROLE_MASTER) { + res_.SetRes(CmdRes::kErrOther, "already master of others, invalid usage"); + return; + } + + master_ip_ = argv_[1]; + std::string str_master_port = argv_[2]; + if (!slash::string2l(str_master_port.data(), str_master_port.size(), &master_port_) || master_port_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + if ((master_ip_ == "127.0.0.1" || master_ip_ == g_pika_server->host()) + && master_port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "you fucked up"); + return; + } + + if (argv_.size() == 4) { + if (!strcasecmp(argv_[3].data(), "force")) { + g_pika_server->SetForceFullSync(true); + } else { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlaveof); + } + } +} + +void SlaveofCmd::Do(std::shared_ptr partition) { + // Check if we are already connected to the specified master + if ((master_ip_ == "127.0.0.1" || g_pika_server->master_ip() == master_ip_) + && g_pika_server->master_port() == master_port_) { + res_.SetRes(CmdRes::kOk); + return; + } + + g_pika_server->RemoveMaster(); + + if (is_noone_) { + res_.SetRes(CmdRes::kOk); + g_pika_conf->SetSlaveof(std::string()); + return; + } + + bool sm_ret = g_pika_server->SetMaster(master_ip_, master_port_); + + if (sm_ret) { + res_.SetRes(CmdRes::kOk); + g_pika_conf->SetSlaveof(master_ip_ + ":" + std::to_string(master_port_)); + } else { + res_.SetRes(CmdRes::kErrOther, "Server is not in correct state for slaveof"); + } +} + +/* + * dbslaveof db[0 ~ 7] + * dbslaveof db[0 ~ 7] force + * dbslaveof db[0 ~ 7] no one + * dbslaveof db[0 ~ 7] filenum offset + */ +void DbSlaveofCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDbSlaveof); + return; + } + if (!g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "DbSlaveof only support on classic mode"); + return; + } + if (g_pika_server->role() ^ PIKA_ROLE_SLAVE + || !g_pika_server->MetaSyncDone()) { + res_.SetRes(CmdRes::kErrOther, "Not currently a slave"); + return; + } + + if (argv_.size() > 4) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDbSlaveof); + return; + } + + db_name_ = argv_[1]; + if (!g_pika_server->IsTableExist(db_name_)) { + res_.SetRes(CmdRes::kErrOther, "Invaild db name"); + return; + } + + if (argv_.size() == 3 + && !strcasecmp(argv_[2].data(), "force")) { + force_sync_ = true; + return; + } + + if (argv_.size() == 4) { + if (!strcasecmp(argv_[2].data(), "no") + && !strcasecmp(argv_[3].data(), "one")) { + is_noone_ = true; + return; + } + + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &filenum_) || filenum_ < 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &offset_) || offset_ < 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + have_offset_ = true; + } +} + +void DbSlaveofCmd::Do(std::shared_ptr partition) { + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName(PartitionInfo(db_name_,0)); + if (!slave_partition) { + res_.SetRes(CmdRes::kErrOther, "Db not found"); + return; + } + + Status s; + if (is_noone_) { + // In classic mode a table has only one partition + s = g_pika_rm->SendRemoveSlaveNodeRequest(db_name_, 0); + } else { + if (slave_partition->State() == ReplState::kNoConnect + || slave_partition->State() == ReplState::kError) { + if (have_offset_) { + std::shared_ptr db_partition = + g_pika_server->GetPartitionByDbName(db_name_); + db_partition->logger()->SetProducerStatus(filenum_, offset_); + } + ReplState state = force_sync_ + ? ReplState::kTryDBSync : ReplState::kTryConnect; + s = g_pika_rm->ActivateSyncSlavePartition( + RmNode(g_pika_server->master_ip(), g_pika_server->master_port(), + db_name_, 0), state); + } + } + + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void AuthCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameAuth); + return; + } + pwd_ = argv_[1]; +} + +void AuthCmd::Do(std::shared_ptr partition) { + std::string root_password(g_pika_conf->requirepass()); + std::string user_password(g_pika_conf->userpass()); + if (user_password.empty() && root_password.empty()) { + res_.SetRes(CmdRes::kErrOther, "Client sent AUTH, but no password is set"); + return; + } + + if (pwd_ == user_password) { + res_.SetRes(CmdRes::kOk, "USER"); + } + if (pwd_ == root_password) { + res_.SetRes(CmdRes::kOk, "ROOT"); + } + if (res_.none()) { + res_.SetRes(CmdRes::kInvalidPwd); + return; + } + + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNamePing); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + cli_conn->auth_stat().ChecknUpdate(res().raw_message()); +} + +void BgsaveCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBgsave); + return; + } + if (argv_.size() == 2) { + std::vector tables; + slash::StringSplit(argv_[1], COMMA, tables); + for (const auto& table : tables) { + if (!g_pika_server->IsTableExist(table)) { + res_.SetRes(CmdRes::kInvalidTable, table); + return; + } else { + bgsave_tables_.insert(table); + } + } + } +} + +void BgsaveCmd::Do(std::shared_ptr partition) { + g_pika_server->DoSameThingSpecificTable(TaskType::kBgSave, bgsave_tables_); + LogCommand(); + res_.AppendContent("+Background saving started"); +} + +void CompactCmd::DoInitial() { + if (!CheckArg(argv_.size()) + || argv_.size() > 3) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameCompact); + return; + } + + if (g_pika_server->IsKeyScaning()) { + res_.SetRes(CmdRes::kErrOther, "The info keyspace operation is executing, Try again later"); + return; + } + + if (argv_.size() == 1) { + struct_type_ = "all"; + } else if (argv_.size() == 2) { + struct_type_ = argv_[1]; + } else if (argv_.size() == 3) { + std::vector tables; + slash::StringSplit(argv_[1], COMMA, tables); + for (const auto& table : tables) { + if (!g_pika_server->IsTableExist(table)) { + res_.SetRes(CmdRes::kInvalidTable, table); + return; + } else { + compact_tables_.insert(table); + } + } + struct_type_ = argv_[2]; + } +} + +void CompactCmd::Do(std::shared_ptr partition) { + if (!strcasecmp(struct_type_.data(), "all")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactAll, compact_tables_); + } else if (!strcasecmp(struct_type_.data(), "string")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactStrings, compact_tables_); + } else if (!strcasecmp(struct_type_.data(), "hash")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactHashes, compact_tables_); + } else if (!strcasecmp(struct_type_.data(), "set")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactSets, compact_tables_); + } else if (!strcasecmp(struct_type_.data(), "zset")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactZSets, compact_tables_); + } else if (!strcasecmp(struct_type_.data(), "list")) { + g_pika_server->DoSameThingSpecificTable(TaskType::kCompactList, compact_tables_); + } else { + res_.SetRes(CmdRes::kInvalidDbType, struct_type_); + return; + } + LogCommand(); + res_.SetRes(CmdRes::kOk); +} + +void PurgelogstoCmd::DoInitial() { + if (!CheckArg(argv_.size()) + || argv_.size() > 3) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePurgelogsto); + return; + } + std::string filename = argv_[1]; + if (filename.size() <= kBinlogPrefixLen || + kBinlogPrefix != filename.substr(0, kBinlogPrefixLen)) { + res_.SetRes(CmdRes::kInvalidParameter); + return; + } + std::string str_num = filename.substr(kBinlogPrefixLen); + int64_t num = 0; + if (!slash::string2l(str_num.data(), str_num.size(), &num) || num < 0) { + res_.SetRes(CmdRes::kInvalidParameter); + return; + } + num_ = num; + + table_ = (argv_.size() == 3) ? argv_[2] :g_pika_conf->default_table(); + if (!g_pika_server->IsTableExist(table_)) { + res_.SetRes(CmdRes::kInvalidTable, table_); + return; + } +} + +void PurgelogstoCmd::Do(std::shared_ptr partition) { + std::shared_ptr table_partition = g_pika_server->GetTablePartitionById(table_, 0); + if (!table_partition) { + res_.SetRes(CmdRes::kErrOther, "Partition not found"); + } else { + table_partition->PurgeLogs(num_, true); + res_.SetRes(CmdRes::kOk); + } +} + +void PingCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePing); + return; + } +} + +void PingCmd::Do(std::shared_ptr partition) { + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNamePing); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + + if (cli_conn->IsPubSub()) { + return res_.SetRes(CmdRes::kNone, ConstructPinginPubSubResp(argv_)); + } + res_.SetRes(CmdRes::kPong); +} + +void SelectCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSelect); + return; + } + if (g_pika_conf->classic_mode()) { + int index = atoi(argv_[1].data()); + if (std::to_string(index) != argv_[1]) { + res_.SetRes(CmdRes::kInvalidIndex, kCmdNameSelect); + return; + } else if (index < 0 || index >= g_pika_conf->databases()) { + res_.SetRes(CmdRes::kInvalidIndex, kCmdNameSelect + " DB index is out of range"); + return; + } else { + table_name_ = "db" + argv_[1]; + } + } else { + // only pika codis use sharding mode currently, but pika + // codis only support single db, so in sharding mode we + // do no thing in select command + table_name_ = g_pika_conf->default_table(); + } + if (!g_pika_server->IsTableExist(table_name_)) { + res_.SetRes(CmdRes::kInvalidTable, kCmdNameSelect); + return; + } +} + +void SelectCmd::Do(std::shared_ptr partition) { + std::shared_ptr conn = + std::dynamic_pointer_cast(GetConn()); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNameSelect); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + conn->SetCurrentTable(table_name_); + res_.SetRes(CmdRes::kOk); +} + +void FlushallCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameFlushall); + return; + } +} +void FlushallCmd::Do(std::shared_ptr partition) { + if (!partition) { + LOG(INFO) << "Flushall, but partition not found"; + } else { + partition->FlushDB(); + } +} + +// flushall convert flushdb writes to every partition binlog +std::string FlushallCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 1, "*"); + + // to flushdb cmd + std::string flushdb_cmd("flushdb"); + RedisAppendLen(content, flushdb_cmd.size(), "$"); + RedisAppendContent(content, flushdb_cmd); + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void FlushdbCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameFlushdb); + return; + } + if (argv_.size() == 1) { + db_name_ = "all"; + } else { + std::string struct_type = argv_[1]; + if (!strcasecmp(struct_type.data(), "string")) { + db_name_ = "strings"; + } else if (!strcasecmp(struct_type.data(), "hash")) { + db_name_ = "hashes"; + } else if (!strcasecmp(struct_type.data(), "set")) { + db_name_ = "sets"; + } else if (!strcasecmp(struct_type.data(), "zset")) { + db_name_ = "zsets"; + } else if (!strcasecmp(struct_type.data(), "list")) { + db_name_ = "lists"; + } else { + res_.SetRes(CmdRes::kInvalidDbType); + } + } +} + +void FlushdbCmd::Do(std::shared_ptr partition) { + if (!partition) { + LOG(INFO) << "Flushdb, but partition not found"; + } else { + if (db_name_ == "all") { + partition->FlushDB(); + } else { + partition->FlushSubDB(db_name_); + } + } +} + +void ClientCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameClient); + return; + } + if (!strcasecmp(argv_[1].data(), "list") && argv_.size() == 2) { + // nothing + } else if (!strcasecmp(argv_[1].data(), "list") && argv_.size() == 5) { + if (!strcasecmp(argv_[2].data(), "order") && + !strcasecmp(argv_[3].data(), "by")) { + info_ = argv_[4]; + } else { + res_.SetRes(CmdRes::kErrOther, + "Syntax error, try CLIENT (LIST [order by [addr|idle])"); + return; + } + } else if (!strcasecmp(argv_[1].data(), "kill") && argv_.size() == 3) { + info_ = argv_[2]; + } else { + res_.SetRes(CmdRes::kErrOther, + "Syntax error, try CLIENT (LIST [order by [addr|idle]| KILL ip:port)"); + return; + } + operation_ = argv_[1]; + return; +} + +void ClientCmd::Do(std::shared_ptr partition) { + if (!strcasecmp(operation_.data(), "list")) { + struct timeval now; + gettimeofday(&now, NULL); + std::vector clients; + g_pika_server->ClientList(&clients); + std::vector::iterator iter = clients.begin(); + std::string reply = ""; + char buf[128]; + if (!strcasecmp(info_.data(), "addr")) { + std::sort(clients.begin(), clients.end(), AddrCompare); + } else if (!strcasecmp(info_.data(), "idle")) { + std::sort(clients.begin(), clients.end(), IdleCompare); + } + while (iter != clients.end()) { + snprintf(buf, sizeof(buf), "addr=%s fd=%d idle=%ld\n", + iter->ip_port.c_str(), iter->fd, + iter->last_interaction == 0 ? 0 : now.tv_sec - iter->last_interaction); + reply.append(buf); + iter++; + } + res_.AppendString(reply); + } else if (!strcasecmp(operation_.data(), "kill") && + !strcasecmp(info_.data(), "all")) { + g_pika_server->ClientKillAll(); + res_.SetRes(CmdRes::kOk); + } else if (g_pika_server->ClientKill(info_) == 1) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, "No such client"); + } + return; +} + +void ShutdownCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameShutdown); + return; + } + + // For now, only shutdown need check local + if (is_local()) { + std::shared_ptr conn = GetConn(); + if (conn) { + if (conn->ip_port().find("127.0.0.1") == std::string::npos + && conn->ip_port().find(g_pika_server->host()) == std::string::npos) { + LOG(WARNING) << "\'shutdown\' should be localhost" << " command from " << conn->ip_port(); + res_.SetRes(CmdRes::kErrOther, kCmdNameShutdown + " should be localhost"); + } + } else { + LOG(WARNING) << name_ << " weak ptr is empty"; + res_.SetRes(CmdRes::kErrOther, kCmdNameShutdown); + return; + } + } +} +// no return +void ShutdownCmd::Do(std::shared_ptr partition) { + DLOG(WARNING) << "handle \'shutdown\'"; + g_pika_server->Exit(); + res_.SetRes(CmdRes::kNone); +} + +const std::string InfoCmd::kInfoSection = "info"; +const std::string InfoCmd::kAllSection = "all"; +const std::string InfoCmd::kServerSection = "server"; +const std::string InfoCmd::kClientsSection = "clients"; +const std::string InfoCmd::kStatsSection = "stats"; +const std::string InfoCmd::kExecCountSection= "command_exec_count"; +const std::string InfoCmd::kCPUSection = "cpu"; +const std::string InfoCmd::kReplicationSection = "replication"; +const std::string InfoCmd::kKeyspaceSection = "keyspace"; +const std::string InfoCmd::kDataSection = "data"; +const std::string InfoCmd::kDebugSection = "debug"; + +void InfoCmd::DoInitial() { + size_t argc = argv_.size(); + if (argc > 4) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (argc == 1) { + info_section_ = kInfo; + return; + } //then the agc is 2 or 3 + + if (!strcasecmp(argv_[1].data(), kAllSection.data())) { + info_section_ = kInfoAll; + } else if (!strcasecmp(argv_[1].data(), kServerSection.data())) { + info_section_ = kInfoServer; + } else if (!strcasecmp(argv_[1].data(), kClientsSection.data())) { + info_section_ = kInfoClients; + } else if (!strcasecmp(argv_[1].data(), kStatsSection.data())) { + info_section_ = kInfoStats; + } else if (!strcasecmp(argv_[1].data(), kExecCountSection.data())) { + info_section_ = kInfoExecCount; + } else if (!strcasecmp(argv_[1].data(), kCPUSection.data())) { + info_section_ = kInfoCPU; + } else if (!strcasecmp(argv_[1].data(), kReplicationSection.data())) { + info_section_ = kInfoReplication; + } else if (!strcasecmp(argv_[1].data(), kKeyspaceSection.data())) { + info_section_ = kInfoKeyspace; + if (argc == 2) { + LogCommand(); + return; + } + // info keyspace [ 0 | 1 | off ] + // info keyspace 1 db0,db1 + // info keyspace 0 db0,db1 + // info keyspace off db0,db1 + if (argv_[2] == "1") { + if (g_pika_server->IsCompacting()) { + res_.SetRes(CmdRes::kErrOther, "The compact operation is executing, Try again later"); + } else { + rescan_ = true; + } + } else if (argv_[2] == "off") { + off_ = true; + } else if (argv_[2] != "0") { + res_.SetRes(CmdRes::kSyntaxErr); + } + + if (argc == 4) { + std::vector tables; + slash::StringSplit(argv_[3], COMMA, tables); + for (const auto& table : tables) { + if (!g_pika_server->IsTableExist(table)) { + res_.SetRes(CmdRes::kInvalidTable, table); + return; + } else { + keyspace_scan_tables_.insert(table); + } + } + } + LogCommand(); + return; + } else if (!strcasecmp(argv_[1].data(), kDataSection.data())) { + info_section_ = kInfoData; + } else if (!strcasecmp(argv_[1].data(), kDebugSection.data())) { + info_section_ = kInfoDebug; + } else { + info_section_ = kInfoErr; + } + if (argc != 2) { + res_.SetRes(CmdRes::kSyntaxErr); + } +} + +void InfoCmd::Do(std::shared_ptr partition) { + std::string info; + switch (info_section_) { + case kInfo: + InfoServer(info); + info.append("\r\n"); + InfoData(info); + info.append("\r\n"); + InfoClients(info); + info.append("\r\n"); + InfoStats(info); + info.append("\r\n"); + InfoCPU(info); + info.append("\r\n"); + InfoReplication(info); + info.append("\r\n"); + InfoKeyspace(info); + break; + case kInfoAll: + InfoServer(info); + info.append("\r\n"); + InfoData(info); + info.append("\r\n"); + InfoClients(info); + info.append("\r\n"); + InfoStats(info); + info.append("\r\n"); + InfoExecCount(info); + info.append("\r\n"); + InfoCPU(info); + info.append("\r\n"); + InfoReplication(info); + info.append("\r\n"); + InfoKeyspace(info); + break; + case kInfoServer: + InfoServer(info); + break; + case kInfoClients: + InfoClients(info); + break; + case kInfoStats: + InfoStats(info); + break; + case kInfoExecCount: + InfoExecCount(info); + break; + case kInfoCPU: + InfoCPU(info); + break; + case kInfoReplication: + InfoReplication(info); + break; + case kInfoKeyspace: + InfoKeyspace(info); + break; + case kInfoData: + InfoData(info); + break; + case kInfoDebug: + InfoDebug(info); + break; + default: + //kInfoErr is nothing + break; + } + + + res_.AppendStringLen(info.size()); + res_.AppendContent(info); + return; +} + +void InfoCmd::InfoServer(std::string& info) { + static struct utsname host_info; + static bool host_info_valid = false; + if (!host_info_valid) { + uname(&host_info); + host_info_valid = true; + } + + time_t current_time_s = time(NULL); + std::stringstream tmp_stream; + char version[32]; + snprintf(version, sizeof(version), "%d.%d.%d", PIKA_MAJOR, + PIKA_MINOR, PIKA_PATCH); + tmp_stream << "# Server\r\n"; + tmp_stream << "pika_version:" << version << "\r\n"; + tmp_stream << pika_build_git_sha << "\r\n"; + tmp_stream << "pika_build_compile_date: " << + pika_build_compile_date << "\r\n"; + tmp_stream << "os:" << host_info.sysname << " " << host_info.release << " " << host_info.machine << "\r\n"; + tmp_stream << "arch_bits:" << (reinterpret_cast(&host_info.machine) + strlen(host_info.machine) - 2) << "\r\n"; + tmp_stream << "process_id:" << getpid() << "\r\n"; + tmp_stream << "tcp_port:" << g_pika_conf->port() << "\r\n"; + tmp_stream << "thread_num:" << g_pika_conf->thread_num() << "\r\n"; + tmp_stream << "sync_thread_num:" << g_pika_conf->sync_thread_num() << "\r\n"; + tmp_stream << "uptime_in_seconds:" << (current_time_s - g_pika_server->start_time_s()) << "\r\n"; + tmp_stream << "uptime_in_days:" << (current_time_s / (24*3600) - g_pika_server->start_time_s() / (24*3600) + 1) << "\r\n"; + tmp_stream << "config_file:" << g_pika_conf->conf_path() << "\r\n"; + tmp_stream << "server_id:" << g_pika_conf->server_id() << "\r\n"; + + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoClients(std::string& info) { + std::stringstream tmp_stream; + tmp_stream << "# Clients\r\n"; + tmp_stream << "connected_clients:" << g_pika_server->ClientList() << "\r\n"; + + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoStats(std::string& info) { + std::stringstream tmp_stream; + tmp_stream << "# Stats\r\n"; + tmp_stream << "total_connections_received:" << g_pika_server->accumulative_connections() << "\r\n"; + tmp_stream << "instantaneous_ops_per_sec:" << g_pika_server->ServerCurrentQps() << "\r\n"; + tmp_stream << "total_commands_processed:" << g_pika_server->ServerQueryNum() << "\r\n"; + tmp_stream << "is_bgsaving:" << (g_pika_server->IsBgSaving() ? "Yes" : "No") << "\r\n"; + tmp_stream << "is_scaning_keyspace:" << (g_pika_server->IsKeyScaning() ? "Yes" : "No") << "\r\n"; + tmp_stream << "is_compact:" << (g_pika_server->IsCompacting() ? "Yes" : "No") << "\r\n"; + tmp_stream << "compact_cron:" << g_pika_conf->compact_cron() << "\r\n"; + tmp_stream << "compact_interval:" << g_pika_conf->compact_interval() << "\r\n"; + + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoExecCount(std::string& info) { + std::stringstream tmp_stream; + tmp_stream << "# Command_Exec_Count\r\n"; + + std::unordered_map command_exec_count_table = g_pika_server->ServerExecCountTable(); + for (const auto& item : command_exec_count_table) { + if (item.second == 0) { + continue; + } + tmp_stream << item.first << ":" << item.second << "\r\n"; + } + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoCPU(std::string& info) { + struct rusage self_ru, c_ru; + getrusage(RUSAGE_SELF, &self_ru); + getrusage(RUSAGE_CHILDREN, &c_ru); + std::stringstream tmp_stream; + tmp_stream << "# CPU\r\n"; + tmp_stream << "used_cpu_sys:" << + setiosflags(std::ios::fixed) << std::setprecision(2) << + (float)self_ru.ru_stime.tv_sec+(float)self_ru.ru_stime.tv_usec/1000000 << + "\r\n"; + tmp_stream << "used_cpu_user:" << + setiosflags(std::ios::fixed) << std::setprecision(2) << + (float)self_ru.ru_utime.tv_sec+(float)self_ru.ru_utime.tv_usec/1000000 << + "\r\n"; + tmp_stream << "used_cpu_sys_children:" << + setiosflags(std::ios::fixed) << std::setprecision(2) << + (float)c_ru.ru_stime.tv_sec+(float)c_ru.ru_stime.tv_usec/1000000 << + "\r\n"; + tmp_stream << "used_cpu_user_children:" << + setiosflags(std::ios::fixed) << std::setprecision(2) << + (float)c_ru.ru_utime.tv_sec+(float)c_ru.ru_utime.tv_usec/1000000 << + "\r\n"; + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoShardingReplication(std::string& info) { + int role = 0; + std::string slave_list_string; + uint32_t slave_num = g_pika_server->GetShardingSlaveListString(slave_list_string); + if (slave_num) { + role |= PIKA_ROLE_MASTER; + } + std::string common_master; + std::string master_ip; + int master_port = 0; + g_pika_rm->FindCommonMaster(&common_master); + if (!common_master.empty()) { + role |= PIKA_ROLE_SLAVE; + if(!slash::ParseIpPortString(common_master, master_ip, master_port)) { + return; + } + } + + std::stringstream tmp_stream; + tmp_stream << "# Replication("; + switch (role) { + case PIKA_ROLE_SINGLE : + case PIKA_ROLE_MASTER : tmp_stream << "MASTER)\r\nrole:master\r\n"; break; + case PIKA_ROLE_SLAVE : tmp_stream << "SLAVE)\r\nrole:slave\r\n"; break; + case PIKA_ROLE_MASTER | PIKA_ROLE_SLAVE : tmp_stream << "Master && SLAVE)\r\nrole:master&&slave\r\n"; break; + default: info.append("ERR: server role is error\r\n"); return; + } + switch (role) { + case PIKA_ROLE_SLAVE : + tmp_stream << "master_host:" << master_ip << "\r\n"; + tmp_stream << "master_port:" << master_port << "\r\n"; + tmp_stream << "master_link_status:up"<< "\r\n"; + tmp_stream << "slave_priority:" << g_pika_conf->slave_priority() << "\r\n"; + break; + case PIKA_ROLE_MASTER | PIKA_ROLE_SLAVE : + tmp_stream << "master_host:" << master_ip << "\r\n"; + tmp_stream << "master_port:" << master_port << "\r\n"; + tmp_stream << "master_link_status:up"<< "\r\n"; + case PIKA_ROLE_SINGLE : + case PIKA_ROLE_MASTER : + tmp_stream << "connected_slaves:" << slave_num << "\r\n" << slave_list_string; + } + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoReplication(std::string& info) { + if (!g_pika_conf->classic_mode()) { + // In Sharding mode, show different replication info + InfoShardingReplication(info); + return; + } + + int host_role = g_pika_server->role(); + std::stringstream tmp_stream; + std::stringstream out_of_sync; + + bool all_partition_sync = true; + slash::RWLock table_rwl(&g_pika_server->tables_rw_, false); + for (const auto& table_item : g_pika_server->tables_) { + slash::RWLock partition_rwl(&table_item.second->partitions_rw_, false); + for (const auto& partition_item : table_item.second->partitions_) { + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_item.second->GetTableName(), + partition_item.second->GetPartitionId())); + if (!slave_partition) { + out_of_sync << "(" << partition_item.second->GetPartitionName() << ": InternalError)"; + continue; + } + if (slave_partition->State() != ReplState::kConnected) { + all_partition_sync = false; + out_of_sync << "(" << partition_item.second->GetPartitionName() << ":"; + if (slave_partition->State() == ReplState::kNoConnect) { + out_of_sync << "NoConnect)"; + } else if (slave_partition->State() == ReplState::kWaitDBSync) { + out_of_sync << "WaitDBSync)"; + } else if (slave_partition->State() == ReplState::kError) { + out_of_sync << "Error)"; + } else { + out_of_sync << "Other)"; + } + } + } + } + + tmp_stream << "# Replication("; + switch (host_role) { + case PIKA_ROLE_SINGLE : + case PIKA_ROLE_MASTER : tmp_stream << "MASTER)\r\nrole:master\r\n"; break; + case PIKA_ROLE_SLAVE : tmp_stream << "SLAVE)\r\nrole:slave\r\n"; break; + case PIKA_ROLE_MASTER | PIKA_ROLE_SLAVE : tmp_stream << "Master && SLAVE)\r\nrole:master&&slave\r\n"; break; + default: info.append("ERR: server role is error\r\n"); return; + } + + std::string slaves_list_str; + switch (host_role) { + case PIKA_ROLE_SLAVE : + tmp_stream << "master_host:" << g_pika_server->master_ip() << "\r\n"; + tmp_stream << "master_port:" << g_pika_server->master_port() << "\r\n"; + tmp_stream << "master_link_status:" << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) + && all_partition_sync) ? "up" : "down") << "\r\n"; + tmp_stream << "slave_priority:" << g_pika_conf->slave_priority() << "\r\n"; + tmp_stream << "slave_read_only:" << g_pika_conf->slave_read_only() << "\r\n"; + if (!all_partition_sync) { + tmp_stream <<"db_repl_error_state:" << out_of_sync.str() << "\r\n"; + } + break; + case PIKA_ROLE_MASTER | PIKA_ROLE_SLAVE : + tmp_stream << "master_host:" << g_pika_server->master_ip() << "\r\n"; + tmp_stream << "master_port:" << g_pika_server->master_port() << "\r\n"; + tmp_stream << "master_link_status:" << (((g_pika_server->repl_state() == PIKA_REPL_META_SYNC_DONE) + && all_partition_sync) ? "up" : "down") << "\r\n"; + tmp_stream << "slave_read_only:" << g_pika_conf->slave_read_only() << "\r\n"; + if (!all_partition_sync) { + tmp_stream <<"db_repl_error_state:" << out_of_sync.str() << "\r\n"; + } + case PIKA_ROLE_SINGLE : + case PIKA_ROLE_MASTER : + tmp_stream << "connected_slaves:" << g_pika_server->GetSlaveListString(slaves_list_str) << "\r\n" << slaves_list_str; + } + + + Status s; + uint32_t filenum = 0; + uint64_t offset = 0; + std::string safety_purge; + for (const auto& t_item : g_pika_server->tables_) { + slash::RWLock partition_rwl(&t_item.second->partitions_rw_, false); + for (const auto& p_item : t_item.second->partitions_) { + p_item.second->logger()->GetProducerStatus(&filenum, &offset); + tmp_stream << p_item.second->GetPartitionName() << " binlog_offset=" << filenum << " " << offset; + s = g_pika_rm->GetSafetyPurgeBinlogFromSMP(p_item.second->GetTableName(), p_item.second->GetPartitionId(), &safety_purge); + tmp_stream << ",safety_purge=" << (s.ok() ? safety_purge : "error") << "\r\n"; + } + } + + info.append(tmp_stream.str()); +} + +void InfoCmd::InfoKeyspace(std::string& info) { + if (off_) { + g_pika_server->DoSameThingSpecificTable(TaskType::kStopKeyScan, keyspace_scan_tables_); + info.append("OK\r\n"); + return; + } + + std::string table_name; + KeyScanInfo key_scan_info; + int32_t duration; + std::vector key_infos; + std::stringstream tmp_stream; + tmp_stream << "# Keyspace\r\n"; + slash::RWLock rwl(&g_pika_server->tables_rw_, false); + for (const auto& table_item : g_pika_server->tables_) { + if (keyspace_scan_tables_.empty() + || keyspace_scan_tables_.find(table_item.first) != keyspace_scan_tables_.end()) { + table_name = table_item.second->GetTableName(); + key_scan_info = table_item.second->GetKeyScanInfo(); + key_infos = key_scan_info.key_infos; + duration = key_scan_info.duration; + if (key_infos.size() != 5) { + info.append("info keyspace error\r\n"); + return; + } + tmp_stream << "# Time:" << key_scan_info.s_start_time << "\r\n"; + if (duration == -2) { + tmp_stream << "# Duration: " << "In Waiting\r\n"; + } else if (duration == -1) { + tmp_stream << "# Duration: " << "In Processing\r\n"; + } else if (duration >= 0) { + tmp_stream << "# Duration: " << std::to_string(duration) + "s" << "\r\n"; + } + + tmp_stream << table_name << " Strings_keys=" << key_infos[0].keys << ", expires=" << key_infos[0].expires << ", invaild_keys=" << key_infos[0].invaild_keys << "\r\n"; + tmp_stream << table_name << " Hashes_keys=" << key_infos[1].keys << ", expires=" << key_infos[1].expires << ", invaild_keys=" << key_infos[1].invaild_keys << "\r\n"; + tmp_stream << table_name << " Lists_keys=" << key_infos[2].keys << ", expires=" << key_infos[2].expires << ", invaild_keys=" << key_infos[2].invaild_keys << "\r\n"; + tmp_stream << table_name << " Zsets_keys=" << key_infos[3].keys << ", expires=" << key_infos[3].expires << ", invaild_keys=" << key_infos[3].invaild_keys << "\r\n"; + tmp_stream << table_name << " Sets_keys=" << key_infos[4].keys << ", expires=" << key_infos[4].expires << ", invaild_keys=" << key_infos[4].invaild_keys << "\r\n\r\n"; + } + } + info.append(tmp_stream.str()); + + if (rescan_) { + g_pika_server->DoSameThingSpecificTable(TaskType::kStartKeyScan, keyspace_scan_tables_); + } + return; +} + +void InfoCmd::InfoData(std::string& info) { + std::stringstream tmp_stream; + std::stringstream db_fatal_msg_stream; + + int64_t db_size = slash::Du(g_pika_conf->db_path()); + tmp_stream << "# Data" << "\r\n"; + tmp_stream << "db_size:" << db_size << "\r\n"; + tmp_stream << "db_size_human:" << (db_size >> 20) << "M\r\n"; + int64_t log_size = slash::Du(g_pika_conf->log_path()); + tmp_stream << "log_size:" << log_size << "\r\n"; + tmp_stream << "log_size_human:" << (log_size >> 20) << "M\r\n"; + tmp_stream << "compression:" << g_pika_conf->compression() << "\r\n"; + + // rocksdb related memory usage + std::map type_result; + uint64_t total_background_errors = 0; + uint64_t total_memtable_usage = 0, memtable_usage = 0; + uint64_t total_table_reader_usage = 0, table_reader_usage = 0; + slash::RWLock table_rwl(&g_pika_server->tables_rw_, false); + for (const auto& table_item : g_pika_server->tables_) { + slash::RWLock partition_rwl(&table_item.second->partitions_rw_, false); + for (const auto& patition_item : table_item.second->partitions_) { + type_result.clear(); + memtable_usage = table_reader_usage = 0; + patition_item.second->DbRWLockReader(); + patition_item.second->db()->GetUsage(blackwidow::PROPERTY_TYPE_ROCKSDB_MEMTABLE, &memtable_usage); + patition_item.second->db()->GetUsage(blackwidow::PROPERTY_TYPE_ROCKSDB_TABLE_READER, &table_reader_usage); + patition_item.second->db()->GetUsage(blackwidow::PROPERTY_TYPE_ROCKSDB_BACKGROUND_ERRORS, &type_result); + patition_item.second->DbRWUnLock(); + total_memtable_usage += memtable_usage; + total_table_reader_usage += table_reader_usage; + for (const auto& item : type_result) { + if (item.second != 0) { + db_fatal_msg_stream << (total_background_errors != 0 ? "," : ""); + db_fatal_msg_stream << patition_item.second->GetPartitionName() << "/" << item.first; + total_background_errors += item.second; + } + } + } + } + + tmp_stream << "used_memory:" << (total_memtable_usage + total_table_reader_usage) << "\r\n"; + tmp_stream << "used_memory_human:" << ((total_memtable_usage + total_table_reader_usage) >> 20) << "M\r\n"; + tmp_stream << "db_memtable_usage:" << total_memtable_usage << "\r\n"; + tmp_stream << "db_tablereader_usage:" << total_table_reader_usage << "\r\n"; + tmp_stream << "db_fatal:" << (total_background_errors != 0 ? "1" : "0") << "\r\n"; + tmp_stream << "db_fatal_msg:" << (total_background_errors != 0 ? db_fatal_msg_stream.str() : "NULL") << "\r\n"; + + info.append(tmp_stream.str()); + return; +} + +void InfoCmd::InfoDebug(std::string& info) { + std::stringstream tmp_stream; + tmp_stream << "# Synchronization Status" << "\r\n"; + info.append(tmp_stream.str()); + g_pika_rm->RmStatus(&info); + return; +} + +void ConfigCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameConfig); + return; + } + size_t argc = argv_.size(); + if (!strcasecmp(argv_[1].data(), "get")) { + if (argc != 3) { + res_.SetRes(CmdRes::kErrOther, "Wrong number of arguments for CONFIG get"); + return; + } + } else if (!strcasecmp(argv_[1].data(), "set")) { + if (argc == 3 && argv_[2] != "*") { + res_.SetRes(CmdRes::kErrOther, "Wrong number of arguments for CONFIG set"); + return; + } else if (argc != 4 && argc != 3) { + res_.SetRes(CmdRes::kErrOther, "Wrong number of arguments for CONFIG set"); + return; + } + } else if (!strcasecmp(argv_[1].data(), "rewrite")) { + if (argc != 2) { + res_.SetRes(CmdRes::kErrOther, "Wrong number of arguments for CONFIG rewrite"); + return; + } + } else if (!strcasecmp(argv_[1].data(), "resetstat")) { + if (argc != 2) { + res_.SetRes(CmdRes::kErrOther, "Wrong number of arguments for CONFIG resetstat"); + return; + } + } else { + res_.SetRes(CmdRes::kErrOther, "CONFIG subcommand must be one of GET, SET, RESETSTAT, REWRITE"); + return; + } + config_args_v_.assign(argv_.begin() + 1, argv_.end()); + return; +} + +void ConfigCmd::Do(std::shared_ptr partition) { + std::string config_ret; + if (!strcasecmp(config_args_v_[0].data(), "get")) { + ConfigGet(config_ret); + } else if (!strcasecmp(config_args_v_[0].data(), "set")) { + ConfigSet(config_ret); + } else if (!strcasecmp(config_args_v_[0].data(), "rewrite")) { + ConfigRewrite(config_ret); + } else if (!strcasecmp(config_args_v_[0].data(), "resetstat")) { + ConfigResetstat(config_ret); + } + res_.AppendStringRaw(config_ret); + return; +} + +static void EncodeString(std::string *dst, const std::string &value) { + dst->append("$"); + dst->append(std::to_string(value.size())); + dst->append("\r\n"); + dst->append(value.data(), value.size()); + dst->append("\r\n"); +} + +static void EncodeInt32(std::string *dst, const int32_t v) { + std::string vstr = std::to_string(v); + dst->append("$"); + dst->append(std::to_string(vstr.length())); + dst->append("\r\n"); + dst->append(vstr); + dst->append("\r\n"); +} + +static void EncodeInt64(std::string *dst, const int64_t v) { + std::string vstr = std::to_string(v); + dst->append("$"); + dst->append(std::to_string(vstr.length())); + dst->append("\r\n"); + dst->append(vstr); + dst->append("\r\n"); +} + +void ConfigCmd::ConfigGet(std::string &ret) { + size_t elements = 0; + std::string config_body; + std::string pattern = config_args_v_[1]; + + if (slash::stringmatch(pattern.data(), "port", 1)) { + elements += 2; + EncodeString(&config_body, "port"); + EncodeInt32(&config_body, g_pika_conf->port()); + } + + if (slash::stringmatch(pattern.data(), "thread-num", 1)) { + elements += 2; + EncodeString(&config_body, "thread-num"); + EncodeInt32(&config_body, g_pika_conf->thread_num()); + } + + if (slash::stringmatch(pattern.data(), "thread-pool-size", 1)) { + elements += 2; + EncodeString(&config_body, "thread-pool-size"); + EncodeInt32(&config_body, g_pika_conf->thread_pool_size()); + } + + if (slash::stringmatch(pattern.data(), "sync-thread-num", 1)) { + elements += 2; + EncodeString(&config_body, "sync-thread-num"); + EncodeInt32(&config_body, g_pika_conf->sync_thread_num()); + } + + if (slash::stringmatch(pattern.data(), "log-path", 1)) { + elements += 2; + EncodeString(&config_body, "log-path"); + EncodeString(&config_body, g_pika_conf->log_path()); + } + + if (slash::stringmatch(pattern.data(), "db-path", 1)) { + elements += 2; + EncodeString(&config_body, "db-path"); + EncodeString(&config_body, g_pika_conf->db_path()); + } + + if (slash::stringmatch(pattern.data(), "maxmemory", 1)) { + elements += 2; + EncodeString(&config_body, "maxmemory"); + EncodeInt64(&config_body, g_pika_conf->write_buffer_size()); + } + + if (slash::stringmatch(pattern.data(), "write-buffer-size", 1)) { + elements += 2; + EncodeString(&config_body, "write-buffer-size"); + EncodeInt64(&config_body, g_pika_conf->write_buffer_size()); + } + + if (slash::stringmatch(pattern.data(), "timeout", 1)) { + elements += 2; + EncodeString(&config_body, "timeout"); + EncodeInt32(&config_body, g_pika_conf->timeout()); + } + + if (slash::stringmatch(pattern.data(), "requirepass", 1)) { + elements += 2; + EncodeString(&config_body, "requirepass"); + EncodeString(&config_body, g_pika_conf->requirepass()); + } + + if (slash::stringmatch(pattern.data(), "masterauth", 1)) { + elements += 2; + EncodeString(&config_body, "masterauth"); + EncodeString(&config_body, g_pika_conf->masterauth()); + } + + if (slash::stringmatch(pattern.data(), "userpass", 1)) { + elements += 2; + EncodeString(&config_body, "userpass"); + EncodeString(&config_body, g_pika_conf->userpass()); + } + + if (slash::stringmatch(pattern.data(), "userblacklist", 1)) { + elements += 2; + EncodeString(&config_body, "userblacklist"); + EncodeString(&config_body, (g_pika_conf->suser_blacklist()).c_str()); + } + + if (slash::stringmatch(pattern.data(), "instance-mode", 1)) { + elements += 2; + EncodeString(&config_body, "instance-mode"); + EncodeString(&config_body, (g_pika_conf->classic_mode() ? "classic" : "sharding")); + } + + if (g_pika_conf->classic_mode() + && slash::stringmatch(pattern.data(), "databases", 1)) { + elements += 2; + EncodeString(&config_body, "databases"); + EncodeInt32(&config_body, g_pika_conf->databases()); + } + + if (!g_pika_conf->classic_mode() + && slash::stringmatch(pattern.data(), "default-slot-num", 1)) { + elements += 2; + EncodeString(&config_body, "default-slot-num"); + EncodeInt32(&config_body, g_pika_conf->default_slot_num()); + } + + if (slash::stringmatch(pattern.data(), "daemonize", 1)) { + elements += 2; + EncodeString(&config_body, "daemonize"); + EncodeString(&config_body, g_pika_conf->daemonize() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "dump-path", 1)) { + elements += 2; + EncodeString(&config_body, "dump-path"); + EncodeString(&config_body, g_pika_conf->bgsave_path()); + } + + if (slash::stringmatch(pattern.data(), "dump-expire", 1)) { + elements += 2; + EncodeString(&config_body, "dump-expire"); + EncodeInt32(&config_body, g_pika_conf->expire_dump_days()); + } + + if (slash::stringmatch(pattern.data(), "dump-prefix", 1)) { + elements += 2; + EncodeString(&config_body, "dump-prefix"); + EncodeString(&config_body, g_pika_conf->bgsave_prefix()); + } + + if (slash::stringmatch(pattern.data(), "pidfile", 1)) { + elements += 2; + EncodeString(&config_body, "pidfile"); + EncodeString(&config_body, g_pika_conf->pidfile()); + } + + if (slash::stringmatch(pattern.data(), "maxclients", 1)) { + elements += 2; + EncodeString(&config_body, "maxclients"); + EncodeInt32(&config_body, g_pika_conf->maxclients()); + } + + if (slash::stringmatch(pattern.data(), "target-file-size-base", 1)) { + elements += 2; + EncodeString(&config_body, "target-file-size-base"); + EncodeInt32(&config_body, g_pika_conf->target_file_size_base()); + } + + if (slash::stringmatch(pattern.data(), "max-cache-statistic-keys", 1)) { + elements += 2; + EncodeString(&config_body, "max-cache-statistic-keys"); + EncodeInt32(&config_body, g_pika_conf->max_cache_statistic_keys()); + } + + if (slash::stringmatch(pattern.data(), "small-compaction-threshold", 1)) { + elements += 2; + EncodeString(&config_body, "small-compaction-threshold"); + EncodeInt32(&config_body, g_pika_conf->small_compaction_threshold()); + } + + if (slash::stringmatch(pattern.data(), "max-background-flushes", 1)) { + elements += 2; + EncodeString(&config_body, "max-background-flushes"); + EncodeInt32(&config_body, g_pika_conf->max_background_flushes()); + } + + if (slash::stringmatch(pattern.data(), "max-background-compactions", 1)) { + elements += 2; + EncodeString(&config_body, "max-background-compactions"); + EncodeInt32(&config_body, g_pika_conf->max_background_compactions()); + } + + if (slash::stringmatch(pattern.data(), "max-cache-files", 1)) { + elements += 2; + EncodeString(&config_body, "max-cache-files"); + EncodeInt32(&config_body, g_pika_conf->max_cache_files()); + } + + if (slash::stringmatch(pattern.data(), "max-bytes-for-level-multiplier", 1)) { + elements += 2; + EncodeString(&config_body, "max-bytes-for-level-multiplier"); + EncodeInt32(&config_body, g_pika_conf->max_bytes_for_level_multiplier()); + } + + if (slash::stringmatch(pattern.data(), "block-size", 1)) { + elements += 2; + EncodeString(&config_body, "block-size"); + EncodeInt64(&config_body, g_pika_conf->block_size()); + } + + if (slash::stringmatch(pattern.data(), "block-cache", 1)) { + elements += 2; + EncodeString(&config_body, "block-cache"); + EncodeInt64(&config_body, g_pika_conf->block_cache()); + } + + if (slash::stringmatch(pattern.data(), "share-block-cache", 1)) { + elements += 2; + EncodeString(&config_body, "share-block-cache"); + EncodeString(&config_body, g_pika_conf->share_block_cache() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "cache-index-and-filter-blocks", 1)) { + elements += 2; + EncodeString(&config_body, "cache-index-and-filter-blocks"); + EncodeString(&config_body, g_pika_conf->cache_index_and_filter_blocks() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "optimize-filters-for-hits", 1)) { + elements += 2; + EncodeString(&config_body, "optimize-filters-for-hits"); + EncodeString(&config_body, g_pika_conf->optimize_filters_for_hits() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "level-compaction-dynamic-level-bytes", 1)) { + elements += 2; + EncodeString(&config_body, "level-compaction-dynamic-level-bytes"); + EncodeString(&config_body, g_pika_conf->level_compaction_dynamic_level_bytes() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "expire-logs-days", 1)) { + elements += 2; + EncodeString(&config_body, "expire-logs-days"); + EncodeInt32(&config_body, g_pika_conf->expire_logs_days()); + } + + if (slash::stringmatch(pattern.data(), "expire-logs-nums", 1)) { + elements += 2; + EncodeString(&config_body, "expire-logs-nums"); + EncodeInt32(&config_body, g_pika_conf->expire_logs_nums()); + } + + if (slash::stringmatch(pattern.data(), "root-connection-num", 1)) { + elements += 2; + EncodeString(&config_body, "root-connection-num"); + EncodeInt32(&config_body, g_pika_conf->root_connection_num()); + } + + if (slash::stringmatch(pattern.data(), "slowlog-write-errorlog", 1)) { + elements += 2; + EncodeString(&config_body, "slowlog-write-errorlog"); + EncodeString(&config_body, g_pika_conf->slowlog_write_errorlog() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "slowlog-log-slower-than", 1)) { + elements += 2; + EncodeString(&config_body, "slowlog-log-slower-than"); + EncodeInt32(&config_body, g_pika_conf->slowlog_slower_than()); + } + + if (slash::stringmatch(pattern.data(), "slowlog-max-len", 1)) { + elements += 2; + EncodeString(&config_body, "slowlog-max-len"); + EncodeInt32(&config_body, g_pika_conf->slowlog_max_len()); + } + + if (slash::stringmatch(pattern.data(), "write-binlog", 1)) { + elements += 2; + EncodeString(&config_body, "write-binlog"); + EncodeString(&config_body, g_pika_conf->write_binlog() ? "yes" : "no"); + } + + if (slash::stringmatch(pattern.data(), "binlog-file-size", 1)) { + elements += 2; + EncodeString(&config_body, "binlog-file-size"); + EncodeInt32(&config_body, g_pika_conf->binlog_file_size()); + } + + if (slash::stringmatch(pattern.data(), "max-cache-statistic-keys", 1)) { + elements += 2; + EncodeString(&config_body, "max-cache-statistic-keys"); + EncodeInt32(&config_body, g_pika_conf->max_cache_statistic_keys()); + } + + if (slash::stringmatch(pattern.data(), "small-compaction-threshold", 1)) { + elements += 2; + EncodeString(&config_body, "small-compaction-threshold"); + EncodeInt32(&config_body, g_pika_conf->small_compaction_threshold()); + } + + if (slash::stringmatch(pattern.data(), "max-write-buffer-size", 1)) { + elements += 2; + EncodeString(&config_body, "max-write-buffer-size"); + EncodeInt64(&config_body, g_pika_conf->max_write_buffer_size()); + } + + if (slash::stringmatch(pattern.data(), "max-client-response-size", 1)) { + elements += 2; + EncodeString(&config_body, "max-client-response-size"); + EncodeInt64(&config_body, g_pika_conf->max_client_response_size()); + } + + if (slash::stringmatch(pattern.data(), "compression", 1)) { + elements += 2; + EncodeString(&config_body, "compression"); + EncodeString(&config_body, g_pika_conf->compression()); + } + + if (slash::stringmatch(pattern.data(), "db-sync-path", 1)) { + elements += 2; + EncodeString(&config_body, "db-sync-path"); + EncodeString(&config_body, g_pika_conf->db_sync_path()); + } + + if (slash::stringmatch(pattern.data(), "db-sync-speed", 1)) { + elements += 2; + EncodeString(&config_body, "db-sync-speed"); + EncodeInt32(&config_body, g_pika_conf->db_sync_speed()); + } + + if (slash::stringmatch(pattern.data(), "compact-cron", 1)) { + elements += 2; + EncodeString(&config_body, "compact-cron"); + EncodeString(&config_body, g_pika_conf->compact_cron()); + } + + if (slash::stringmatch(pattern.data(), "compact-interval", 1)) { + elements += 2; + EncodeString(&config_body, "compact-interval"); + EncodeString(&config_body, g_pika_conf->compact_interval()); + } + + if (slash::stringmatch(pattern.data(), "network-interface", 1)) { + elements += 2; + EncodeString(&config_body, "network-interface"); + EncodeString(&config_body, g_pika_conf->network_interface()); + } + + if (slash::stringmatch(pattern.data(), "slaveof", 1)) { + elements += 2; + EncodeString(&config_body, "slaveof"); + EncodeString(&config_body, g_pika_conf->slaveof()); + } + + if (slash::stringmatch(pattern.data(), "slave-priority", 1)) { + elements += 2; + EncodeString(&config_body, "slave-priority"); + EncodeInt32(&config_body, g_pika_conf->slave_priority()); + } + + if (slash::stringmatch(pattern.data(), "sync-window-size", 1)) { + elements += 2; + EncodeString(&config_body, "sync-window-size"); + EncodeInt32(&config_body, g_pika_conf->sync_window_size()); + } + + std::stringstream resp; + resp << "*" << std::to_string(elements) << "\r\n" << config_body; + ret = resp.str(); +} + +// Remember to sync change PikaConf::ConfigRewrite(); +void ConfigCmd::ConfigSet(std::string& ret) { + std::string set_item = config_args_v_[1]; + if (set_item == "*") { + ret = "*23\r\n"; + EncodeString(&ret, "timeout"); + EncodeString(&ret, "requirepass"); + EncodeString(&ret, "masterauth"); + EncodeString(&ret, "userpass"); + EncodeString(&ret, "userblacklist"); + EncodeString(&ret, "dump-prefix"); + EncodeString(&ret, "maxclients"); + EncodeString(&ret, "dump-expire"); + EncodeString(&ret, "expire-logs-days"); + EncodeString(&ret, "expire-logs-nums"); + EncodeString(&ret, "root-connection-num"); + EncodeString(&ret, "slowlog-write-errorlog"); + EncodeString(&ret, "slowlog-log-slower-than"); + EncodeString(&ret, "slowlog-max-len"); + EncodeString(&ret, "write-binlog"); + EncodeString(&ret, "max-cache-statistic-keys"); + EncodeString(&ret, "small-compaction-threshold"); + EncodeString(&ret, "max-client-response-size"); + EncodeString(&ret, "db-sync-speed"); + EncodeString(&ret, "compact-cron"); + EncodeString(&ret, "compact-interval"); + EncodeString(&ret, "slave-priority"); + EncodeString(&ret, "sync-window-size"); + return; + } + long int ival; + std::string value = config_args_v_[2]; + if (set_item == "timeout") { + if (!slash::string2l(value.data(), value.size(), &ival)) { + ret = "-ERR Invalid argument " + value + " for CONFIG SET 'timeout'\r\n"; + return; + } + g_pika_conf->SetTimeout(ival); + ret = "+OK\r\n"; + } else if (set_item == "requirepass") { + g_pika_conf->SetRequirePass(value); + ret = "+OK\r\n"; + } else if (set_item == "masterauth") { + g_pika_conf->SetMasterAuth(value); + ret = "+OK\r\n"; + } else if (set_item == "userpass") { + g_pika_conf->SetUserPass(value); + ret = "+OK\r\n"; + } else if (set_item == "userblacklist") { + g_pika_conf->SetUserBlackList(value); + ret = "+OK\r\n"; + } else if (set_item == "dump-prefix") { + g_pika_conf->SetBgsavePrefix(value); + ret = "+OK\r\n"; + } else if (set_item == "maxclients") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival <= 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'maxclients'\r\n"; + return; + } + g_pika_conf->SetMaxConnection(ival); + g_pika_server->SetDispatchQueueLimit(ival); + ret = "+OK\r\n"; + } else if (set_item == "dump-expire") { + if (!slash::string2l(value.data(), value.size(), &ival)) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'dump-expire'\r\n"; + return; + } + g_pika_conf->SetExpireDumpDays(ival); + ret = "+OK\r\n"; + } else if (set_item == "slave-priority") { + if (!slash::string2l(value.data(), value.size(), &ival)) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'slave-priority'\r\n"; + return; + } + g_pika_conf->SetSlavePriority(ival); + ret = "+OK\r\n"; + } else if (set_item == "expire-logs-days") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival <= 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'expire-logs-days'\r\n"; + return; + } + g_pika_conf->SetExpireLogsDays(ival); + ret = "+OK\r\n"; + } else if (set_item == "expire-logs-nums") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival <= 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'expire-logs-nums'\r\n"; + return; + } + g_pika_conf->SetExpireLogsNums(ival); + ret = "+OK\r\n"; + } else if (set_item == "root-connection-num") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival <= 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'root-connection-num'\r\n"; + return; + } + g_pika_conf->SetRootConnectionNum(ival); + ret = "+OK\r\n"; + } else if (set_item == "slowlog-write-errorlog") { + bool is_write_errorlog; + if (value == "yes") { + is_write_errorlog = true; + } else if (value == "no") { + is_write_errorlog = false; + } else { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'slowlog-write-errorlog'\r\n"; + return; + } + g_pika_conf->SetSlowlogWriteErrorlog(is_write_errorlog); + ret = "+OK\r\n"; + } else if (set_item == "slowlog-log-slower-than") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival < 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'slowlog-log-slower-than'\r\n"; + return; + } + g_pika_conf->SetSlowlogSlowerThan(ival); + ret = "+OK\r\n"; + } else if (set_item == "slowlog-max-len") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival < 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'slowlog-max-len'\r\n"; + return; + } + g_pika_conf->SetSlowlogMaxLen(ival); + g_pika_server->SlowlogTrim(); + ret = "+OK\r\n"; + } else if (set_item == "max-cache-statistic-keys") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival < 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-cache-statistic-keys'\r\n"; + return; + } + g_pika_conf->SetMaxCacheStatisticKeys(ival); + g_pika_server->PartitionSetMaxCacheStatisticKeys(ival); + ret = "+OK\r\n"; + } else if (set_item == "small-compaction-threshold") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival < 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'small-compaction-threshold'\r\n"; + return; + } + g_pika_conf->SetSmallCompactionThreshold(ival); + g_pika_server->PartitionSetSmallCompactionThreshold(ival); + ret = "+OK\r\n"; + } else if (set_item == "max-client-response-size") { + if (!slash::string2l(value.data(), value.size(), &ival) || ival < 0) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'max-client-response-size'\r\n"; + return; + } + g_pika_conf->SetMaxClientResponseSize(ival); + ret = "+OK\r\n"; + } else if (set_item == "write-binlog") { + int role = g_pika_server->role(); + if (role == PIKA_ROLE_SLAVE) { + ret = "-ERR need to close master-slave mode first\r\n"; + return; + } else if (value != "yes" && value != "no") { + ret = "-ERR invalid write-binlog (yes or no)\r\n"; + return; + } else { + g_pika_conf->SetWriteBinlog(value); + ret = "+OK\r\n"; + } + } else if (set_item == "db-sync-speed") { + if (!slash::string2l(value.data(), value.size(), &ival)) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'db-sync-speed(MB)'\r\n"; + return; + } + if (ival < 0 || ival > 1024) { + ival = 1024; + } + g_pika_conf->SetDbSyncSpeed(ival); + ret = "+OK\r\n"; + } else if (set_item == "compact-cron") { + bool invalid = false; + if (value != "") { + bool have_week = false; + std::string compact_cron, week_str; + int slash_num = count(value.begin(), value.end(), '/'); + if (slash_num == 2) { + have_week = true; + std::string::size_type first_slash = value.find("/"); + week_str = value.substr(0, first_slash); + compact_cron = value.substr(first_slash + 1); + } else { + compact_cron = value; + } + + std::string::size_type len = compact_cron.length(); + std::string::size_type colon = compact_cron.find("-"); + std::string::size_type underline = compact_cron.find("/"); + if (colon == std::string::npos || underline == std::string::npos || + colon >= underline || colon + 1 >= len || + colon + 1 == underline || underline + 1 >= len) { + invalid = true; + } else { + int week = std::atoi(week_str.c_str()); + int start = std::atoi(compact_cron.substr(0, colon).c_str()); + int end = std::atoi(compact_cron.substr(colon + 1, underline).c_str()); + int usage = std::atoi(compact_cron.substr(underline + 1).c_str()); + if ((have_week && (week < 1 || week > 7)) || start < 0 || start > 23 || end < 0 || end > 23 || usage < 0 || usage > 100) { + invalid = true; + } + } + } + if (invalid) { + ret = "-ERR invalid compact-cron\r\n"; + return; + } else { + g_pika_conf->SetCompactCron(value); + ret = "+OK\r\n"; + } + } else if (set_item == "compact-interval") { + bool invalid = false; + if (value != "") { + std::string::size_type len = value.length(); + std::string::size_type slash = value.find("/"); + if (slash == std::string::npos || slash + 1 >= len) { + invalid = true; + } else { + int interval = std::atoi(value.substr(0, slash).c_str()); + int usage = std::atoi(value.substr(slash+1).c_str()); + if (interval <= 0 || usage < 0 || usage > 100) { + invalid = true; + } + } + } + if (invalid) { + ret = "-ERR invalid compact-interval\r\n"; + return; + } else { + g_pika_conf->SetCompactInterval(value); + ret = "+OK\r\n"; + } + } else if (set_item == "sync-window-size") { + if (!slash::string2l(value.data(), value.size(), &ival)) { + ret = "-ERR Invalid argument \'" + value + "\' for CONFIG SET 'sync-window-size'\r\n"; + return; + } + if (ival <= 0 || ival > kBinlogReadWinMaxSize) { + ret = "-ERR Argument exceed range \'" + value + "\' for CONFIG SET 'sync-window-size'\r\n"; + return; + } + g_pika_conf->SetSyncWindowSize(ival); + ret = "+OK\r\n"; + } else { + ret = "-ERR Unsupported CONFIG parameter: " + set_item + "\r\n"; + } +} + +void ConfigCmd::ConfigRewrite(std::string &ret) { + g_pika_conf->ConfigRewrite(); + ret = "+OK\r\n"; +} + +void ConfigCmd::ConfigResetstat(std::string &ret) { + g_pika_server->ResetStat(); + ret = "+OK\r\n"; +} + +void MonitorCmd::DoInitial() { + if (argv_.size() != 1) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMonitor); + return; + } +} + +void MonitorCmd::Do(std::shared_ptr partition) { + std::shared_ptr conn_repl = GetConn(); + if (!conn_repl) { + res_.SetRes(CmdRes::kErrOther, kCmdNameMonitor); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr conn = + std::dynamic_pointer_cast(conn_repl)->server_thread()->MoveConnOut(conn_repl->fd()); + assert(conn.get() == conn_repl.get()); + g_pika_server->AddMonitorClient(std::dynamic_pointer_cast(conn)); + g_pika_server->AddMonitorMessage("OK"); + return; // Monitor thread will return "OK" +} + +void DbsizeCmd::DoInitial() { + if (argv_.size() != 1) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDbsize); + return; + } +} + +void DbsizeCmd::Do(std::shared_ptr partition) { + std::shared_ptr
table = g_pika_server->GetTable(table_name_); + if (!table) { + res_.SetRes(CmdRes::kInvalidTable); + } else { + KeyScanInfo key_scan_info = table->GetKeyScanInfo(); + std::vector key_infos = key_scan_info.key_infos; + if (key_infos.size() != 5) { + res_.SetRes(CmdRes::kErrOther, "keyspace error"); + return; + } + int64_t dbsize = key_infos[0].keys + + key_infos[1].keys + + key_infos[2].keys + + key_infos[3].keys + + key_infos[4].keys; + res_.AppendInteger(dbsize); + } +} + +void TimeCmd::DoInitial() { + if (argv_.size() != 1) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameTime); + return; + } +} + +void TimeCmd::Do(std::shared_ptr partition) { + struct timeval tv; + if (gettimeofday(&tv, NULL) == 0) { + res_.AppendArrayLen(2); + char buf[32]; + int32_t len = slash::ll2string(buf, sizeof(buf), tv.tv_sec); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + len = slash::ll2string(buf, sizeof(buf), tv.tv_usec); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } else { + res_.SetRes(CmdRes::kErrOther, strerror(errno)); + } +} + +void DelbackupCmd::DoInitial() { + if (argv_.size() != 1) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDelbackup); + return; + } +} + +void DelbackupCmd::Do(std::shared_ptr partition) { + std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); + std::string db_sync_path = g_pika_conf->bgsave_path(); + std::vector dump_dir; + + // Dump file is not exist + if (!slash::FileExists(db_sync_path)) { + res_.SetRes(CmdRes::kOk); + return; + } + // Directory traversal + if (slash::GetChildren(db_sync_path, dump_dir) != 0) { + res_.SetRes(CmdRes::kOk); + return; + } + + int len = dump_dir.size(); + for (size_t i = 0; i < dump_dir.size(); i++) { + if (dump_dir[i].substr(0, db_sync_prefix.size()) != db_sync_prefix || dump_dir[i].size() != (db_sync_prefix.size() + 8)) { + continue; + } + + std::string str_date = dump_dir[i].substr(db_sync_prefix.size(), (dump_dir[i].size() - db_sync_prefix.size())); + char *end = NULL; + std::strtol(str_date.c_str(), &end, 10); + if (*end != 0) { + continue; + } + + std::string dump_dir_name = db_sync_path + dump_dir[i] + "/" + table_name_; + if (g_pika_server->CountSyncSlaves() == 0) { + LOG(INFO) << "Not syncing, delete dump file: " << dump_dir_name; + slash::DeleteDirIfExist(dump_dir_name); + len--; + } else { + LOG(INFO) << "Syncing, can not delete " << dump_dir_name << " dump file" << std::endl; + } + } + res_.SetRes(CmdRes::kOk); + return; +} + +void EchoCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameEcho); + return; + } + body_ = argv_[1]; + return; +} + +void EchoCmd::Do(std::shared_ptr partition) { + res_.AppendString(body_); + return; +} + +void ScandbCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameEcho); + return; + } + if (argv_.size() == 1) { + type_ = blackwidow::kAll; + } else { + if (!strcasecmp(argv_[1].data(),"string")) { + type_ = blackwidow::kStrings; + } else if (!strcasecmp(argv_[1].data(), "hash")) { + type_ = blackwidow::kHashes; + } else if (!strcasecmp(argv_[1].data(), "set")) { + type_ = blackwidow::kSets; + } else if (!strcasecmp(argv_[1].data(), "zset")) { + type_ = blackwidow::kZSets; + } else if (!strcasecmp(argv_[1].data(), "list")) { + type_ = blackwidow::kLists; + } else { + res_.SetRes(CmdRes::kInvalidDbType); + } + } + return; +} + +void ScandbCmd::Do(std::shared_ptr partition) { + std::shared_ptr
table = g_pika_server->GetTable(table_name_); + if (!table) { + res_.SetRes(CmdRes::kInvalidTable); + } else { + table->ScanDatabase(type_); + res_.SetRes(CmdRes::kOk); + } + return; +} + +void SlowlogCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlowlog); + return; + } + if (argv_.size() == 2 && !strcasecmp(argv_[1].data(), "reset")) { + condition_ = SlowlogCmd::kRESET; + } else if (argv_.size() == 2 && !strcasecmp(argv_[1].data(), "len")) { + condition_ = SlowlogCmd::kLEN; + } else if ((argv_.size() == 2 || argv_.size() == 3) && !strcasecmp(argv_[1].data(), "get")) { + condition_ = SlowlogCmd::kGET; + if (argv_.size() == 3 && !slash::string2l(argv_[2].data(), argv_[2].size(), &number_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kErrOther, "Unknown SLOWLOG subcommand or wrong # of args. Try GET, RESET, LEN."); + return; + } +} + +void SlowlogCmd::Do(std::shared_ptr partition) { + if (condition_ == SlowlogCmd::kRESET) { + g_pika_server->SlowlogReset(); + res_.SetRes(CmdRes::kOk); + } else if (condition_ == SlowlogCmd::kLEN) { + res_.AppendInteger(g_pika_server->SlowlogLen()); + } else { + std::vector slowlogs; + g_pika_server->SlowlogObtain(number_, &slowlogs); + res_.AppendArrayLen(slowlogs.size()); + for (const auto& slowlog : slowlogs) { + res_.AppendArrayLen(4); + res_.AppendInteger(slowlog.id); + res_.AppendInteger(slowlog.start_time); + res_.AppendInteger(slowlog.duration); + res_.AppendArrayLen(slowlog.argv.size()); + for (const auto& arg : slowlog.argv) { + res_.AppendString(arg); + } + } + } + return; +} + +void PaddingCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePadding); + return; + } +} + +void PaddingCmd::Do(std::shared_ptr partition) { + res_.SetRes(CmdRes::kOk); +} + +std::string PaddingCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + return PikaBinlogTransverter::ConstructPaddingBinlog( + BinlogType::TypeFirst, argv_[1].size() + BINLOG_ITEM_HEADER_SIZE + + PADDING_BINLOG_PROTOCOL_SIZE + SPACE_STROE_PARAMETER_LENGTH); +} + +#ifdef TCMALLOC_EXTENSION +void TcmallocCmd::DoInitial() { + if (argv_.size() != 2 && argv_.size() != 3) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameTcmalloc); + return; + } + rate_ = 0; + std::string type = argv_[1]; + if (!strcasecmp(type.data(), "stats")) { + type_ = 0; + } else if (!strcasecmp(type.data(), "rate")) { + type_ = 1; + if (argv_.size() == 3) { + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &rate_)) { + res_.SetRes(CmdRes::kSyntaxErr, kCmdNameTcmalloc); + } + } + } else if (!strcasecmp(type.data(), "list")) { + type_ = 2; + } else if (!strcasecmp(type.data(), "free")) { + type_ = 3; + } else { + res_.SetRes(CmdRes::kInvalidParameter, kCmdNameTcmalloc); + return; + } +} + +void TcmallocCmd::Do(std::shared_ptr partition) { + std::vector fli; + std::vector elems; + switch(type_) { + case 0: + char stats[1024]; + MallocExtension::instance()->GetStats(stats, 1024); + slash::StringSplit(stats, '\n', elems); + res_.AppendArrayLen(elems.size()); + for (auto& i : elems) { + res_.AppendString(i); + } + break; + case 1: + if (rate_) { + MallocExtension::instance()->SetMemoryReleaseRate(rate_); + } + res_.AppendInteger(MallocExtension::instance()->GetMemoryReleaseRate()); + break; + case 2: + MallocExtension::instance()->GetFreeListSizes(&fli); + res_.AppendArrayLen(fli.size()); + for (auto& i : fli) { + res_.AppendString("type: " + std::string(i.type) + ", min: " + std::to_string(i.min_object_size) + + ", max: " + std::to_string(i.max_object_size) + ", total: " + std::to_string(i.total_bytes_free)); + } + break; + case 3: + MallocExtension::instance()->ReleaseFreeMemory(); + res_.SetRes(CmdRes::kOk); + } +} +#endif + +void PKPatternMatchDelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKPatternMatchDel); + return; + } + pattern_ = argv_[1]; + if (!strcasecmp(argv_[2].data(), "set")) { + type_ = blackwidow::kSets; + } else if (!strcasecmp(argv_[2].data(), "list")) { + type_ = blackwidow::kLists; + } else if (!strcasecmp(argv_[2].data(), "string")) { + type_ = blackwidow::kStrings; + } else if (!strcasecmp(argv_[2].data(), "zset")) { + type_ = blackwidow::kZSets; + } else if (!strcasecmp(argv_[2].data(), "hash")) { + type_ = blackwidow::kHashes; + } else { + res_.SetRes(CmdRes::kInvalidDbType, kCmdNamePKPatternMatchDel); + return; + } +} + +void PKPatternMatchDelCmd::Do(std::shared_ptr partition) { + int ret = 0; + rocksdb::Status s = partition->db()->PKPatternMatchDel(type_, pattern_, &ret); + if (s.ok()) { + res_.AppendInteger(ret); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} diff --git a/tools/pika_migrate/src/pika_auxiliary_thread.cc b/tools/pika_migrate/src/pika_auxiliary_thread.cc new file mode 100644 index 0000000000..62a2b22941 --- /dev/null +++ b/tools/pika_migrate/src/pika_auxiliary_thread.cc @@ -0,0 +1,55 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_auxiliary_thread.h" + +#include "include/pika_server.h" +#include "include/pika_define.h" +#include "include/pika_rm.h" + +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaAuxiliaryThread::~PikaAuxiliaryThread() { + StopThread(); + LOG(INFO) << "PikaAuxiliary thread " << thread_id() << " exit!!!"; +} + +void* PikaAuxiliaryThread::ThreadMain() { + while (!should_stop()) { + if (g_pika_conf->classic_mode()) { + if (g_pika_server->ShouldMetaSync()) { + g_pika_rm->SendMetaSyncRequest(); + } else if (g_pika_server->MetaSyncDone()) { + g_pika_rm->RunSyncSlavePartitionStateMachine(); + } + } else { + g_pika_rm->RunSyncSlavePartitionStateMachine(); + } + + Status s = g_pika_rm->CheckSyncTimeout(slash::NowMicros()); + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + + // TODO(whoiami) timeout + s = g_pika_server->TriggerSendBinlogSync(); + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + // send to peer + int res = g_pika_server->SendToPeer(); + if (!res) { + // sleep 100 ms + mu_.Lock(); + cv_.TimedWait(100); + mu_.Unlock(); + } else { + //LOG_EVERY_N(INFO, 1000) << "Consume binlog number " << res; + } + } + return NULL; +} + diff --git a/tools/pika_migrate/src/pika_binlog.cc b/tools/pika_migrate/src/pika_binlog.cc new file mode 100644 index 0000000000..7c71b32bba --- /dev/null +++ b/tools/pika_migrate/src/pika_binlog.cc @@ -0,0 +1,357 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_binlog.h" + +#include +#include + +#include "include/pika_binlog_transverter.h" + +using slash::RWLock; + +std::string NewFileName(const std::string name, const uint32_t current) { + char buf[256]; + snprintf(buf, sizeof(buf), "%s%u", name.c_str(), current); + return std::string(buf); +} + +/* + * Version + */ +Version::Version(slash::RWFile *save) + : pro_num_(0), + pro_offset_(0), + logic_id_(0), + save_(save) { + assert(save_ != NULL); + + pthread_rwlock_init(&rwlock_, NULL); +} + +Version::~Version() { + StableSave(); + pthread_rwlock_destroy(&rwlock_); +} + +Status Version::StableSave() { + char *p = save_->GetData(); + memcpy(p, &pro_num_, sizeof(uint32_t)); + p += 4; + memcpy(p, &pro_offset_, sizeof(uint64_t)); + p += 8; + memcpy(p, &logic_id_, sizeof(uint64_t)); + p += 8; + return Status::OK(); +} + +Status Version::Init() { + Status s; + if (save_->GetData() != NULL) { + memcpy((char*)(&pro_num_), save_->GetData(), sizeof(uint32_t)); + memcpy((char*)(&pro_offset_), save_->GetData() + 4, sizeof(uint64_t)); + memcpy((char*)(&logic_id_), save_->GetData() + 12, sizeof(uint64_t)); + return Status::OK(); + } else { + return Status::Corruption("version init error"); + } +} + +/* + * Binlog + */ +Binlog::Binlog(const std::string& binlog_path, const int file_size) : + consumer_num_(0), + version_(NULL), + queue_(NULL), + versionfile_(NULL), + pro_num_(0), + pool_(NULL), + exit_all_consume_(false), + binlog_path_(binlog_path), + file_size_(file_size) { + + // To intergrate with old version, we don't set mmap file size to 100M; + //slash::SetMmapBoundSize(file_size); + //slash::kMmapBoundSize = 1024 * 1024 * 100; + + Status s; + + slash::CreateDir(binlog_path_); + + filename = binlog_path_ + kBinlogPrefix; + const std::string manifest = binlog_path_ + kManifest; + std::string profile; + + if (!slash::FileExists(manifest)) { + LOG(INFO) << "Binlog: Manifest file not exist, we create a new one."; + + profile = NewFileName(filename, pro_num_); + s = slash::NewWritableFile(profile, &queue_); + if (!s.ok()) { + LOG(FATAL) << "Binlog: new " << filename << " " << s.ToString(); + } + + + s = slash::NewRWFile(manifest, &versionfile_); + if (!s.ok()) { + LOG(FATAL) << "Binlog: new versionfile error " << s.ToString(); + } + + version_ = new Version(versionfile_); + version_->StableSave(); + } else { + LOG(INFO) << "Binlog: Find the exist file."; + + s = slash::NewRWFile(manifest, &versionfile_); + if (s.ok()) { + version_ = new Version(versionfile_); + version_->Init(); + pro_num_ = version_->pro_num_; + + // Debug + //version_->debug(); + } else { + LOG(FATAL) << "Binlog: open versionfile error"; + } + + profile = NewFileName(filename, pro_num_); + DLOG(INFO) << "Binlog: open profile " << profile; + s = slash::AppendWritableFile(profile, &queue_, version_->pro_offset_); + if (!s.ok()) { + LOG(FATAL) << "Binlog: Open file " << profile << " error " << s.ToString(); + } + + uint64_t filesize = queue_->Filesize(); + DLOG(INFO) << "Binlog: filesize is " << filesize; + } + + InitLogFile(); +} + +Binlog::~Binlog() { + delete version_; + delete versionfile_; + + delete queue_; +} + +void Binlog::InitLogFile() { + assert(queue_ != NULL); + + uint64_t filesize = queue_->Filesize(); + block_offset_ = filesize % kBlockSize; +} + +Status Binlog::GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset, uint64_t* logic_id) { + slash::RWLock(&(version_->rwlock_), false); + + *filenum = version_->pro_num_; + *pro_offset = version_->pro_offset_; + if (logic_id != NULL) { + *logic_id = version_->logic_id_; + } + + return Status::OK(); +} + +// Note: mutex lock should be held +Status Binlog::Put(const std::string &item) { + return Put(item.c_str(), item.size()); +} + +// Note: mutex lock should be held +Status Binlog::Put(const char* item, int len) { + Status s; + + /* Check to roll log file */ + uint64_t filesize = queue_->Filesize(); + if (filesize > file_size_) { + delete queue_; + queue_ = NULL; + + pro_num_++; + std::string profile = NewFileName(filename, pro_num_); + slash::NewWritableFile(profile, &queue_); + + { + slash::RWLock(&(version_->rwlock_), true); + version_->pro_offset_ = 0; + version_->pro_num_ = pro_num_; + version_->StableSave(); + } + InitLogFile(); + } + + int pro_offset; + s = Produce(Slice(item, len), &pro_offset); + if (s.ok()) { + slash::RWLock(&(version_->rwlock_), true); + version_->pro_offset_ = pro_offset; + version_->logic_id_++; + version_->StableSave(); + } + + return s; +} + +Status Binlog::EmitPhysicalRecord(RecordType t, const char *ptr, size_t n, int *temp_pro_offset) { + Status s; + assert(n <= 0xffffff); + assert(block_offset_ + kHeaderSize + n <= kBlockSize); + + char buf[kHeaderSize]; + + uint64_t now; + struct timeval tv; + gettimeofday(&tv, NULL); + now = tv.tv_sec; + buf[0] = static_cast(n & 0xff); + buf[1] = static_cast((n & 0xff00) >> 8); + buf[2] = static_cast(n >> 16); + buf[3] = static_cast(now & 0xff); + buf[4] = static_cast((now & 0xff00) >> 8); + buf[5] = static_cast((now & 0xff0000) >> 16); + buf[6] = static_cast((now & 0xff000000) >> 24); + buf[7] = static_cast(t); + + s = queue_->Append(Slice(buf, kHeaderSize)); + if (s.ok()) { + s = queue_->Append(Slice(ptr, n)); + if (s.ok()) { + s = queue_->Flush(); + } + } + block_offset_ += static_cast(kHeaderSize + n); + + *temp_pro_offset += kHeaderSize + n; + return s; +} + +Status Binlog::Produce(const Slice &item, int *temp_pro_offset) { + Status s; + const char *ptr = item.data(); + size_t left = item.size(); + bool begin = true; + + *temp_pro_offset = version_->pro_offset_; + do { + const int leftover = static_cast(kBlockSize) - block_offset_; + assert(leftover >= 0); + if (static_cast(leftover) < kHeaderSize) { + if (leftover > 0) { + s = queue_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover)); + if (!s.ok()) { + return s; + } + *temp_pro_offset += leftover; + } + block_offset_ = 0; + } + + const size_t avail = kBlockSize - block_offset_ - kHeaderSize; + const size_t fragment_length = (left < avail) ? left : avail; + RecordType type; + const bool end = (left == fragment_length); + if (begin && end) { + type = kFullType; + } else if (begin) { + type = kFirstType; + } else if (end) { + type = kLastType; + } else { + type = kMiddleType; + } + + s = EmitPhysicalRecord(type, ptr, fragment_length, temp_pro_offset); + ptr += fragment_length; + left -= fragment_length; + begin = false; + } while (s.ok() && left > 0); + + return s; +} + +Status Binlog::AppendPadding(slash::WritableFile* file, uint64_t* len) { + if (*len < kHeaderSize) { + return Status::OK(); + } + + Status s; + char buf[kBlockSize]; + uint64_t now; + struct timeval tv; + gettimeofday(&tv, NULL); + now = tv.tv_sec; + + uint64_t left = *len; + while (left > 0 && s.ok()) { + uint32_t size = (left >= kBlockSize) ? kBlockSize : left; + if (size < kHeaderSize) { + break; + } else { + uint32_t bsize = size - kHeaderSize; + std::string binlog = PikaBinlogTransverter::ConstructPaddingBinlog( + BinlogType::TypeFirst, bsize); + if (binlog.empty()) { + break; + } + buf[0] = static_cast(bsize & 0xff); + buf[1] = static_cast((bsize & 0xff00) >> 8); + buf[2] = static_cast(bsize >> 16); + buf[3] = static_cast(now & 0xff); + buf[4] = static_cast((now & 0xff00) >> 8); + buf[5] = static_cast((now & 0xff0000) >> 16); + buf[6] = static_cast((now & 0xff000000) >> 24); + buf[7] = static_cast(kFullType); + s = file->Append(Slice(buf, kHeaderSize)); + if (s.ok()) { + s = file->Append(Slice(binlog.data(), binlog.size())); + if (s.ok()) { + s = file->Flush(); + left -= size; + } + } + } + } + *len -= left; + return s; +} + +Status Binlog::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { + slash::MutexLock l(&mutex_); + + // offset smaller than the first header + if (pro_offset < 4) { + pro_offset = 0; + } + + delete queue_; + + std::string init_profile = NewFileName(filename, 0); + if (slash::FileExists(init_profile)) { + slash::DeleteFile(init_profile); + } + + std::string profile = NewFileName(filename, pro_num); + if (slash::FileExists(profile)) { + slash::DeleteFile(profile); + } + + slash::NewWritableFile(profile, &queue_); + Binlog::AppendPadding(queue_, &pro_offset); + + pro_num_ = pro_num; + + { + slash::RWLock(&(version_->rwlock_), true); + version_->pro_num_ = pro_num; + version_->pro_offset_ = pro_offset; + version_->StableSave(); + } + + InitLogFile(); + return Status::OK(); +} diff --git a/tools/pika_migrate/src/pika_binlog_reader.cc b/tools/pika_migrate/src/pika_binlog_reader.cc new file mode 100644 index 0000000000..46c7e8c604 --- /dev/null +++ b/tools/pika_migrate/src/pika_binlog_reader.cc @@ -0,0 +1,267 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_binlog_reader.h" + +#include + +PikaBinlogReader::PikaBinlogReader(uint32_t cur_filenum, + uint64_t cur_offset) + : cur_filenum_(cur_filenum), + cur_offset_(cur_offset), + logger_(nullptr), + queue_(nullptr), + backing_store_(new char[kBlockSize]), + buffer_() { + last_record_offset_ = cur_offset % kBlockSize; + pthread_rwlock_init(&rwlock_, NULL); +} + +PikaBinlogReader::PikaBinlogReader() + : cur_filenum_(0), + cur_offset_(0), + logger_(nullptr), + queue_(nullptr), + backing_store_(new char[kBlockSize]), + buffer_() { + last_record_offset_ = 0 % kBlockSize; + pthread_rwlock_init(&rwlock_, NULL); +} + + +PikaBinlogReader::~PikaBinlogReader() { + delete[] backing_store_; + delete queue_; + pthread_rwlock_destroy(&rwlock_); +} + +void PikaBinlogReader::GetReaderStatus(uint32_t* cur_filenum, uint64_t* cur_offset) { + slash::RWLock(&(rwlock_), false); + *cur_filenum = cur_filenum_; + *cur_offset = cur_offset_; +} + +bool PikaBinlogReader::ReadToTheEnd() { + uint32_t pro_num; + uint64_t pro_offset; + logger_->GetProducerStatus(&pro_num, &pro_offset); + slash::RWLock(&(rwlock_), false); + return (pro_num == cur_filenum_ && pro_offset == cur_offset_); +} + +int PikaBinlogReader::Seek(std::shared_ptr logger, uint32_t filenum, uint64_t offset) { + std::string confile = NewFileName(logger->filename, filenum); + if (!slash::FileExists(confile)) { + return -1; + } + slash::SequentialFile* readfile; + if (!slash::NewSequentialFile(confile, &readfile).ok()) { + return -1; + } + if (queue_) { + delete queue_; + } + queue_ = readfile; + logger_ = logger; + + slash::RWLock(&(rwlock_), true); + cur_filenum_ = filenum; + cur_offset_ = offset; + last_record_offset_ = cur_filenum_ % kBlockSize; + + slash::Status s; + uint64_t start_block = (cur_offset_ / kBlockSize) * kBlockSize; + s = queue_->Skip((cur_offset_ / kBlockSize) * kBlockSize); + uint64_t block_offset = cur_offset_ % kBlockSize; + uint64_t ret = 0; + uint64_t res = 0; + bool is_error = false; + + while (true) { + if (res >= block_offset) { + cur_offset_ = start_block + res; + break; + } + ret = 0; + is_error = GetNext(&ret); + if (is_error == true) { + return -1; + } + res += ret; + } + last_record_offset_ = cur_offset_ % kBlockSize; + return 0; +} + +bool PikaBinlogReader::GetNext(uint64_t* size) { + uint64_t offset = 0; + slash::Status s; + bool is_error = false; + + while (true) { + buffer_.clear(); + s = queue_->Read(kHeaderSize, &buffer_, backing_store_); + if (!s.ok()) { + is_error = true; + return is_error; + } + + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[0]) & 0xff; + const uint32_t b = static_cast(header[1]) & 0xff; + const uint32_t c = static_cast(header[2]) & 0xff; + const unsigned int type = header[7]; + const uint32_t length = a | (b << 8) | (c << 16); + + if (type == kFullType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + break; + } else if (type == kFirstType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + } else if (type == kMiddleType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + } else if (type == kLastType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + break; + } else { + is_error = true; + break; + } + } + *size = offset; + return is_error; +} + +unsigned int PikaBinlogReader::ReadPhysicalRecord(slash::Slice *result, uint32_t* filenum, uint64_t* offset) { + slash::Status s; + if (kBlockSize - last_record_offset_ <= kHeaderSize) { + queue_->Skip(kBlockSize - last_record_offset_); + slash::RWLock(&(rwlock_), true); + cur_offset_ += (kBlockSize - last_record_offset_); + last_record_offset_ = 0; + } + buffer_.clear(); + s = queue_->Read(kHeaderSize, &buffer_, backing_store_); + if (s.IsEndFile()) { + return kEof; + } else if (!s.ok()) { + return kBadRecord; + } + + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[0]) & 0xff; + const uint32_t b = static_cast(header[1]) & 0xff; + const uint32_t c = static_cast(header[2]) & 0xff; + const unsigned int type = header[7]; + const uint32_t length = a | (b << 8) | (c << 16); + if (type == kZeroType || length == 0) { + buffer_.clear(); + return kOldRecord; + } + + buffer_.clear(); + s = queue_->Read(length, &buffer_, backing_store_); + *result = slash::Slice(buffer_.data(), buffer_.size()); + last_record_offset_ += kHeaderSize + length; + if (s.ok()) { + slash::RWLock(&(rwlock_), true); + *filenum = cur_filenum_; + cur_offset_ += (kHeaderSize + length); + *offset = cur_offset_; + } + return type; +} + +Status PikaBinlogReader::Consume(std::string* scratch, uint32_t* filenum, uint64_t* offset) { + Status s; + + slash::Slice fragment; + while (true) { + const unsigned int record_type = ReadPhysicalRecord(&fragment, filenum, offset); + + switch (record_type) { + case kFullType: + *scratch = std::string(fragment.data(), fragment.size()); + s = Status::OK(); + break; + case kFirstType: + scratch->assign(fragment.data(), fragment.size()); + s = Status::NotFound("Middle Status"); + break; + case kMiddleType: + scratch->append(fragment.data(), fragment.size()); + s = Status::NotFound("Middle Status"); + break; + case kLastType: + scratch->append(fragment.data(), fragment.size()); + s = Status::OK(); + break; + case kEof: + return Status::EndFile("Eof"); + case kBadRecord: + return Status::IOError("Data Corruption"); + case kOldRecord: + return Status::EndFile("Eof"); + default: + return Status::IOError("Unknow reason"); + } + if (s.ok()) { + break; + } + } + // DLOG(INFO) << "Binlog Sender consumer a msg: " << scratch; + return Status::OK(); +} + +// Get a whole message; +// Append to scratch; +// the status will be OK, IOError or Corruption, EndFile; +Status PikaBinlogReader::Get(std::string* scratch, uint32_t* filenum, uint64_t* offset) { + if (logger_ == nullptr || queue_ == NULL) { + return Status::Corruption("Not seek"); + } + scratch->clear(); + Status s = Status::OK(); + + do { + if (ReadToTheEnd()) { + return Status::EndFile("End of cur log file"); + } + s = Consume(scratch, filenum, offset); + if (s.IsEndFile()) { + std::string confile = NewFileName(logger_->filename, cur_filenum_ + 1); + + // sleep 10ms wait produce thread generate the new binlog + usleep(10000); + + // Roll to next file need retry; + if (slash::FileExists(confile)) { + DLOG(INFO) << "BinlogSender roll to new binlog" << confile; + delete queue_; + queue_ = NULL; + + slash::NewSequentialFile(confile, &(queue_)); + { + slash::RWLock(&(rwlock_), true); + cur_filenum_++; + cur_offset_ = 0; + } + last_record_offset_ = 0; + } else { + return Status::IOError("File Does Not Exists"); + } + } else { + break; + } + } while (s.IsEndFile()); + + return Status::OK(); +} + + diff --git a/tools/pika_migrate/src/pika_binlog_transverter.cc b/tools/pika_migrate/src/pika_binlog_transverter.cc new file mode 100644 index 0000000000..702fd6ca5d --- /dev/null +++ b/tools/pika_migrate/src/pika_binlog_transverter.cc @@ -0,0 +1,200 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_binlog_transverter.h" + +#include +#include +#include + +#include "slash/include/slash_coding.h" + +#include "include/pika_command.h" + +uint32_t BinlogItem::exec_time() const { + return exec_time_; +} + +uint32_t BinlogItem::server_id() const { + return server_id_; +} + +uint64_t BinlogItem::logic_id() const { + return logic_id_; +} + +uint32_t BinlogItem::filenum() const { + return filenum_; +} + +uint64_t BinlogItem::offset() const { + return offset_; +} + +std::string BinlogItem::content() const { + return content_; +} + +void BinlogItem::set_exec_time(uint32_t exec_time) { + exec_time_ = exec_time; +} + +void BinlogItem::set_server_id(uint32_t server_id) { + server_id_ = server_id; +} + +void BinlogItem::set_logic_id(uint64_t logic_id) { + logic_id_ = logic_id; +} + +void BinlogItem::set_filenum(uint32_t filenum) { + filenum_ = filenum; +} + +void BinlogItem::set_offset(uint64_t offset) { + offset_ = offset; +} + +std::string BinlogItem::ToString() const { + std::string str; + str.append("exec_time: " + std::to_string(exec_time_)); + str.append(",server_id: " + std::to_string(server_id_)); + str.append(",logic_id: " + std::to_string(logic_id_)); + str.append(",filenum: " + std::to_string(filenum_)); + str.append(",offset: " + std::to_string(offset_)); + str.append("\ncontent: "); + for (size_t idx = 0; idx < content_.size(); ++idx) { + if (content_[idx] == '\n') { + str.append("\\n"); + } else if (content_[idx] == '\r') { + str.append("\\r"); + } else { + str.append(1, content_[idx]); + } + } + str.append("\n"); + return str; +} + +std::string PikaBinlogTransverter::BinlogEncode(BinlogType type, + uint32_t exec_time, + uint32_t server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset, + const std::string& content, + const std::vector& extends) { + std::string binlog; + slash::PutFixed16(&binlog, type); + slash::PutFixed32(&binlog, exec_time); + slash::PutFixed32(&binlog, server_id); + slash::PutFixed64(&binlog, logic_id); + slash::PutFixed32(&binlog, filenum); + slash::PutFixed64(&binlog, offset); + uint32_t content_length = content.size(); + slash::PutFixed32(&binlog, content_length); + binlog.append(content); + return binlog; +} + +bool PikaBinlogTransverter::BinlogDecode(BinlogType type, + const std::string& binlog, + BinlogItem* binlog_item) { + uint16_t binlog_type = 0; + uint32_t content_length = 0; + std::string binlog_str = binlog; + slash::GetFixed16(&binlog_str, &binlog_type); + if (binlog_type != type) { + LOG(ERROR) << "Binlog Item type error, expect type:" << type << " actualy type: " << binlog_type; + return false; + } + slash::GetFixed32(&binlog_str, &binlog_item->exec_time_); + slash::GetFixed32(&binlog_str, &binlog_item->server_id_); + slash::GetFixed64(&binlog_str, &binlog_item->logic_id_); + slash::GetFixed32(&binlog_str, &binlog_item->filenum_); + slash::GetFixed64(&binlog_str, &binlog_item->offset_); + slash::GetFixed32(&binlog_str, &content_length); + if (binlog_str.size() == content_length) { + binlog_item->content_.assign(binlog_str.data(), content_length); + } else { + LOG(ERROR) << "Binlog Item get content error, expect length:" << content_length << " left length:" << binlog_str.size(); + return false; + } + return true; +} + +/* + * *************************************************Type First Binlog Item Format************************************************** + * | | | | | | | | | + * | 2 Bytes | 4 Bytes | 4 Bytes | 8 Bytes | 4 Bytes | 8 Bytes | 4 Bytes | content length Bytes | + * |---------------------------------------------- 34 Bytes -----------------------------------------------| + * + * content: *2\r\n$7\r\npadding\r\n$00001\r\n***\r\n + * length of *** -> total_len - PADDING_BINLOG_PROTOCOL_SIZE - SPACE_STROE_PARAMETER_LENGTH; + * + * We allocate five bytes to store the length of the parameter + */ +std::string PikaBinlogTransverter::ConstructPaddingBinlog(BinlogType type, + uint32_t size) { + assert(size <= kBlockSize - kHeaderSize); + assert(BINLOG_ITEM_HEADER_SIZE + PADDING_BINLOG_PROTOCOL_SIZE + + SPACE_STROE_PARAMETER_LENGTH <= size); + + std::string binlog; + slash::PutFixed16(&binlog, type); + slash::PutFixed32(&binlog, 0); + slash::PutFixed32(&binlog, 0); + slash::PutFixed64(&binlog, 0); + slash::PutFixed32(&binlog, 0); + slash::PutFixed64(&binlog, 0); + int32_t content_len = size - BINLOG_ITEM_HEADER_SIZE; + int32_t parameter_len = content_len - PADDING_BINLOG_PROTOCOL_SIZE + - SPACE_STROE_PARAMETER_LENGTH; + if (parameter_len < 0) { + return std::string(); + } + + std::string content; + RedisAppendLen(content, 2, "*"); + RedisAppendLen(content, 7, "$"); + RedisAppendContent(content, "padding"); + + std::string parameter_len_str; + std::ostringstream os; + os << parameter_len; + std::istringstream is(os.str()); + is >> parameter_len_str; + if (parameter_len_str.size() > SPACE_STROE_PARAMETER_LENGTH) { + return std::string(); + } + + content.append("$"); + content.append(SPACE_STROE_PARAMETER_LENGTH - parameter_len_str.size(), '0'); + content.append(parameter_len_str); + content.append(kNewLine); + RedisAppendContent(content, std::string(parameter_len, '*')); + + slash::PutFixed32(&binlog, content_len); + binlog.append(content); + return binlog; +} + +bool PikaBinlogTransverter::BinlogItemWithoutContentDecode(BinlogType type, + const std::string& binlog, + BinlogItem* binlog_item) { + uint16_t binlog_type = 0; + std::string binlog_str = binlog; + slash::GetFixed16(&binlog_str, &binlog_type); + if (binlog_type != type) { + LOG(ERROR) << "Binlog Item type error, expect type:" << type << " actualy type: " << binlog_type; + return false; + } + slash::GetFixed32(&binlog_str, &binlog_item->exec_time_); + slash::GetFixed32(&binlog_str, &binlog_item->server_id_); + slash::GetFixed64(&binlog_str, &binlog_item->logic_id_); + slash::GetFixed32(&binlog_str, &binlog_item->filenum_); + slash::GetFixed64(&binlog_str, &binlog_item->offset_); + return true; +} diff --git a/tools/pika_migrate/src/pika_bit.cc b/tools/pika_migrate/src/pika_bit.cc new file mode 100644 index 0000000000..0815acb040 --- /dev/null +++ b/tools/pika_migrate/src/pika_bit.cc @@ -0,0 +1,221 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_bit.h" + +#include "slash/include/slash_string.h" + +#include "include/pika_define.h" + +void BitSetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitSet); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &bit_offset_)) { + res_.SetRes(CmdRes::kInvalidBitOffsetInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &on_)) { + res_.SetRes(CmdRes::kInvalidBitInt); + return; + } + if (bit_offset_ < 0) { + res_.SetRes(CmdRes::kInvalidBitOffsetInt); + return; + } + // value no bigger than 2^18 + if ( (bit_offset_ >> kMaxBitOpInputBit) > 0) { + res_.SetRes(CmdRes::kInvalidBitOffsetInt); + return; + } + if (on_ & ~1) { + res_.SetRes(CmdRes::kInvalidBitInt); + return; + } + return; +} + +void BitSetCmd::Do(std::shared_ptr partition) { + std::string value; + int32_t bit_val = 0; + rocksdb::Status s = partition->db()->SetBit(key_, bit_offset_, on_, &bit_val); + if (s.ok()){ + res_.AppendInteger((int)bit_val); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void BitGetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitGet); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &bit_offset_)) { + res_.SetRes(CmdRes::kInvalidBitOffsetInt); + return; + } + if (bit_offset_ < 0) { + res_.SetRes(CmdRes::kInvalidBitOffsetInt); + return; + } + return; +} + +void BitGetCmd::Do(std::shared_ptr partition) { + int32_t bit_val = 0; + rocksdb::Status s = partition->db()->GetBit(key_, bit_offset_, &bit_val); + if (s.ok()) { + res_.AppendInteger((int)bit_val); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void BitCountCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitCount); + return; + } + key_ = argv_[1]; + if (argv_.size() == 4) { + count_all_ = false; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &start_offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &end_offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else if (argv_.size() == 2) { + count_all_ = true; + } else { + res_.SetRes(CmdRes::kSyntaxErr, kCmdNameBitCount); + } + return; +} + +void BitCountCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s; + if (count_all_) { + s = partition->db()->BitCount(key_, start_offset_, end_offset_, &count, false); + } else { + s = partition->db()->BitCount(key_, start_offset_, end_offset_, &count, true); + } + + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void BitPosCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitPos); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &bit_val_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (bit_val_ & ~1) { + res_.SetRes(CmdRes::kInvalidBitPosArgument); + return; + } + if (argv_.size() == 3) { + pos_all_ = true; + endoffset_set_ = false; + } else if (argv_.size() == 4) { + pos_all_ = false; + endoffset_set_ = false; + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &start_offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else if (argv_.size() == 5) { + pos_all_ = false; + endoffset_set_ = true; + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &start_offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[4].data(), argv_[4].size(), &end_offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else + res_.SetRes(CmdRes::kSyntaxErr, kCmdNameBitPos); + return; +} + +void BitPosCmd::Do(std::shared_ptr partition) { + int64_t pos = 0; + rocksdb::Status s; + if (pos_all_) { + s = partition->db()->BitPos(key_, bit_val_, &pos); + } else if (!pos_all_ && !endoffset_set_) { + s = partition->db()->BitPos(key_, bit_val_, start_offset_, &pos); + } else if (!pos_all_ && endoffset_set_) { + s = partition->db()->BitPos(key_, bit_val_, start_offset_, end_offset_, &pos); + } + if (s.ok()) { + res_.AppendInteger((int)pos); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void BitOpCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitOp); + return; + } + std::string op_str = argv_[1]; + if (!strcasecmp(op_str.data(), "not")) { + op_ = blackwidow::kBitOpNot; + } else if (!strcasecmp(op_str.data(), "and")) { + op_ = blackwidow::kBitOpAnd; + } else if (!strcasecmp(op_str.data(), "or")) { + op_ = blackwidow::kBitOpOr; + } else if (!strcasecmp(op_str.data(), "xor")) { + op_ = blackwidow::kBitOpXor; + } else { + res_.SetRes(CmdRes::kSyntaxErr, kCmdNameBitOp); + return; + } + if (op_ == blackwidow::kBitOpNot && argv_.size() != 4) { + res_.SetRes(CmdRes::kWrongBitOpNotNum, kCmdNameBitOp); + return; + } else if (op_ != blackwidow::kBitOpNot && argv_.size() < 4) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitOp); + return; + } else if (argv_.size() >= kMaxBitOpInputKey) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameBitOp); + return; + } + + dest_key_ = argv_[2].data(); + for(unsigned int i = 3; i <= argv_.size() - 1; i++) { + src_keys_.push_back(argv_[i].data()); + } + return; +} + +void BitOpCmd::Do(std::shared_ptr partition) { + int64_t result_length; + rocksdb::Status s = partition->db()->BitOp(op_, dest_key_, src_keys_, &result_length); + if (s.ok()) { + res_.AppendInteger(result_length); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} diff --git a/tools/pika_migrate/src/pika_client_conn.cc b/tools/pika_migrate/src/pika_client_conn.cc new file mode 100644 index 0000000000..fd51331438 --- /dev/null +++ b/tools/pika_migrate/src/pika_client_conn.cc @@ -0,0 +1,256 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_client_conn.h" + +#include +#include + +#include + +#include "include/pika_conf.h" +#include "include/pika_server.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; +extern PikaCmdTableManager* g_pika_cmd_table_manager; + +PikaClientConn::PikaClientConn(int fd, std::string ip_port, + pink::Thread* thread, + pink::PinkEpoll* pink_epoll, + const pink::HandleType& handle_type) + : RedisConn(fd, ip_port, thread, pink_epoll, handle_type), + server_thread_(reinterpret_cast(thread)), + current_table_(g_pika_conf->default_table()), + is_pubsub_(false) { + auth_stat_.Init(); +} + +std::string PikaClientConn::DoCmd(const PikaCmdArgsType& argv, + const std::string& opt) { + // Get command info + std::shared_ptr c_ptr = g_pika_cmd_table_manager->GetCmd(opt); + if (!c_ptr) { + return "-Err unknown or unsupported command \'" + opt + "\'\r\n"; + } + c_ptr->SetConn(std::dynamic_pointer_cast(shared_from_this())); + + // Check authed + // AuthCmd will set stat_ + if (!auth_stat_.IsAuthed(c_ptr)) { + return "-ERR NOAUTH Authentication required.\r\n"; + } + + uint64_t start_us = 0; + if (g_pika_conf->slowlog_slower_than() >= 0) { + start_us = slash::NowMicros(); + } + + bool is_monitoring = g_pika_server->HasMonitorClients(); + if (is_monitoring) { + ProcessMonitor(argv); + } + + // Initial + c_ptr->Initial(argv, current_table_); + if (!c_ptr->res().ok()) { + return c_ptr->res().message(); + } + + g_pika_server->UpdateQueryNumAndExecCountTable(opt); + + // PubSub connection + // (P)SubscribeCmd will set is_pubsub_ + if (this->IsPubSub()) { + if (opt != kCmdNameSubscribe && + opt != kCmdNameUnSubscribe && + opt != kCmdNamePing && + opt != kCmdNamePSubscribe && + opt != kCmdNamePUnSubscribe) { + return "-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in this context\r\n"; + } + } + + if (!g_pika_server->IsCommandSupport(opt)) { + return "-ERR This command only support in classic mode\r\n"; + } + + if (!g_pika_server->IsTableExist(current_table_)) { + return "-ERR Table not found\r\n"; + } + + // TODO: Consider special commands, like flushall, flushdb? + if (c_ptr->is_write()) { + if (g_pika_server->IsTableBinlogIoError(current_table_)) { + return "-ERR Writing binlog failed, maybe no space left on device\r\n"; + } + std::vector cur_key = c_ptr->current_key(); + if (cur_key.empty()) { + return "-ERR Internal ERROR\r\n"; + } + if (g_pika_server->readonly(current_table_, cur_key.front())) { + return "-ERR Server in read-only\r\n"; + } + } + + // Process Command + c_ptr->Execute(); + + if (g_pika_conf->slowlog_slower_than() >= 0) { + ProcessSlowlog(argv, start_us); + } + + return c_ptr->res().message(); +} + +void PikaClientConn::ProcessSlowlog(const PikaCmdArgsType& argv, uint64_t start_us) { + int32_t start_time = start_us / 1000000; + int64_t duration = slash::NowMicros() - start_us; + if (duration > g_pika_conf->slowlog_slower_than()) { + g_pika_server->SlowlogPushEntry(argv, start_time, duration); + if (g_pika_conf->slowlog_write_errorlog()) { + bool trim = false; + std::string slow_log; + uint32_t cmd_size = 0; + for (unsigned int i = 0; i < argv.size(); i++) { + cmd_size += 1 + argv[i].size(); // blank space and argument length + if (!trim) { + slow_log.append(" "); + slow_log.append(slash::ToRead(argv[i])); + if (slow_log.size() >= 1000) { + trim = true; + slow_log.resize(1000); + slow_log.append("...\""); + } + } + } + LOG(ERROR) << "ip_port: " << ip_port() << ", table: " << current_table_ + << ", command:" << slow_log << ", command_size: " << cmd_size - 1 + << ", arguments: " << argv.size() << ", start_time(s): " << start_time + << ", duration(us): " << duration; + } + } +} + +void PikaClientConn::ProcessMonitor(const PikaCmdArgsType& argv) { + std::string monitor_message; + std::string table_name = g_pika_conf->classic_mode() + ? current_table_.substr(2) : current_table_; + monitor_message = std::to_string(1.0*slash::NowMicros()/1000000) + + " [" + table_name + " " + this->ip_port() + "]"; + for (PikaCmdArgsType::const_iterator iter = argv.begin(); iter != argv.end(); iter++) { + monitor_message += " " + slash::ToRead(*iter); + } + g_pika_server->AddMonitorMessage(monitor_message); +} + +void PikaClientConn::AsynProcessRedisCmds(const std::vector& argvs, std::string* response) { + BgTaskArg* arg = new BgTaskArg(); + arg->redis_cmds = argvs; + arg->response = response; + arg->pcc = std::dynamic_pointer_cast(shared_from_this()); + g_pika_server->Schedule(&DoBackgroundTask, arg); +} + +void PikaClientConn::BatchExecRedisCmd(const std::vector& argvs, std::string* response) { + bool success = true; + for (const auto& argv : argvs) { + if (DealMessage(argv, response) != 0) { + success = false; + break; + } + } + if (!response->empty()) { + set_is_reply(true); + NotifyEpoll(success); + } +} + +int PikaClientConn::DealMessage(const PikaCmdArgsType& argv, std::string* response) { + + if (argv.empty()) return -2; + std::string opt = argv[0]; + if (opt == kClusterPrefix) { + if (argv.size() >=2 ) { + opt += argv[1]; + } + } + slash::StringToLower(opt); + + if (response->empty()) { + // Avoid memory copy + *response = std::move(DoCmd(argv, opt)); + } else { + // Maybe pipeline + response->append(DoCmd(argv, opt)); + } + return 0; +} + +void PikaClientConn::DoBackgroundTask(void* arg) { + BgTaskArg* bg_arg = reinterpret_cast(arg); + bg_arg->pcc->BatchExecRedisCmd(bg_arg->redis_cmds, bg_arg->response); + delete bg_arg; +} + +// Initial permission status +void PikaClientConn::AuthStat::Init() { + // Check auth required + stat_ = g_pika_conf->userpass() == "" ? + kLimitAuthed : kNoAuthed; + if (stat_ == kLimitAuthed + && g_pika_conf->requirepass() == "") { + stat_ = kAdminAuthed; + } +} + +// Check permission for current command +bool PikaClientConn::AuthStat::IsAuthed(const std::shared_ptr cmd_ptr) { + std::string opt = cmd_ptr->name(); + if (opt == kCmdNameAuth) { + return true; + } + const std::vector& blacklist = g_pika_conf->vuser_blacklist(); + switch (stat_) { + case kNoAuthed: + return false; + case kAdminAuthed: + break; + case kLimitAuthed: + if (cmd_ptr->is_admin_require() + || find(blacklist.begin(), blacklist.end(), opt) != blacklist.end()) { + return false; + } + break; + default: + LOG(WARNING) << "Invalid auth stat : " << static_cast(stat_); + return false; + } + return true; +} + +// Update permission status +bool PikaClientConn::AuthStat::ChecknUpdate(const std::string& message) { + // Situations to change auth status + if (message == "USER") { + stat_ = kLimitAuthed; + } else if (message == "ROOT") { + stat_ = kAdminAuthed; + } else { + return false; + } + return true; +} + +// compare addr in ClientInfo +bool AddrCompare(const ClientInfo& lhs, const ClientInfo& rhs) { + return rhs.ip_port < lhs.ip_port; +} + +bool IdleCompare(const ClientInfo& lhs, const ClientInfo& rhs) { + return lhs.last_interaction < rhs.last_interaction; +} + diff --git a/tools/pika_migrate/src/pika_cluster.cc b/tools/pika_migrate/src/pika_cluster.cc new file mode 100644 index 0000000000..34d5b1630d --- /dev/null +++ b/tools/pika_migrate/src/pika_cluster.cc @@ -0,0 +1,495 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_rm.h" +#include "include/pika_table.h" +#include "include/pika_server.h" +#include "include/pika_cluster.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaReplicaManager* g_pika_rm; +extern PikaServer* g_pika_server; +extern PikaConf* g_pika_conf; + +const std::string PkClusterInfoCmd::kSlotSection = "slot"; + +// pkcluster info slot table:slot +// pkcluster info table +// pkcluster info node +// pkcluster info cluster +void PkClusterInfoCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePkClusterInfo); + return; + } + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "PkClusterInfo only support on sharding mode"); + return; + } + if (!strcasecmp(argv_[2].data(), kSlotSection.data())) { + info_section_ = kInfoSlot; + if (!ParseInfoSlotSubCmd()) { + return; + } + } else { + info_section_ = kInfoErr; + } + return; +} + +void PkClusterInfoCmd::Do(std::shared_ptr partition) { + std::string info; + switch (info_section_) { + case kInfoSlot: + if (info_range_ == kAll) { + ClusterInfoSlotAll(&info); + } else if (info_range_ == kSingle) { + // doesn't process error, if error return nothing + GetSlotInfo(table_name_, partition_id_, &info); + } + break; + default: + break; + } + res_.AppendStringLen(info.size()); + res_.AppendContent(info); + return; +} + +bool PkClusterInfoCmd::ParseInfoSlotSubCmd() { + if (argv_.size() > 3) { + if (argv_.size() == 4) { + info_range_ = kSingle; + std::string tmp(argv_[3]); + size_t pos = tmp.find(':'); + std::string slot_num_str; + if (pos == std::string::npos) { + table_name_ = g_pika_conf->default_table(); + slot_num_str = tmp; + } else { + table_name_ = tmp.substr(0, pos); + slot_num_str = tmp.substr(pos + 1); + } + unsigned long partition_id; + if (!slash::string2ul(slot_num_str.c_str(), slot_num_str.size(), &partition_id)) { + res_.SetRes(CmdRes::kInvalidParameter, kCmdNamePkClusterInfo); + return false; + } + partition_id_ = partition_id; + } else { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePkClusterInfo); + return false; + } + } + return true; +} + +void PkClusterInfoCmd::ClusterInfoSlotAll(std::string* info) { + std::stringstream tmp_stream; + for (const auto& table_item : g_pika_server->tables_) { + slash::RWLock partition_rwl(&table_item.second->partitions_rw_, false); + for (const auto& partition_item : table_item.second->partitions_) { + std::string table_name = table_item.second->GetTableName(); + uint32_t partition_id = partition_item.second->GetPartitionId(); + std::string p_info; + Status s = GetSlotInfo(table_name, partition_id, &p_info); + if (!s.ok()) { + continue; + } + tmp_stream << p_info; + } + } + info->append(tmp_stream.str()); +} + +Status PkClusterInfoCmd::GetSlotInfo(const std::string table_name, + uint32_t partition_id, + std::string* info) { + std::shared_ptr partition = + g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition) { + return Status::NotFound("not found"); + } + Status s; + std::stringstream tmp_stream; + + // binlog offset section + uint32_t filenum = 0; + uint64_t offset = 0; + partition->logger()->GetProducerStatus(&filenum, &offset); + tmp_stream << partition->GetPartitionName() << " binlog_offset=" + << filenum << " " << offset; + + // safety purge section + std::string safety_purge; + s = g_pika_rm->GetSafetyPurgeBinlogFromSMP(table_name, partition_id, &safety_purge); + tmp_stream << ",safety_purge=" << (s.ok() ? safety_purge : "error") << "\r\n"; + + // partition info section + std::string p_info; + s = g_pika_rm->GetPartitionInfo(table_name, partition_id, &p_info); + if (!s.ok()) { + return s; + } + tmp_stream << p_info; + info->append(tmp_stream.str()); + return Status::OK(); +} + +Status ParseSlotGroup(const std::string& slot_group, + std::set* slots) { + std::set tmp_slots; + int64_t slot_idx, start_idx, end_idx; + std::string::size_type pos; + std::vector elems; + slash::StringSplit(slot_group, COMMA, elems); + for (const auto& elem : elems) { + if ((pos = elem.find("-")) == std::string::npos) { + if (!slash::string2l(elem.data(), elem.size(), &slot_idx) + || slot_idx < 0) { + return Status::Corruption("syntax error"); + } else { + tmp_slots.insert(static_cast(slot_idx)); + } + } else { + if (pos == 0 || pos == (elem.size() - 1)) { + return Status::Corruption("syntax error"); + } else { + std::string start_pos = elem.substr(0, pos); + std::string end_pos = elem.substr(pos + 1, elem.size() - pos); + if (!slash::string2l(start_pos.data(), start_pos.size(), &start_idx) + || !slash::string2l(end_pos.data(), end_pos.size(), &end_idx) + || start_idx < 0 || end_idx < 0 || start_idx > end_idx) { + return Status::Corruption("syntax error"); + } + for (int64_t idx = start_idx; idx <= end_idx; ++idx) { + tmp_slots.insert(static_cast(idx)); + } + } + } + } + slots->swap(tmp_slots); + return Status::OK(); +} + +void SlotParentCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "PkClusterAddSlots/PkClusterDelSlots only support on sharding mode"); + return; + } + + Status s = ParseSlotGroup(argv_[2], &slots_); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + + std::string table_name = g_pika_conf->default_table(); + for (const auto& slot_id : slots_) { + p_infos_.insert(PartitionInfo(table_name, slot_id)); + } +} + +/* + * pkcluster addslots 0-3,8-11 + * pkcluster addslots 0-3,8,9,10,11 + * pkcluster addslots 0,2,4,6,8,10,12,14 + */ +void PkClusterAddSlotsCmd::DoInitial() { + SlotParentCmd::DoInitial(); + if (!res_.ok()) { + return; + } +} + +void PkClusterAddSlotsCmd::Do(std::shared_ptr partition) { + std::string table_name = g_pika_conf->default_table(); + std::shared_ptr
table_ptr = g_pika_server->GetTable(table_name); + if (!table_ptr) { + res_.SetRes(CmdRes::kErrOther, "Internal error: table not found!"); + return; + } + + SlotState expected = INFREE; + if (!std::atomic_compare_exchange_strong(&g_pika_server->slot_state_, + &expected, INBUSY)) { + res_.SetRes(CmdRes::kErrOther, + "Slot in syncing or a change operation is under way, retry later"); + return; + } + + bool pre_success = true; + Status s = AddSlotsSanityCheck(table_name); + if (!s.ok()) { + LOG(WARNING) << "Addslots sanity check failed: " << s.ToString(); + pre_success = false; + } + if (pre_success) { + s = g_pika_conf->AddTablePartitions(table_name, slots_); + if (!s.ok()) { + LOG(WARNING) << "Addslots add to pika conf failed: " << s.ToString(); + pre_success = false; + } + } + if (pre_success) { + s = table_ptr->AddPartitions(slots_); + if (!s.ok()) { + LOG(WARNING) << "Addslots add to table partition failed: " << s.ToString(); + pre_success = false; + } + } + if (pre_success) { + s = g_pika_rm->AddSyncPartition(p_infos_); + if (!s.ok()) { + LOG(WARNING) << "Addslots add to sync partition failed: " << s.ToString(); + pre_success = false; + } + } + + g_pika_server->slot_state_.store(INFREE); + + if (!pre_success) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + + res_.SetRes(CmdRes::kOk); + LOG(INFO) << "Pika meta file overwrite success"; +} + +Status PkClusterAddSlotsCmd::AddSlotsSanityCheck(const std::string& table_name) { + Status s = g_pika_conf->TablePartitionsSanityCheck(table_name, slots_, true); + if (!s.ok()) { + return s; + } + + std::shared_ptr
table_ptr = g_pika_server->GetTable(table_name); + if (!table_ptr) { + return Status::NotFound("table not found!"); + } + + for (uint32_t id : slots_) { + std::shared_ptr partition_ptr = table_ptr->GetPartitionById(id); + if (partition_ptr != nullptr) { + return Status::Corruption("partition " + std::to_string(id) + " already exist"); + } + } + s = g_pika_rm->AddSyncPartitionSanityCheck(p_infos_); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +/* pkcluster delslots 0-3,8-11 + * pkcluster delslots 0-3,8,9,10,11 + * pkcluster delslots 0,2,4,6,8,10,12,14 + */ +void PkClusterDelSlotsCmd::DoInitial() { + SlotParentCmd::DoInitial(); + if (!res_.ok()) { + return; + } +} + +void PkClusterDelSlotsCmd::Do(std::shared_ptr partition) { + std::string table_name = g_pika_conf->default_table(); + std::shared_ptr
table_ptr = g_pika_server->GetTable(table_name); + if (!table_ptr) { + res_.SetRes(CmdRes::kErrOther, "Internal error: default table not found!"); + return; + } + + SlotState expected = INFREE; + if (!std::atomic_compare_exchange_strong(&g_pika_server->slot_state_, + &expected, INBUSY)) { + res_.SetRes(CmdRes::kErrOther, + "Slot in syncing or a change operation is under way, retry later"); + return; + } + + bool pre_success = true; + Status s = RemoveSlotsSanityCheck(table_name); + if (!s.ok()) { + LOG(WARNING) << "Removeslots sanity check failed: " << s.ToString(); + pre_success = false; + } + if (pre_success) { + s = g_pika_conf->RemoveTablePartitions(table_name, slots_); + if (!s.ok()) { + LOG(WARNING) << "Removeslots remove from pika conf failed: " << s.ToString(); + pre_success = false; + } + } + if (pre_success) { + s = table_ptr->RemovePartitions(slots_); + if (!s.ok()) { + LOG(WARNING) << "Removeslots remove from table partition failed: " << s.ToString(); + pre_success = false; + } + } + if (pre_success) { + s = g_pika_rm->RemoveSyncPartition(p_infos_); + if (!s.ok()) { + LOG(WARNING) << "Remvoeslots remove from sync partition failed: " << s.ToString(); + pre_success = false; + } + } + + g_pika_server->slot_state_.store(INFREE); + + if (!pre_success) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res_.SetRes(CmdRes::kOk); + LOG(INFO) << "Pika meta file overwrite success"; +} + +Status PkClusterDelSlotsCmd::RemoveSlotsSanityCheck(const std::string& table_name) { + Status s = g_pika_conf->TablePartitionsSanityCheck(table_name, slots_, false); + if (!s.ok()) { + return s; + } + + std::shared_ptr
table_ptr = g_pika_server->GetTable(table_name); + if (!table_ptr) { + return Status::NotFound("table not found"); + } + + for (uint32_t id : slots_) { + std::shared_ptr partition_ptr = table_ptr->GetPartitionById(id); + if (partition_ptr == nullptr) { + return Status::Corruption("partition " + std::to_string(id) + " not found"); + } + } + s = g_pika_rm->RemoveSyncPartitionSanityCheck(p_infos_); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +/* pkcluster slotsslaveof no one [0-3,8-11 | all] + * pkcluster slotsslaveof ip port [0-3,8,9,10,11 | all] + * pkcluster slotsslaveof ip port [0,2,4,6,7,8,9 | all] force + */ +void PkClusterSlotsSlaveofCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePkClusterSlotsSlaveof); + return; + } + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "PkClusterSlotSync only support on sharding mode"); + return; + } + + if (!strcasecmp(argv_[2].data(), "no") + && !strcasecmp(argv_[3].data(), "one")) { + is_noone_ = true; + } else { + ip_ = argv_[2]; + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &port_) + || port_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + + if ((ip_ == "127.0.0.1" || ip_ == g_pika_server->host()) + && port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "You fucked up"); + return; + } + } + + if (!strcasecmp(argv_[4].data(), "all")) { + std::string table_name = g_pika_conf->default_table(); + slots_ = g_pika_server->GetTablePartitionIds(table_name); + } else { + Status s = ParseSlotGroup(argv_[4], &slots_); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + } + + if (slots_.empty()) { + res_.SetRes(CmdRes::kErrOther, "Slots set empty"); + } + + if (argv_.size() == 5) { + // do nothing + } else if (argv_.size() == 6 + && !strcasecmp(argv_[5].data(), "force")) { + force_sync_ = true; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + } +} + +void PkClusterSlotsSlaveofCmd::Do(std::shared_ptr partition) { + std::string table_name = g_pika_conf->default_table(); + std::vector to_del_slots; + for (const auto& slot : slots_) { + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name, slot)); + if (!slave_partition) { + res_.SetRes(CmdRes::kErrOther, "Slot " + std::to_string(slot) + " not found!"); + return; + } + if (is_noone_) { + // check okay + } else if (slave_partition->State() == ReplState::kConnected + && slave_partition->MasterIp() == ip_ && slave_partition->MasterPort() == port_) { + to_del_slots.push_back(slot); + } + } + + for (auto to_del : to_del_slots) { + slots_.erase(to_del); + } + + Status s = Status::OK(); + ReplState state = force_sync_ + ? ReplState::kTryDBSync : ReplState::kTryConnect; + for (const auto& slot : slots_) { + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name, slot)); + if (slave_partition->State() == ReplState::kConnected) { + s = g_pika_rm->SendRemoveSlaveNodeRequest(table_name, slot); + } + if (!s.ok()) { + break; + } + if (slave_partition->State() != ReplState::kNoConnect) { + // reset state + s = g_pika_rm->SetSlaveReplState( + PartitionInfo(table_name, slot), ReplState::kNoConnect); + if (!s.ok()) { + break; + } + } + if (is_noone_) { + } else { + s = g_pika_rm->ActivateSyncSlavePartition( + RmNode(ip_, port_, table_name, slot), state); + if (!s.ok()) { + break; + } + } + } + + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + diff --git a/tools/pika_migrate/src/pika_cmd_table_manager.cc b/tools/pika_migrate/src/pika_cmd_table_manager.cc new file mode 100644 index 0000000000..b046de878f --- /dev/null +++ b/tools/pika_migrate/src/pika_cmd_table_manager.cc @@ -0,0 +1,88 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_cmd_table_manager.h" + +#include +#include + +#include "include/pika_conf.h" +#include "slash/include/slash_mutex.h" + + +#define gettid() syscall(__NR_gettid) + +extern PikaConf* g_pika_conf; + +PikaCmdTableManager::PikaCmdTableManager() { + pthread_rwlock_init(&map_protector_, NULL); + cmds_ = new CmdTable(); + cmds_->reserve(300); + InitCmdTable(cmds_); +} + +PikaCmdTableManager::~PikaCmdTableManager() { + pthread_rwlock_destroy(&map_protector_); + for (const auto&item : thread_distribution_map_) { + delete item.second; + } + DestoryCmdTable(cmds_); + delete cmds_; +} + +std::shared_ptr PikaCmdTableManager::GetCmd(const std::string& opt) { + std::string internal_opt = opt; + if (!g_pika_conf->classic_mode()) { + TryChangeToAlias(&internal_opt); + } + return NewCommand(internal_opt); +} + +std::shared_ptr PikaCmdTableManager::NewCommand(const std::string& opt) { + Cmd* cmd = GetCmdFromTable(opt, *cmds_); + if (cmd) { + return std::shared_ptr(cmd->Clone()); + } + return nullptr; +} + +void PikaCmdTableManager::TryChangeToAlias(std::string *internal_opt) { + if (!strcasecmp(internal_opt->c_str(), kCmdNameSlaveof.c_str())) { + *internal_opt = kCmdNamePkClusterSlotsSlaveof; + } +} + +bool PikaCmdTableManager::CheckCurrentThreadDistributionMapExist(const pid_t& tid) { + slash::RWLock l(&map_protector_, false); + if (thread_distribution_map_.find(tid) == thread_distribution_map_.end()) { + return false; + } + return true; +} + +void PikaCmdTableManager::InsertCurrentThreadDistributionMap() { + pid_t tid = gettid(); + PikaDataDistribution* distribution = nullptr; + if (g_pika_conf->classic_mode()) { + distribution = new HashModulo(); + } else { + distribution = new Crc32(); + } + distribution->Init(); + slash::RWLock l(&map_protector_, true); + thread_distribution_map_.insert(std::make_pair(tid, distribution)); +} + +uint32_t PikaCmdTableManager::DistributeKey(const std::string& key, uint32_t partition_num) { + pid_t tid = gettid(); + PikaDataDistribution* data_dist = nullptr; + if (!CheckCurrentThreadDistributionMapExist(tid)) { + InsertCurrentThreadDistributionMap(); + } + + slash::RWLock l(&map_protector_, false); + data_dist = thread_distribution_map_[tid]; + return data_dist->Distribute(key, partition_num); +} diff --git a/tools/pika_migrate/src/pika_command.cc b/tools/pika_migrate/src/pika_command.cc new file mode 100644 index 0000000000..5e40cf6416 --- /dev/null +++ b/tools/pika_migrate/src/pika_command.cc @@ -0,0 +1,763 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_command.h" + +#include "include/pika_kv.h" +#include "include/pika_bit.h" +#include "include/pika_set.h" +#include "include/pika_geo.h" +#include "include/pika_list.h" +#include "include/pika_zset.h" +#include "include/pika_hash.h" +#include "include/pika_admin.h" +#include "include/pika_pubsub.h" +#include "include/pika_server.h" +#include "include/pika_hyperloglog.h" +#include "include/pika_slot.h" +#include "include/pika_cluster.h" + +extern PikaServer* g_pika_server; + +void InitCmdTable(std::unordered_map *cmd_table) { + //Admin + ////Slaveof + Cmd* slaveofptr = new SlaveofCmd(kCmdNameSlaveof, -3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlaveof, slaveofptr)); + Cmd* dbslaveofptr = new DbSlaveofCmd(kCmdNameDbSlaveof, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameDbSlaveof, dbslaveofptr)); + Cmd* authptr = new AuthCmd(kCmdNameAuth, 2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameAuth, authptr)); + Cmd* bgsaveptr = new BgsaveCmd(kCmdNameBgsave, -1, kCmdFlagsRead | kCmdFlagsAdmin | kCmdFlagsSuspend); + cmd_table->insert(std::pair(kCmdNameBgsave, bgsaveptr)); + Cmd* compactptr = new CompactCmd(kCmdNameCompact, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameCompact, compactptr)); + Cmd* purgelogsto = new PurgelogstoCmd(kCmdNamePurgelogsto, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePurgelogsto, purgelogsto)); + Cmd* pingptr = new PingCmd(kCmdNamePing, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePing, pingptr)); + Cmd* selectptr = new SelectCmd(kCmdNameSelect, 2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSelect, selectptr)); + Cmd* flushallptr = new FlushallCmd(kCmdNameFlushall, 1, kCmdFlagsWrite | kCmdFlagsSuspend | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameFlushall, flushallptr)); + Cmd* flushdbptr = new FlushdbCmd(kCmdNameFlushdb, -1, kCmdFlagsWrite | kCmdFlagsSuspend | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameFlushdb, flushdbptr)); + Cmd* clientptr = new ClientCmd(kCmdNameClient, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameClient, clientptr)); + Cmd* shutdownptr = new ShutdownCmd(kCmdNameShutdown, 1, kCmdFlagsRead | kCmdFlagsLocal | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameShutdown, shutdownptr)); + Cmd* infoptr = new InfoCmd(kCmdNameInfo, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameInfo, infoptr)); + Cmd* configptr = new ConfigCmd(kCmdNameConfig, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameConfig, configptr)); + Cmd* monitorptr = new MonitorCmd(kCmdNameMonitor, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameMonitor, monitorptr)); + Cmd* dbsizeptr = new DbsizeCmd(kCmdNameDbsize, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameDbsize, dbsizeptr)); + Cmd* timeptr = new TimeCmd(kCmdNameTime, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameTime, timeptr)); + Cmd* delbackupptr = new DelbackupCmd(kCmdNameDelbackup, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameDelbackup, delbackupptr)); + Cmd* echoptr = new EchoCmd(kCmdNameEcho, 2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameEcho, echoptr)); + Cmd* scandbptr = new ScandbCmd(kCmdNameScandb, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameScandb, scandbptr)); + Cmd* slowlogptr = new SlowlogCmd(kCmdNameSlowlog, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlowlog, slowlogptr)); + Cmd* paddingptr = new PaddingCmd(kCmdNamePadding, 2, kCmdFlagsWrite | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePadding, paddingptr)); + Cmd* pkpatternmatchdelptr = new PKPatternMatchDelCmd(kCmdNamePKPatternMatchDel, 3, kCmdFlagsWrite | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePKPatternMatchDel, pkpatternmatchdelptr)); + + // Slots related + Cmd* slotsinfoptr = new SlotsInfoCmd(kCmdNameSlotsInfo, -1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsInfo, slotsinfoptr)); + Cmd* slotshashkeyptr = new SlotsHashKeyCmd(kCmdNameSlotsHashKey, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsHashKey, slotshashkeyptr)); + Cmd* slotmgrtslotasyncptr = new SlotsMgrtSlotAsyncCmd(kCmdNameSlotsMgrtSlotAsync, 8, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtSlotAsync, slotmgrtslotasyncptr)); + Cmd* slotmgrttagslotasyncptr = new SlotsMgrtTagSlotAsyncCmd(kCmdNameSlotsMgrtTagSlotAsync, 8, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtTagSlotAsync, slotmgrttagslotasyncptr)); + Cmd* slotsdelptr = new SlotsDelCmd(kCmdNameSlotsDel, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsDel, slotsdelptr)); + Cmd* slotsscanptr = new SlotsScanCmd(kCmdNameSlotsScan, -3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsScan, slotsscanptr)); + Cmd* slotmgrtexecwrapper = new SlotsMgrtExecWrapperCmd(kCmdNameSlotsMgrtExecWrapper, -3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtExecWrapper, slotmgrtexecwrapper)); + Cmd* slotmgrtasyncstatus = new SlotsMgrtAsyncStatusCmd(kCmdNameSlotsMgrtAsyncStatus, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtAsyncStatus, slotmgrtasyncstatus)); + Cmd* slotmgrtasynccancel = new SlotsMgrtAsyncCancelCmd(kCmdNameSlotsMgrtAsyncCancel, 1, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtAsyncCancel, slotmgrtasynccancel)); + Cmd* slotmgrtslotptr = new SlotsMgrtSlotCmd(kCmdNameSlotsMgrtSlot, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtSlot, slotmgrtslotptr)); + Cmd* slotmgrttagslotptr = new SlotsMgrtTagSlotCmd(kCmdNameSlotsMgrtTagSlot, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtTagSlot, slotmgrttagslotptr)); + Cmd* slotmgrtoneptr = new SlotsMgrtOneCmd(kCmdNameSlotsMgrtOne, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtOne, slotmgrtoneptr)); + Cmd* slotmgrttagoneptr = new SlotsMgrtTagOneCmd(kCmdNameSlotsMgrtTagOne, 5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameSlotsMgrtTagOne, slotmgrttagoneptr)); + + // Cluster related + Cmd* pkclusterinfoptr = new PkClusterInfoCmd(kCmdNamePkClusterInfo, -3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePkClusterInfo, pkclusterinfoptr)); + Cmd* pkclusteraddslotsptr = new PkClusterAddSlotsCmd(kCmdNamePkClusterAddSlots, 3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePkClusterAddSlots, pkclusteraddslotsptr)); + Cmd* pkclusterdelslotsptr = new PkClusterDelSlotsCmd(kCmdNamePkClusterDelSlots, 3, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePkClusterDelSlots, pkclusterdelslotsptr)); + Cmd* pkclusterslotsslaveofptr = new PkClusterSlotsSlaveofCmd(kCmdNamePkClusterSlotsSlaveof, -5, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNamePkClusterSlotsSlaveof, pkclusterslotsslaveofptr)); + +#ifdef TCMALLOC_EXTENSION + Cmd* tcmallocptr = new TcmallocCmd(kCmdNameTcmalloc, -2, kCmdFlagsRead | kCmdFlagsAdmin); + cmd_table->insert(std::pair(kCmdNameTcmalloc, tcmallocptr)); +#endif + + //Kv + ////SetCmd + Cmd* setptr = new SetCmd(kCmdNameSet, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameSet, setptr)); + ////GetCmd + Cmd* getptr = new GetCmd(kCmdNameGet, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameGet, getptr)); + ////DelCmd + Cmd* delptr = new DelCmd(kCmdNameDel, -2, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameDel, delptr)); + ////IncrCmd + Cmd* incrptr = new IncrCmd(kCmdNameIncr, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameIncr, incrptr)); + ////IncrbyCmd + Cmd* incrbyptr = new IncrbyCmd(kCmdNameIncrby, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameIncrby, incrbyptr)); + ////IncrbyfloatCmd + Cmd* incrbyfloatptr = new IncrbyfloatCmd(kCmdNameIncrbyfloat, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameIncrbyfloat, incrbyfloatptr)); + ////DecrCmd + Cmd* decrptr = new DecrCmd(kCmdNameDecr, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameDecr, decrptr)); + ////DecrbyCmd + Cmd* decrbyptr = new DecrbyCmd(kCmdNameDecrby, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameDecrby, decrbyptr)); + ////GetsetCmd + Cmd* getsetptr = new GetsetCmd(kCmdNameGetset, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameGetset, getsetptr)); + ////AppendCmd + Cmd* appendptr = new AppendCmd(kCmdNameAppend, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameAppend, appendptr)); + ////MgetCmd + Cmd* mgetptr = new MgetCmd(kCmdNameMget, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameMget, mgetptr)); + ////KeysCmd + Cmd* keysptr = new KeysCmd(kCmdNameKeys, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameKeys, keysptr)); + ////SetnxCmd + Cmd* setnxptr = new SetnxCmd(kCmdNameSetnx, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameSetnx, setnxptr)); + ////SetexCmd + Cmd* setexptr = new SetexCmd(kCmdNameSetex, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameSetex, setexptr)); + ////PsetexCmd + Cmd* psetexptr = new PsetexCmd(kCmdNamePsetex, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePsetex, psetexptr)); + ////DelvxCmd + Cmd* delvxptr = new DelvxCmd(kCmdNameDelvx, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameDelvx, delvxptr)); + ////MSetCmd + Cmd* msetptr = new MsetCmd(kCmdNameMset, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameMset, msetptr)); + ////MSetnxCmd + Cmd* msetnxptr = new MsetnxCmd(kCmdNameMsetnx, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameMsetnx, msetnxptr)); + ////GetrangeCmd + Cmd* getrangeptr = new GetrangeCmd(kCmdNameGetrange, 4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameGetrange, getrangeptr)); + ////SetrangeCmd + Cmd* setrangeptr = new SetrangeCmd(kCmdNameSetrange, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameSetrange, setrangeptr)); + ////StrlenCmd + Cmd* strlenptr = new StrlenCmd(kCmdNameStrlen, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameStrlen, strlenptr)); + ////ExistsCmd + Cmd* existsptr = new ExistsCmd(kCmdNameExists, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameExists, existsptr)); + ////ExpireCmd + Cmd* expireptr = new ExpireCmd(kCmdNameExpire, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameExpire, expireptr)); + ////PexpireCmd + Cmd* pexpireptr = new PexpireCmd(kCmdNamePexpire, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePexpire, pexpireptr)); + ////ExpireatCmd + Cmd* expireatptr = new ExpireatCmd(kCmdNameExpireat, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameExpireat, expireatptr)); + ////PexpireatCmd + Cmd* pexpireatptr = new PexpireatCmd(kCmdNamePexpireat, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePexpireat, pexpireatptr)); + ////TtlCmd + Cmd* ttlptr = new TtlCmd(kCmdNameTtl, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameTtl, ttlptr)); + ////PttlCmd + Cmd* pttlptr = new PttlCmd(kCmdNamePttl, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePttl, pttlptr)); + ////PersistCmd + Cmd* persistptr = new PersistCmd(kCmdNamePersist, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePersist, persistptr)); + ////TypeCmd + Cmd* typeptr = new TypeCmd(kCmdNameType, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameType, typeptr)); + ////ScanCmd + Cmd* scanptr = new ScanCmd(kCmdNameScan, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameScan, scanptr)); + ////ScanxCmd + Cmd* scanxptr = new ScanxCmd(kCmdNameScanx, -3, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNameScanx, scanxptr)); + ////PKSetexAtCmd + Cmd* pksetexatptr = new PKSetexAtCmd(kCmdNamePKSetexAt, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePKSetexAt, pksetexatptr)); + ////PKScanRange + Cmd* pkscanrangeptr = new PKScanRangeCmd(kCmdNamePKScanRange, -4, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePKScanRange, pkscanrangeptr)); + ////PKRScanRange + Cmd* pkrscanrangeptr = new PKRScanRangeCmd(kCmdNamePKRScanRange, -4, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsKv); + cmd_table->insert(std::pair(kCmdNamePKRScanRange, pkrscanrangeptr)); + + //Hash + ////HDelCmd + Cmd* hdelptr = new HDelCmd(kCmdNameHDel, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHDel, hdelptr)); + ////HSetCmd + Cmd* hsetptr = new HSetCmd(kCmdNameHSet, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHSet, hsetptr)); + ////HGetCmd + Cmd* hgetptr = new HGetCmd(kCmdNameHGet, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHGet, hgetptr)); + ////HGetallCmd + Cmd* hgetallptr = new HGetallCmd(kCmdNameHGetall, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHGetall, hgetallptr)); + ////HExistsCmd + Cmd* hexistsptr = new HExistsCmd(kCmdNameHExists, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHExists, hexistsptr)); + ////HIncrbyCmd + Cmd* hincrbyptr = new HIncrbyCmd(kCmdNameHIncrby, 4, kCmdFlagsWrite |kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHIncrby, hincrbyptr)); + ////HIncrbyfloatCmd + Cmd* hincrbyfloatptr = new HIncrbyfloatCmd(kCmdNameHIncrbyfloat, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHIncrbyfloat, hincrbyfloatptr)); + ////HKeysCmd + Cmd* hkeysptr = new HKeysCmd(kCmdNameHKeys, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHKeys, hkeysptr)); + ////HLenCmd + Cmd* hlenptr = new HLenCmd(kCmdNameHLen, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHLen, hlenptr)); + ////HMgetCmd + Cmd* hmgetptr = new HMgetCmd(kCmdNameHMget, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHMget, hmgetptr)); + ////HMsetCmd + Cmd* hmsetptr = new HMsetCmd(kCmdNameHMset, -4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHMset, hmsetptr)); + ////HSetnxCmd + Cmd* hsetnxptr = new HSetnxCmd(kCmdNameHSetnx, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHSetnx, hsetnxptr)); + ////HStrlenCmd + Cmd* hstrlenptr = new HStrlenCmd(kCmdNameHStrlen, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHStrlen, hstrlenptr)); + ////HValsCmd + Cmd* hvalsptr = new HValsCmd(kCmdNameHVals, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHVals, hvalsptr)); + ////HScanCmd + Cmd* hscanptr = new HScanCmd(kCmdNameHScan, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHScan, hscanptr)); + ////HScanxCmd + Cmd* hscanxptr = new HScanxCmd(kCmdNameHScanx, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNameHScanx, hscanxptr)); + ////PKHScanRange + Cmd* pkhscanrangeptr = new PKHScanRangeCmd(kCmdNamePKHScanRange, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNamePKHScanRange, pkhscanrangeptr)); + ////PKHRScanRange + Cmd* pkhrscanrangeptr = new PKHRScanRangeCmd(kCmdNamePKHRScanRange, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsHash); + cmd_table->insert(std::pair(kCmdNamePKHRScanRange, pkhrscanrangeptr)); + + //List + Cmd* lindexptr = new LIndexCmd(kCmdNameLIndex, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLIndex, lindexptr)); + Cmd* linsertptr = new LInsertCmd(kCmdNameLInsert, 5, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLInsert, linsertptr)); + Cmd* llenptr = new LLenCmd(kCmdNameLLen, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLLen, llenptr)); + Cmd* lpopptr = new LPopCmd(kCmdNameLPop, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLPop, lpopptr)); + Cmd* lpushptr = new LPushCmd(kCmdNameLPush, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLPush, lpushptr)); + Cmd* lpushxptr = new LPushxCmd(kCmdNameLPushx, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLPushx, lpushxptr)); + Cmd* lrangeptr = new LRangeCmd(kCmdNameLRange, 4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLRange, lrangeptr)); + Cmd* lremptr = new LRemCmd(kCmdNameLRem, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLRem, lremptr)); + Cmd* lsetptr = new LSetCmd(kCmdNameLSet, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLSet, lsetptr)); + Cmd* ltrimptr = new LTrimCmd(kCmdNameLTrim, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameLTrim, ltrimptr)); + Cmd* rpopptr = new RPopCmd(kCmdNameRPop, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameRPop, rpopptr)); + Cmd* rpoplpushptr = new RPopLPushCmd(kCmdNameRPopLPush, 3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameRPopLPush, rpoplpushptr)); + Cmd* rpushptr = new RPushCmd(kCmdNameRPush, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameRPush, rpushptr)); + Cmd* rpushxptr = new RPushxCmd(kCmdNameRPushx, 3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsList); + cmd_table->insert(std::pair(kCmdNameRPushx, rpushxptr)); + + //Zset + ////ZAddCmd + Cmd* zaddptr = new ZAddCmd(kCmdNameZAdd, -4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZAdd, zaddptr)); + ////ZCardCmd + Cmd* zcardptr = new ZCardCmd(kCmdNameZCard, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZCard, zcardptr)); + ////ZScanCmd + Cmd* zscanptr = new ZScanCmd(kCmdNameZScan, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZScan, zscanptr)); + ////ZIncrbyCmd + Cmd* zincrbyptr = new ZIncrbyCmd(kCmdNameZIncrby, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZIncrby, zincrbyptr)); + ////ZRangeCmd + Cmd* zrangeptr = new ZRangeCmd(kCmdNameZRange, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRange, zrangeptr)); + ////ZRevrangeCmd + Cmd* zrevrangeptr = new ZRevrangeCmd(kCmdNameZRevrange, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRevrange, zrevrangeptr)); + ////ZRangebyscoreCmd + Cmd* zrangebyscoreptr = new ZRangebyscoreCmd(kCmdNameZRangebyscore, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRangebyscore, zrangebyscoreptr)); + ////ZRevrangebyscoreCmd + Cmd* zrevrangebyscoreptr = new ZRevrangebyscoreCmd(kCmdNameZRevrangebyscore, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRevrangebyscore, zrevrangebyscoreptr)); + ////ZCountCmd + Cmd* zcountptr = new ZCountCmd(kCmdNameZCount, 4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZCount, zcountptr)); + ////ZRemCmd + Cmd* zremptr = new ZRemCmd(kCmdNameZRem, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRem, zremptr)); + ////ZUnionstoreCmd + Cmd* zunionstoreptr = new ZUnionstoreCmd(kCmdNameZUnionstore, -4, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZUnionstore, zunionstoreptr)); + ////ZInterstoreCmd + Cmd* zinterstoreptr = new ZInterstoreCmd(kCmdNameZInterstore, -4, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZInterstore, zinterstoreptr)); + ////ZRankCmd + Cmd* zrankptr = new ZRankCmd(kCmdNameZRank, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRank, zrankptr)); + ////ZRevrankCmd + Cmd* zrevrankptr = new ZRevrankCmd(kCmdNameZRevrank, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRevrank, zrevrankptr)); + ////ZScoreCmd + Cmd* zscoreptr = new ZScoreCmd(kCmdNameZScore, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZScore, zscoreptr)); + ////ZRangebylexCmd + Cmd* zrangebylexptr = new ZRangebylexCmd(kCmdNameZRangebylex, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRangebylex, zrangebylexptr)); + ////ZRevrangebylexCmd + Cmd* zrevrangebylexptr = new ZRevrangebylexCmd(kCmdNameZRevrangebylex, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRevrangebylex, zrevrangebylexptr)); + ////ZLexcountCmd + Cmd* zlexcountptr = new ZLexcountCmd(kCmdNameZLexcount, 4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZLexcount, zlexcountptr)); + ////ZRemrangebyrankCmd + Cmd* zremrangebyrankptr = new ZRemrangebyrankCmd(kCmdNameZRemrangebyrank, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRemrangebyrank, zremrangebyrankptr)); + ////ZRemrangebyscoreCmd + Cmd* zremrangebyscoreptr = new ZRemrangebyscoreCmd(kCmdNameZRemrangebyscore, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRemrangebyscore, zremrangebyscoreptr)); + ////ZRemrangebylexCmd + Cmd* zremrangebylexptr = new ZRemrangebylexCmd(kCmdNameZRemrangebylex, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZRemrangebylex, zremrangebylexptr)); + ////ZPopmax + Cmd* zpopmaxptr = new ZPopmaxCmd(kCmdNameZPopmax, -2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZPopmax, zpopmaxptr)); + ////ZPopmin + Cmd* zpopminptr = new ZPopminCmd(kCmdNameZPopmin, -2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsZset); + cmd_table->insert(std::pair(kCmdNameZPopmin, zpopminptr)); + + //Set + ////SAddCmd + Cmd* saddptr = new SAddCmd(kCmdNameSAdd, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSAdd, saddptr)); + ////SPopCmd + Cmd* spopptr = new SPopCmd(kCmdNameSPop, 2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSPop, spopptr)); + ////SCardCmd + Cmd* scardptr = new SCardCmd(kCmdNameSCard, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSCard, scardptr)); + ////SMembersCmd + Cmd* smembersptr = new SMembersCmd(kCmdNameSMembers, 2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSMembers, smembersptr)); + ////SScanCmd + Cmd* sscanptr = new SScanCmd(kCmdNameSScan, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSScan, sscanptr)); + ////SRemCmd + Cmd* sremptr = new SRemCmd(kCmdNameSRem, -3, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSRem, sremptr)); + ////SUnionCmd + Cmd* sunionptr = new SUnionCmd(kCmdNameSUnion, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSUnion, sunionptr)); + ////SUnionstoreCmd + Cmd* sunionstoreptr = new SUnionstoreCmd(kCmdNameSUnionstore, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSUnionstore, sunionstoreptr)); + ////SInterCmd + Cmd* sinterptr = new SInterCmd(kCmdNameSInter, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSInter, sinterptr)); + ////SInterstoreCmd + Cmd* sinterstoreptr = new SInterstoreCmd(kCmdNameSInterstore, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSInterstore, sinterstoreptr)); + ////SIsmemberCmd + Cmd* sismemberptr = new SIsmemberCmd(kCmdNameSIsmember, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSIsmember, sismemberptr)); + ////SDiffCmd + Cmd* sdiffptr = new SDiffCmd(kCmdNameSDiff, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSDiff, sdiffptr)); + ////SDiffstoreCmd + Cmd* sdiffstoreptr = new SDiffstoreCmd(kCmdNameSDiffstore, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSDiffstore, sdiffstoreptr)); + ////SMoveCmd + Cmd* smoveptr = new SMoveCmd(kCmdNameSMove, 4, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSMove, smoveptr)); + ////SRandmemberCmd + Cmd* srandmemberptr = new SRandmemberCmd(kCmdNameSRandmember, -2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsSet); + cmd_table->insert(std::pair(kCmdNameSRandmember, srandmemberptr)); + + //BitMap + ////bitsetCmd + Cmd* bitsetptr = new BitSetCmd(kCmdNameBitSet, 4, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsBit); + cmd_table->insert(std::pair(kCmdNameBitSet, bitsetptr)); + ////bitgetCmd + Cmd* bitgetptr = new BitGetCmd(kCmdNameBitGet, 3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsBit); + cmd_table->insert(std::pair(kCmdNameBitGet, bitgetptr)); + ////bitcountCmd + Cmd* bitcountptr = new BitCountCmd(kCmdNameBitCount, -2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsBit); + cmd_table->insert(std::pair(kCmdNameBitCount, bitcountptr)); + ////bitposCmd + Cmd* bitposptr = new BitPosCmd(kCmdNameBitPos, -3, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsBit); + cmd_table->insert(std::pair(kCmdNameBitPos, bitposptr)); + ////bitopCmd + Cmd* bitopptr = new BitOpCmd(kCmdNameBitOp, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsBit); + cmd_table->insert(std::pair(kCmdNameBitOp, bitopptr)); + + //HyperLogLog + ////pfaddCmd + Cmd * pfaddptr = new PfAddCmd(kCmdNamePfAdd, -2, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsHyperLogLog); + cmd_table->insert(std::pair(kCmdNamePfAdd, pfaddptr)); + ////pfcountCmd + Cmd * pfcountptr = new PfCountCmd(kCmdNamePfCount, -2, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsHyperLogLog); + cmd_table->insert(std::pair(kCmdNamePfCount, pfcountptr)); + ////pfmergeCmd + Cmd * pfmergeptr = new PfMergeCmd(kCmdNamePfMerge, -3, kCmdFlagsWrite | kCmdFlagsMultiPartition | kCmdFlagsHyperLogLog); + cmd_table->insert(std::pair(kCmdNamePfMerge, pfmergeptr)); + + //GEO + ////GepAdd + Cmd * geoaddptr = new GeoAddCmd(kCmdNameGeoAdd, -5, kCmdFlagsWrite | kCmdFlagsSinglePartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoAdd, geoaddptr)); + ////GeoPos + Cmd * geoposptr = new GeoPosCmd(kCmdNameGeoPos, -2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoPos, geoposptr)); + ////GeoDist + Cmd * geodistptr = new GeoDistCmd(kCmdNameGeoDist, -4, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoDist, geodistptr)); + ////GeoHash + Cmd * geohashptr = new GeoHashCmd(kCmdNameGeoHash, -2, kCmdFlagsRead | kCmdFlagsSinglePartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoHash, geohashptr)); + ////GeoRadius + Cmd * georadiusptr = new GeoRadiusCmd(kCmdNameGeoRadius, -6, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoRadius, georadiusptr)); + ////GeoRadiusByMember + Cmd * georadiusbymemberptr = new GeoRadiusByMemberCmd(kCmdNameGeoRadiusByMember, -5, kCmdFlagsRead | kCmdFlagsMultiPartition | kCmdFlagsGeo); + cmd_table->insert(std::pair(kCmdNameGeoRadiusByMember, georadiusbymemberptr)); + + //PubSub + ////Publish + Cmd * publishptr = new PublishCmd(kCmdNamePublish, 3, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNamePublish, publishptr)); + ////Subscribe + Cmd * subscribeptr = new SubscribeCmd(kCmdNameSubscribe, -2, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNameSubscribe, subscribeptr)); + ////UnSubscribe + Cmd * unsubscribeptr = new UnSubscribeCmd(kCmdNameUnSubscribe, -1, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNameUnSubscribe, unsubscribeptr)); + ////PSubscribe + Cmd * psubscribeptr = new PSubscribeCmd(kCmdNamePSubscribe, -2, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNamePSubscribe, psubscribeptr)); + ////PUnSubscribe + Cmd * punsubscribeptr = new PUnSubscribeCmd(kCmdNamePUnSubscribe, -1, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNamePUnSubscribe, punsubscribeptr)); + ////PubSub + Cmd * pubsubptr = new PubSubCmd(kCmdNamePubSub, -2, kCmdFlagsRead | kCmdFlagsPubSub); + cmd_table->insert(std::pair(kCmdNamePubSub, pubsubptr)); +} + +Cmd* GetCmdFromTable(const std::string& opt, const CmdTable& cmd_table) { + CmdTable::const_iterator it = cmd_table.find(opt); + if (it != cmd_table.end()) { + return it->second; + } + return NULL; +} + +void DestoryCmdTable(CmdTable* cmd_table) { + CmdTable::const_iterator it = cmd_table->begin(); + for (; it != cmd_table->end(); ++it) { + delete it->second; + } +} + +void TryAliasChange(std::vector* argv) { + if (argv->empty()) { + return; + } + if (!strcasecmp(argv->front().c_str(), kCmdNameSlaveof.c_str())) { + argv->front() = "slotsslaveof"; + argv->insert(argv->begin(), kClusterPrefix); + if (!strcasecmp(argv->back().c_str(), "force")) { + argv->back() = "all"; + argv->push_back("force"); + } else { + argv->push_back("all"); + } + } +} + +void Cmd::Initial(const PikaCmdArgsType& argv, + const std::string& table_name) { + argv_ = argv; + if (!g_pika_conf->classic_mode()) { + TryAliasChange(&argv_); + } + table_name_ = table_name; + res_.clear(); // Clear res content + Clear(); // Clear cmd, Derived class can has own implement + DoInitial(); +}; + +std::vector Cmd::current_key() const { + std::vector res; + res.push_back(""); + return res; +} + +void Cmd::Execute() { + if (name_ == kCmdNameFlushdb) { + ProcessFlushDBCmd(); + } else if (name_ == kCmdNameFlushall) { + ProcessFlushAllCmd(); + } else if (name_ == kCmdNameInfo || name_ == kCmdNameConfig) { + ProcessDoNotSpecifyPartitionCmd(); + } else if (is_single_partition() || g_pika_conf->classic_mode()) { + ProcessSinglePartitionCmd(); + } else if (is_multi_partition()) { + ProcessMultiPartitionCmd(); + } else { + ProcessDoNotSpecifyPartitionCmd(); + } +} + +void Cmd::ProcessFlushDBCmd() { + std::shared_ptr
table = g_pika_server->GetTable(table_name_); + if (!table) { + res_.SetRes(CmdRes::kInvalidTable); + } else { + if (table->IsKeyScaning()) { + res_.SetRes(CmdRes::kErrOther, "The keyscan operation is executing, Try again later"); + } else { + slash::RWLock l_prw(&table->partitions_rw_, true); + for (const auto& partition_item : table->partitions_) { + ProcessCommand(partition_item.second); + } + res_.SetRes(CmdRes::kOk); + } + } +} + +void Cmd::ProcessFlushAllCmd() { + slash::RWLock l_trw(&g_pika_server->tables_rw_, true); + for (const auto& table_item : g_pika_server->tables_) { + if (table_item.second->IsKeyScaning()) { + res_.SetRes(CmdRes::kErrOther, "The keyscan operation is executing, Try again later"); + return; + } + } + + for (const auto& table_item : g_pika_server->tables_) { + slash::RWLock l_prw(&table_item.second->partitions_rw_, true); + for (const auto& partition_item : table_item.second->partitions_) { + ProcessCommand(partition_item.second); + } + } + res_.SetRes(CmdRes::kOk); +} + +void Cmd::ProcessSinglePartitionCmd() { + std::shared_ptr partition; + if (g_pika_conf->classic_mode()) { + // in classic mode a table has only one partition + partition = g_pika_server->GetPartitionByDbName(table_name_); + } else { + std::vector cur_key = current_key(); + if (cur_key.empty()) { + res_.SetRes(CmdRes::kErrOther, "Internal Error"); + return; + } + // in sharding mode we select partition by key + partition = g_pika_server->GetTablePartitionByKey(table_name_, cur_key.front()); + } + + if (!partition) { + res_.SetRes(CmdRes::kErrOther, "Partition not found"); + return; + } + ProcessCommand(partition); +} + +void Cmd::ProcessCommand(std::shared_ptr partition) { + slash::lock::MultiRecordLock record_lock(partition->LockMgr()); + if (is_write()) { + record_lock.Lock(current_key()); + } + + DoCommand(partition); + + DoBinlog(partition); + + if (is_write()) { + record_lock.Unlock(current_key()); + } + +} + +void Cmd::DoCommand(std::shared_ptr partition) { + if (!is_suspend()) { + partition->DbRWLockReader(); + } + + Do(partition); + + if (!is_suspend()) { + partition->DbRWUnLock(); + } + +} + +void Cmd::DoBinlog(std::shared_ptr partition) { + if (res().ok() + && is_write() + && g_pika_conf->write_binlog()) { + + uint32_t filenum = 0; + uint64_t offset = 0; + uint64_t logic_id = 0; + + partition->logger()->Lock(); + partition->logger()->GetProducerStatus(&filenum, &offset, &logic_id); + uint32_t exec_time = time(nullptr); + std::string binlog = ToBinlog(exec_time, + g_pika_conf->server_id(), + logic_id, + filenum, + offset); + + Status s = partition->WriteBinlog(binlog); + partition->logger()->Unlock(); + + if (!s.ok()) { + res().SetRes(CmdRes::kErrOther, s.ToString()); + } + } +} + +void Cmd::ProcessMultiPartitionCmd() { + if (argv_.size() == static_cast(arity_ < 0 ? -arity_ : arity_)) { + ProcessSinglePartitionCmd(); + } else { + res_.SetRes(CmdRes::kErrOther, "This command usage only support in classic mode\r\n"); + return; + } +} + +void Cmd::ProcessDoNotSpecifyPartitionCmd() { + Do(); +} + +bool Cmd::is_write() const { + return ((flag_ & kCmdFlagsMaskRW) == kCmdFlagsWrite); +} +bool Cmd::is_local() const { + return ((flag_ & kCmdFlagsMaskLocal) == kCmdFlagsLocal); +} +// Others need to be suspended when a suspend command run +bool Cmd::is_suspend() const { + return ((flag_ & kCmdFlagsMaskSuspend) == kCmdFlagsSuspend); +} +// Must with admin auth +bool Cmd::is_admin_require() const { + return ((flag_ & kCmdFlagsMaskAdminRequire) == kCmdFlagsAdminRequire); +} +bool Cmd::is_single_partition() const { + return ((flag_ & kCmdFlagsMaskPartition) == kCmdFlagsSinglePartition); +} +bool Cmd::is_multi_partition() const { + return ((flag_ & kCmdFlagsMaskPartition) == kCmdFlagsMultiPartition); +} + +std::string Cmd::name() const { + return name_; +} +CmdRes& Cmd::res() { + return res_; +} + +std::string Cmd::ToBinlog(uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, argv_.size(), "*"); + + for (const auto& v : argv_) { + RedisAppendLen(content, v.size(), "$"); + RedisAppendContent(content, v); + } + + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +bool Cmd::CheckArg(int num) const { + if ((arity_ > 0 && num != arity_) + || (arity_ < 0 && num < -arity_)) { + return false; + } + return true; +} + +void Cmd::LogCommand() const { + std::string command; + for (const auto& item : argv_) { + command.append(" "); + command.append(item); + } + LOG(INFO) << "command:" << command; +} + +void Cmd::SetConn(const std::shared_ptr conn) { + conn_ = conn; +} + +std::shared_ptr Cmd::GetConn() { + return conn_.lock(); +} diff --git a/tools/pika_migrate/src/pika_conf.cc b/tools/pika_migrate/src/pika_conf.cc new file mode 100644 index 0000000000..ca19667eb7 --- /dev/null +++ b/tools/pika_migrate/src/pika_conf.cc @@ -0,0 +1,502 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_conf.h" + +#include + +#include +#include + +#include "slash/include/env.h" + +#include "include/pika_define.h" + +PikaConf::PikaConf(const std::string& path) + : slash::BaseConf(path), conf_path_(path) { + pthread_rwlock_init(&rwlock_, NULL); + local_meta_ = new PikaMeta(); +} + +PikaConf::~PikaConf() { + pthread_rwlock_destroy(&rwlock_); + delete local_meta_; +} + +Status PikaConf::InternalGetTargetTable(const std::string& table_name, uint32_t* const target) { + int32_t table_index = -1; + for (size_t idx = 0; table_structs_.size(); ++idx) { + if (table_structs_[idx].table_name == table_name) { + table_index = idx; + break; + } + } + if (table_index == -1) { + return Status::NotFound("table : " + table_name + " not found"); + } + *target = table_index; + return Status::OK(); +} + +Status PikaConf::TablePartitionsSanityCheck(const std::string& table_name, + const std::set& partition_ids, + bool is_add) { + RWLock l(&rwlock_, false); + uint32_t table_index = 0; + Status s = InternalGetTargetTable(table_name, &table_index); + if (!s.ok()) { + return s; + } + // Sanity Check + for (const auto& id : partition_ids) { + if (id >= table_structs_[table_index].partition_num) { + return Status::Corruption("partition index out of range"); + } else if (is_add && table_structs_[table_index].partition_ids.count(id) != 0) { + return Status::Corruption("partition : " + std::to_string(id) + " exist"); + } else if (!is_add && table_structs_[table_index].partition_ids.count(id) == 0) { + return Status::Corruption("partition : " + std::to_string(id) + " not exist"); + } + } + return Status::OK(); +} + +Status PikaConf::AddTablePartitions(const std::string& table_name, + const std::set& partition_ids) { + Status s = TablePartitionsSanityCheck(table_name, partition_ids, true); + if (!s.ok()) { + return s; + } + + RWLock l(&rwlock_, true); + uint32_t index = 0; + s = InternalGetTargetTable(table_name, &index); + if (s.ok()) { + for (const auto& id : partition_ids) { + table_structs_[index].partition_ids.insert(id); + } + s = local_meta_->StableSave(table_structs_); + } + return s; +} + +Status PikaConf::RemoveTablePartitions(const std::string& table_name, + const std::set& partition_ids) { + Status s = TablePartitionsSanityCheck(table_name, partition_ids, false); + if (!s.ok()) { + return s; + } + + RWLock l(&rwlock_, true); + uint32_t index = 0; + s = InternalGetTargetTable(table_name, &index); + if (s.ok()) { + for (const auto& id : partition_ids) { + table_structs_[index].partition_ids.erase(id); + } + s = local_meta_->StableSave(table_structs_); + } + return s; +} + +int PikaConf::Load() +{ + int ret = LoadConf(); + if (ret != 0) { + return ret; + } + + GetConfInt("timeout", &timeout_); + if (timeout_ < 0) { + timeout_ = 60; // 60s + } + GetConfStr("server-id", &server_id_); + if (server_id_.empty()) { + server_id_ = "1"; + } + GetConfStr("requirepass", &requirepass_); + GetConfStr("masterauth", &masterauth_); + GetConfStr("userpass", &userpass_); + GetConfInt("maxclients", &maxclients_); + if (maxclients_ <= 0) { + maxclients_ = 20000; + } + GetConfInt("root-connection-num", &root_connection_num_); + if (root_connection_num_ < 0) { + root_connection_num_ = 2; + } + + std::string swe; + GetConfStr("slowlog-write-errorlog", &swe); + slowlog_write_errorlog_.store(swe == "yes" ? true : false); + + int tmp_slowlog_log_slower_than; + GetConfInt("slowlog-log-slower-than", &tmp_slowlog_log_slower_than); + slowlog_log_slower_than_.store(tmp_slowlog_log_slower_than); + GetConfInt("slowlog-max-len", &slowlog_max_len_); + if (slowlog_max_len_ == 0) { + slowlog_max_len_ = 128; + } + std::string user_blacklist; + GetConfStr("userblacklist", &user_blacklist); + slash::StringSplit(user_blacklist, COMMA, user_blacklist_); + for (auto& item : user_blacklist_) { + slash::StringToLower(item); + } + + GetConfStr("dump-path", &bgsave_path_); + bgsave_path_ = bgsave_path_.empty() ? "./dump/" : bgsave_path_; + if (bgsave_path_[bgsave_path_.length() - 1] != '/') { + bgsave_path_ += "/"; + } + GetConfInt("dump-expire", &expire_dump_days_); + if (expire_dump_days_ < 0 ) { + expire_dump_days_ = 0; + } + GetConfStr("dump-prefix", &bgsave_prefix_); + + GetConfInt("expire-logs-nums", &expire_logs_nums_); + if (expire_logs_nums_ <= 10 ) { + expire_logs_nums_ = 10; + } + GetConfInt("expire-logs-days", &expire_logs_days_); + if (expire_logs_days_ <= 0 ) { + expire_logs_days_ = 1; + } + GetConfStr("compression", &compression_); + // set slave read only true as default + slave_read_only_ = true; + GetConfInt("slave-priority", &slave_priority_); + + // + // Immutable Sections + // + GetConfInt("port", &port_); + GetConfStr("log-path", &log_path_); + log_path_ = log_path_.empty() ? "./log/" : log_path_; + if (log_path_[log_path_.length() - 1] != '/') { + log_path_ += "/"; + } + GetConfStr("db-path", &db_path_); + db_path_ = db_path_.empty() ? "./db/" : db_path_; + if (db_path_[db_path_.length() - 1] != '/') { + db_path_ += "/"; + } + local_meta_->SetPath(db_path_); + + GetConfInt("thread-num", &thread_num_); + if (thread_num_ <= 0) { + thread_num_ = 12; + } + if (thread_num_ > 24) { + thread_num_ = 24; + } + GetConfInt("thread-pool-size", &thread_pool_size_); + if (thread_pool_size_ <= 0) { + thread_pool_size_ = 12; + } + if (thread_pool_size_ > 24) { + thread_pool_size_ = 24; + } + GetConfInt("sync-thread-num", &sync_thread_num_); + if (sync_thread_num_ <= 0) { + sync_thread_num_ = 3; + } + if (sync_thread_num_ > 24) { + sync_thread_num_ = 24; + } + + std::string instance_mode; + GetConfStr("instance-mode", &instance_mode); + classic_mode_.store(instance_mode.empty() + || !strcasecmp(instance_mode.data(), "classic")); + + if (classic_mode_.load()) { + GetConfInt("databases", &databases_); + if (databases_ < 1 || databases_ > 8) { + LOG(FATAL) << "config databases error, limit [1 ~ 8], the actual is: " + << databases_; + } + for (int idx = 0; idx < databases_; ++idx) { + table_structs_.push_back({"db" + std::to_string(idx), 1, {0}}); + } + } else { + GetConfInt("default-slot-num", &default_slot_num_); + if (default_slot_num_ <= 0) { + LOG(FATAL) << "config default-slot-num error," + << " it should greater than zero, the actual is: " + << default_slot_num_; + } + std::string pika_meta_path = db_path_ + kPikaMeta; + if (!slash::FileExists(pika_meta_path)) { + local_meta_->StableSave({{"db0", static_cast(default_slot_num_), {}}}); + } + Status s = local_meta_->ParseMeta(&table_structs_); + if (!s.ok()) { + LOG(FATAL) << "parse meta file error"; + } + } + default_table_ = table_structs_[0].table_name; + + compact_cron_ = ""; + GetConfStr("compact-cron", &compact_cron_); + if (compact_cron_ != "") { + bool have_week = false; + std::string compact_cron, week_str; + int slash_num = count(compact_cron_.begin(), compact_cron_.end(), '/'); + if (slash_num == 2) { + have_week = true; + std::string::size_type first_slash = compact_cron_.find("/"); + week_str = compact_cron_.substr(0, first_slash); + compact_cron = compact_cron_.substr(first_slash + 1); + } else { + compact_cron = compact_cron_; + } + + std::string::size_type len = compact_cron.length(); + std::string::size_type colon = compact_cron.find("-"); + std::string::size_type underline = compact_cron.find("/"); + if (colon == std::string::npos || underline == std::string::npos || + colon >= underline || colon + 1 >= len || + colon + 1 == underline || underline + 1 >= len) { + compact_cron_ = ""; + } else { + int week = std::atoi(week_str.c_str()); + int start = std::atoi(compact_cron.substr(0, colon).c_str()); + int end = std::atoi(compact_cron.substr(colon + 1, underline).c_str()); + int usage = std::atoi(compact_cron.substr(underline + 1).c_str()); + if ((have_week && (week < 1 || week > 7)) || start < 0 || start > 23 || end < 0 || end > 23 || usage < 0 || usage > 100) { + compact_cron_ = ""; + } + } + } + + compact_interval_ = ""; + GetConfStr("compact-interval", &compact_interval_); + if (compact_interval_ != "") { + std::string::size_type len = compact_interval_.length(); + std::string::size_type slash = compact_interval_.find("/"); + if (slash == std::string::npos || slash + 1 >= len) { + compact_interval_ = ""; + } else { + int interval = std::atoi(compact_interval_.substr(0, slash).c_str()); + int usage = std::atoi(compact_interval_.substr(slash+1).c_str()); + if (interval <= 0 || usage < 0 || usage > 100) { + compact_interval_ = ""; + } + } + } + + // write_buffer_size + GetConfInt64("write-buffer-size", &write_buffer_size_); + if (write_buffer_size_ <= 0 ) { + write_buffer_size_ = 268435456; // 256Mb + } + + // max_write_buffer_size + GetConfInt64("max-write-buffer-size", &max_write_buffer_size_); + if (max_write_buffer_size_ <= 0) { + max_write_buffer_size_ = 10737418240; // 10Gb + } + + // max_client_response_size + GetConfInt64("max-client-response-size", &max_client_response_size_); + if (max_client_response_size_ <= 0) { + max_client_response_size_ = 1073741824; // 1Gb + } + + // target_file_size_base + GetConfInt("target-file-size-base", &target_file_size_base_); + if (target_file_size_base_ <= 0) { + target_file_size_base_ = 1048576; // 10Mb + } + + max_cache_statistic_keys_ = 0; + GetConfInt("max-cache-statistic-keys", &max_cache_statistic_keys_); + if (max_cache_statistic_keys_ <= 0) { + max_cache_statistic_keys_ = 0; + } + + small_compaction_threshold_ = 5000; + GetConfInt("small-compaction-threshold", &small_compaction_threshold_); + if (small_compaction_threshold_ <= 0 + || small_compaction_threshold_ >= 100000) { + small_compaction_threshold_ = 5000; + } + + max_background_flushes_ = 1; + GetConfInt("max-background-flushes", &max_background_flushes_); + if (max_background_flushes_ <= 0) { + max_background_flushes_ = 1; + } + if (max_background_flushes_ >= 4) { + max_background_flushes_ = 4; + } + + max_background_compactions_ = 2; + GetConfInt("max-background-compactions", &max_background_compactions_); + if (max_background_compactions_ <= 0) { + max_background_compactions_ = 2; + } + if (max_background_compactions_ >= 8) { + max_background_compactions_ = 8; + } + + max_cache_files_ = 5000; + GetConfInt("max-cache-files", &max_cache_files_); + if (max_cache_files_ < -1) { + max_cache_files_ = 5000; + } + max_bytes_for_level_multiplier_ = 10; + GetConfInt("max-bytes-for-level-multiplier", &max_bytes_for_level_multiplier_); + if (max_bytes_for_level_multiplier_ < 10) { + max_bytes_for_level_multiplier_ = 5; + } + + block_size_ = 4 * 1024; + GetConfInt64("block-size", &block_size_); + if (block_size_ <= 0) { + block_size_ = 4 * 1024; + } + + block_cache_ = 8 * 1024 * 1024; + GetConfInt64("block-cache", &block_cache_); + if (block_cache_ < 0) { + block_cache_ = 8 * 1024 * 1024; + } + + std::string sbc; + GetConfStr("share-block-cache", &sbc); + share_block_cache_ = (sbc == "yes") ? true : false; + + std::string ciafb; + GetConfStr("cache-index-and-filter-blocks", &ciafb); + cache_index_and_filter_blocks_ = (ciafb == "yes") ? true : false; + + std::string offh; + GetConfStr("optimize-filters-for-hits", &offh); + optimize_filters_for_hits_ = (offh == "yes") ? true : false; + + std::string lcdlb; + GetConfStr("level-compaction-dynamic-level-bytes", &lcdlb); + level_compaction_dynamic_level_bytes_ = (lcdlb == "yes") ? true : false; + + // daemonize + std::string dmz; + GetConfStr("daemonize", &dmz); + daemonize_ = (dmz == "yes") ? true : false; + + // binlog + std::string wb; + GetConfStr("write-binlog", &wb); + write_binlog_ = (wb == "no") ? false : true; + GetConfInt("binlog-file-size", &binlog_file_size_); + if (binlog_file_size_ < 1024 + || static_cast(binlog_file_size_) > (1024LL * 1024 * 1024)) { + binlog_file_size_ = 100 * 1024 * 1024; // 100M + } + GetConfStr("pidfile", &pidfile_); + + // db sync + GetConfStr("db-sync-path", &db_sync_path_); + db_sync_path_ = db_sync_path_.empty() ? "./dbsync/" : db_sync_path_; + if (db_sync_path_[db_sync_path_.length() - 1] != '/') { + db_sync_path_ += "/"; + } + GetConfInt("db-sync-speed", &db_sync_speed_); + if (db_sync_speed_ < 0 || db_sync_speed_ > 1024) { + db_sync_speed_ = 1024; + } + // network interface + network_interface_ = ""; + GetConfStr("network-interface", &network_interface_); + + // slaveof + slaveof_ = ""; + GetConfStr("slaveof", &slaveof_); + + // sync window size + int tmp_sync_window_size = kBinlogReadWinDefaultSize; + GetConfInt("sync-window-size", &tmp_sync_window_size); + if (tmp_sync_window_size <= 0) { + sync_window_size_.store(kBinlogReadWinDefaultSize); + } else if (tmp_sync_window_size > kBinlogReadWinMaxSize) { + sync_window_size_.store(kBinlogReadWinMaxSize); + } else { + sync_window_size_.store(tmp_sync_window_size); + } + + target_redis_host_ = "127.0.0.1"; + GetConfStr("target-redis-host", &target_redis_host_); + + target_redis_port_ = 6379; + GetConfInt("target-redis-port", &target_redis_port_); + + target_redis_pwd_ = ""; + GetConfStr("target-redis-pwd" , &target_redis_pwd_); + + sync_batch_num_ = 100; + GetConfInt("sync-batch-num", &sync_batch_num_); + + redis_sender_num_ = 8; + GetConfInt("redis-sender-num", &redis_sender_num_); + return ret; +} + +void PikaConf::TryPushDiffCommands(const std::string& command, const std::string& value) { + if (!CheckConfExist(command)) { + diff_commands_[command] = value; + } +} + +int PikaConf::ConfigRewrite() { + std::string userblacklist = suser_blacklist(); + + RWLock l(&rwlock_, true); + // Only set value for config item that can be config set. + SetConfInt("timeout", timeout_); + SetConfStr("requirepass", requirepass_); + SetConfStr("masterauth", masterauth_); + SetConfStr("userpass", userpass_); + SetConfStr("userblacklist", userblacklist); + SetConfStr("dump-prefix", bgsave_prefix_); + SetConfInt("maxclients", maxclients_); + SetConfInt("dump-expire", expire_dump_days_); + SetConfInt("expire-logs-days", expire_logs_days_); + SetConfInt("expire-logs-nums", expire_logs_nums_); + SetConfInt("root-connection-num", root_connection_num_); + SetConfStr("slowlog-write-errorlog", slowlog_write_errorlog_.load() ? "yes" : "no"); + SetConfInt("slowlog-log-slower-than", slowlog_log_slower_than_.load()); + SetConfInt("slowlog-max-len", slowlog_max_len_); + SetConfStr("write-binlog", write_binlog_ ? "yes" : "no"); + SetConfInt("max-cache-statistic-keys", max_cache_statistic_keys_); + SetConfInt("small-compaction-threshold", small_compaction_threshold_); + SetConfInt("max-client-response-size", max_client_response_size_); + SetConfInt("db-sync-speed", db_sync_speed_); + SetConfStr("compact-cron", compact_cron_); + SetConfStr("compact-interval", compact_interval_); + SetConfInt("slave-priority", slave_priority_); + SetConfInt("sync-window-size", sync_window_size_.load()); + // slaveof config item is special + SetConfStr("slaveof", slaveof_); + + if (!diff_commands_.empty()) { + std::vector filtered_items; + for (const auto& diff_command : diff_commands_) { + if (!diff_command.second.empty()) { + slash::BaseConf::Rep::ConfItem item(slash::BaseConf::Rep::kConf, diff_command.first, diff_command.second); + filtered_items.push_back(item); + } + } + if (!filtered_items.empty()) { + slash::BaseConf::Rep::ConfItem comment_item(slash::BaseConf::Rep::kComment, "# Generated by CONFIG REWRITE\n"); + PushConfItem(comment_item); + for (const auto& item : filtered_items) { + PushConfItem(item); + } + } + diff_commands_.clear(); + } + return WriteBack(); +} diff --git a/tools/pika_migrate/src/pika_data_distribution.cc b/tools/pika_migrate/src/pika_data_distribution.cc new file mode 100644 index 0000000000..e5e55dc51a --- /dev/null +++ b/tools/pika_migrate/src/pika_data_distribution.cc @@ -0,0 +1,48 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_data_distribution.h" + +void HashModulo::Init() { +} + +uint32_t HashModulo::Distribute(const std::string& str, uint32_t partition_num) { + return std::hash()(str) % partition_num; +} + +void Crc32::Init() { + Crc32TableInit(IEEE_POLY); +} + +void Crc32::Crc32TableInit(uint32_t poly) { + int i, j; + for (i = 0; i < 256; i ++) { + uint32_t crc = i; + for (j = 0; j < 8; j ++) { + if (crc & 1) { + crc = (crc >> 1) ^ poly; + } else { + crc = (crc >> 1); + } + } + crc32tab[i] = crc; + } +} + +uint32_t Crc32::Distribute(const std::string &str, uint32_t partition_num) { + uint32_t crc = Crc32Update(0, str.data(), (int)str.size()); + // partition_num need to minus 1 + assert(partition_num > 1); + return (int)(crc & (partition_num == 0 ? 0 : (partition_num - 1))); +} + +uint32_t Crc32::Crc32Update(uint32_t crc, const char* buf, int len) { + int i; + crc = ~crc; + for (i = 0; i < len; i ++) { + crc = crc32tab[(uint8_t)((char)crc ^ buf[i])] ^ (crc >> 8); + } + return ~crc; +} diff --git a/tools/pika_migrate/src/pika_dispatch_thread.cc b/tools/pika_migrate/src/pika_dispatch_thread.cc new file mode 100644 index 0000000000..676f34843e --- /dev/null +++ b/tools/pika_migrate/src/pika_dispatch_thread.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_dispatch_thread.h" + +#include + +#include "include/pika_conf.h" +#include "include/pika_server.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; + +PikaDispatchThread::PikaDispatchThread(std::set &ips, int port, int work_num, + int cron_interval, int queue_limit) + : handles_(this) { + thread_rep_ = pink::NewDispatchThread(ips, port, work_num, &conn_factory_, + cron_interval, queue_limit, &handles_); + thread_rep_->set_thread_name("Dispatcher"); +} + +PikaDispatchThread::~PikaDispatchThread() { + thread_rep_->StopThread(); + LOG(INFO) << "dispatch thread " << thread_rep_->thread_id() << " exit!!!"; + delete thread_rep_; +} + +int PikaDispatchThread::StartThread() { + return thread_rep_->StartThread(); +} + +int64_t PikaDispatchThread::ThreadClientList(std::vector *clients) { + std::vector conns_info = + thread_rep_->conns_info(); + if (clients != nullptr) { + for (auto& info : conns_info) { + clients->push_back({ + info.fd, + info.ip_port, + info.last_interaction.tv_sec, + nullptr /* PinkConn pointer, doesn't need here */ + }); + } + } + return conns_info.size(); +} + +bool PikaDispatchThread::ClientKill(const std::string& ip_port) { + return thread_rep_->KillConn(ip_port); +} + +void PikaDispatchThread::ClientKillAll() { + thread_rep_->KillAllConns(); +} + +bool PikaDispatchThread::Handles::AccessHandle(std::string& ip) const { + if (ip == "127.0.0.1") { + ip = g_pika_server->host(); + } + + int client_num = pika_disptcher_->thread_rep_->conn_num(); + if ((client_num >= g_pika_conf->maxclients() + g_pika_conf->root_connection_num()) + || (client_num >= g_pika_conf->maxclients() && ip != g_pika_server->host())) { + LOG(WARNING) << "Max connections reach, Deny new comming: " << ip; + return false; + } + + DLOG(INFO) << "new clinet comming, ip: " << ip; + g_pika_server->incr_accumulative_connections(); + return true; +} + +void PikaDispatchThread::Handles::CronHandle() const { + pika_disptcher_->thread_rep_->set_keepalive_timeout(g_pika_conf->timeout()); + g_pika_server->ResetLastSecQuerynum(); +} diff --git a/tools/pika_migrate/src/pika_geo.cc b/tools/pika_migrate/src/pika_geo.cc new file mode 100644 index 0000000000..64005920c3 --- /dev/null +++ b/tools/pika_migrate/src/pika_geo.cc @@ -0,0 +1,551 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_geo.h" + +#include + +#include "slash/include/slash_string.h" + +#include "include/pika_geohash_helper.h" + +void GeoAddCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoAdd); + return; + } + size_t argc = argv_.size(); + if ((argc - 2) % 3 != 0) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoAdd); + return; + } + key_ = argv_[1]; + pos_.clear(); + struct GeoPoint point; + double longitude, latitude; + for (size_t index = 2; index < argc; index += 3) { + if (!slash::string2d(argv_[index].data(), argv_[index].size(), &longitude)) { + res_.SetRes(CmdRes::kInvalidFloat); + return; + } + if (!slash::string2d(argv_[index + 1].data(), argv_[index + 1].size(), &latitude)) { + res_.SetRes(CmdRes::kInvalidFloat); + return; + } + point.member = argv_[index + 2]; + point.longitude = longitude; + point.latitude = latitude; + pos_.push_back(point); + } + return; +} + +void GeoAddCmd::Do(std::shared_ptr partition) { + std::vector score_members; + for (const auto& geo_point : pos_) { + // Convert coordinates to geohash + GeoHashBits hash; + geohashEncodeWGS84(geo_point.longitude, geo_point.latitude, GEO_STEP_MAX, &hash); + GeoHashFix52Bits bits = geohashAlign52Bits(hash); + // Convert uint64 to double + double score; + std::string str_bits = std::to_string(bits); + slash::string2d(str_bits.data(), str_bits.size(), &score); + score_members.push_back({score, geo_point.member}); + } + int32_t count = 0; + rocksdb::Status s = partition->db()->ZAdd(key_, score_members, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void GeoPosCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoPos); + return; + } + key_ = argv_[1]; + members_.clear(); + size_t pos = 2; + while (pos < argv_.size()) { + members_.push_back(argv_[pos++]); + } +} + +void GeoPosCmd::Do(std::shared_ptr partition) { + double score; + res_.AppendArrayLen(members_.size()); + for (const auto& member : members_) { + rocksdb::Status s = partition->db()->ZScore(key_, member, &score); + if (s.ok()) { + double xy[2]; + GeoHashBits hash = { .bits = (uint64_t)score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + + res_.AppendArrayLen(2); + char longitude[32]; + int64_t len = slash::d2string(longitude, sizeof(longitude), xy[0]); + res_.AppendStringLen(len); + res_.AppendContent(longitude); + + char latitude[32]; + len = slash::d2string(latitude, sizeof(latitude), xy[1]); + res_.AppendStringLen(len); + res_.AppendContent(latitude); + + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + continue; + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + continue; + } + } +} + +static double length_converter(double meters, const std::string & unit) { + if (unit == "m") { + return meters; + } else if (unit == "km") { + return meters / 1000; + } else if (unit == "ft") { + return meters / 0.3048; + } else if (unit == "mi") { + return meters / 1609.34; + } else { + return -1; + } +} + +static bool check_unit(const std::string & unit) { + if (unit == "m" || unit == "km" || unit == "ft" || unit == "mi") { + return true; + } else { + return false; + } +} + +void GeoDistCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoDist); + return; + } + if (argv_.size() < 4) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoDist); + return; + } else if (argv_.size() > 5) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + key_ = argv_[1]; + first_pos_ = argv_[2]; + second_pos_ = argv_[3]; + if (argv_.size() == 5) { + unit_ = argv_[4]; + } else { + unit_ = "m"; + } + if (!check_unit(unit_)) { + res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi"); + return; + } +} + +void GeoDistCmd::Do(std::shared_ptr partition) { + double first_score, second_score, first_xy[2], second_xy[2]; + rocksdb::Status s = partition->db()->ZScore(key_, first_pos_, &first_score); + if (s.ok()) { + GeoHashBits hash = { .bits = (uint64_t)first_score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, first_xy); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + return; + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + + s = partition->db()->ZScore(key_, second_pos_, &second_score); + if (s.ok()) { + GeoHashBits hash = { .bits = (uint64_t)second_score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, second_xy); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + return; + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + + double distance = geohashGetDistance(first_xy[0], first_xy[1], second_xy[0], second_xy[1]); + distance = length_converter(distance, unit_); + char buf[32]; + sprintf(buf, "%.4f", distance); + res_.AppendStringLen(strlen(buf)); + res_.AppendContent(buf); +} + +void GeoHashCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoHash); + return; + } + key_ = argv_[1]; + members_.clear(); + size_t pos = 2; + while (pos < argv_.size()) { + members_.push_back(argv_[pos++]); + } +} + +void GeoHashCmd::Do(std::shared_ptr partition) { + const char * geoalphabet= "0123456789bcdefghjkmnpqrstuvwxyz"; + res_.AppendArrayLen(members_.size()); + for (const auto& member : members_) { + double score; + rocksdb::Status s = partition->db()->ZScore(key_, member, &score); + if (s.ok()) { + double xy[2]; + GeoHashBits hash = { .bits = (uint64_t)score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + GeoHashRange r[2]; + GeoHashBits encode_hash; + r[0].min = -180; + r[0].max = 180; + r[1].min = -90; + r[1].max = 90; + geohashEncode(&r[0], &r[1], xy[0], xy[1], 26, &encode_hash); + + char buf[12]; + int i; + for (i = 0; i < 11; i++) { + int idx = (encode_hash.bits >> (52-((i+1)*5))) & 0x1f; + buf[i] = geoalphabet[idx]; + } + buf[11] = '\0'; + res_.AppendStringLen(11); + res_.AppendContent(buf); + continue; + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + continue; + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + continue; + } + } +} + +static bool sort_distance_asc(const NeighborPoint & pos1, const NeighborPoint & pos2) { + return pos1.distance < pos2.distance; +} + +static bool sort_distance_desc(const NeighborPoint & pos1, const NeighborPoint & pos2) { + return pos1.distance > pos2.distance; +} + +static void GetAllNeighbors(std::shared_ptr partition, std::string & key, GeoRange & range, CmdRes & res) { + rocksdb::Status s; + double longitude = range.longitude, latitude = range.latitude, distance = range.distance; + int count_limit = 0; + // Convert other units to meters + if (range.unit == "m") { + distance = distance; + } else if (range.unit == "km") { + distance = distance * 1000; + } else if (range.unit == "ft") { + distance = distance * 0.3048; + } else if (range.unit == "mi") { + distance = distance * 1609.34; + } else { + distance = -1; + } + // Search the zset for all matching points + GeoHashRadius georadius = geohashGetAreasByRadiusWGS84(longitude, latitude, distance); + GeoHashBits neighbors[9]; + neighbors[0] = georadius.hash; + neighbors[1] = georadius.neighbors.north; + neighbors[2] = georadius.neighbors.south; + neighbors[3] = georadius.neighbors.east; + neighbors[4] = georadius.neighbors.west; + neighbors[5] = georadius.neighbors.north_east; + neighbors[6] = georadius.neighbors.north_west; + neighbors[7] = georadius.neighbors.south_east; + neighbors[8] = georadius.neighbors.south_west; + + // For each neighbor, get all the matching + // members and add them to the potential result list. + std::vector result; + int last_processed = 0; + for (size_t i = 0; i < sizeof(neighbors) / sizeof(*neighbors); i++) { + GeoHashFix52Bits min, max; + if (HASHISZERO(neighbors[i])) + continue; + min = geohashAlign52Bits(neighbors[i]); + neighbors[i].bits++; + max = geohashAlign52Bits(neighbors[i]); + // When a huge Radius (in the 5000 km range or more) is used, + // adjacent neighbors can be the same, so need to remove duplicated elements + if(last_processed && neighbors[i].bits == neighbors[last_processed].bits && neighbors[i].step == neighbors[last_processed].step) { + continue; + } + std::vector score_members; + s = partition->db()->ZRangebyscore(key, (double)min, (double)max, true, true, &score_members); + if (!s.ok() && !s.IsNotFound()) { + res.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + // Insert into result only if the point is within the search area. + for (size_t i = 0; i < score_members.size(); ++i) { + double xy[2], real_distance; + GeoHashBits hash = { .bits = (uint64_t)score_members[i].score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + if(geohashGetDistanceIfInRadiusWGS84(longitude, latitude, xy[0], xy[1], distance, &real_distance)) { + NeighborPoint item; + item.member = score_members[i].member; + item.score = score_members[i].score; + item.distance = real_distance; + result.push_back(item); + } + } + last_processed = i; + } + + // If using the count opiton + if (range.count) { + count_limit = static_cast(result.size()) < range.count_limit ? result.size() : range.count_limit; + } else { + count_limit = result.size(); + } + // If using sort option + if (range.sort == Asc) { + std::sort(result.begin(), result.end(), sort_distance_asc); + } else if(range.sort == Desc) { + std::sort(result.begin(), result.end(), sort_distance_desc); + } + + if (range.store || range.storedist) { + // Target key, create a sorted set with the results. + std::vector score_members; + for (int i = 0; i < count_limit; ++i) { + double distance = length_converter(result[i].distance, range.unit); + double score = range.store ? result[i].score : distance; + score_members.push_back({score, result[i].member}); + } + int32_t count = 0; + s = partition->db()->ZAdd(range.storekey, score_members, &count); + if (!s.ok()) { + res.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res.AppendInteger(count_limit); + return; + } else { + // No target key, return results to user. + + // For each the result + res.AppendArrayLen(count_limit); + for (int i = 0; i < count_limit; ++i) { + if (range.option_num != 0) { + res.AppendArrayLen(range.option_num+1); + } + // Member + res.AppendStringLen(result[i].member.size()); + res.AppendContent(result[i].member); + + // If using withdist option + if (range.withdist) { + double xy[2]; + GeoHashBits hash = { .bits = (uint64_t)result[i].score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + double distance = geohashGetDistance(longitude, latitude, xy[0], xy[1]); + distance = length_converter(distance, range.unit); + char buf[32]; + sprintf(buf, "%.4f", distance); + res.AppendStringLen(strlen(buf)); + res.AppendContent(buf); + } + // If using withhash option + if (range.withhash) { + res.AppendInteger(result[i].score); + } + // If using withcoord option + if (range.withcoord) { + res.AppendArrayLen(2); + double xy[2]; + GeoHashBits hash = { .bits = (uint64_t)result[i].score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + + char longitude[32]; + int64_t len = slash::d2string(longitude, sizeof(longitude), xy[0]); + res.AppendStringLen(len); + res.AppendContent(longitude); + + char latitude[32]; + len = slash::d2string(latitude, sizeof(latitude), xy[1]); + res.AppendStringLen(len); + res.AppendContent(latitude); + } + } + } +} + +void GeoRadiusCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius); + return; + } + key_ = argv_[1]; + slash::string2d(argv_[2].data(), argv_[2].size(), &range_.longitude); + slash::string2d(argv_[3].data(), argv_[3].size(), &range_.latitude); + slash::string2d(argv_[4].data(), argv_[4].size(), &range_.distance); + range_.unit = argv_[5]; + if (!check_unit(range_.unit)) { + res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi"); + return; + } + size_t pos = 6; + while (pos < argv_.size()) { + if (!strcasecmp(argv_[pos].c_str(), "withdist")) { + range_.withdist = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "withhash")) { + range_.withhash = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "withcoord")) { + range_.withcoord = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "count")) { + range_.count = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + std::string str_count = argv_[++pos]; + for (auto s : str_count) { + if (!isdigit(s)) { + res_.SetRes(CmdRes::kErrOther, "value is not an integer or out of range"); + return; + } + } + range_.count_limit = std::stoi(str_count); + } else if (!strcasecmp(argv_[pos].c_str(), "store")) { + range_.store = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + range_.storekey = argv_[++pos]; + } else if (!strcasecmp(argv_[pos].c_str(), "storedist")) { + range_.storedist = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + range_.storekey = argv_[++pos]; + } else if (!strcasecmp(argv_[pos].c_str(), "asc")) { + range_.sort = Asc; + } else if (!strcasecmp(argv_[pos].c_str(), "desc")) { + range_.sort = Desc; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + pos++; + } + if (range_.store && (range_.withdist || range_.withcoord || range_.withhash)) { + res_.SetRes(CmdRes::kErrOther, "STORE option in GEORADIUS is not compatible with WITHDIST, WITHHASH and WITHCOORDS options"); + return; + } +} + +void GeoRadiusCmd::Do(std::shared_ptr partition) { + GetAllNeighbors(partition, key_, range_, this->res_); +} + +void GeoRadiusByMemberCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius); + return; + } + key_ = argv_[1]; + range_.member = argv_[2]; + slash::string2d(argv_[3].data(), argv_[3].size(), &range_.distance); + range_.unit = argv_[4]; + if (!check_unit(range_.unit)) { + res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi"); + return; + } + size_t pos = 5; + while (pos < argv_.size()) { + if (!strcasecmp(argv_[pos].c_str(), "withdist")) { + range_.withdist = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "withhash")) { + range_.withhash = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "withcoord")) { + range_.withcoord = true; + range_.option_num++; + } else if (!strcasecmp(argv_[pos].c_str(), "count")) { + range_.count = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + std::string str_count = argv_[++pos]; + for (auto s : str_count) { + if (!isdigit(s)) { + res_.SetRes(CmdRes::kErrOther, "value is not an integer or out of range"); + return; + } + } + range_.count_limit = std::stoi(str_count); + } else if (!strcasecmp(argv_[pos].c_str(), "store")) { + range_.store = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + range_.storekey = argv_[++pos]; + } else if (!strcasecmp(argv_[pos].c_str(), "storedist")) { + range_.storedist = true; + if (argv_.size() < (pos+2)) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + range_.storekey = argv_[++pos]; + } else if (!strcasecmp(argv_[pos].c_str(), "asc")) { + range_.sort = Asc; + } else if (!strcasecmp(argv_[pos].c_str(), "desc")) { + range_.sort = Desc; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + pos++; + } + if (range_.store && (range_.withdist || range_.withcoord || range_.withhash)) { + res_.SetRes(CmdRes::kErrOther, "STORE option in GEORADIUS is not compatible with WITHDIST, WITHHASH and WITHCOORDS options"); + return; + } +} + +void GeoRadiusByMemberCmd::Do(std::shared_ptr partition) { + double score; + rocksdb::Status s = partition->db()->ZScore(key_, range_.member, &score); + if (s.ok()) { + double xy[2]; + GeoHashBits hash = { .bits = (uint64_t)score, .step = GEO_STEP_MAX }; + geohashDecodeToLongLatWGS84(hash, xy); + range_.longitude = xy[0]; + range_.latitude = xy[1]; + } + GetAllNeighbors(partition, key_, range_, this->res_); +} diff --git a/tools/pika_migrate/src/pika_geohash.cc b/tools/pika_migrate/src/pika_geohash.cc new file mode 100644 index 0000000000..2ad66314b8 --- /dev/null +++ b/tools/pika_migrate/src/pika_geohash.cc @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2013-2014, yinqiwen + * Copyright (c) 2014, Matt Stancliff . + * Copyright (c) 2015-2016, Salvatore Sanfilippo . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "include/pika_geohash.h" + +/** + * Hashing works like this: + * Divide the world into 4 buckets. Label each one as such: + * ----------------- + * | | | + * | | | + * | 0,1 | 1,1 | + * ----------------- + * | | | + * | | | + * | 0,0 | 1,0 | + * ----------------- + */ + +/* Interleave lower bits of x and y, so the bits of x + * are in the even positions and bits from y in the odd; + * x and y must initially be less than 2**32 (65536). + * From: https://graphics.stanford.edu/~seander/bithacks.html#InterleaveBMN + */ +static inline uint64_t interleave64(uint32_t xlo, uint32_t ylo) { + static const uint64_t B[] = {0x5555555555555555ULL, 0x3333333333333333ULL, + 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, + 0x0000FFFF0000FFFFULL}; + static const unsigned int S[] = {1, 2, 4, 8, 16}; + + uint64_t x = xlo; + uint64_t y = ylo; + + x = (x | (x << S[4])) & B[4]; + y = (y | (y << S[4])) & B[4]; + + x = (x | (x << S[3])) & B[3]; + y = (y | (y << S[3])) & B[3]; + + x = (x | (x << S[2])) & B[2]; + y = (y | (y << S[2])) & B[2]; + + x = (x | (x << S[1])) & B[1]; + y = (y | (y << S[1])) & B[1]; + + x = (x | (x << S[0])) & B[0]; + y = (y | (y << S[0])) & B[0]; + + return x | (y << 1); +} + +/* reverse the interleave process + * derived from http://stackoverflow.com/questions/4909263 + */ +static inline uint64_t deinterleave64(uint64_t interleaved) { + static const uint64_t B[] = {0x5555555555555555ULL, 0x3333333333333333ULL, + 0x0F0F0F0F0F0F0F0FULL, 0x00FF00FF00FF00FFULL, + 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL}; + static const unsigned int S[] = {0, 1, 2, 4, 8, 16}; + + uint64_t x = interleaved; + uint64_t y = interleaved >> 1; + + x = (x | (x >> S[0])) & B[0]; + y = (y | (y >> S[0])) & B[0]; + + x = (x | (x >> S[1])) & B[1]; + y = (y | (y >> S[1])) & B[1]; + + x = (x | (x >> S[2])) & B[2]; + y = (y | (y >> S[2])) & B[2]; + + x = (x | (x >> S[3])) & B[3]; + y = (y | (y >> S[3])) & B[3]; + + x = (x | (x >> S[4])) & B[4]; + y = (y | (y >> S[4])) & B[4]; + + x = (x | (x >> S[5])) & B[5]; + y = (y | (y >> S[5])) & B[5]; + + return x | (y << 32); +} + +void geohashGetCoordRange(GeoHashRange *long_range, GeoHashRange *lat_range) { + /* These are constraints from EPSG:900913 / EPSG:3785 / OSGEO:41001 */ + /* We can't geocode at the north/south pole. */ + long_range->max = GEO_LONG_MAX; + long_range->min = GEO_LONG_MIN; + lat_range->max = GEO_LAT_MAX; + lat_range->min = GEO_LAT_MIN; +} + +int geohashEncode(const GeoHashRange *long_range, const GeoHashRange *lat_range, + double longitude, double latitude, uint8_t step, + GeoHashBits *hash) { + /* Check basic arguments sanity. */ + if (hash == NULL || step > 32 || step == 0 || + RANGEPISZERO(lat_range) || RANGEPISZERO(long_range)) return 0; + + /* Return an error when trying to index outside the supported + * constraints. */ + if (longitude > 180 || longitude < -180 || + latitude > 85.05112878 || latitude < -85.05112878) return 0; + + hash->bits = 0; + hash->step = step; + + if (latitude < lat_range->min || latitude > lat_range->max || + longitude < long_range->min || longitude > long_range->max) { + return 0; + } + + double lat_offset = + (latitude - lat_range->min) / (lat_range->max - lat_range->min); + double long_offset = + (longitude - long_range->min) / (long_range->max - long_range->min); + + /* convert to fixed point based on the step size */ + lat_offset *= (1ULL << step); + long_offset *= (1ULL << step); + hash->bits = interleave64(lat_offset, long_offset); + return 1; +} + +int geohashEncodeType(double longitude, double latitude, uint8_t step, GeoHashBits *hash) { + GeoHashRange r[2] = {{0}}; + geohashGetCoordRange(&r[0], &r[1]); + return geohashEncode(&r[0], &r[1], longitude, latitude, step, hash); +} + +int geohashEncodeWGS84(double longitude, double latitude, uint8_t step, + GeoHashBits *hash) { + return geohashEncodeType(longitude, latitude, step, hash); +} + +int geohashDecode(const GeoHashRange long_range, const GeoHashRange lat_range, + const GeoHashBits hash, GeoHashArea *area) { + if (HASHISZERO(hash) || NULL == area || RANGEISZERO(lat_range) || + RANGEISZERO(long_range)) { + return 0; + } + + area->hash = hash; + uint8_t step = hash.step; + uint64_t hash_sep = deinterleave64(hash.bits); /* hash = [LAT][LONG] */ + + double lat_scale = lat_range.max - lat_range.min; + double long_scale = long_range.max - long_range.min; + + uint32_t ilato = hash_sep; /* get lat part of deinterleaved hash */ + uint32_t ilono = hash_sep >> 32; /* shift over to get long part of hash */ + + /* divide by 2**step. + * Then, for 0-1 coordinate, multiply times scale and add + to the min to get the absolute coordinate. */ + area->latitude.min = + lat_range.min + (ilato * 1.0 / (1ull << step)) * lat_scale; + area->latitude.max = + lat_range.min + ((ilato + 1) * 1.0 / (1ull << step)) * lat_scale; + area->longitude.min = + long_range.min + (ilono * 1.0 / (1ull << step)) * long_scale; + area->longitude.max = + long_range.min + ((ilono + 1) * 1.0 / (1ull << step)) * long_scale; + + return 1; +} + +int geohashDecodeType(const GeoHashBits hash, GeoHashArea *area) { + GeoHashRange r[2] = {{0}}; + geohashGetCoordRange(&r[0], &r[1]); + return geohashDecode(r[0], r[1], hash, area); +} + +int geohashDecodeWGS84(const GeoHashBits hash, GeoHashArea *area) { + return geohashDecodeType(hash, area); +} + +int geohashDecodeAreaToLongLat(const GeoHashArea *area, double *xy) { + if (!xy) return 0; + xy[0] = (area->longitude.min + area->longitude.max) / 2; + xy[1] = (area->latitude.min + area->latitude.max) / 2; + return 1; +} + +int geohashDecodeToLongLatType(const GeoHashBits hash, double *xy) { + GeoHashArea area = {{0}}; + if (!xy || !geohashDecodeType(hash, &area)) + return 0; + return geohashDecodeAreaToLongLat(&area, xy); +} + +int geohashDecodeToLongLatWGS84(const GeoHashBits hash, double *xy) { + return geohashDecodeToLongLatType(hash, xy); +} + +static void geohash_move_x(GeoHashBits *hash, int8_t d) { + if (d == 0) + return; + + uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaULL; + uint64_t y = hash->bits & 0x5555555555555555ULL; + + uint64_t zz = 0x5555555555555555ULL >> (64 - hash->step * 2); + + if (d > 0) { + x = x + (zz + 1); + } else { + x = x | zz; + x = x - (zz + 1); + } + + x &= (0xaaaaaaaaaaaaaaaaULL >> (64 - hash->step * 2)); + hash->bits = (x | y); +} + +static void geohash_move_y(GeoHashBits *hash, int8_t d) { + if (d == 0) + return; + + uint64_t x = hash->bits & 0xaaaaaaaaaaaaaaaaULL; + uint64_t y = hash->bits & 0x5555555555555555ULL; + + uint64_t zz = 0xaaaaaaaaaaaaaaaaULL >> (64 - hash->step * 2); + if (d > 0) { + y = y + (zz + 1); + } else { + y = y | zz; + y = y - (zz + 1); + } + y &= (0x5555555555555555ULL >> (64 - hash->step * 2)); + hash->bits = (x | y); +} + +void geohashNeighbors(const GeoHashBits *hash, GeoHashNeighbors *neighbors) { + neighbors->east = *hash; + neighbors->west = *hash; + neighbors->north = *hash; + neighbors->south = *hash; + neighbors->south_east = *hash; + neighbors->south_west = *hash; + neighbors->north_east = *hash; + neighbors->north_west = *hash; + + geohash_move_x(&neighbors->east, 1); + geohash_move_y(&neighbors->east, 0); + + geohash_move_x(&neighbors->west, -1); + geohash_move_y(&neighbors->west, 0); + + geohash_move_x(&neighbors->south, 0); + geohash_move_y(&neighbors->south, -1); + + geohash_move_x(&neighbors->north, 0); + geohash_move_y(&neighbors->north, 1); + + geohash_move_x(&neighbors->north_west, -1); + geohash_move_y(&neighbors->north_west, 1); + + geohash_move_x(&neighbors->north_east, 1); + geohash_move_y(&neighbors->north_east, 1); + + geohash_move_x(&neighbors->south_east, 1); + geohash_move_y(&neighbors->south_east, -1); + + geohash_move_x(&neighbors->south_west, -1); + geohash_move_y(&neighbors->south_west, -1); +} diff --git a/tools/pika_migrate/src/pika_geohash_helper.cc b/tools/pika_migrate/src/pika_geohash_helper.cc new file mode 100644 index 0000000000..a2f18d7090 --- /dev/null +++ b/tools/pika_migrate/src/pika_geohash_helper.cc @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2013-2014, yinqiwen + * Copyright (c) 2014, Matt Stancliff . + * Copyright (c) 2015-2016, Salvatore Sanfilippo . + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + * THE POSSIBILITY OF SUCH DAMAGE. + */ + +/* This is a C++ to C conversion from the ardb project. + * This file started out as: + * https://github.com/yinqiwen/ardb/blob/d42503/src/geo/geohash_helper.cpp + */ + +//#include "fmacros.h" +#include "include/pika_geohash_helper.h" +//#include "debugmacro.h" +#include + +#define D_R (M_PI / 180.0) +#define R_MAJOR 6378137.0 +#define R_MINOR 6356752.3142 +#define RATIO (R_MINOR / R_MAJOR) +#define ECCENT (sqrt(1.0 - (RATIO *RATIO))) +#define COM (0.5 * ECCENT) + +/// @brief The usual PI/180 constant +const double DEG_TO_RAD = 0.017453292519943295769236907684886; +/// @brief Earth's quatratic mean radius for WGS-84 +const double EARTH_RADIUS_IN_METERS = 6372797.560856; + +const double MERCATOR_MAX = 20037726.37; +const double MERCATOR_MIN = -20037726.37; + +static inline double deg_rad(double ang) { return ang * D_R; } +static inline double rad_deg(double ang) { return ang / D_R; } + +/* This function is used in order to estimate the step (bits precision) + * of the 9 search area boxes during radius queries. */ +uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) { + if (range_meters == 0) return 26; + int step = 1; + while (range_meters < MERCATOR_MAX) { + range_meters *= 2; + step++; + } + step -= 2; /* Make sure range is included in most of the base cases. */ + + /* Wider range torwards the poles... Note: it is possible to do better + * than this approximation by computing the distance between meridians + * at this latitude, but this does the trick for now. */ + if (lat > 66 || lat < -66) { + step--; + if (lat > 80 || lat < -80) step--; + } + + /* Frame to valid range. */ + if (step < 1) step = 1; + if (step > 26) step = 26; + return step; +} + +/* Return the bounding box of the search area centered at latitude,longitude + * having a radius of radius_meter. bounds[0] - bounds[2] is the minimum + * and maxium longitude, while bounds[1] - bounds[3] is the minimum and + * maximum latitude. + * + * This function does not behave correctly with very large radius values, for + * instance for the coordinates 81.634948934258375 30.561509253718668 and a + * radius of 7083 kilometers, it reports as bounding boxes: + * + * min_lon 7.680495, min_lat -33.119473, max_lon 155.589402, max_lat 94.242491 + * + * However, for instance, a min_lon of 7.680495 is not correct, because the + * point -1.27579540014266968 61.33421815228281559 is at less than 7000 + * kilometers away. + * + * Since this function is currently only used as an optimization, the + * optimization is not used for very big radiuses, however the function + * should be fixed. */ +int geohashBoundingBox(double longitude, double latitude, double radius_meters, + double *bounds) { + if (!bounds) return 0; + + bounds[0] = longitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude))); + bounds[2] = longitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS/cos(deg_rad(latitude))); + bounds[1] = latitude - rad_deg(radius_meters/EARTH_RADIUS_IN_METERS); + bounds[3] = latitude + rad_deg(radius_meters/EARTH_RADIUS_IN_METERS); + return 1; +} + +/* Return a set of areas (center + 8) that are able to cover a range query + * for the specified position and radius. */ +GeoHashRadius geohashGetAreasByRadius(double longitude, double latitude, double radius_meters) { + GeoHashRange long_range, lat_range; + GeoHashRadius radius; + GeoHashBits hash; + GeoHashNeighbors neighbors; + GeoHashArea area; + double min_lon, max_lon, min_lat, max_lat; + double bounds[4]; + int steps; + + geohashBoundingBox(longitude, latitude, radius_meters, bounds); + min_lon = bounds[0]; + min_lat = bounds[1]; + max_lon = bounds[2]; + max_lat = bounds[3]; + + steps = geohashEstimateStepsByRadius(radius_meters,latitude); + + geohashGetCoordRange(&long_range,&lat_range); + geohashEncode(&long_range,&lat_range,longitude,latitude,steps,&hash); + geohashNeighbors(&hash,&neighbors); + geohashDecode(long_range,lat_range,hash,&area); + + /* Check if the step is enough at the limits of the covered area. + * Sometimes when the search area is near an edge of the + * area, the estimated step is not small enough, since one of the + * north / south / west / east square is too near to the search area + * to cover everything. */ + int decrease_step = 0; + { + GeoHashArea north, south, east, west; + + geohashDecode(long_range, lat_range, neighbors.north, &north); + geohashDecode(long_range, lat_range, neighbors.south, &south); + geohashDecode(long_range, lat_range, neighbors.east, &east); + geohashDecode(long_range, lat_range, neighbors.west, &west); + + if (geohashGetDistance(longitude,latitude,longitude,north.latitude.max) + < radius_meters) decrease_step = 1; + if (geohashGetDistance(longitude,latitude,longitude,south.latitude.min) + < radius_meters) decrease_step = 1; + if (geohashGetDistance(longitude,latitude,east.longitude.max,latitude) + < radius_meters) decrease_step = 1; + if (geohashGetDistance(longitude,latitude,west.longitude.min,latitude) + < radius_meters) decrease_step = 1; + } + + if (steps > 1 && decrease_step) { + steps--; + geohashEncode(&long_range,&lat_range,longitude,latitude,steps,&hash); + geohashNeighbors(&hash,&neighbors); + geohashDecode(long_range,lat_range,hash,&area); + } + + /* Exclude the search areas that are useless. */ + if (steps >= 2) { + if (area.latitude.min < min_lat) { + GZERO(neighbors.south); + GZERO(neighbors.south_west); + GZERO(neighbors.south_east); + } + if (area.latitude.max > max_lat) { + GZERO(neighbors.north); + GZERO(neighbors.north_east); + GZERO(neighbors.north_west); + } + if (area.longitude.min < min_lon) { + GZERO(neighbors.west); + GZERO(neighbors.south_west); + GZERO(neighbors.north_west); + } + if (area.longitude.max > max_lon) { + GZERO(neighbors.east); + GZERO(neighbors.south_east); + GZERO(neighbors.north_east); + } + } + radius.hash = hash; + radius.neighbors = neighbors; + radius.area = area; + return radius; +} + +GeoHashRadius geohashGetAreasByRadiusWGS84(double longitude, double latitude, + double radius_meters) { + return geohashGetAreasByRadius(longitude, latitude, radius_meters); +} + +GeoHashFix52Bits geohashAlign52Bits(const GeoHashBits hash) { + uint64_t bits = hash.bits; + bits <<= (52 - hash.step * 2); + return bits; +} + +/* Calculate distance using haversin great circle distance formula. */ +double geohashGetDistance(double lon1d, double lat1d, double lon2d, double lat2d) { + double lat1r, lon1r, lat2r, lon2r, u, v; + lat1r = deg_rad(lat1d); + lon1r = deg_rad(lon1d); + lat2r = deg_rad(lat2d); + lon2r = deg_rad(lon2d); + u = sin((lat2r - lat1r) / 2); + v = sin((lon2r - lon1r) / 2); + return 2.0 * EARTH_RADIUS_IN_METERS * + asin(sqrt(u * u + cos(lat1r) * cos(lat2r) * v * v)); +} + +int geohashGetDistanceIfInRadius(double x1, double y1, + double x2, double y2, double radius, + double *distance) { + *distance = geohashGetDistance(x1, y1, x2, y2); + if (*distance > radius) return 0; + return 1; +} + +int geohashGetDistanceIfInRadiusWGS84(double x1, double y1, double x2, + double y2, double radius, + double *distance) { + return geohashGetDistanceIfInRadius(x1, y1, x2, y2, radius, distance); +} diff --git a/tools/pika_migrate/src/pika_hash.cc b/tools/pika_migrate/src/pika_hash.cc new file mode 100644 index 0000000000..2f0e4fdf7d --- /dev/null +++ b/tools/pika_migrate/src/pika_hash.cc @@ -0,0 +1,609 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_hash.h" + +#include "slash/include/slash_string.h" + +#include "include/pika_conf.h" + +extern PikaConf *g_pika_conf; + +void HDelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHDel); + return; + } + key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + iter++; + fields_.assign(iter, argv_.end()); + return; +} + +void HDelCmd::Do(std::shared_ptr partition) { + int32_t num = 0; + rocksdb::Status s = partition->db()->HDel(key_, fields_, &num); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(num); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HSetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHSet); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + value_ = argv_[3]; + return; +} + +void HSetCmd::Do(std::shared_ptr partition) { + int32_t ret = 0; + rocksdb::Status s = partition->db()->HSet(key_, field_, value_, &ret); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(ret)); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HGetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHGet); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + return; +} + +void HGetCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->HGet(key_, field_, &value); + if (s.ok()) { + res_.AppendStringLen(value.size()); + res_.AppendContent(value); + } else if (s.IsNotFound()) { + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void HGetallCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHGetall); + return; + } + key_ = argv_[1]; + return; +} + +void HGetallCmd::Do(std::shared_ptr partition) { + int64_t total_fv = 0; + int64_t cursor = 0, next_cursor = 0; + size_t raw_limit = g_pika_conf->max_client_response_size(); + std::string raw; + rocksdb::Status s; + std::vector fvs; + + do { + fvs.clear(); + s = partition->db()->HScan(key_, cursor, "*", PIKA_SCAN_STEP_LENGTH, &fvs, &next_cursor); + if (!s.ok()) { + raw.clear(); + total_fv = 0; + break; + } else { + for (const auto& fv : fvs) { + RedisAppendLen(raw, fv.field.size(), "$"); + RedisAppendContent(raw, fv.field); + RedisAppendLen(raw, fv.value.size(), "$"); + RedisAppendContent(raw, fv.value); + } + if (raw.size() >= raw_limit) { + res_.SetRes(CmdRes::kErrOther, "Response exceeds the max-client-response-size limit"); + return; + } + total_fv += fvs.size(); + cursor = next_cursor; + } + } while (cursor != 0); + + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(total_fv * 2); + res_.AppendStringRaw(raw); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + + +void HExistsCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHExists); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + return; +} + +void HExistsCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->HExists(key_, field_); + if (s.ok()) { + res_.AppendContent(":1"); + } else if (s.IsNotFound()) { + res_.AppendContent(":0"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void HIncrbyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHIncrby); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + if (argv_[3].find(" ") != std::string::npos || !slash::string2l(argv_[3].data(), argv_[3].size(), &by_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void HIncrbyCmd::Do(std::shared_ptr partition) { + int64_t new_value; + rocksdb::Status s = partition->db()->HIncrby(key_, field_, by_, &new_value); + if (s.ok() || s.IsNotFound()) { + res_.AppendContent(":" + std::to_string(new_value)); + } else if (s.IsCorruption() && s.ToString() == "Corruption: hash value is not an integer") { + res_.SetRes(CmdRes::kInvalidInt); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HIncrbyfloatCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHIncrbyfloat); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + by_ = argv_[3]; + return; +} + +void HIncrbyfloatCmd::Do(std::shared_ptr partition) { + std::string new_value; + rocksdb::Status s = partition->db()->HIncrbyfloat(key_, field_, by_, &new_value); + if (s.ok()) { + res_.AppendStringLen(new_value.size()); + res_.AppendContent(new_value); + } else if (s.IsCorruption() && s.ToString() == "Corruption: value is not a vaild float") { + res_.SetRes(CmdRes::kInvalidFloat); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HKeysCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHKeys); + return; + } + key_ = argv_[1]; + return; +} + +void HKeysCmd::Do(std::shared_ptr partition) { + std::vector fields; + rocksdb::Status s = partition->db()->HKeys(key_, &fields); + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(fields.size()); + for (const auto& field : fields) { + res_.AppendString(field); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HLenCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHLen); + return; + } + key_ = argv_[1]; + return; +} + +void HLenCmd::Do(std::shared_ptr partition) { + int32_t len = 0; + rocksdb::Status s = partition->db()->HLen(key_, &len); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(len); + } else { + res_.SetRes(CmdRes::kErrOther, "something wrong in hlen"); + } + return; +} + +void HMgetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHMget); + return; + } + key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + iter++; + fields_.assign(iter, argv_.end()); + return; +} + +void HMgetCmd::Do(std::shared_ptr partition) { + std::vector vss; + rocksdb::Status s = partition->db()->HMGet(key_, fields_, &vss); + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(vss.size()); + for (const auto& vs : vss) { + if (vs.status.ok()) { + res_.AppendStringLen(vs.value.size()); + res_.AppendContent(vs.value); + } else { + res_.AppendContent("$-1"); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HMsetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHMset); + return; + } + key_ = argv_[1]; + size_t argc = argv_.size(); + if (argc % 2 != 0) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHMset); + return; + } + size_t index = 2; + fvs_.clear(); + for (; index < argc; index += 2) { + fvs_.push_back({argv_[index], argv_[index + 1]}); + } + return; +} + +void HMsetCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->HMSet(key_, fvs_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HSetnxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHSetnx); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + value_ = argv_[3]; + return; +} + +void HSetnxCmd::Do(std::shared_ptr partition) { + int32_t ret = 0; + rocksdb::Status s = partition->db()->HSetnx(key_, field_, value_, &ret); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(ret)); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void HStrlenCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHStrlen); + return; + } + key_ = argv_[1]; + field_ = argv_[2]; + return; +} + +void HStrlenCmd::Do(std::shared_ptr partition) { + int32_t len = 0; + rocksdb::Status s = partition->db()->HStrlen(key_, field_, &len); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(len); + } else { + res_.SetRes(CmdRes::kErrOther, "something wrong in hstrlen"); + } + return; +} + +void HValsCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHVals); + return; + } + key_ = argv_[1]; + return; +} + +void HValsCmd::Do(std::shared_ptr partition) { + std::vector values; + rocksdb::Status s = partition->db()->HVals(key_, &values); + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(values.size()); + for (const auto& value : values) { + res_.AppendStringLen(value.size()); + res_.AppendContent(value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHScan); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &cursor_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + size_t index = 3, argc = argv_.size(); + + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + if (count_ < 0) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + return; +} + +void HScanCmd::Do(std::shared_ptr partition) { + int64_t next_cursor = 0; + std::vector field_values; + rocksdb::Status s = partition->db()->HScan(key_, cursor_, pattern_, count_, &field_values, &next_cursor); + + if (s.ok() || s.IsNotFound()) { + res_.AppendContent("*2"); + char buf[32]; + int32_t len = slash::ll2string(buf, sizeof(buf), next_cursor); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(field_values.size()*2); + for (const auto& field_value : field_values) { + res_.AppendString(field_value.field); + res_.AppendString(field_value.value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void HScanxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameHScan); + return; + } + key_ = argv_[1]; + start_field_ = argv_[2]; + + size_t index = 3, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + if (count_ < 0) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + return; +} + +void HScanxCmd::Do(std::shared_ptr partition) { + std::string next_field; + std::vector field_values; + rocksdb::Status s = partition->db()->HScanx(key_, start_field_, pattern_, count_, &field_values, &next_field); + + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(2); + res_.AppendStringLen(next_field.size()); + res_.AppendContent(next_field); + + res_.AppendArrayLen(2 * field_values.size()); + for (const auto& field_value : field_values) { + res_.AppendString(field_value.field); + res_.AppendString(field_value.value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void PKHScanRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKHScanRange); + return; + } + key_ = argv_[1]; + field_start_ = argv_[2]; + field_end_ = argv_[3]; + + size_t index = 4, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "limit")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &limit_) || limit_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void PKHScanRangeCmd::Do(std::shared_ptr partition) { + std::string next_field; + std::vector field_values; + rocksdb::Status s = partition->db()->PKHScanRange(key_, field_start_, field_end_, + pattern_, limit_, &field_values, &next_field); + + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(2); + res_.AppendString(next_field); + + res_.AppendArrayLen(2 * field_values.size()); + for (const auto& field_value : field_values) { + res_.AppendString(field_value.field); + res_.AppendString(field_value.value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void PKHRScanRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKHRScanRange); + return; + } + key_ = argv_[1]; + field_start_ = argv_[2]; + field_end_ = argv_[3]; + + size_t index = 4, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "limit")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &limit_) || limit_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void PKHRScanRangeCmd::Do(std::shared_ptr partition) { + std::string next_field; + std::vector field_values; + rocksdb::Status s = partition->db()->PKHRScanRange(key_, field_start_, field_end_, + pattern_, limit_, &field_values, &next_field); + + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(2); + res_.AppendString(next_field); + + res_.AppendArrayLen(2 * field_values.size()); + for (const auto& field_value : field_values) { + res_.AppendString(field_value.field); + res_.AppendString(field_value.value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} diff --git a/tools/pika_migrate/src/pika_hyperloglog.cc b/tools/pika_migrate/src/pika_hyperloglog.cc new file mode 100644 index 0000000000..e36cff7d81 --- /dev/null +++ b/tools/pika_migrate/src/pika_hyperloglog.cc @@ -0,0 +1,73 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_hyperloglog.h" + +void PfAddCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePfAdd); + return; + } + if (argv_.size() > 1) { + key_ = argv_[1]; + size_t pos = 2; + while (pos < argv_.size()) { + values_.push_back(argv_[pos++]); + } + } +} + +void PfAddCmd::Do(std::shared_ptr partition) { + bool update = false; + rocksdb::Status s = partition->db()->PfAdd(key_, values_, &update); + if (s.ok() && update) { + res_.AppendInteger(1); + } else if (s.ok() && !update) { + res_.AppendInteger(0); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void PfCountCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePfCount); + return; + } + size_t pos = 1; + while (pos < argv_.size()) { + keys_.push_back(argv_[pos++]); + } +} + +void PfCountCmd::Do(std::shared_ptr partition) { + int64_t value_ = 0; + rocksdb::Status s = partition->db()->PfCount(keys_, &value_); + if (s.ok()) { + res_.AppendInteger(value_); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void PfMergeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePfMerge); + return; + } + size_t pos = 1; + while (pos < argv_.size()) { + keys_.push_back(argv_[pos++]); + } +} + +void PfMergeCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->PfMerge(keys_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} diff --git a/tools/pika_migrate/src/pika_inner_message.proto b/tools/pika_migrate/src/pika_inner_message.proto new file mode 100644 index 0000000000..713c05077a --- /dev/null +++ b/tools/pika_migrate/src/pika_inner_message.proto @@ -0,0 +1,145 @@ +package InnerMessage; + +enum Type { + kMetaSync = 1; + kTrySync = 2; + kDBSync = 3; + kBinlogSync = 4; + kHeatBeat = 5; + kRemoveSlaveNode = 6; +} + +enum StatusCode { + kOk = 1; + kError = 2; +} + +message BinlogOffset { + required uint32 filenum = 1; + required uint64 offset = 2; +} + +message Node { + required string ip = 1; + required int32 port = 2; +} + +message Partition { + required string table_name = 1; + required uint32 partition_id = 2; +} + +message TableInfo { + required string table_name = 1; + required uint32 partition_num = 2; + repeated uint32 partition_ids = 3; +} + +message PikaMeta { + repeated TableInfo table_infos = 1; +} + +// Request message +message InnerRequest { + // slave to master + message MetaSync { + required Node node = 1; + optional string auth = 2; + } + + // slave to master + message TrySync { + required Node node = 1; + required Partition partition = 2; + required BinlogOffset binlog_offset = 3; + } + + // slave to master + message DBSync { + required Node node = 1; + required Partition partition = 2; + required BinlogOffset binlog_offset = 3; + } + + message BinlogSync { + required Node node = 1; + required string table_name = 2; + required uint32 partition_id = 3; + required BinlogOffset ack_range_start = 4; + required BinlogOffset ack_range_end = 5; + required int32 session_id = 6; + required bool first_send = 7; + } + + message RemoveSlaveNode { + required Node node = 1; + required Partition partition = 2; + } + + required Type type = 1; + optional MetaSync meta_sync = 2; + optional TrySync try_sync = 3; + optional DBSync db_sync = 4; + optional BinlogSync binlog_sync = 5; + repeated RemoveSlaveNode remove_slave_node = 6; +} + +message PartitionInfo { + required uint32 partition_id = 1; + required Node master = 2; + repeated Node slaves = 3; +} + +// Response message +message InnerResponse { + // master to slave + message MetaSync { + message TableInfo { + required string table_name = 1; + required int32 partition_num = 2; + } + required bool classic_mode = 1; + repeated TableInfo tables_info = 2; + } + + // master to slave + message TrySync { + enum ReplyCode { + kOk = 1; + kSyncPointBePurged = 2; + kSyncPointLarger = 3; + kError = 4; + } + required ReplyCode reply_code = 1; + required Partition partition = 2; + optional BinlogOffset binlog_offset = 3; + optional int32 session_id = 4; + } + + message DBSync { + required Partition partition = 1; + required int32 session_id = 2; + } + + // master to slave + message BinlogSync { + required Partition partition = 1; + required BinlogOffset binlog_offset = 2; + required bytes binlog = 3; + required int32 session_id = 4; + } + + message RemoveSlaveNode { + required Node node = 1; + required Partition partition = 2; + } + + required Type type = 1; + required StatusCode code = 2; + optional string reply = 3; + optional MetaSync meta_sync = 4; + optional DBSync db_sync = 5; + optional TrySync try_sync = 6; + repeated BinlogSync binlog_sync = 7; + repeated RemoveSlaveNode remove_slave_node = 8; +} diff --git a/tools/pika_migrate/src/pika_kv.cc b/tools/pika_migrate/src/pika_kv.cc new file mode 100644 index 0000000000..732878b05b --- /dev/null +++ b/tools/pika_migrate/src/pika_kv.cc @@ -0,0 +1,1447 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_kv.h" + +#include "slash/include/slash_string.h" + +#include "include/pika_conf.h" +#include "include/pika_binlog_transverter.h" + +extern PikaConf *g_pika_conf; + +/* SET key value [NX] [XX] [EX ] [PX ] */ +void SetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSet); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; + condition_ = SetCmd::kNONE; + sec_ = 0; + size_t index = 3; + while (index != argv_.size()) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "xx")) { + condition_ = SetCmd::kXX; + } else if (!strcasecmp(opt.data(), "nx")) { + condition_ = SetCmd::kNX; + } else if (!strcasecmp(opt.data(), "vx")) { + condition_ = SetCmd::kVX; + index++; + if (index == argv_.size()) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } else { + target_ = argv_[index]; + } + } else if (!strcasecmp(opt.data(), "ex") || !strcasecmp(opt.data(), "px")) { + condition_ = (condition_ == SetCmd::kNONE) ? SetCmd::kEXORPX : condition_; + index++; + if (index == argv_.size()) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!slash::string2l(argv_[index].data(), argv_[index].size(), &sec_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } else if (sec_ <= 0) { + res_.SetRes(CmdRes::kErrOther, "invalid expire time in set"); + return; + } + + if (!strcasecmp(opt.data(), "px")) { + sec_ /= 1000; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void SetCmd::Do(std::shared_ptr partition) { + rocksdb::Status s; + int32_t res = 1; + switch (condition_) { + case SetCmd::kXX: + s = partition->db()->Setxx(key_, value_, &res, sec_); + break; + case SetCmd::kNX: + s = partition->db()->Setnx(key_, value_, &res, sec_); + break; + case SetCmd::kVX: + s = partition->db()->Setvx(key_, target_, value_, &success_, sec_); + break; + case SetCmd::kEXORPX: + s = partition->db()->Setex(key_, value_, sec_); + break; + default: + s = partition->db()->Set(key_, value_); + break; + } + + if (s.ok() || s.IsNotFound()) { + if (condition_ == SetCmd::kVX) { + res_.AppendInteger(success_); + } else { + if (res == 1) { + res_.SetRes(CmdRes::kOk); + } else { + res_.AppendArrayLen(-1);; + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +std::string SetCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + if (condition_ == SetCmd::kEXORPX) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 4, "*"); + + // to pksetexat cmd + std::string pksetexat_cmd("pksetexat"); + RedisAppendLen(content, pksetexat_cmd.size(), "$"); + RedisAppendContent(content, pksetexat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // time_stamp + char buf[100]; + int32_t time_stamp = time(nullptr) + sec_; + slash::ll2string(buf, 100, time_stamp); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + // value + RedisAppendLen(content, value_.size(), "$"); + RedisAppendContent(content, value_); + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); + } else { + return Cmd::ToBinlog(exec_time, server_id, logic_id, filenum, offset); + } +} + +void GetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGet); + return; + } + key_ = argv_[1]; + return; +} + +void GetCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->Get(key_, &value); + if (s.ok()) { + res_.AppendStringLen(value.size()); + res_.AppendContent(value); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void DelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDel); + return; + } + std::vector::iterator iter = argv_.begin(); + keys_.assign(++iter, argv_.end()); + return; +} + +void DelCmd::Do(std::shared_ptr partition) { + std::map type_status; + int64_t count = partition->db()->Del(keys_, &type_status); + if (count >= 0) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, "delete error"); + } + return; +} + +void IncrCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameIncr); + return; + } + key_ = argv_[1]; + return; +} + +void IncrCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Incrby(key_, 1, &new_value_); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(new_value_)); + } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { + res_.SetRes(CmdRes::kInvalidInt); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void IncrbyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameIncrby); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &by_)) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameIncrby); + return; + } + return; +} + +void IncrbyCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Incrby(key_, by_, &new_value_); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(new_value_)); + } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { + res_.SetRes(CmdRes::kInvalidInt); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void IncrbyfloatCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameIncrbyfloat); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; + if (!slash::string2d(argv_[2].data(), argv_[2].size(), &by_)) { + res_.SetRes(CmdRes::kInvalidFloat); + return; + } + return; +} + +void IncrbyfloatCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Incrbyfloat(key_, value_, &new_value_); + if (s.ok()) { + res_.AppendStringLen(new_value_.size()); + res_.AppendContent(new_value_); + } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a vaild float"){ + res_.SetRes(CmdRes::kInvalidFloat); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void DecrCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDecr); + return; + } + key_ = argv_[1]; + return; +} + +void DecrCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Decrby(key_, 1, &new_value_); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(new_value_)); + } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { + res_.SetRes(CmdRes::kInvalidInt); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void DecrbyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDecrby); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &by_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void DecrbyCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Decrby(key_, by_, &new_value_); + if (s.ok()) { + res_.AppendContent(":" + std::to_string(new_value_)); + } else if (s.IsCorruption() && s.ToString() == "Corruption: Value is not a integer") { + res_.SetRes(CmdRes::kInvalidInt); + } else if (s.IsInvalidArgument()) { + res_.SetRes(CmdRes::kOverFlow); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void GetsetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGetset); + return; + } + key_ = argv_[1]; + new_value_ = argv_[2]; + return; +} + +void GetsetCmd::Do(std::shared_ptr partition) { + std::string old_value; + rocksdb::Status s = partition->db()->GetSet(key_, new_value_, &old_value); + if (s.ok()) { + if (old_value.empty()) { + res_.AppendContent("$-1"); + } else { + res_.AppendStringLen(old_value.size()); + res_.AppendContent(old_value); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void AppendCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameAppend); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; + return; +} + +void AppendCmd::Do(std::shared_ptr partition) { + int32_t new_len = 0; + rocksdb::Status s = partition->db()->Append(key_, value_, &new_len); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(new_len); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void MgetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMget); + return; + } + keys_ = argv_; + keys_.erase(keys_.begin()); + return; +} + +void MgetCmd::Do(std::shared_ptr partition) { + std::vector vss; + rocksdb::Status s = partition->db()->MGet(keys_, &vss); + if (s.ok()) { + res_.AppendArrayLen(vss.size()); + for (const auto& vs : vss) { + if (vs.status.ok()) { + res_.AppendStringLen(vs.value.size()); + res_.AppendContent(vs.value); + } else { + res_.AppendContent("$-1"); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void KeysCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameKeys); + return; + } + pattern_ = argv_[1]; + if (argv_.size() == 3) { + std::string opt = argv_[2]; + if (!strcasecmp(opt.data(), "string")) { + type_ = blackwidow::DataType::kStrings; + } else if (!strcasecmp(opt.data(), "zset")) { + type_ = blackwidow::DataType::kZSets; + } else if (!strcasecmp(opt.data(), "set")) { + type_ = blackwidow::DataType::kSets; + } else if (!strcasecmp(opt.data(), "list")) { + type_ = blackwidow::DataType::kLists; + } else if (!strcasecmp(opt.data(), "hash")) { + type_ = blackwidow::DataType::kHashes; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + } + } else if (argv_.size() > 3) { + res_.SetRes(CmdRes::kSyntaxErr); + } + return; +} + +void KeysCmd::Do(std::shared_ptr partition) { + int64_t total_key = 0; + int64_t cursor = 0; + size_t raw_limit = g_pika_conf->max_client_response_size(); + std::string raw; + std::vector keys; + do { + keys.clear(); + cursor = partition->db()->Scan(type_, cursor, pattern_, PIKA_SCAN_STEP_LENGTH, &keys); + for (const auto& key : keys) { + RedisAppendLen(raw, key.size(), "$"); + RedisAppendContent(raw, key); + } + if (raw.size() >= raw_limit) { + res_.SetRes(CmdRes::kErrOther, "Response exceeds the max-client-response-size limit"); + return; + } + total_key += keys.size(); + } while (cursor != 0); + + res_.AppendArrayLen(total_key); + res_.AppendStringRaw(raw); + return; +} + +void SetnxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSetnx); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; + return; +} + +void SetnxCmd::Do(std::shared_ptr partition) { + success_ = 0; + rocksdb::Status s = partition->db()->Setnx(key_, value_, &success_); + if (s.ok()) { + res_.AppendInteger(success_); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +std::string SetnxCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + if (success_) { + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 3, "*"); + + // to set cmd + std::string set_cmd("set"); + RedisAppendLen(content, set_cmd.size(), "$"); + RedisAppendContent(content, set_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // value + RedisAppendLen(content, value_.size(), "$"); + RedisAppendContent(content, value_); + + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); + } + return content; +} + +void SetexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSetex); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &sec_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + value_ = argv_[3]; + return; +} + +void SetexCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Setex(key_, value_, sec_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +std::string SetexCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 4, "*"); + + // to pksetexat cmd + std::string pksetexat_cmd("pksetexat"); + RedisAppendLen(content, pksetexat_cmd.size(), "$"); + RedisAppendContent(content, pksetexat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // time_stamp + char buf[100]; + int32_t time_stamp = time(nullptr) + sec_; + slash::ll2string(buf, 100, time_stamp); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + // value + RedisAppendLen(content, value_.size(), "$"); + RedisAppendContent(content, value_); + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void PsetexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePsetex); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &usec_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + value_ = argv_[3]; + return; +} + +void PsetexCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Setex(key_, value_, usec_ / 1000); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +std::string PsetexCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 4, "*"); + + // to pksetexat cmd + std::string pksetexat_cmd("pksetexat"); + RedisAppendLen(content, pksetexat_cmd.size(), "$"); + RedisAppendContent(content, pksetexat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // time_stamp + char buf[100]; + int32_t time_stamp = time(nullptr) + usec_ / 1000; + slash::ll2string(buf, 100, time_stamp); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + // value + RedisAppendLen(content, value_.size(), "$"); + RedisAppendContent(content, value_); + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void DelvxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameDelvx); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; + return; +} + +void DelvxCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->Delvx(key_, value_, &success_); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(success_); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void MsetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMset); + return; + } + size_t argc = argv_.size(); + if (argc % 2 == 0) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMset); + return; + } + kvs_.clear(); + for (size_t index = 1; index != argc; index += 2) { + kvs_.push_back({argv_[index], argv_[index + 1]}); + } + return; +} + +void MsetCmd::Do(std::shared_ptr partition) { + blackwidow::Status s = partition->db()->MSet(kvs_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void MsetnxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMsetnx); + return; + } + size_t argc = argv_.size(); + if (argc % 2 == 0) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameMsetnx); + return; + } + kvs_.clear(); + for (size_t index = 1; index != argc; index += 2) { + kvs_.push_back({argv_[index], argv_[index + 1]}); + } + return; +} + +void MsetnxCmd::Do(std::shared_ptr partition) { + success_ = 0; + rocksdb::Status s = partition->db()->MSetnx(kvs_, &success_); + if (s.ok()) { + res_.AppendInteger(success_); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void GetrangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameGetrange); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &start_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &end_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void GetrangeCmd::Do(std::shared_ptr partition) { + std::string substr; + rocksdb::Status s = partition->db()->Getrange(key_, start_, end_, &substr); + if (s.ok() || s.IsNotFound()) { + res_.AppendStringLen(substr.size()); + res_.AppendContent(substr); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void SetrangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSetrange); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + value_ = argv_[3]; + return; +} + +void SetrangeCmd::Do(std::shared_ptr partition) { + int32_t new_len; + rocksdb::Status s = partition->db()->Setrange(key_, offset_, value_, &new_len); + if (s.ok()) { + res_.AppendInteger(new_len); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void StrlenCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameStrlen); + return; + } + key_ = argv_[1]; + return; +} + +void StrlenCmd::Do(std::shared_ptr partition) { + int32_t len = 0; + rocksdb::Status s = partition->db()->Strlen(key_, &len); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(len); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ExistsCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameExists); + return; + } + keys_ = argv_; + keys_.erase(keys_.begin()); + return; +} + +void ExistsCmd::Do(std::shared_ptr partition) { + std::map type_status; + int64_t res = partition->db()->Exists(keys_, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "exists internal error"); + } + return; +} + +void ExpireCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameExpire); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &sec_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void ExpireCmd::Do(std::shared_ptr partition) { + std::map type_status; + int64_t res = partition->db()->Expire(key_, sec_, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "expire internal error"); + } + return; +} + +std::string ExpireCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, 3, "*"); + + // to expireat cmd + std::string expireat_cmd("expireat"); + RedisAppendLen(content, expireat_cmd.size(), "$"); + RedisAppendContent(content, expireat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // sec + char buf[100]; + int64_t expireat = time(nullptr) + sec_; + slash::ll2string(buf, 100, expireat); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void PexpireCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePexpire); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &msec_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void PexpireCmd::Do(std::shared_ptr partition) { + std::map type_status; + int64_t res = partition->db()->Expire(key_, msec_/1000, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "expire internal error"); + } + return; +} + +std::string PexpireCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, argv_.size(), "*"); + + // to expireat cmd + std::string expireat_cmd("expireat"); + RedisAppendLen(content, expireat_cmd.size(), "$"); + RedisAppendContent(content, expireat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // sec + char buf[100]; + int64_t expireat = time(nullptr) + msec_ / 1000; + slash::ll2string(buf, 100, expireat); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void ExpireatCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameExpireat); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &time_stamp_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void ExpireatCmd::Do(std::shared_ptr partition) { + std::map type_status; + int32_t res = partition->db()->Expireat(key_, time_stamp_, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "expireat internal error"); + } +} + +void PexpireatCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePexpireat); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &time_stamp_ms_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +std::string PexpireatCmd::ToBinlog( + uint32_t exec_time, + const std::string& server_id, + uint64_t logic_id, + uint32_t filenum, + uint64_t offset) { + std::string content; + content.reserve(RAW_ARGS_LEN); + RedisAppendLen(content, argv_.size(), "*"); + + // to expireat cmd + std::string expireat_cmd("expireat"); + RedisAppendLen(content, expireat_cmd.size(), "$"); + RedisAppendContent(content, expireat_cmd); + // key + RedisAppendLen(content, key_.size(), "$"); + RedisAppendContent(content, key_); + // sec + char buf[100]; + int64_t expireat = time_stamp_ms_ / 1000; + slash::ll2string(buf, 100, expireat); + std::string at(buf); + RedisAppendLen(content, at.size(), "$"); + RedisAppendContent(content, at); + + return PikaBinlogTransverter::BinlogEncode(BinlogType::TypeFirst, + exec_time, + std::stoi(server_id), + logic_id, + filenum, + offset, + content, + {}); +} + +void PexpireatCmd::Do(std::shared_ptr partition) { + std::map type_status; + int32_t res = partition->db()->Expireat(key_, time_stamp_ms_/1000, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "pexpireat internal error"); + } + return; +} + +void TtlCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameTtl); + return; + } + key_ = argv_[1]; + return; +} + +void TtlCmd::Do(std::shared_ptr partition) { + std::map type_timestamp; + std::map type_status; + type_timestamp = partition->db()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[blackwidow::kStrings] != -2) { + res_.AppendInteger(type_timestamp[blackwidow::kStrings]); + } else if (type_timestamp[blackwidow::kHashes] != -2) { + res_.AppendInteger(type_timestamp[blackwidow::kHashes]); + } else if (type_timestamp[blackwidow::kLists] != -2) { + res_.AppendInteger(type_timestamp[blackwidow::kLists]); + } else if (type_timestamp[blackwidow::kZSets] != -2) { + res_.AppendInteger(type_timestamp[blackwidow::kZSets]); + } else if (type_timestamp[blackwidow::kSets] != -2) { + res_.AppendInteger(type_timestamp[blackwidow::kSets]); + } else { + // mean this key not exist + res_.AppendInteger(-2); + } + return; +} + +void PttlCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePttl); + return; + } + key_ = argv_[1]; + return; +} + +void PttlCmd::Do(std::shared_ptr partition) { + std::map type_timestamp; + std::map type_status; + type_timestamp = partition->db()->TTL(key_, &type_status); + for (const auto& item : type_timestamp) { + // mean operation exception errors happen in database + if (item.second == -3) { + res_.SetRes(CmdRes::kErrOther, "ttl internal error"); + return; + } + } + if (type_timestamp[blackwidow::kStrings] != -2) { + if (type_timestamp[blackwidow::kStrings] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[blackwidow::kStrings] * 1000); + } + } else if (type_timestamp[blackwidow::kHashes] != -2) { + if (type_timestamp[blackwidow::kHashes] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[blackwidow::kHashes] * 1000); + } + } else if (type_timestamp[blackwidow::kLists] != -2) { + if (type_timestamp[blackwidow::kLists] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[blackwidow::kLists] * 1000); + } + } else if (type_timestamp[blackwidow::kSets] != -2) { + if (type_timestamp[blackwidow::kSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[blackwidow::kSets] * 1000); + } + } else if (type_timestamp[blackwidow::kZSets] != -2) { + if (type_timestamp[blackwidow::kZSets] == -1) { + res_.AppendInteger(-1); + } else { + res_.AppendInteger(type_timestamp[blackwidow::kZSets] * 1000); + } + } else { + // mean this key not exist + res_.AppendInteger(-2); + } + return; +} + +void PersistCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePersist); + return; + } + key_ = argv_[1]; + return; +} + +void PersistCmd::Do(std::shared_ptr partition) { + std::map type_status; + int32_t res = partition->db()->Persist(key_, &type_status); + if (res != -1) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, "persist internal error"); + } + return; +} + +void TypeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameType); + return; + } + key_ = argv_[1]; + return; +} + +void TypeCmd::Do(std::shared_ptr partition) { + std::string res; + rocksdb::Status s = partition->db()->Type(key_, &res); + if (s.ok()) { + res_.AppendContent("+" + res); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameScan); + return; + } + if (!slash::string2l(argv_[1].data(), argv_[1].size(), &cursor_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + size_t index = 2, argc = argv_.size(); + + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_) || count_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void ScanCmd::Do(std::shared_ptr partition) { + int64_t total_key = 0; + int64_t batch_count = 0; + int64_t left = count_; + int64_t cursor_ret = cursor_; + size_t raw_limit = g_pika_conf->max_client_response_size(); + std::string raw; + std::vector keys; + // To avoid memory overflow, we call the Scan method in batches + do { + keys.clear(); + batch_count = left < PIKA_SCAN_STEP_LENGTH ? left : PIKA_SCAN_STEP_LENGTH; + left = left > PIKA_SCAN_STEP_LENGTH ? left - PIKA_SCAN_STEP_LENGTH : 0; + cursor_ret = partition->db()->Scan(blackwidow::DataType::kAll, cursor_ret, + pattern_, batch_count, &keys); + for (const auto& key : keys) { + RedisAppendLen(raw, key.size(), "$"); + RedisAppendContent(raw, key); + } + if (raw.size() >= raw_limit) { + res_.SetRes(CmdRes::kErrOther, "Response exceeds the max-client-response-size limit"); + return; + } + total_key += keys.size(); + } while (cursor_ret != 0 && left); + + res_.AppendArrayLen(2); + + char buf[32]; + int len = slash::ll2string(buf, sizeof(buf), cursor_ret); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(total_key); + res_.AppendStringRaw(raw); + return; +} + +void ScanxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameScanx); + return; + } + if (!strcasecmp(argv_[1].data(), "string")) { + type_ = blackwidow::kStrings; + } else if (!strcasecmp(argv_[1].data(), "hash")) { + type_ = blackwidow::kHashes; + } else if (!strcasecmp(argv_[1].data(), "set")) { + type_ = blackwidow::kSets; + } else if (!strcasecmp(argv_[1].data(), "zset")) { + type_ = blackwidow::kZSets; + } else if (!strcasecmp(argv_[1].data(), "list")) { + type_ = blackwidow::kLists; + } else { + res_.SetRes(CmdRes::kInvalidDbType); + return; + } + + start_key_ = argv_[2]; + size_t index = 3, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_) || count_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void ScanxCmd::Do(std::shared_ptr partition) { + std::string next_key; + std::vector keys; + rocksdb::Status s = partition->db()->Scanx(type_, start_key_, pattern_, count_, &keys, &next_key); + + if (s.ok()) { + res_.AppendArrayLen(2); + res_.AppendStringLen(next_key.size()); + res_.AppendContent(next_key); + + res_.AppendArrayLen(keys.size()); + std::vector::iterator iter; + for (const auto& key : keys){ + res_.AppendString(key); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void PKSetexAtCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKSetexAt); + return; + } + key_ = argv_[1]; + value_ = argv_[3]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &time_stamp_) + || time_stamp_ >= INT32_MAX) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void PKSetexAtCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->PKSetexAt(key_, value_, time_stamp_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void PKScanRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKScanRange); + return; + } + if (!strcasecmp(argv_[1].data(), "string_with_value")) { + type_ = blackwidow::kStrings; + string_with_value = true; + } else if (!strcasecmp(argv_[1].data(), "string")) { + type_ = blackwidow::kStrings; + } else if (!strcasecmp(argv_[1].data(), "hash")) { + type_ = blackwidow::kHashes; + } else if (!strcasecmp(argv_[1].data(), "set")) { + type_ = blackwidow::kSets; + } else if (!strcasecmp(argv_[1].data(), "zset")) { + type_ = blackwidow::kZSets; + } else if (!strcasecmp(argv_[1].data(), "list")) { + type_ = blackwidow::kLists; + } else { + res_.SetRes(CmdRes::kInvalidDbType); + return; + } + + key_start_ = argv_[2]; + key_end_ = argv_[3]; + size_t index = 4, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "limit")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &limit_) || limit_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void PKScanRangeCmd::Do(std::shared_ptr partition) { + std::string next_key; + std::vector keys; + std::vector kvs; + rocksdb::Status s = partition->db()->PKScanRange(type_, key_start_, key_end_, pattern_, limit_, &keys, &kvs, &next_key); + + if (s.ok()) { + res_.AppendArrayLen(2); + res_.AppendStringLen(next_key.size()); + res_.AppendContent(next_key); + + if (type_ == blackwidow::kStrings) { + res_.AppendArrayLen(string_with_value ? 2 * kvs.size() : kvs.size()); + for (const auto& kv : kvs) { + res_.AppendString(kv.key); + if (string_with_value) { + res_.AppendString(kv.value); + } + } + } else { + res_.AppendArrayLen(keys.size()); + for (const auto& key : keys){ + res_.AppendString(key); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void PKRScanRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePKRScanRange); + return; + } + if (!strcasecmp(argv_[1].data(), "string_with_value")) { + type_ = blackwidow::kStrings; + string_with_value = true; + } else if (!strcasecmp(argv_[1].data(), "string")) { + type_ = blackwidow::kStrings; + } else if (!strcasecmp(argv_[1].data(), "hash")) { + type_ = blackwidow::kHashes; + } else if (!strcasecmp(argv_[1].data(), "set")) { + type_ = blackwidow::kSets; + } else if (!strcasecmp(argv_[1].data(), "zset")) { + type_ = blackwidow::kZSets; + } else if (!strcasecmp(argv_[1].data(), "list")) { + type_ = blackwidow::kLists; + } else { + res_.SetRes(CmdRes::kInvalidDbType); + return; + } + + key_start_ = argv_[2]; + key_end_ = argv_[3]; + size_t index = 4, argc = argv_.size(); + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "limit")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &limit_) || limit_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void PKRScanRangeCmd::Do(std::shared_ptr partition) { + std::string next_key; + std::vector keys; + std::vector kvs; + rocksdb::Status s = partition->db()->PKRScanRange(type_, key_start_, key_end_, pattern_, limit_, &keys, &kvs, &next_key); + + if (s.ok()) { + res_.AppendArrayLen(2); + res_.AppendStringLen(next_key.size()); + res_.AppendContent(next_key); + + if (type_ == blackwidow::kStrings) { + res_.AppendArrayLen(string_with_value ? 2 * kvs.size() : kvs.size()); + for (const auto& kv : kvs) { + res_.AppendString(kv.key); + if (string_with_value) { + res_.AppendString(kv.value); + } + } + } else { + res_.AppendArrayLen(keys.size()); + for (const auto& key : keys){ + res_.AppendString(key); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} diff --git a/tools/pika_migrate/src/pika_list.cc b/tools/pika_migrate/src/pika_list.cc new file mode 100644 index 0000000000..cf4442dab4 --- /dev/null +++ b/tools/pika_migrate/src/pika_list.cc @@ -0,0 +1,321 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_list.h" + +#include "slash/include/slash_string.h" + +void LIndexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLIndex); + return; + } + key_ = argv_[1]; + std::string index = argv_[2]; + if (!slash::string2l(index.data(), index.size(), &index_)) { + res_.SetRes(CmdRes::kInvalidInt); + } + return; +} +void LIndexCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->LIndex(key_, index_, &value); + if (s.ok()) { + res_.AppendString(value); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LInsertCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLInsert); + return; + } + key_ = argv_[1]; + std::string dir = argv_[2]; + if (!strcasecmp(dir.data(), "before")) { + dir_ = blackwidow::Before; + } else if (!strcasecmp(dir.data(), "after")) { + dir_ = blackwidow::After; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + pivot_ = argv_[3]; + value_ = argv_[4]; +} +void LInsertCmd::Do(std::shared_ptr partition) { + int64_t llen = 0; + rocksdb::Status s = partition->db()->LInsert(key_, dir_, pivot_, value_, &llen); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LLenCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLLen); + return; + } + key_ = argv_[1]; +} +void LLenCmd::Do(std::shared_ptr partition) { + uint64_t llen = 0; + rocksdb::Status s = partition->db()->LLen(key_, &llen); + if (s.ok() || s.IsNotFound()){ + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LPushCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLPush); + return; + } + key_ = argv_[1]; + size_t pos = 2; + while (pos < argv_.size()) { + values_.push_back(argv_[pos++]); + } +} +void LPushCmd::Do(std::shared_ptr partition) { + uint64_t llen = 0; + rocksdb::Status s = partition->db()->LPush(key_, values_, &llen); + if (s.ok()) { + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LPopCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLPop); + return; + } + key_ = argv_[1]; +} +void LPopCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->LPop(key_, &value); + if (s.ok()) { + res_.AppendString(value); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LPushxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLPushx); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; +} +void LPushxCmd::Do(std::shared_ptr partition) { + uint64_t llen = 0; + rocksdb::Status s = partition->db()->LPushx(key_, value_, &llen); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLRange); + return; + } + key_ = argv_[1]; + std::string left = argv_[2]; + if (!slash::string2l(left.data(), left.size(), &left_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + std::string right = argv_[3]; + if (!slash::string2l(right.data(), right.size(), &right_)) { + res_.SetRes(CmdRes::kInvalidInt); + } + return; +} +void LRangeCmd::Do(std::shared_ptr partition) { + std::vector values; + rocksdb::Status s = partition->db()->LRange(key_, left_, right_, &values); + if (s.ok()) { + res_.AppendArrayLen(values.size()); + for (const auto& value : values) { + res_.AppendString(value); + } + } else if (s.IsNotFound()) { + res_.AppendArrayLen(0); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LRemCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLRem); + return; + } + key_ = argv_[1]; + std::string count = argv_[2]; + if (!slash::string2l(count.data(), count.size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + value_ = argv_[3]; +} +void LRemCmd::Do(std::shared_ptr partition) { + uint64_t res = 0; + rocksdb::Status s = partition->db()->LRem(key_, count_, value_, &res); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LSetCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLSet); + return; + } + key_ = argv_[1]; + std::string index = argv_[2]; + if (!slash::string2l(index.data(), index.size(), &index_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + value_ = argv_[3]; +} +void LSetCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->LSet(key_, index_, value_); + if (s.ok()) { + res_.SetRes(CmdRes::kOk); + } else if (s.IsNotFound()) { + res_.SetRes(CmdRes::kNotFound); + } else if (s.IsCorruption() && s.ToString() == "Corruption: index out of range") { + //TODO refine return value + res_.SetRes(CmdRes::kOutOfRange); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void LTrimCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameLSet); + return; + } + key_ = argv_[1]; + std::string start = argv_[2]; + if (!slash::string2l(start.data(), start.size(), &start_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + std::string stop = argv_[3]; + if (!slash::string2l(stop.data(), stop.size(), &stop_)) { + res_.SetRes(CmdRes::kInvalidInt); + } + return; +} +void LTrimCmd::Do(std::shared_ptr partition) { + rocksdb::Status s = partition->db()->LTrim(key_, start_, stop_); + if (s.ok() || s.IsNotFound()) { + res_.SetRes(CmdRes::kOk); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void RPopCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameRPop); + return; + } + key_ = argv_[1]; +} +void RPopCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->RPop(key_, &value); + if (s.ok()) { + res_.AppendString(value); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void RPopLPushCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameRPopLPush); + return; + } + source_ = argv_[1]; + receiver_ = argv_[2]; +} +void RPopLPushCmd::Do(std::shared_ptr partition) { + std::string value; + rocksdb::Status s = partition->db()->RPoplpush(source_, receiver_, &value); + if (s.ok()) { + res_.AppendString(value); + } else if (s.IsNotFound()) { + res_.AppendStringLen(-1); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void RPushCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameRPush); + return; + } + key_ = argv_[1]; + size_t pos = 2; + while (pos < argv_.size()) { + values_.push_back(argv_[pos++]); + } +} +void RPushCmd::Do(std::shared_ptr partition) { + uint64_t llen = 0; + rocksdb::Status s = partition->db()->RPush(key_, values_, &llen); + if (s.ok()) { + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void RPushxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameRPushx); + return; + } + key_ = argv_[1]; + value_ = argv_[2]; +} +void RPushxCmd::Do(std::shared_ptr partition) { + uint64_t llen = 0; + rocksdb::Status s = partition->db()->RPushx(key_, value_, &llen); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(llen); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} diff --git a/tools/pika_migrate/src/pika_meta.cc b/tools/pika_migrate/src/pika_meta.cc new file mode 100644 index 0000000000..48e11de7e2 --- /dev/null +++ b/tools/pika_migrate/src/pika_meta.cc @@ -0,0 +1,131 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_meta.h" +#include "src/pika_inner_message.pb.h" + +const uint32_t VERSION = 1; + +PikaMeta::PikaMeta() + : local_meta_path_("") { + pthread_rwlock_init(&rwlock_, NULL); +} + +PikaMeta::~PikaMeta() { + pthread_rwlock_destroy(&rwlock_); +} + +void PikaMeta::SetPath(const std::string& path) { + local_meta_path_ = path; +} + +/* + * ******************* Meta File Format ****************** + * | | | | + * 4 Bytes 4 Bytes meta size Bytes + */ +Status PikaMeta::StableSave(const std::vector& table_structs) { + slash::RWLock l(&rwlock_, true); + if (local_meta_path_.empty()) { + LOG(WARNING) << "Local meta file path empty"; + return Status::Corruption("local meta file path empty"); + } + std::string local_meta_file = local_meta_path_ + kPikaMeta; + std::string tmp_file = local_meta_file; + tmp_file.append("_tmp"); + + slash::RWFile* saver = NULL; + slash::CreatePath(local_meta_path_); + Status s = slash::NewRWFile(tmp_file, &saver); + if (!s.ok()) { + delete saver; + LOG(WARNING) << "Open local meta file failed"; + return Status::Corruption("open local meta file failed"); + } + + InnerMessage::PikaMeta meta; + for (const auto& ts : table_structs) { + InnerMessage::TableInfo* table_info = meta.add_table_infos(); + table_info->set_table_name(ts.table_name); + table_info->set_partition_num(ts.partition_num); + for (const auto& id : ts.partition_ids) { + table_info->add_partition_ids(id); + } + } + + std::string meta_str; + if (!meta.SerializeToString(&meta_str)) { + delete saver; + LOG(WARNING) << "Serialize meta string failed"; + return Status::Corruption("serialize meta string failed"); + } + uint32_t meta_str_size = meta_str.size(); + + char *p = saver->GetData(); + memcpy(p, &VERSION, sizeof(uint32_t)); + p += sizeof(uint32_t); + memcpy(p, &meta_str_size, sizeof(uint32_t)); + p += sizeof(uint32_t); + memcpy(p, meta_str.data(), meta_str.size()); + delete saver; + + slash::DeleteFile(local_meta_file); + if (slash::RenameFile(tmp_file, local_meta_file)) { + LOG(WARNING) << "Failed to rename file, error: " << strerror(errno); + return Status::Corruption("faild to rename file"); + } + return Status::OK(); +} + +Status PikaMeta::ParseMeta(std::vector* const table_structs) { + slash::RWLock l(&rwlock_, false); + std::string local_meta_file = local_meta_path_ + kPikaMeta; + if (!slash::FileExists(local_meta_file)) { + LOG(WARNING) << "Local meta file not found, path: " << local_meta_file; + return Status::Corruption("meta file not found"); + } + + slash::RWFile* reader = NULL; + Status s = slash::NewRWFile(local_meta_file, &reader); + if (!s.ok()) { + delete reader; + LOG(WARNING) << "Open local meta file failed"; + return Status::Corruption("open local meta file failed"); + } + + if (reader->GetData() == NULL) { + delete reader; + LOG(WARNING) << "Meta file init error"; + return Status::Corruption("meta file init error"); + } + + uint32_t version = 0; + uint32_t meta_size = 0; + memcpy((char*)(&version), reader->GetData(), sizeof(uint32_t)); + memcpy((char*)(&meta_size), reader->GetData() + sizeof(uint32_t), sizeof(uint32_t)); + char* const buf = new char[meta_size]; + memcpy(buf, reader->GetData() + 2 * sizeof(uint32_t), meta_size); + + InnerMessage::PikaMeta meta; + if (!meta.ParseFromArray(buf, meta_size)) { + delete[] buf; + delete reader; + LOG(WARNING) << "Parse meta string failed"; + return Status::Corruption("parse meta string failed"); + } + delete[] buf; + delete reader; + + table_structs->clear(); + for (int idx = 0; idx < meta.table_infos_size(); ++idx) { + InnerMessage::TableInfo ti = meta.table_infos(idx); + std::set partition_ids; + for (int sidx = 0; sidx < ti.partition_ids_size(); ++sidx) { + partition_ids.insert(ti.partition_ids(sidx)); + } + table_structs->emplace_back(ti.table_name(), ti.partition_num(), partition_ids); + } + return Status::OK(); +} diff --git a/tools/pika_migrate/src/pika_monitor_thread.cc b/tools/pika_migrate/src/pika_monitor_thread.cc new file mode 100644 index 0000000000..746aa09080 --- /dev/null +++ b/tools/pika_migrate/src/pika_monitor_thread.cc @@ -0,0 +1,203 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_monitor_thread.h" + +#include + +PikaMonitorThread::PikaMonitorThread() + : pink::Thread(), + monitor_cond_(&monitor_mutex_protector_) { + set_thread_name("MonitorThread"); + has_monitor_clients_.store(false); +} + +PikaMonitorThread::~PikaMonitorThread() { + set_should_stop(); + if (is_running()) { + monitor_cond_.SignalAll(); + StopThread(); + } + for (std::list::iterator iter = monitor_clients_.begin(); + iter != monitor_clients_.end(); + ++iter) { + close(iter->fd); + } + LOG(INFO) << "PikaMonitorThread " << pthread_self() << " exit!!!"; +} + +void PikaMonitorThread::AddMonitorClient(std::shared_ptr client_ptr) { + StartThread(); + slash::MutexLock lm(&monitor_mutex_protector_); + monitor_clients_.push_back(ClientInfo{client_ptr->fd(), client_ptr->ip_port(), 0, client_ptr}); + has_monitor_clients_.store(true); +} + +void PikaMonitorThread::RemoveMonitorClient(const std::string& ip_port) { + std::list::iterator iter = monitor_clients_.begin(); + for (; iter != monitor_clients_.end(); ++iter) { + if (ip_port == "all") { + close(iter->fd); + continue; + } + if (iter->ip_port == ip_port) { + close(iter->fd); + break; + } + } + if (ip_port == "all") { + monitor_clients_.clear(); + } else if (iter != monitor_clients_.end()) { + monitor_clients_.erase(iter); + } + has_monitor_clients_.store(!monitor_clients_.empty()); +} + +void PikaMonitorThread::AddMonitorMessage(const std::string &monitor_message) { + slash::MutexLock lm(&monitor_mutex_protector_); + if (monitor_messages_.empty() && cron_tasks_.empty()) { + monitor_messages_.push_back(monitor_message); + monitor_cond_.Signal(); + } else { + monitor_messages_.push_back(monitor_message); + } +} + +int32_t PikaMonitorThread::ThreadClientList(std::vector* clients_ptr) { + if (clients_ptr != NULL) { + for (std::list::iterator iter = monitor_clients_.begin(); + iter != monitor_clients_.end(); + iter++) { + clients_ptr->push_back(*iter); + } + } + return monitor_clients_.size(); +} + +void PikaMonitorThread::AddCronTask(MonitorCronTask task) { + slash::MutexLock lm(&monitor_mutex_protector_); + if (monitor_messages_.empty() && cron_tasks_.empty()) { + cron_tasks_.push(task); + monitor_cond_.Signal(); + } else { + cron_tasks_.push(task); + } +} + +bool PikaMonitorThread::FindClient(const std::string &ip_port) { + slash::MutexLock lm(&monitor_mutex_protector_); + for (std::list::iterator iter = monitor_clients_.begin(); + iter != monitor_clients_.end(); + ++iter) { + if (iter->ip_port == ip_port) { + return true; + } + } + return false; +} + +bool PikaMonitorThread::ThreadClientKill(const std::string& ip_port) { + if (is_running()) { + if (ip_port == "all") { + AddCronTask({TASK_KILLALL, "all"}); + } else if (FindClient(ip_port)) { + AddCronTask({TASK_KILL, ip_port}); + } else { + return false; + } + } + return true; +} + +bool PikaMonitorThread::HasMonitorClients() { + return has_monitor_clients_.load(); +} + +pink::WriteStatus PikaMonitorThread::SendMessage(int32_t fd, std::string& message) { + size_t retry = 0; + ssize_t nwritten = 0, message_len_sended = 0, message_len_left = message.size(); + while (message_len_left > 0) { + nwritten = write(fd, message.data() + message_len_sended, message_len_left); + if (nwritten == -1 && errno == EAGAIN) { + // If the write buffer is full, but the client no longer consumes, it will + // get stuck in the loop and cause the entire Pika to block becase of monitor_mutex_protector_. + // So we put a limit on the number of retries + if (++retry >= 10) { + return pink::kWriteError; + } else { + // Sleep one second wait for client consume message + sleep(1); + continue; + } + } else if (nwritten == -1) { + return pink::kWriteError; + } + if (retry > 0) retry = 0; + message_len_sended += nwritten; + message_len_left -= nwritten; + } + return pink::kWriteAll; +} + +void* PikaMonitorThread::ThreadMain() { + std::deque messages_deque; + std::string messages_transfer; + MonitorCronTask task; + pink::WriteStatus write_status; + while (!should_stop()) { + { + slash::MutexLock lm(&monitor_mutex_protector_); + while (monitor_messages_.empty() && cron_tasks_.empty() && !should_stop()) { + monitor_cond_.Wait(); + } + } + if (should_stop()) { + break; + } + { + slash::MutexLock lm(&monitor_mutex_protector_); + while (!cron_tasks_.empty()) { + task = cron_tasks_.front(); + cron_tasks_.pop(); + RemoveMonitorClient(task.ip_port); + if (task.task == TASK_KILLALL) { + std::queue empty_queue; + cron_tasks_.swap(empty_queue); + } + } + } + + messages_deque.clear(); + { + slash::MutexLock lm(&monitor_mutex_protector_); + messages_deque.swap(monitor_messages_); + if (monitor_clients_.empty() || messages_deque.empty()) { + continue; + } + } + messages_transfer = "+"; + for (std::deque::iterator iter = messages_deque.begin(); + iter != messages_deque.end(); + ++iter) { + messages_transfer.append(iter->data(), iter->size()); + messages_transfer.append("\n"); + } + if (messages_transfer == "+") { + continue; + } + messages_transfer.replace(messages_transfer.size()-1, 1, "\r\n", 0, 2); + monitor_mutex_protector_.Lock(); + for (std::list::iterator iter = monitor_clients_.begin(); + iter != monitor_clients_.end(); + ++iter) { + write_status = SendMessage(iter->fd, messages_transfer); + if (write_status == pink::kWriteError) { + cron_tasks_.push({TASK_KILL, iter->ip_port}); + } + } + monitor_mutex_protector_.Unlock(); + } + return NULL; +} diff --git a/tools/pika_migrate/src/pika_partition.cc b/tools/pika_migrate/src/pika_partition.cc new file mode 100644 index 0000000000..5d4c014135 --- /dev/null +++ b/tools/pika_migrate/src/pika_partition.cc @@ -0,0 +1,679 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_partition.h" + +#include + +#include "include/pika_conf.h" +#include "include/pika_server.h" +#include "include/pika_rm.h" + +#include "slash/include/mutex_impl.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +std::string PartitionPath(const std::string& table_path, + uint32_t partition_id) { + char buf[100]; + snprintf(buf, sizeof(buf), "%u/", partition_id); + return table_path + buf; +} + +std::string PartitionName(const std::string& table_name, + uint32_t partition_id) { + char buf[256]; + snprintf(buf, sizeof(buf), "(%s:%u)", table_name.data(), partition_id); + return std::string(buf); +} + +std::string BgsaveSubPath(const std::string& table_name, + uint32_t partition_id) { + char buf[256]; + std::string partition_id_str = std::to_string(partition_id); + snprintf(buf, sizeof(buf), "%s/%s", table_name.data(), partition_id_str.data()); + return std::string(buf); +} + +std::string DbSyncPath(const std::string& sync_path, + const std::string& table_name, + const uint32_t partition_id, + bool classic_mode) { + char buf[256]; + std::string partition_id_str = std::to_string(partition_id); + if (classic_mode) { + snprintf(buf, sizeof(buf), "%s/", table_name.data()); + } else { + snprintf(buf, sizeof(buf), "%s/%s/", table_name.data(), partition_id_str.data()); + } + return sync_path + buf; +} + +Partition::Partition(const std::string& table_name, + uint32_t partition_id, + const std::string& table_db_path, + const std::string& table_log_path) : + table_name_(table_name), + partition_id_(partition_id), + binlog_io_error_(false), + bgsave_engine_(NULL), + purging_(false) { + + db_path_ = g_pika_conf->classic_mode() ? + table_db_path : PartitionPath(table_db_path, partition_id_); + log_path_ = g_pika_conf->classic_mode() ? + table_log_path : PartitionPath(table_log_path, partition_id_); + bgsave_sub_path_ = g_pika_conf->classic_mode() ? + table_name : BgsaveSubPath(table_name_, partition_id_); + dbsync_path_ = DbSyncPath(g_pika_conf->db_sync_path(), table_name_, + partition_id_, g_pika_conf->classic_mode()); + partition_name_ = g_pika_conf->classic_mode() ? + table_name : PartitionName(table_name_, partition_id_); + + pthread_rwlockattr_t attr; + pthread_rwlockattr_init(&attr); + pthread_rwlockattr_setkind_np(&attr, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + + pthread_rwlock_init(&db_rwlock_, &attr); + + db_ = std::shared_ptr(new blackwidow::BlackWidow()); + rocksdb::Status s = db_->Open(g_pika_server->bw_options(), db_path_); + + lock_mgr_ = new slash::lock::LockMgr(1000, 0, std::make_shared()); + + opened_ = s.ok() ? true : false; + assert(db_); + assert(s.ok()); + LOG(INFO) << partition_name_ << " DB Success"; + + logger_ = std::shared_ptr( + new Binlog(log_path_, g_pika_conf->binlog_file_size())); +} + +Partition::~Partition() { + Close(); + delete bgsave_engine_; + pthread_rwlock_destroy(&db_rwlock_); + delete lock_mgr_; +} + +void Partition::Leave() { + Close(); + MoveToTrash(); +} + +void Partition::Close() { + if (!opened_) { + return; + } + slash::RWLock rwl(&db_rwlock_, true); + db_.reset(); + logger_.reset(); + opened_ = false; +} + +// Before call this function, should +// close db and log first +void Partition::MoveToTrash() { + if (opened_) { + return; + } + + std::string dbpath = db_path_; + if (dbpath[dbpath.length() - 1] == '/') { + dbpath.erase(dbpath.length() - 1); + } + dbpath.append("_deleting/"); + if (slash::RenameFile(db_path_, dbpath.c_str())) { + LOG(WARNING) << "Failed to move db to trash, error: " << strerror(errno); + return; + } + g_pika_server->PurgeDir(dbpath); + + std::string logpath = log_path_; + if (logpath[logpath.length() - 1] == '/') { + logpath.erase(logpath.length() - 1); + } + logpath.append("_deleting/"); + if (slash::RenameFile(log_path_, logpath.c_str())) { + LOG(WARNING) << "Failed to move log to trash, error: " << strerror(errno); + return; + } + g_pika_server->PurgeDir(logpath); + + LOG(WARNING) << "Partition: " << partition_name_ << " move to trash success"; +} + +std::string Partition::GetTableName() const { + return table_name_; +} + +uint32_t Partition::GetPartitionId() const { + return partition_id_; +} + +std::string Partition::GetPartitionName() const { + return partition_name_; +} + +std::shared_ptr Partition::logger() const { + return logger_; +} + +std::shared_ptr Partition::db() const { + return db_; +} + +Status Partition::WriteBinlog(const std::string& binlog) { + if (!opened_) { + LOG(WARNING) << partition_name_ << " not opened, failed to exec command"; + return Status::Corruption("Partition Not Opened"); + } + slash::Status s; + if (!binlog.empty()) { + s = logger_->Put(binlog); + } + + if (!s.ok()) { + LOG(WARNING) << partition_name_ << " Writing binlog failed, maybe no space left on device"; + SetBinlogIoError(true); + return Status::Corruption("Writing binlog failed, maybe no space left on device"); + } + return Status::OK(); +} + +void Partition::Compact(const blackwidow::DataType& type) { + if (!opened_) return; + db_->Compact(type); +} + +void Partition::DbRWLockWriter() { + pthread_rwlock_wrlock(&db_rwlock_); +} + +void Partition::DbRWLockReader() { + pthread_rwlock_rdlock(&db_rwlock_); +} + +void Partition::DbRWUnLock() { + pthread_rwlock_unlock(&db_rwlock_); +} + +slash::lock::LockMgr* Partition::LockMgr() { + return lock_mgr_; +} + +void Partition::SetBinlogIoError(bool error) { + binlog_io_error_ = error; +} + +bool Partition::IsBinlogIoError() { + return binlog_io_error_; +} + +bool Partition::GetBinlogOffset(BinlogOffset* const boffset) { + if (opened_) { + logger_->GetProducerStatus(&boffset->filenum, &boffset->offset); + return true; + } + return false; +} + +bool Partition::SetBinlogOffset(const BinlogOffset& boffset) { + if (opened_) { + logger_->SetProducerStatus(boffset.filenum, boffset.offset); + return true; + } + return false; +} + +void Partition::PrepareRsync() { + slash::DeleteDirIfExist(dbsync_path_); + slash::CreatePath(dbsync_path_ + "strings"); + slash::CreatePath(dbsync_path_ + "hashes"); + slash::CreatePath(dbsync_path_ + "lists"); + slash::CreatePath(dbsync_path_ + "sets"); + slash::CreatePath(dbsync_path_ + "zsets"); +} + +// Try to update master offset +// This may happend when dbsync from master finished +// Here we do: +// 1, Check dbsync finished, got the new binlog offset +// 2, Replace the old db +// 3, Update master offset, and the PikaAuxiliaryThread cron will connect and do slaveof task with master +bool Partition::TryUpdateMasterOffset() { + std::string info_path = dbsync_path_ + kBgsaveInfoFile; + if (!slash::FileExists(info_path)) { + return false; + } + + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name_, partition_id_)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << partition_name_ << " not exist"; + return false; + } + + // Got new binlog offset + std::ifstream is(info_path); + if (!is) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Failed to open info file after db sync"; + slave_partition->SetReplState(ReplState::kError); + return false; + } + std::string line, master_ip; + int lineno = 0; + int64_t filenum = 0, offset = 0, tmp = 0, master_port = 0; + while (std::getline(is, line)) { + lineno++; + if (lineno == 2) { + master_ip = line; + } else if (lineno > 2 && lineno < 6) { + if (!slash::string2l(line.data(), line.size(), &tmp) || tmp < 0) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Format of info file after db sync error, line : " << line; + is.close(); + slave_partition->SetReplState(ReplState::kError); + return false; + } + if (lineno == 3) { master_port = tmp; } + else if (lineno == 4) { filenum = tmp; } + else { offset = tmp; } + + } else if (lineno > 5) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Format of info file after db sync error, line : " << line; + is.close(); + slave_partition->SetReplState(ReplState::kError); + return false; + } + } + is.close(); + + LOG(INFO) << "Partition: " << partition_name_ << " Information from dbsync info" + << ", master_ip: " << master_ip + << ", master_port: " << master_port + << ", filenum: " << filenum + << ", offset: " << offset; + + // Sanity check + if (master_ip != slave_partition->MasterIp() + || master_port != slave_partition->MasterPort()) { + LOG(WARNING) << "Partition: " << partition_name_ + << " Error master node ip port: " << master_ip << ":" << master_port; + slave_partition->SetReplState(ReplState::kError); + return false; + } + + // Retransmit Data to target redis + g_pika_server->RetransmitData(dbsync_path_); + + slash::DeleteFile(info_path); + if (!ChangeDb(dbsync_path_)) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Failed to change db"; + slave_partition->SetReplState(ReplState::kError); + return false; + } + + // Update master offset + logger_->SetProducerStatus(filenum, offset); + slave_partition->SetReplState(ReplState::kTryConnect); + return true; +} + +/* + * Change a new db locate in new_path + * return true when change success + * db remain the old one if return false + */ +bool Partition::ChangeDb(const std::string& new_path) { + + std::string tmp_path(db_path_); + if (tmp_path.back() == '/') { + tmp_path.resize(tmp_path.size() - 1); + } + tmp_path += "_bak"; + slash::DeleteDirIfExist(tmp_path); + + RWLock l(&db_rwlock_, true); + LOG(INFO) << "Partition: "<< partition_name_ + << ", Prepare change db from: " << tmp_path; + db_.reset(); + + if (0 != slash::RenameFile(db_path_.c_str(), tmp_path)) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Failed to rename db path when change db, error: " << strerror(errno); + return false; + } + + if (0 != slash::RenameFile(new_path.c_str(), db_path_.c_str())) { + LOG(WARNING) << "Partition: " << partition_name_ + << ", Failed to rename new db path when change db, error: " << strerror(errno); + return false; + } + + db_.reset(new blackwidow::BlackWidow()); + rocksdb::Status s = db_->Open(g_pika_server->bw_options(), db_path_); + assert(db_); + assert(s.ok()); + slash::DeleteDirIfExist(tmp_path); + LOG(INFO) << "Partition: " << partition_name_ << ", Change db success"; + return true; +} + +bool Partition::IsBgSaving() { + slash::MutexLock ml(&bgsave_protector_); + return bgsave_info_.bgsaving; +} + +void Partition::BgSavePartition() { + slash::MutexLock l(&bgsave_protector_); + if (bgsave_info_.bgsaving) { + return; + } + bgsave_info_.bgsaving = true; + BgTaskArg* bg_task_arg = new BgTaskArg(); + bg_task_arg->partition = shared_from_this(); + g_pika_server->BGSaveTaskSchedule(&DoBgSave, static_cast(bg_task_arg)); +} + +BgSaveInfo Partition::bgsave_info() { + slash::MutexLock l(&bgsave_protector_); + return bgsave_info_; +} + +void Partition::DoBgSave(void* arg) { + BgTaskArg* bg_task_arg = static_cast(arg); + + // Do BgSave + bool success = bg_task_arg->partition->RunBgsaveEngine(); + + // Some output + BgSaveInfo info = bg_task_arg->partition->bgsave_info(); + std::ofstream out; + out.open(info.path + "/" + kBgsaveInfoFile, std::ios::in | std::ios::trunc); + if (out.is_open()) { + out << (time(NULL) - info.start_time) << "s\n" + << g_pika_server->host() << "\n" + << g_pika_server->port() << "\n" + << info.filenum << "\n" + << info.offset << "\n"; + out.close(); + } + if (!success) { + std::string fail_path = info.path + "_FAILED"; + slash::RenameFile(info.path.c_str(), fail_path.c_str()); + } + bg_task_arg->partition->FinishBgsave(); + + delete bg_task_arg; +} + +bool Partition::RunBgsaveEngine() { + // Prepare for Bgsaving + if (!InitBgsaveEnv() || !InitBgsaveEngine()) { + ClearBgsave(); + return false; + } + LOG(INFO) << partition_name_ << " after prepare bgsave"; + + BgSaveInfo info = bgsave_info(); + LOG(INFO) << partition_name_ << " bgsave_info: path=" << info.path + << ", filenum=" << info.filenum + << ", offset=" << info.offset; + + // Backup to tmp dir + rocksdb::Status s = bgsave_engine_->CreateNewBackup(info.path); + LOG(INFO) << partition_name_ << " create new backup finished."; + + if (!s.ok()) { + LOG(WARNING) << partition_name_ << " create new backup failed :" << s.ToString(); + return false; + } + return true; +} + +// Prepare engine, need bgsave_protector protect +bool Partition::InitBgsaveEnv() { + slash::MutexLock l(&bgsave_protector_); + // Prepare for bgsave dir + bgsave_info_.start_time = time(NULL); + char s_time[32]; + int len = strftime(s_time, sizeof(s_time), "%Y%m%d%H%M%S", localtime(&bgsave_info_.start_time)); + bgsave_info_.s_start_time.assign(s_time, len); + std::string time_sub_path = g_pika_conf->bgsave_prefix() + std::string(s_time, 8); + bgsave_info_.path = g_pika_conf->bgsave_path() + time_sub_path + "/" + bgsave_sub_path_; + if (!slash::DeleteDirIfExist(bgsave_info_.path)) { + LOG(WARNING) << partition_name_ << " remove exist bgsave dir failed"; + return false; + } + slash::CreatePath(bgsave_info_.path, 0755); + // Prepare for failed dir + if (!slash::DeleteDirIfExist(bgsave_info_.path + "_FAILED")) { + LOG(WARNING) << partition_name_ << " remove exist fail bgsave dir failed :"; + return false; + } + return true; +} + +// Prepare bgsave env, need bgsave_protector protect +bool Partition::InitBgsaveEngine() { + delete bgsave_engine_; + rocksdb::Status s = blackwidow::BackupEngine::Open(db().get(), &bgsave_engine_); + if (!s.ok()) { + LOG(WARNING) << partition_name_ << " open backup engine failed " << s.ToString(); + return false; + } + + { + RWLock l(&db_rwlock_, true); + { + slash::MutexLock l(&bgsave_protector_); + logger_->GetProducerStatus(&bgsave_info_.filenum, &bgsave_info_.offset); + } + s = bgsave_engine_->SetBackupContent(); + if (!s.ok()) { + LOG(WARNING) << partition_name_ << " set backup content failed " << s.ToString(); + return false; + } + } + return true; +} + +void Partition::ClearBgsave() { + slash::MutexLock l(&bgsave_protector_); + bgsave_info_.Clear(); +} + +void Partition::FinishBgsave() { + slash::MutexLock l(&bgsave_protector_); + bgsave_info_.bgsaving = false; +} + +bool Partition::FlushDB() { + slash::RWLock rwl(&db_rwlock_, true); + slash::MutexLock ml(&bgsave_protector_); + if (bgsave_info_.bgsaving) { + return false; + } + + LOG(INFO) << partition_name_ << " Delete old db..."; + db_.reset(); + + std::string dbpath = db_path_; + if (dbpath[dbpath.length() - 1] == '/') { + dbpath.erase(dbpath.length() - 1); + } + dbpath.append("_deleting/"); + slash::RenameFile(db_path_, dbpath.c_str()); + + db_ = std::shared_ptr(new blackwidow::BlackWidow()); + rocksdb::Status s = db_->Open(g_pika_server->bw_options(), db_path_); + assert(db_); + assert(s.ok()); + LOG(INFO) << partition_name_ << " Open new db success"; + g_pika_server->PurgeDir(dbpath); + return true; +} + +bool Partition::FlushSubDB(const std::string& db_name) { + slash::RWLock rwl(&db_rwlock_, true); + slash::MutexLock ml(&bgsave_protector_); + if (bgsave_info_.bgsaving) { + return false; + } + + LOG(INFO) << partition_name_ << " Delete old " + db_name + " db..."; + db_.reset(); + + std::string dbpath = db_path_; + if (dbpath[dbpath.length() - 1] != '/') { + dbpath.append("/"); + } + + std::string sub_dbpath = dbpath + db_name; + std::string del_dbpath = dbpath + db_name + "_deleting"; + slash::RenameFile(sub_dbpath, del_dbpath); + + db_ = std::shared_ptr(new blackwidow::BlackWidow()); + rocksdb::Status s = db_->Open(g_pika_server->bw_options(), db_path_); + assert(db_); + assert(s.ok()); + LOG(INFO) << partition_name_ << " open new " + db_name + " db success"; + g_pika_server->PurgeDir(del_dbpath); + return true; +} + +bool Partition::PurgeLogs(uint32_t to, bool manual) { + // Only one thread can go through + bool expect = false; + if (!purging_.compare_exchange_strong(expect, true)) { + LOG(WARNING) << "purge process already exist"; + return false; + } + PurgeArg *arg = new PurgeArg(); + arg->to = to; + arg->manual = manual; + arg->partition = shared_from_this(); + g_pika_server->PurgelogsTaskSchedule(&DoPurgeLogs, static_cast(arg)); + return true; +} + +void Partition::ClearPurge() { + purging_ = false; +} + +void Partition::DoPurgeLogs(void* arg) { + PurgeArg* purge = static_cast(arg); + purge->partition->PurgeFiles(purge->to, purge->manual); + purge->partition->ClearPurge(); + delete (PurgeArg*)arg; +} + +bool Partition::PurgeFiles(uint32_t to, bool manual) { + std::map binlogs; + if (!GetBinlogFiles(binlogs)) { + LOG(WARNING) << partition_name_ << " Could not get binlog files!"; + return false; + } + + int delete_num = 0; + struct stat file_stat; + int remain_expire_num = binlogs.size() - g_pika_conf->expire_logs_nums(); + std::map::iterator it; + for (it = binlogs.begin(); it != binlogs.end(); ++it) { + if ((manual && it->first <= to) // Manual purgelogsto + || (remain_expire_num > 0) // Expire num trigger + || (binlogs.size() - delete_num > 10 // At lease remain 10 files + && stat(((log_path_ + it->second)).c_str(), &file_stat) == 0 + && file_stat.st_mtime < time(NULL) - g_pika_conf->expire_logs_days() * 24 * 3600)) { // Expire time trigger + // We check this every time to avoid lock when we do file deletion + if (!g_pika_rm->BinlogCloudPurgeFromSMP(table_name_, partition_id_, it->first)) { + LOG(WARNING) << partition_name_ << " Could not purge "<< (it->first) << ", since it is already be used"; + return false; + } + + // Do delete + slash::Status s = slash::DeleteFile(log_path_ + it->second); + if (s.ok()) { + ++delete_num; + --remain_expire_num; + } else { + LOG(WARNING) << partition_name_ << " Purge log file : " << (it->second) << " failed! error:" << s.ToString(); + } + } else { + // Break when face the first one not satisfied + // Since the binlogs is order by the file index + break; + } + } + if (delete_num) { + LOG(INFO) << partition_name_ << " Success purge "<< delete_num; + } + return true; +} + +bool Partition::GetBinlogFiles(std::map& binlogs) { + std::vector children; + int ret = slash::GetChildren(log_path_, children); + if (ret != 0) { + LOG(WARNING) << partition_name_ << " Get all files in log path failed! error:" << ret; + return false; + } + + int64_t index = 0; + std::string sindex; + std::vector::iterator it; + for (it = children.begin(); it != children.end(); ++it) { + if ((*it).compare(0, kBinlogPrefixLen, kBinlogPrefix) != 0) { + continue; + } + sindex = (*it).substr(kBinlogPrefixLen); + if (slash::string2l(sindex.c_str(), sindex.size(), &index) == 1) { + binlogs.insert(std::pair(static_cast(index), *it)); + } + } + return true; +} + +void Partition::InitKeyScan() { + key_scan_info_.start_time = time(NULL); + char s_time[32]; + int len = strftime(s_time, sizeof(s_time), "%Y-%m-%d %H:%M:%S", localtime(&key_scan_info_.start_time)); + key_scan_info_.s_start_time.assign(s_time, len); + key_scan_info_.duration = -1; // duration -1 mean the task in processing +} + +KeyScanInfo Partition::GetKeyScanInfo() { + slash::MutexLock l(&key_info_protector_); + return key_scan_info_; +} + +Status Partition::GetKeyNum(std::vector* key_info) { + slash::MutexLock l(&key_info_protector_); + if (key_scan_info_.key_scaning_) { + *key_info = key_scan_info_.key_infos; + return Status::OK(); + } + InitKeyScan(); + key_scan_info_.key_scaning_ = true; + key_scan_info_.duration = -2; // duration -2 mean the task in waiting status, + // has not been scheduled for exec + rocksdb::Status s = db_->GetKeyNum(key_info); + if (!s.ok()) { + return Status::Corruption(s.ToString()); + } + key_scan_info_.key_infos = *key_info; + key_scan_info_.duration = time(NULL) - key_scan_info_.start_time; + key_scan_info_.key_scaning_ = false; + return Status::OK(); +} diff --git a/tools/pika_migrate/src/pika_pubsub.cc b/tools/pika_migrate/src/pika_pubsub.cc new file mode 100644 index 0000000000..c3a0127d3f --- /dev/null +++ b/tools/pika_migrate/src/pika_pubsub.cc @@ -0,0 +1,222 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_pubsub.h" + +#include "include/pika_server.h" + +extern PikaServer *g_pika_server; + + +static std::string ConstructPubSubResp( + const std::string& cmd, + const std::vector>& result) { + std::stringstream resp; + if (result.size() == 0) { + resp << "*3\r\n" << "$" << cmd.length() << "\r\n" << cmd << "\r\n" << + "$" << -1 << "\r\n" << ":" << 0 << "\r\n"; + } + for (auto it = result.begin(); it != result.end(); it++) { + resp << "*3\r\n" << "$" << cmd.length() << "\r\n" << cmd << "\r\n" << + "$" << it->first.length() << "\r\n" << it->first << "\r\n" << + ":" << it->second << "\r\n"; + } + return resp.str(); +} + + +void PublishCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePublish); + return; + } + channel_ = argv_[1]; + msg_ = argv_[2]; +} + +void PublishCmd::Do(std::shared_ptr partition) { + int receivers = g_pika_server->Publish(channel_, msg_); + res_.AppendInteger(receivers); + return; +} + +void SubscribeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSubscribe); + return; + } +} + +void SubscribeCmd::Do(std::shared_ptr partition) { + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNameSubscribe); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + + if (!cli_conn->IsPubSub()) { + cli_conn->server_thread()->MoveConnOut(conn->fd()); + } + std::vector channels; + for (size_t i = 1; i < argv_.size(); i++) { + channels.push_back(argv_[i]); + } + std::vector> result; + cli_conn->SetIsPubSub(true); + cli_conn->SetHandleType(pink::HandleType::kSynchronous); + g_pika_server->Subscribe(conn, channels, name_ == kCmdNamePSubscribe, &result); + return res_.SetRes(CmdRes::kNone, ConstructPubSubResp(name_, result)); +} + +void UnSubscribeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameUnSubscribe); + return; + } +} + +void UnSubscribeCmd::Do(std::shared_ptr partition) { + std::vector channels; + for (size_t i = 1; i < argv_.size(); i++) { + channels.push_back(argv_[i]); + } + + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNameUnSubscribe); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + + std::vector> result; + int subscribed = g_pika_server->UnSubscribe(conn, channels, name_ == kCmdNamePUnSubscribe, &result); + if (subscribed == 0 && cli_conn->IsPubSub()) { + /* + * if the number of client subscribed is zero, + * the client will exit the Pub/Sub state + */ + cli_conn->server_thread()->HandleNewConn(conn->fd(), conn->ip_port()); + cli_conn->SetIsPubSub(false); + } + return res_.SetRes(CmdRes::kNone, ConstructPubSubResp(name_, result)); +} + +void PSubscribeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePSubscribe); + return; + } +} + +void PSubscribeCmd::Do(std::shared_ptr partition) { + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNamePSubscribe); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + + if (!cli_conn->IsPubSub()) { + cli_conn->server_thread()->MoveConnOut(conn->fd()); + } + std::vector channels; + for (size_t i = 1; i < argv_.size(); i++) { + channels.push_back(argv_[i]); + } + std::vector> result; + cli_conn->SetIsPubSub(true); + cli_conn->SetHandleType(pink::HandleType::kSynchronous); + g_pika_server->Subscribe(conn, channels, name_ == kCmdNamePSubscribe, &result); + return res_.SetRes(CmdRes::kNone, ConstructPubSubResp(name_, result)); +} + +void PUnSubscribeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePUnSubscribe); + return; + } +} + +void PUnSubscribeCmd::Do(std::shared_ptr partition) { + std::vector channels; + for (size_t i = 1; i < argv_.size(); i++) { + channels.push_back(argv_[i]); + } + + std::shared_ptr conn = GetConn(); + if (!conn) { + res_.SetRes(CmdRes::kErrOther, kCmdNamePUnSubscribe); + LOG(WARNING) << name_ << " weak ptr is empty"; + return; + } + std::shared_ptr cli_conn = std::dynamic_pointer_cast(conn); + + std::vector> result; + int subscribed = g_pika_server->UnSubscribe(conn, channels, name_ == kCmdNamePUnSubscribe, &result); + if (subscribed == 0 && cli_conn->IsPubSub()) { + /* + * if the number of client subscribed is zero, + * the client will exit the Pub/Sub state + */ + cli_conn->server_thread()->HandleNewConn(conn->fd(), conn->ip_port()); + cli_conn->SetIsPubSub(false); + } + return res_.SetRes(CmdRes::kNone, ConstructPubSubResp(name_, result)); +} + +void PubSubCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNamePubSub); + return; + } + subcommand_ = argv_[1]; + if (strcasecmp(subcommand_.data(), "channels") + && strcasecmp(subcommand_.data(), "numsub") + && strcasecmp(subcommand_.data(), "numpat")) { + res_.SetRes(CmdRes::kErrOther, "Unknown PUBSUB subcommand or wrong number of arguments for '" + subcommand_ + "'"); + } + for (size_t i = 2; i < argv_.size(); i++) { + arguments_.push_back(argv_[i]); + } +} + +void PubSubCmd::Do(std::shared_ptr partition) { + if (!strcasecmp(subcommand_.data(), "channels")) { + std::string pattern = ""; + std::vector result; + if (arguments_.size() == 1) { + pattern = arguments_[0]; + } else if (arguments_.size() > 1) { + res_.SetRes(CmdRes::kErrOther, "Unknown PUBSUB subcommand or wrong number of arguments for '" + subcommand_ + "'"); + return; + } + g_pika_server->PubSubChannels(pattern, &result); + + res_.AppendArrayLen(result.size()); + for (auto it = result.begin(); it != result.end(); ++it) { + res_.AppendStringLen((*it).length()); + res_.AppendContent(*it); + } + } else if (!strcasecmp(subcommand_.data(), "numsub")) { + std::vector> result; + g_pika_server->PubSubNumSub(arguments_, &result); + res_.AppendArrayLen(result.size() * 2); + for (auto it = result.begin(); it != result.end(); ++it) { + res_.AppendStringLen(it->first.length()); + res_.AppendContent(it->first); + res_.AppendInteger(it->second); + } + return; + } else if (!strcasecmp(subcommand_.data(), "numpat")) { + int subscribed = g_pika_server->PubSubNumPat(); + res_.AppendInteger(subscribed); + } + return; +} + diff --git a/tools/pika_migrate/src/pika_repl_bgworker.cc b/tools/pika_migrate/src/pika_repl_bgworker.cc new file mode 100644 index 0000000000..f68db2d288 --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_bgworker.cc @@ -0,0 +1,301 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_bgworker.h" + +#include + +#include "pink/include/redis_cli.h" + +#include "include/pika_rm.h" +#include "include/pika_conf.h" +#include "include/pika_server.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; +extern PikaCmdTableManager* g_pika_cmd_table_manager; + +PikaReplBgWorker::PikaReplBgWorker(int queue_size) + : bg_thread_(queue_size) { + bg_thread_.set_thread_name("ReplBgWorker"); + pink::RedisParserSettings settings; + settings.DealMessage = &(PikaReplBgWorker::HandleWriteBinlog); + redis_parser_.RedisParserInit(REDIS_PARSER_REQUEST, settings); + redis_parser_.data = this; + table_name_ = g_pika_conf->default_table(); + partition_id_ = 0; + +} + +PikaReplBgWorker::~PikaReplBgWorker() { +} + +int PikaReplBgWorker::StartThread() { + return bg_thread_.StartThread(); +} + +int PikaReplBgWorker::StopThread() { + return bg_thread_.StopThread(); +} + +void PikaReplBgWorker::Schedule(pink::TaskFunc func, void* arg) { + bg_thread_.Schedule(func, arg); +} + +void PikaReplBgWorker::QueueClear() { + bg_thread_.QueueClear(); +} + +void PikaReplBgWorker::HandleBGWorkerWriteBinlog(void* arg) { + ReplClientWriteBinlogTaskArg* task_arg = static_cast(arg); + const std::shared_ptr res = task_arg->res; + std::shared_ptr conn = task_arg->conn; + std::vector* index = static_cast* >(task_arg->res_private_data); + PikaReplBgWorker* worker = task_arg->worker; + worker->ip_port_ = conn->ip_port(); + + std::string table_name; + uint32_t partition_id = 0; + BinlogOffset ack_start, ack_end; + // find the first not keepalive binlogsync + for (size_t i = 0; i < index->size(); ++i) { + const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync((*index)[i]); + if (i == 0) { + table_name = binlog_res.partition().table_name(); + partition_id = binlog_res.partition().partition_id(); + } + if (!binlog_res.binlog().empty()) { + ack_start.filenum = binlog_res.binlog_offset().filenum(); + ack_start.offset = binlog_res.binlog_offset().offset(); + break; + } + } + worker->table_name_ = table_name; + worker->partition_id_ = partition_id; + + std::shared_ptr partition = g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition) { + LOG(WARNING) << "Partition " << table_name << "_" << partition_id << " Not Found"; + delete index; + delete task_arg; + return; + } + + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition " << table_name << "_" << partition_id << " Not Found"; + delete index; + delete task_arg; + return; + } + + for (size_t i = 0; i < index->size(); ++i) { + const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync((*index)[i]); + // if pika are not current a slave or partition not in + // BinlogSync state, we drop remain write binlog task + if ((g_pika_conf->classic_mode() && !(g_pika_server->role() & PIKA_ROLE_SLAVE)) + || ((slave_partition->State() != ReplState::kConnected) + && (slave_partition->State() != ReplState::kWaitDBSync))) { + delete index; + delete task_arg; + return; + } + + if (!g_pika_rm->CheckSlavePartitionSessionId( + binlog_res.partition().table_name(), + binlog_res.partition().partition_id(), + binlog_res.session_id())) { + LOG(WARNING) << "Check Session failed " + << binlog_res.partition().table_name() + << "_" << binlog_res.partition().partition_id(); + slave_partition->SetReplState(ReplState::kTryConnect); + delete index; + delete task_arg; + return; + } + + // empty binlog treated as keepalive packet + if (binlog_res.binlog().empty()) { + continue; + } + if (!PikaBinlogTransverter::BinlogItemWithoutContentDecode(TypeFirst, binlog_res.binlog(), &worker->binlog_item_)) { + LOG(WARNING) << "Binlog item decode failed"; + slave_partition->SetReplState(ReplState::kTryConnect); + delete index; + delete task_arg; + return; + } + const char* redis_parser_start = binlog_res.binlog().data() + BINLOG_ENCODE_LEN; + int redis_parser_len = static_cast(binlog_res.binlog().size()) - BINLOG_ENCODE_LEN; + int processed_len = 0; + pink::RedisParserStatus ret = worker->redis_parser_.ProcessInputBuffer( + redis_parser_start, redis_parser_len, &processed_len); + if (ret != pink::kRedisParserDone) { + LOG(WARNING) << "Redis parser failed"; + slave_partition->SetReplState(ReplState::kTryConnect); + delete index; + delete task_arg; + return; + } + } + delete index; + delete task_arg; + + // Reply Ack to master immediately + std::shared_ptr logger = partition->logger(); + logger->GetProducerStatus(&ack_end.filenum, &ack_end.offset); + // keepalive case + if (ack_start == BinlogOffset()) { + // set ack_end as 0 + ack_end = ack_start; + } + g_pika_rm->SendPartitionBinlogSyncAckRequest(table_name, partition_id, ack_start, ack_end); +} + +int PikaReplBgWorker::HandleWriteBinlog(pink::RedisParser* parser, const pink::RedisCmdArgsType& argv) { + PikaReplBgWorker* worker = static_cast(parser->data); + const BinlogItem& binlog_item = worker->binlog_item_; + g_pika_server->UpdateQueryNumAndExecCountTable(argv[0]); + + // Monitor related + std::string monitor_message; + if (g_pika_server->HasMonitorClients()) { + std::string table_name = g_pika_conf->classic_mode() + ? worker->table_name_.substr(2) : worker->table_name_; + std::string monitor_message = std::to_string(1.0 * slash::NowMicros() / 1000000) + + " [" + table_name + " " + worker->ip_port_ + "]"; + for (const auto& item : argv) { + monitor_message += " " + slash::ToRead(item); + } + g_pika_server->AddMonitorMessage(monitor_message); + } + + std::string opt = argv[0]; + std::shared_ptr c_ptr = g_pika_cmd_table_manager->GetCmd(slash::StringToLower(opt)); + if (!c_ptr) { + LOG(WARNING) << "Command " << opt << " not in the command table"; + return -1; + } + // Initial + c_ptr->Initial(argv, worker->table_name_); + if (!c_ptr->res().ok()) { + LOG(WARNING) << "Fail to initial command from binlog: " << opt; + return -1; + } + + std::shared_ptr partition = g_pika_server->GetTablePartitionById(worker->table_name_, worker->partition_id_); + std::shared_ptr logger = partition->logger(); + + logger->Lock(); + logger->Put(c_ptr->ToBinlog(binlog_item.exec_time(), + std::to_string(binlog_item.server_id()), + binlog_item.logic_id(), + binlog_item.filenum(), + binlog_item.offset())); + uint32_t filenum; + uint64_t offset; + logger->GetProducerStatus(&filenum, &offset); + logger->Unlock(); + + PikaCmdArgsType *v = new PikaCmdArgsType(argv); + BinlogItem *b = new BinlogItem(binlog_item); + std::string dispatch_key = argv.size() >= 2 ? argv[1] : argv[0]; + g_pika_rm->ScheduleWriteDBTask(dispatch_key, v, b, worker->table_name_, worker->partition_id_); + return 0; +} + +void PikaReplBgWorker::HandleBGWorkerWriteDB(void* arg) { + ReplClientWriteDBTaskArg* task_arg = static_cast(arg); + PikaCmdArgsType* argv = task_arg->argv; + BinlogItem binlog_item = *(task_arg->binlog_item); + std::string table_name = task_arg->table_name; + uint32_t partition_id = task_arg->partition_id; + std::string opt = (*argv)[0]; + slash::StringToLower(opt); + + // Get command + std::shared_ptr c_ptr = g_pika_cmd_table_manager->GetCmd(slash::StringToLower(opt)); + if (!c_ptr) { + LOG(WARNING) << "Error operation from binlog: " << opt; + delete task_arg; + return; + } + + // Initial + c_ptr->Initial(*argv, table_name); + if (!c_ptr->res().ok()) { + LOG(WARNING) << "Fail to initial command from binlog: " << opt; + delete task_arg; + return; + } + + uint64_t start_us = 0; + if (g_pika_conf->slowlog_slower_than() >= 0) { + start_us = slash::NowMicros(); + } + std::shared_ptr partition = g_pika_server->GetTablePartitionById(table_name, partition_id); + + if (strcmp(table_name.data(), "db0") || partition_id != 0) { + LOG(FATAL) << "table_name: " << table_name << ", partition_id: " + << std::to_string(partition_id) << ", but only single DB data is support transfer"; + return; + } + + /* convert Pika custom command to Redis standard command */ + if (!strcasecmp((*argv)[0].data(), "pksetexat")) { + if (argv->size() != 4) { + LOG(WARNING) << "find invaild command, command size: " << argv->size(); + return; + } else { + std::string key = (*argv)[1]; + int timestamp = std::atoi((*argv)[2].data()); + std::string value = (*argv)[3]; + + int seconds = timestamp - time(NULL); + PikaCmdArgsType tmp_argv; + tmp_argv.push_back("setex"); + tmp_argv.push_back(key); + tmp_argv.push_back(std::to_string(seconds)); + tmp_argv.push_back(value); + + std::string command; + pink::SerializeRedisCommand(tmp_argv, &command); + g_pika_server->SendRedisCommand(command, key); + } + } else { + std::string key = argv->size() > 1 ? (*argv)[1] : ""; + std::string command; + pink::SerializeRedisCommand(*argv, &command); + g_pika_server->SendRedisCommand(command, key); + } + + // Add read lock for no suspend command + if (!c_ptr->is_suspend()) { + partition->DbRWLockReader(); + } + + c_ptr->Do(partition); + + if (!c_ptr->is_suspend()) { + partition->DbRWUnLock(); + } + + if (g_pika_conf->slowlog_slower_than() >= 0) { + int32_t start_time = start_us / 1000000; + int64_t duration = slash::NowMicros() - start_us; + if (duration > g_pika_conf->slowlog_slower_than()) { + g_pika_server->SlowlogPushEntry(*argv, start_time, duration); + if (g_pika_conf->slowlog_write_errorlog()) { + LOG(ERROR) << "command: " << opt << ", start_time(s): " << start_time << ", duration(us): " << duration; + } + } + } + delete task_arg; +} + diff --git a/tools/pika_migrate/src/pika_repl_client.cc b/tools/pika_migrate/src/pika_repl_client.cc new file mode 100644 index 0000000000..5c78dfab7d --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_client.cc @@ -0,0 +1,271 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_client.h" + +#include +#include +#include + +#include "pink/include/pink_cli.h" +#include "pink/include/redis_cli.h" +#include "slash/include/slash_coding.h" +#include "slash/include/env.h" +#include "slash/include/slash_string.h" + +#include "include/pika_rm.h" +#include "include/pika_server.h" + +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaReplClient::PikaReplClient(int cron_interval, int keepalive_timeout) : next_avail_(0) { + client_thread_ = new PikaReplClientThread(cron_interval, keepalive_timeout); + client_thread_->set_thread_name("PikaReplClient"); + for (int i = 0; i < 2 * g_pika_conf->sync_thread_num(); ++i) { + bg_workers_.push_back(new PikaReplBgWorker(PIKA_SYNC_BUFFER_SIZE)); + } +} + +PikaReplClient::~PikaReplClient() { + client_thread_->StopThread(); + delete client_thread_; + for (size_t i = 0; i < bg_workers_.size(); ++i) { + delete bg_workers_[i]; + } + LOG(INFO) << "PikaReplClient exit!!!"; +} + +int PikaReplClient::Start() { + int res = client_thread_->StartThread(); + if (res != pink::kSuccess) { + LOG(FATAL) << "Start ReplClient ClientThread Error: " << res << (res == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + for (size_t i = 0; i < bg_workers_.size(); ++i) { + res = bg_workers_[i]->StartThread(); + if (res != pink::kSuccess) { + LOG(FATAL) << "Start Pika Repl Worker Thread Error: " << res + << (res == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + } + return res; +} + +int PikaReplClient::Stop() { + client_thread_->StopThread(); + for (size_t i = 0; i < bg_workers_.size(); ++i) { + bg_workers_[i]->StopThread(); + } + return 0; +} + +void PikaReplClient::Schedule(pink::TaskFunc func, void* arg) { + bg_workers_[next_avail_]->Schedule(func, arg); + UpdateNextAvail(); +} + +void PikaReplClient::ScheduleWriteBinlogTask(std::string table_partition, + const std::shared_ptr res, + std::shared_ptr conn, void* res_private_data) { + size_t index = GetHashIndex(table_partition, true); + ReplClientWriteBinlogTaskArg* task_arg = + new ReplClientWriteBinlogTaskArg(res, conn, res_private_data, bg_workers_[index]); + bg_workers_[index]->Schedule(&PikaReplBgWorker::HandleBGWorkerWriteBinlog, static_cast(task_arg)); +} + +void PikaReplClient::ScheduleWriteDBTask(const std::string& dispatch_key, + PikaCmdArgsType* argv, BinlogItem* binlog_item, + const std::string& table_name, uint32_t partition_id) { + size_t index = GetHashIndex(dispatch_key, false); + ReplClientWriteDBTaskArg* task_arg = + new ReplClientWriteDBTaskArg(argv, binlog_item, table_name, partition_id); + bg_workers_[index]->Schedule(&PikaReplBgWorker::HandleBGWorkerWriteDB, static_cast(task_arg)); +} + +size_t PikaReplClient::GetHashIndex(std::string key, bool upper_half) { + size_t hash_base = bg_workers_.size() / 2; + return (str_hash(key) % hash_base) + (upper_half ? 0 : hash_base); +} + +Status PikaReplClient::Write(const std::string& ip, const int port, const std::string& msg) { + return client_thread_->Write(ip, port, msg); +} + +Status PikaReplClient::Close(const std::string& ip, const int port) { + return client_thread_->Close(ip, port); +} + + +Status PikaReplClient::SendMetaSync() { + std::string local_ip; + pink::PinkCli* cli = pink::NewRedisCli(); + cli->set_connect_timeout(1500); + if ((cli->Connect(g_pika_server->master_ip(), g_pika_server->master_port(), "")).ok()) { + struct sockaddr_in laddr; + socklen_t llen = sizeof(laddr); + getsockname(cli->fd(), (struct sockaddr*) &laddr, &llen); + std::string tmp_local_ip(inet_ntoa(laddr.sin_addr)); + local_ip = tmp_local_ip; + cli->Close(); + delete cli; + } else { + LOG(WARNING) << "Failed to connect master, Master (" + << g_pika_server->master_ip() << ":" << g_pika_server->master_port() << "), try reconnect"; + // Sleep three seconds to avoid frequent try Meta Sync + // when the connection fails + sleep(3); + g_pika_server->ResetMetaSyncStatus(); + delete cli; + return Status::Corruption("Connect master error"); + } + + InnerMessage::InnerRequest request; + request.set_type(InnerMessage::kMetaSync); + InnerMessage::InnerRequest::MetaSync* meta_sync = request.mutable_meta_sync(); + InnerMessage::Node* node = meta_sync->mutable_node(); + node->set_ip(local_ip); + node->set_port(g_pika_server->port()); + + std::string masterauth = g_pika_conf->masterauth(); + if (!masterauth.empty()) { + meta_sync->set_auth(masterauth); + } + + std::string to_send; + std::string master_ip = g_pika_server->master_ip(); + int master_port = g_pika_server->master_port(); + if (!request.SerializeToString(&to_send)) { + LOG(WARNING) << "Serialize Meta Sync Request Failed, to Master (" + << master_ip << ":" << master_port << ")"; + return Status::Corruption("Serialize Failed"); + } + + LOG(INFO) << "Try Send Meta Sync Request to Master (" + << master_ip << ":" << master_port << ")"; + return client_thread_->Write(master_ip, master_port + kPortShiftReplServer, to_send); +} + +Status PikaReplClient::SendPartitionDBSync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& boffset, + const std::string& local_ip) { + InnerMessage::InnerRequest request; + request.set_type(InnerMessage::kDBSync); + InnerMessage::InnerRequest::DBSync* db_sync = request.mutable_db_sync(); + InnerMessage::Node* node = db_sync->mutable_node(); + node->set_ip(local_ip); + node->set_port(g_pika_server->port()); + InnerMessage::Partition* partition = db_sync->mutable_partition(); + partition->set_table_name(table_name); + partition->set_partition_id(partition_id); + + InnerMessage::BinlogOffset* binlog_offset = db_sync->mutable_binlog_offset(); + binlog_offset->set_filenum(boffset.filenum); + binlog_offset->set_offset(boffset.offset); + + std::string to_send; + if (!request.SerializeToString(&to_send)) { + LOG(WARNING) << "Serialize Partition DBSync Request Failed, to Master (" + << ip << ":" << port << ")"; + return Status::Corruption("Serialize Failed"); + } + return client_thread_->Write(ip, port + kPortShiftReplServer, to_send); +} + + +Status PikaReplClient::SendPartitionTrySync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& boffset, + const std::string& local_ip) { + InnerMessage::InnerRequest request; + request.set_type(InnerMessage::kTrySync); + InnerMessage::InnerRequest::TrySync* try_sync = request.mutable_try_sync(); + InnerMessage::Node* node = try_sync->mutable_node(); + node->set_ip(local_ip); + node->set_port(g_pika_server->port()); + InnerMessage::Partition* partition = try_sync->mutable_partition(); + partition->set_table_name(table_name); + partition->set_partition_id(partition_id); + + InnerMessage::BinlogOffset* binlog_offset = try_sync->mutable_binlog_offset(); + binlog_offset->set_filenum(boffset.filenum); + binlog_offset->set_offset(boffset.offset); + + std::string to_send; + if (!request.SerializeToString(&to_send)) { + LOG(WARNING) << "Serialize Partition TrySync Request Failed, to Master (" + << ip << ":" << port << ")"; + return Status::Corruption("Serialize Failed"); + } + return client_thread_->Write(ip, port + kPortShiftReplServer, to_send); +} + +Status PikaReplClient::SendPartitionBinlogSync(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const BinlogOffset& ack_start, + const BinlogOffset& ack_end, + const std::string& local_ip, + bool is_first_send) { + InnerMessage::InnerRequest request; + request.set_type(InnerMessage::kBinlogSync); + InnerMessage::InnerRequest::BinlogSync* binlog_sync = request.mutable_binlog_sync(); + InnerMessage::Node* node = binlog_sync->mutable_node(); + node->set_ip(local_ip); + node->set_port(g_pika_server->port()); + binlog_sync->set_table_name(table_name); + binlog_sync->set_partition_id(partition_id); + binlog_sync->set_first_send(is_first_send); + + InnerMessage::BinlogOffset* ack_range_start = binlog_sync->mutable_ack_range_start(); + ack_range_start->set_filenum(ack_start.filenum); + ack_range_start->set_offset(ack_start.offset); + + InnerMessage::BinlogOffset* ack_range_end = binlog_sync->mutable_ack_range_end(); + ack_range_end->set_filenum(ack_end.filenum); + ack_range_end->set_offset(ack_end.offset); + + int32_t session_id = g_pika_rm->GetSlavePartitionSessionId(table_name, partition_id); + binlog_sync->set_session_id(session_id); + + std::string to_send; + if (!request.SerializeToString(&to_send)) { + LOG(WARNING) << "Serialize Partition BinlogSync Request Failed, to Master (" + << ip << ":" << port << ")"; + return Status::Corruption("Serialize Failed"); + } + return client_thread_->Write(ip, port + kPortShiftReplServer, to_send); +} + +Status PikaReplClient::SendRemoveSlaveNode(const std::string& ip, + uint32_t port, + const std::string& table_name, + uint32_t partition_id, + const std::string& local_ip) { + InnerMessage::InnerRequest request; + request.set_type(InnerMessage::kRemoveSlaveNode); + InnerMessage::InnerRequest::RemoveSlaveNode* remove_slave_node = + request.add_remove_slave_node(); + InnerMessage::Node* node = remove_slave_node->mutable_node(); + node->set_ip(local_ip); + node->set_port(g_pika_server->port()); + + InnerMessage::Partition* partition = remove_slave_node->mutable_partition(); + partition->set_table_name(table_name); + partition->set_partition_id(partition_id); + + std::string to_send; + if (!request.SerializeToString(&to_send)) { + LOG(WARNING) << "Serialize Remove Slave Node Failed, to Master (" + << ip << ":" << port << "), " << table_name << "_" << partition_id; + return Status::Corruption("Serialize Failed"); + } + return client_thread_->Write(ip, port + kPortShiftReplServer, to_send); +} diff --git a/tools/pika_migrate/src/pika_repl_client_conn.cc b/tools/pika_migrate/src/pika_repl_client_conn.cc new file mode 100644 index 0000000000..dce825afbf --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_client_conn.cc @@ -0,0 +1,261 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_client_conn.h" + +#include + +#include "include/pika_server.h" +#include "include/pika_rm.h" +#include "slash/include/slash_string.h" + +#include "include/pika_rm.h" +#include "include/pika_server.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaReplClientConn::PikaReplClientConn(int fd, + const std::string& ip_port, + pink::Thread* thread, + void* worker_specific_data, + pink::PinkEpoll* epoll) + : pink::PbConn(fd, ip_port, thread, epoll) { +} + +bool PikaReplClientConn::IsTableStructConsistent( + const std::vector& current_tables, + const std::vector& expect_tables) { + if (current_tables.size() != expect_tables.size()) { + return false; + } + for (const auto& table_struct : current_tables) { + if (find(expect_tables.begin(), expect_tables.end(), + table_struct) == expect_tables.end()) { + return false; + } + } + return true; +} + +int PikaReplClientConn::DealMessage() { + std::shared_ptr response = std::make_shared(); + response->ParseFromArray(rbuf_ + cur_pos_ - header_len_, header_len_); + switch (response->type()) { + case InnerMessage::kMetaSync: + { + ReplClientTaskArg* task_arg = new ReplClientTaskArg(response, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplClientBGTask(&PikaReplClientConn::HandleMetaSyncResponse, static_cast(task_arg)); + break; + } + case InnerMessage::kDBSync: + { + ReplClientTaskArg* task_arg = new ReplClientTaskArg(response, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplClientBGTask(&PikaReplClientConn::HandleDBSyncResponse, static_cast(task_arg)); + break; + } + case InnerMessage::kTrySync: + { + ReplClientTaskArg* task_arg = new ReplClientTaskArg(response, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplClientBGTask(&PikaReplClientConn::HandleTrySyncResponse, static_cast(task_arg)); + break; + } + case InnerMessage::kBinlogSync: + { + DispatchBinlogRes(response); + break; + } + case InnerMessage::kRemoveSlaveNode: + { + ReplClientTaskArg* task_arg = new ReplClientTaskArg(response, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplClientBGTask(&PikaReplClientConn::HandleRemoveSlaveNodeResponse, static_cast(task_arg)); + break; + } + default: + break; + } + return 0; +} + +void PikaReplClientConn::HandleMetaSyncResponse(void* arg) { + ReplClientTaskArg* task_arg = static_cast(arg); + std::shared_ptr conn = task_arg->conn; + std::shared_ptr response = task_arg->res; + + if (response->code() != InnerMessage::kOk) { + std::string reply = response->has_reply() ? response->reply() : ""; + LOG(WARNING) << "Meta Sync Failed: " << reply; + g_pika_server->SyncError(); + conn->NotifyClose(); + delete task_arg; + return; + } + + const InnerMessage::InnerResponse_MetaSync meta_sync = response->meta_sync(); + if (g_pika_conf->classic_mode() != meta_sync.classic_mode()) { + LOG(WARNING) << "Self in " << (g_pika_conf->classic_mode() ? "classic" : "sharding") + << " mode, but master in " << (meta_sync.classic_mode() ? "classic" : "sharding") + << " mode, failed to establish master-slave relationship"; + g_pika_server->SyncError(); + conn->NotifyClose(); + delete task_arg; + return; + } + + std::vector master_table_structs; + for (int idx = 0; idx < meta_sync.tables_info_size(); ++idx) { + InnerMessage::InnerResponse_MetaSync_TableInfo table_info = meta_sync.tables_info(idx); + master_table_structs.push_back({table_info.table_name(), + static_cast(table_info.partition_num()), {0}}); + } + + std::vector self_table_structs = g_pika_conf->table_structs(); + if (!PikaReplClientConn::IsTableStructConsistent(self_table_structs, master_table_structs)) { + LOG(WARNING) << "Self table structs(number of databases: " << self_table_structs.size() + << ") inconsistent with master(number of databases: " << master_table_structs.size() + << "), failed to establish master-slave relationship"; + g_pika_server->SyncError(); + conn->NotifyClose(); + delete task_arg; + return; + } + + g_pika_conf->SetWriteBinlog("yes"); + g_pika_server->PreparePartitionTrySync(); + g_pika_server->FinishMetaSync(); + LOG(INFO) << "Finish to handle meta sync response"; + delete task_arg; +} + +void PikaReplClientConn::HandleDBSyncResponse(void* arg) { + ReplClientTaskArg* task_arg = static_cast(arg); + std::shared_ptr conn = task_arg->conn; + std::shared_ptr response = task_arg->res; + + const InnerMessage::InnerResponse_DBSync db_sync_response = response->db_sync(); + int32_t session_id = db_sync_response.session_id(); + const InnerMessage::Partition partition_response = db_sync_response.partition(); + std::string table_name = partition_response.table_name(); + uint32_t partition_id = partition_response.partition_id(); + + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << table_name << ":" << partition_id << " Not Found"; + delete task_arg; + return; + } + + if (response->code() != InnerMessage::kOk) { + slave_partition->SetReplState(ReplState::kError); + std::string reply = response->has_reply() ? response->reply() : ""; + LOG(WARNING) << "DBSync Failed: " << reply; + delete task_arg; + return; + } + + g_pika_rm->UpdateSyncSlavePartitionSessionId( + PartitionInfo(table_name, partition_id), session_id); + + std::string partition_name = slave_partition->SyncPartitionInfo().ToString(); + slave_partition->SetReplState(ReplState::kWaitDBSync); + LOG(INFO) << "Partition: " << partition_name << " Need Wait To Sync"; + delete task_arg; +} + +void PikaReplClientConn::HandleTrySyncResponse(void* arg) { + ReplClientTaskArg* task_arg = static_cast(arg); + std::shared_ptr conn = task_arg->conn; + std::shared_ptr response = task_arg->res; + + if (response->code() != InnerMessage::kOk) { + std::string reply = response->has_reply() ? response->reply() : ""; + LOG(WARNING) << "TrySync Failed: " << reply; + delete task_arg; + return; + } + + const InnerMessage::InnerResponse_TrySync& try_sync_response = response->try_sync(); + const InnerMessage::Partition& partition_response = try_sync_response.partition(); + std::string table_name = partition_response.table_name(); + uint32_t partition_id = partition_response.partition_id(); + std::shared_ptr partition = g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition) { + LOG(WARNING) << "Partition: " << table_name << ":" << partition_id << " Not Found"; + delete task_arg; + return; + } + + std::shared_ptr slave_partition = + g_pika_rm->GetSyncSlavePartitionByName( + PartitionInfo(table_name, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << table_name << ":" << partition_id << " Not Found"; + delete task_arg; + return; + } + + std::string partition_name = partition->GetPartitionName(); + if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kOk) { + BinlogOffset boffset; + int32_t session_id = try_sync_response.session_id(); + partition->logger()->GetProducerStatus(&boffset.filenum, &boffset.offset); + g_pika_rm->UpdateSyncSlavePartitionSessionId(PartitionInfo(table_name, partition_id), session_id); + g_pika_rm->SendPartitionBinlogSyncAckRequest(table_name, partition_id, boffset, boffset, true); + slave_partition->SetReplState(ReplState::kConnected); + LOG(INFO) << "Partition: " << partition_name << " TrySync Ok"; + } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kSyncPointBePurged) { + slave_partition->SetReplState(ReplState::kTryDBSync); + LOG(INFO) << "Partition: " << partition_name << " Need To Try DBSync"; + } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kSyncPointLarger) { + slave_partition->SetReplState(ReplState::kError); + LOG(WARNING) << "Partition: " << partition_name << " TrySync Error, Because the invalid filenum and offset"; + } else if (try_sync_response.reply_code() == InnerMessage::InnerResponse::TrySync::kError) { + slave_partition->SetReplState(ReplState::kError); + LOG(WARNING) << "Partition: " << partition_name << " TrySync Error"; + } + delete task_arg; +} + +void PikaReplClientConn::DispatchBinlogRes(const std::shared_ptr res) { + // partition to a bunch of binlog chips + std::unordered_map*, hash_partition_info> par_binlog; + for (int i = 0; i < res->binlog_sync_size(); ++i) { + const InnerMessage::InnerResponse::BinlogSync& binlog_res = res->binlog_sync(i); + // hash key: table + partition_id + PartitionInfo p_info(binlog_res.partition().table_name(), + binlog_res.partition().partition_id()); + if (par_binlog.find(p_info) == par_binlog.end()) { + par_binlog[p_info] = new std::vector(); + } + par_binlog[p_info]->push_back(i); + } + + for (auto& binlog_nums : par_binlog) { + RmNode node(binlog_nums.first.table_name_, binlog_nums.first.partition_id_); + g_pika_rm->SetSlaveLastRecvTime(node, slash::NowMicros()); + g_pika_rm->ScheduleWriteBinlogTask( + binlog_nums.first.table_name_ + std::to_string(binlog_nums.first.partition_id_), + res, + std::dynamic_pointer_cast(shared_from_this()), + reinterpret_cast(binlog_nums.second)); + } +} + +void PikaReplClientConn::HandleRemoveSlaveNodeResponse(void* arg) { + ReplClientTaskArg* task_arg = static_cast(arg); + std::shared_ptr conn = task_arg->conn; + std::shared_ptr response = task_arg->res; + if (response->code() != InnerMessage::kOk) { + std::string reply = response->has_reply() ? response->reply() : ""; + LOG(WARNING) << "Remove slave node Failed: " << reply; + delete task_arg; + return; + } + delete task_arg; +} + diff --git a/tools/pika_migrate/src/pika_repl_client_thread.cc b/tools/pika_migrate/src/pika_repl_client_thread.cc new file mode 100644 index 0000000000..cfb8de7500 --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_client_thread.cc @@ -0,0 +1,48 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_client_thread.h" + +#include "include/pika_server.h" + +#include "slash/include/slash_string.h" + +extern PikaServer* g_pika_server; + +PikaReplClientThread::PikaReplClientThread(int cron_interval, int keepalive_timeout) : + ClientThread(&conn_factory_, cron_interval, keepalive_timeout, &handle_, NULL) { +} + +void PikaReplClientThread::ReplClientHandle::FdClosedHandle(int fd, const std::string& ip_port) const { + LOG(INFO) << "ReplClient Close conn, fd=" << fd << ", ip_port=" << ip_port; + std::string ip; + int port = 0; + if (!slash::ParseIpPortString(ip_port, ip, port)) { + LOG(WARNING) << "Parse ip_port error " << ip_port; + return; + } + if (ip == g_pika_server->master_ip() + && port == g_pika_server->master_port() + kPortShiftReplServer + && PIKA_REPL_ERROR != g_pika_server->repl_state()) { // if state machine in error state, no retry + LOG(WARNING) << "Master conn disconnect : " << ip_port << " try reconnect"; + g_pika_server->ResetMetaSyncStatus(); + } +}; + +void PikaReplClientThread::ReplClientHandle::FdTimeoutHandle(int fd, const std::string& ip_port) const { + LOG(INFO) << "ReplClient Timeout conn, fd=" << fd << ", ip_port=" << ip_port; + std::string ip; + int port = 0; + if (!slash::ParseIpPortString(ip_port, ip, port)) { + LOG(WARNING) << "Parse ip_port error " << ip_port; + return; + } + if (ip == g_pika_server->master_ip() + && port == g_pika_server->master_port() + kPortShiftReplServer + && PIKA_REPL_ERROR != g_pika_server->repl_state()) { // if state machine in error state, no retry + LOG(WARNING) << "Master conn timeout : " << ip_port << " try reconnect"; + g_pika_server->ResetMetaSyncStatus(); + } +}; diff --git a/tools/pika_migrate/src/pika_repl_server.cc b/tools/pika_migrate/src/pika_repl_server.cc new file mode 100644 index 0000000000..6587780561 --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_server.cc @@ -0,0 +1,126 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_server.h" + +#include + +#include "include/pika_rm.h" +#include "include/pika_conf.h" +#include "include/pika_server.h" + +extern PikaConf* g_pika_conf; +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaReplServer::PikaReplServer(const std::set& ips, + int port, + int cron_interval) { + server_tp_ = new pink::ThreadPool(PIKA_REPL_SERVER_TP_SIZE, 100000); + pika_repl_server_thread_ = new PikaReplServerThread(ips, port, cron_interval); + pika_repl_server_thread_->set_thread_name("PikaReplServer"); + pthread_rwlock_init(&client_conn_rwlock_, NULL); +} + +PikaReplServer::~PikaReplServer() { + delete pika_repl_server_thread_; + delete server_tp_; + pthread_rwlock_destroy(&client_conn_rwlock_); + LOG(INFO) << "PikaReplServer exit!!!"; +} + +int PikaReplServer::Start() { + int res = pika_repl_server_thread_->StartThread(); + if (res != pink::kSuccess) { + LOG(FATAL) << "Start Pika Repl Server Thread Error: " << res + << (res == pink::kBindError ? ": bind port " + std::to_string(pika_repl_server_thread_->ListenPort()) + " conflict" : ": create thread error ") + << ", Listen on this port to handle the request sent by the Slave"; + } + res = server_tp_->start_thread_pool(); + if (res != pink::kSuccess) { + LOG(FATAL) << "Start ThreadPool Error: " << res << (res == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + return res; +} + +int PikaReplServer::Stop() { + server_tp_->stop_thread_pool(); + pika_repl_server_thread_->StopThread(); + return 0; +} + +slash::Status PikaReplServer::SendSlaveBinlogChips(const std::string& ip, + int port, + const std::vector& tasks) { + InnerMessage::InnerResponse response; + response.set_code(InnerMessage::kOk); + response.set_type(InnerMessage::Type::kBinlogSync); + for (const auto task :tasks) { + InnerMessage::InnerResponse::BinlogSync* binlog_sync = response.add_binlog_sync(); + binlog_sync->set_session_id(task.rm_node_.SessionId()); + InnerMessage::Partition* partition = binlog_sync->mutable_partition(); + partition->set_table_name(task.rm_node_.TableName()); + partition->set_partition_id(task.rm_node_.PartitionId()); + InnerMessage::BinlogOffset* boffset = binlog_sync->mutable_binlog_offset(); + boffset->set_filenum(task.binlog_chip_.offset_.filenum); + boffset->set_offset(task.binlog_chip_.offset_.offset); + binlog_sync->set_binlog(task.binlog_chip_.binlog_); + } + + std::string binlog_chip_pb; + if (!response.SerializeToString(&binlog_chip_pb)) { + return Status::Corruption("Serialized Failed"); + } + return Write(ip, port, binlog_chip_pb); +} + +slash::Status PikaReplServer::Write(const std::string& ip, + const int port, + const std::string& msg) { + slash::RWLock l(&client_conn_rwlock_, false); + const std::string ip_port = slash::IpPortString(ip, port); + if (client_conn_map_.find(ip_port) == client_conn_map_.end()) { + return Status::NotFound("The " + ip_port + " fd cannot be found"); + } + int fd = client_conn_map_[ip_port]; + std::shared_ptr conn = + std::dynamic_pointer_cast(pika_repl_server_thread_->get_conn(fd)); + if (conn == nullptr) { + return Status::NotFound("The" + ip_port + " conn cannot be found"); + } + + if (conn->WriteResp(msg)) { + conn->NotifyClose(); + return Status::Corruption("The" + ip_port + " conn, Write Resp Failed"); + } + conn->NotifyWrite(); + return Status::OK(); +} + +void PikaReplServer::Schedule(pink::TaskFunc func, void* arg){ + server_tp_->Schedule(func, arg); +} + +void PikaReplServer::UpdateClientConnMap(const std::string& ip_port, int fd) { + slash::RWLock l(&client_conn_rwlock_, true); + client_conn_map_[ip_port] = fd; +} + +void PikaReplServer::RemoveClientConn(int fd) { + slash::RWLock l(&client_conn_rwlock_, true); + std::map::const_iterator iter = client_conn_map_.begin(); + while (iter != client_conn_map_.end()) { + if (iter->second == fd) { + iter = client_conn_map_.erase(iter); + break; + } + iter++; + } +} + +void PikaReplServer::KillAllConns() { + return pika_repl_server_thread_->KillAllConns(); +} + diff --git a/tools/pika_migrate/src/pika_repl_server_conn.cc b/tools/pika_migrate/src/pika_repl_server_conn.cc new file mode 100644 index 0000000000..85b3273741 --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_server_conn.cc @@ -0,0 +1,448 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_server_conn.h" + +#include + +#include "include/pika_rm.h" +#include "include/pika_server.h" + +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaReplServerConn::PikaReplServerConn(int fd, + std::string ip_port, + pink::Thread* thread, + void* worker_specific_data, pink::PinkEpoll* epoll) + : PbConn(fd, ip_port, thread, epoll) { +} + +PikaReplServerConn::~PikaReplServerConn() { +} + +void PikaReplServerConn::HandleMetaSyncRequest(void* arg) { + ReplServerTaskArg* task_arg = static_cast(arg); + const std::shared_ptr req = task_arg->req; + std::shared_ptr conn = task_arg->conn; + + InnerMessage::InnerRequest::MetaSync meta_sync_request = req->meta_sync(); + InnerMessage::Node node = meta_sync_request.node(); + std::string masterauth = meta_sync_request.has_auth() ? meta_sync_request.auth() : ""; + + InnerMessage::InnerResponse response; + response.set_type(InnerMessage::kMetaSync); + if (!g_pika_conf->requirepass().empty() + && g_pika_conf->requirepass() != masterauth) { + response.set_code(InnerMessage::kError); + response.set_reply("Auth with master error, Invalid masterauth"); + } else { + std::vector table_structs = g_pika_conf->table_structs(); + bool success = g_pika_server->TryAddSlave(node.ip(), node.port(), conn->fd(), table_structs); + const std::string ip_port = slash::IpPortString(node.ip(), node.port()); + g_pika_rm->ReplServerUpdateClientConnMap(ip_port, conn->fd()); + if (!success) { + response.set_code(InnerMessage::kError); + response.set_reply("Slave AlreadyExist"); + } else { + g_pika_server->BecomeMaster(); + response.set_code(InnerMessage::kOk); + InnerMessage::InnerResponse_MetaSync* meta_sync = response.mutable_meta_sync(); + meta_sync->set_classic_mode(g_pika_conf->classic_mode()); + for (const auto& table_struct : table_structs) { + InnerMessage::InnerResponse_MetaSync_TableInfo* table_info = meta_sync->add_tables_info(); + table_info->set_table_name(table_struct.table_name); + table_info->set_partition_num(table_struct.partition_num); + } + } + } + + std::string reply_str; + if (!response.SerializeToString(&reply_str) + || conn->WriteResp(reply_str)) { + LOG(WARNING) << "Process MetaSync request serialization failed"; + conn->NotifyClose(); + delete task_arg; + return; + } + conn->NotifyWrite(); + delete task_arg; +} + +void PikaReplServerConn::HandleTrySyncRequest(void* arg) { + ReplServerTaskArg* task_arg = static_cast(arg); + const std::shared_ptr req = task_arg->req; + std::shared_ptr conn = task_arg->conn; + + InnerMessage::InnerRequest::TrySync try_sync_request = req->try_sync(); + InnerMessage::Partition partition_request = try_sync_request.partition(); + InnerMessage::BinlogOffset slave_boffset = try_sync_request.binlog_offset(); + InnerMessage::Node node = try_sync_request.node(); + + InnerMessage::InnerResponse response; + InnerMessage::InnerResponse::TrySync* try_sync_response = response.mutable_try_sync(); + InnerMessage::Partition* partition_response = try_sync_response->mutable_partition(); + InnerMessage::BinlogOffset* master_partition_boffset = try_sync_response->mutable_binlog_offset(); + + std::string table_name = partition_request.table_name(); + uint32_t partition_id = partition_request.partition_id(); + + bool pre_success = true; + response.set_type(InnerMessage::Type::kTrySync); + std::shared_ptr partition = g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition) { + response.set_code(InnerMessage::kError); + response.set_reply("Partition not found"); + LOG(WARNING) << "Table Name: " << table_name << " Partition ID: " + << partition_id << " Not Found, TrySync Error"; + pre_success = false; + } + + BinlogOffset boffset; + std::string partition_name; + if (pre_success) { + partition_name = partition->GetPartitionName(); + LOG(INFO) << "Receive Trysync, Slave ip: " << node.ip() << ", Slave port:" + << node.port() << ", Partition: " << partition_name << ", filenum: " + << slave_boffset.filenum() << ", pro_offset: " << slave_boffset.offset(); + + response.set_code(InnerMessage::kOk); + partition_response->set_table_name(table_name); + partition_response->set_partition_id(partition_id); + if (!partition->GetBinlogOffset(&boffset)) { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); + LOG(WARNING) << "Handle TrySync, Partition: " + << partition_name << " Get binlog offset error, TrySync failed"; + pre_success = false; + } + } + + if (pre_success) { + master_partition_boffset->set_filenum(boffset.filenum); + master_partition_boffset->set_offset(boffset.offset); + if (boffset.filenum < slave_boffset.filenum() + || (boffset.filenum == slave_boffset.filenum() && boffset.offset < slave_boffset.offset())) { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kSyncPointLarger); + LOG(WARNING) << "Slave offset is larger than mine, Slave ip: " + << node.ip() << ", Slave port: " << node.port() << ", Partition: " + << partition_name << ", filenum: " << slave_boffset.filenum() + << ", pro_offset_: " << slave_boffset.offset(); + pre_success = false; + } + if (pre_success) { + std::string confile = NewFileName(partition->logger()->filename, slave_boffset.filenum()); + if (!slash::FileExists(confile)) { + LOG(INFO) << "Partition: " << partition_name << " binlog has been purged, may need full sync"; + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kSyncPointBePurged); + pre_success = false; + } + } + if (pre_success) { + PikaBinlogReader reader; + reader.Seek(partition->logger(), slave_boffset.filenum(), slave_boffset.offset()); + BinlogOffset seeked_offset; + reader.GetReaderStatus(&(seeked_offset.filenum), &(seeked_offset.offset)); + if (seeked_offset.filenum != slave_boffset.filenum() || seeked_offset.offset != slave_boffset.offset()) { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); + LOG(WARNING) << "Slave offset is not a start point of cur log, Slave ip: " + << node.ip() << ", Slave port: " << node.port() << ", Partition: " + << partition_name << ", cloest start point, filenum: " << seeked_offset.filenum + << ", offset: " << seeked_offset.offset; + pre_success = false; + } + } + } + + if (pre_success) { + if (!g_pika_rm->CheckPartitionSlaveExist(RmNode(node.ip(), node.port(), table_name, partition_id))) { + int32_t session_id = g_pika_rm->GenPartitionSessionId(table_name, partition_id); + if (session_id != -1) { + try_sync_response->set_session_id(session_id); + // incremental sync + Status s = g_pika_rm->AddPartitionSlave(RmNode(node.ip(), node.port(), table_name, partition_id, session_id)); + if (!s.ok()) { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); + LOG(WARNING) << "Partition: " << partition_name << " TrySync Failed, " << s.ToString(); + pre_success = false; + } else { + const std::string ip_port = slash::IpPortString(node.ip(), node.port()); + g_pika_rm->ReplServerUpdateClientConnMap(ip_port, conn->fd()); + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kOk); + LOG(INFO) << "Partition: " << partition_name << " TrySync Success, Session: " << session_id; + } + } else { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); + LOG(WARNING) << "Partition: " << partition_name << ", Gen Session id Failed"; + pre_success = false; + } + } else { + int32_t session_id; + Status s = g_pika_rm->GetPartitionSlaveSession( + RmNode(node.ip(), node.port(), table_name, partition_id), &session_id); + if (!s.ok()) { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kError); + LOG(WARNING) << "Partition: " << partition_name << ", Get Session id Failed" << s.ToString(); + pre_success = false; + } else { + try_sync_response->set_reply_code(InnerMessage::InnerResponse::TrySync::kOk); + try_sync_response->set_session_id(session_id); + LOG(INFO) << "Partition: " << partition_name << " TrySync Success, Session: " << session_id; + } + } + } + + std::string reply_str; + if (!response.SerializeToString(&reply_str) + || conn->WriteResp(reply_str)) { + LOG(WARNING) << "Handle Try Sync Failed"; + conn->NotifyClose(); + delete task_arg; + return; + } + conn->NotifyWrite(); + delete task_arg; +} + + +void PikaReplServerConn::HandleDBSyncRequest(void* arg) { + ReplServerTaskArg* task_arg = static_cast(arg); + const std::shared_ptr req = task_arg->req; + std::shared_ptr conn = task_arg->conn; + + InnerMessage::InnerRequest::DBSync db_sync_request = req->db_sync(); + InnerMessage::Partition partition_request = db_sync_request.partition(); + InnerMessage::Node node = db_sync_request.node(); + InnerMessage::BinlogOffset slave_boffset = db_sync_request.binlog_offset(); + std::string table_name = partition_request.table_name(); + uint32_t partition_id = partition_request.partition_id(); + std::string partition_name = table_name + "_" + std::to_string(partition_id); + + InnerMessage::InnerResponse response; + response.set_code(InnerMessage::kOk); + response.set_type(InnerMessage::Type::kDBSync); + InnerMessage::InnerResponse::DBSync* db_sync_response = response.mutable_db_sync(); + InnerMessage::Partition* partition_response = db_sync_response->mutable_partition(); + partition_response->set_table_name(table_name); + partition_response->set_partition_id(partition_id); + + LOG(INFO) << "Handle partition DBSync Request"; + bool prior_success = true; + if (!g_pika_rm->CheckPartitionSlaveExist(RmNode(node.ip(), node.port(), table_name, partition_id))) { + int32_t session_id = g_pika_rm->GenPartitionSessionId(table_name, partition_id); + if (session_id == -1) { + response.set_code(InnerMessage::kError); + LOG(WARNING) << "Partition: " << partition_name << ", Gen Session id Failed"; + prior_success = false; + } + if (prior_success) { + db_sync_response->set_session_id(session_id); + Status s = g_pika_rm->AddPartitionSlave(RmNode(node.ip(), node.port(), table_name, partition_id, session_id)); + if (s.ok()) { + const std::string ip_port = slash::IpPortString(node.ip(), node.port()); + g_pika_rm->ReplServerUpdateClientConnMap(ip_port, conn->fd()); + LOG(INFO) << "Partition: " << partition_name << " Handle DBSync Request Success, Session: " << session_id; + } else { + response.set_code(InnerMessage::kError); + LOG(WARNING) << "Partition: " << partition_name << " Handle DBSync Request Failed, " << s.ToString(); + prior_success = false; + } + } else { + db_sync_response->set_session_id(-1); + } + } else { + int32_t session_id; + Status s = g_pika_rm->GetPartitionSlaveSession( + RmNode(node.ip(), node.port(), table_name, partition_id), &session_id); + if (!s.ok()) { + response.set_code(InnerMessage::kError); + LOG(WARNING) << "Partition: " << partition_name << ", Get Session id Failed" << s.ToString(); + prior_success = false; + db_sync_response->set_session_id(-1); + } else { + db_sync_response->set_session_id(session_id); + LOG(INFO) << "Partition: " << partition_name << " Handle DBSync Request Success, Session: " << session_id; + } + } + + g_pika_server->TryDBSync(node.ip(), node.port() + kPortShiftRSync, + table_name, partition_id, slave_boffset.filenum()); + + std::string reply_str; + if (!response.SerializeToString(&reply_str) + || conn->WriteResp(reply_str)) { + LOG(WARNING) << "Handle DBSync Failed"; + conn->NotifyClose(); + delete task_arg; + return; + } + conn->NotifyWrite(); + delete task_arg; +} + +void PikaReplServerConn::HandleBinlogSyncRequest(void* arg) { + ReplServerTaskArg* task_arg = static_cast(arg); + const std::shared_ptr req = task_arg->req; + std::shared_ptr conn = task_arg->conn; + if (!req->has_binlog_sync()) { + LOG(WARNING) << "Pb parse error"; + //conn->NotifyClose(); + delete task_arg; + return; + } + const InnerMessage::InnerRequest::BinlogSync& binlog_req = req->binlog_sync(); + const InnerMessage::Node& node = binlog_req.node(); + const std::string& table_name = binlog_req.table_name(); + uint32_t partition_id = binlog_req.partition_id(); + + bool is_first_send = binlog_req.first_send(); + int32_t session_id = binlog_req.session_id(); + const InnerMessage::BinlogOffset& ack_range_start = binlog_req.ack_range_start(); + const InnerMessage::BinlogOffset& ack_range_end = binlog_req.ack_range_end(); + BinlogOffset range_start(ack_range_start.filenum(), ack_range_start.offset()); + BinlogOffset range_end(ack_range_end.filenum(), ack_range_end.offset()); + + if (!g_pika_rm->CheckMasterPartitionSessionId(node.ip(), + node.port(), table_name, partition_id, session_id)) { + LOG(WARNING) << "Check Session failed " << node.ip() << ":" << node.port() + << ", " << table_name << "_" << partition_id; + //conn->NotifyClose(); + delete task_arg; + return; + } + + // Set ack info from slave + RmNode slave_node = RmNode(node.ip(), node.port(), table_name, partition_id); + + Status s = g_pika_rm->SetMasterLastRecvTime(slave_node, slash::NowMicros()); + if (!s.ok()) { + LOG(WARNING) << "SetMasterLastRecvTime failed " << node.ip() << ":" << node.port() + << ", " << table_name << "_" << partition_id << " " << s.ToString(); + conn->NotifyClose(); + delete task_arg; + return; + } + + if (is_first_send) { + if (!(range_start == range_end)) { + LOG(WARNING) << "first binlogsync request pb argument invalid"; + conn->NotifyClose(); + delete task_arg; + return; + } + Status s = g_pika_rm->ActivateBinlogSync(slave_node, range_start); + if (!s.ok()) { + LOG(WARNING) << "Activate Binlog Sync failed " << slave_node.ToString() << " " << s.ToString(); + conn->NotifyClose(); + delete task_arg; + return; + } + delete task_arg; + return; + } + + // not the first_send the range_ack cant be 0 + // set this case as ping + if (range_start == BinlogOffset() && range_end == BinlogOffset()) { + delete task_arg; + return; + } + s = g_pika_rm->UpdateSyncBinlogStatus(slave_node, range_start, range_end); + if (!s.ok()) { + LOG(WARNING) << "Update binlog ack failed " << table_name << " " << partition_id << " " << s.ToString(); + conn->NotifyClose(); + delete task_arg; + return; + } + delete task_arg; + g_pika_server->SignalAuxiliary(); + return; +} + +void PikaReplServerConn::HandleRemoveSlaveNodeRequest(void* arg) { + ReplServerTaskArg* task_arg = static_cast(arg); + const std::shared_ptr req = task_arg->req; + std::shared_ptr conn = task_arg->conn; + if (!req->remove_slave_node_size()) { + LOG(WARNING) << "Pb parse error"; + conn->NotifyClose(); + delete task_arg; + return; + } + const InnerMessage::InnerRequest::RemoveSlaveNode& remove_slave_node_req = req->remove_slave_node(0); + const InnerMessage::Node& node = remove_slave_node_req.node(); + const InnerMessage::Partition& partition = remove_slave_node_req.partition(); + + std::string table_name = partition.table_name(); + uint32_t partition_id = partition.partition_id(); + Status s = g_pika_rm->RemovePartitionSlave(RmNode(node.ip(), + node.port(), table_name, partition_id)); + + InnerMessage::InnerResponse response; + response.set_code(InnerMessage::kOk); + response.set_type(InnerMessage::Type::kRemoveSlaveNode); + InnerMessage::InnerResponse::RemoveSlaveNode* remove_slave_node_response = response.add_remove_slave_node(); + InnerMessage::Partition* partition_response = remove_slave_node_response->mutable_partition(); + partition_response->set_table_name(table_name); + partition_response->set_partition_id(partition_id); + InnerMessage::Node* node_response = remove_slave_node_response->mutable_node(); + node_response->set_ip(g_pika_server->host()); + node_response->set_port(g_pika_server->port()); + + std::string reply_str; + if (!response.SerializeToString(&reply_str) + || conn->WriteResp(reply_str)) { + LOG(WARNING) << "Remove Slave Node Failed"; + conn->NotifyClose(); + delete task_arg; + return; + } + conn->NotifyWrite(); + delete task_arg; +} + +int PikaReplServerConn::DealMessage() { + std::shared_ptr req = std::make_shared(); + bool parse_res = req->ParseFromArray(rbuf_ + cur_pos_ - header_len_, header_len_); + if (!parse_res) { + LOG(WARNING) << "Pika repl server connection pb parse error."; + return -1; + } + int res = 0; + switch (req->type()) { + case InnerMessage::kMetaSync: + { + ReplServerTaskArg* task_arg = new ReplServerTaskArg(req, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplServerBGTask(&PikaReplServerConn::HandleMetaSyncRequest, task_arg); + break; + } + case InnerMessage::kTrySync: + { + ReplServerTaskArg* task_arg = new ReplServerTaskArg(req, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplServerBGTask(&PikaReplServerConn::HandleTrySyncRequest, task_arg); + break; + } + case InnerMessage::kDBSync: + { + ReplServerTaskArg* task_arg = new ReplServerTaskArg(req, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplServerBGTask(&PikaReplServerConn::HandleDBSyncRequest, task_arg); + break; + } + case InnerMessage::kBinlogSync: + { + ReplServerTaskArg* task_arg = new ReplServerTaskArg(req, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplServerBGTask(&PikaReplServerConn::HandleBinlogSyncRequest, task_arg); + break; + } + case InnerMessage::kRemoveSlaveNode: + { + ReplServerTaskArg* task_arg = new ReplServerTaskArg(req, std::dynamic_pointer_cast(shared_from_this())); + g_pika_rm->ScheduleReplServerBGTask(&PikaReplServerConn::HandleRemoveSlaveNodeRequest, task_arg); + break; + } + default: + break; + } + return res; +} diff --git a/tools/pika_migrate/src/pika_repl_server_thread.cc b/tools/pika_migrate/src/pika_repl_server_thread.cc new file mode 100644 index 0000000000..edc33f8fd0 --- /dev/null +++ b/tools/pika_migrate/src/pika_repl_server_thread.cc @@ -0,0 +1,32 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_repl_server_thread.h" + +#include "include/pika_rm.h" +#include "include/pika_server.h" + +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; + +PikaReplServerThread::PikaReplServerThread(const std::set& ips, + int port, + int cron_interval) : + HolyThread(ips, port, &conn_factory_, cron_interval, &handle_, true), + conn_factory_(this), + port_(port), + serial_(0) { + set_keepalive_timeout(180); +} + +int PikaReplServerThread::ListenPort() { + return port_; +} + +void PikaReplServerThread::ReplServerHandle::FdClosedHandle(int fd, const std::string& ip_port) const { + LOG(INFO) << "ServerThread Close Slave Conn, fd: " << fd << ", ip_port: " << ip_port; + g_pika_server->DeleteSlave(fd); + g_pika_rm->ReplServerRemoveClientConn(fd); +} diff --git a/tools/pika_migrate/src/pika_rm.cc b/tools/pika_migrate/src/pika_rm.cc new file mode 100644 index 0000000000..2c240ba9a4 --- /dev/null +++ b/tools/pika_migrate/src/pika_rm.cc @@ -0,0 +1,1634 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "set" + +#include +#include +#include +#include + +#include "pink/include/pink_cli.h" + +#include "include/pika_rm.h" +#include "include/pika_conf.h" +#include "include/pika_server.h" +#include "include/pika_repl_client.h" +#include "include/pika_repl_server.h" + + +extern PikaConf *g_pika_conf; +extern PikaReplicaManager* g_pika_rm; +extern PikaServer *g_pika_server; + +/* BinlogReaderManager */ + +BinlogReaderManager::~BinlogReaderManager() { +} + +Status BinlogReaderManager::FetchBinlogReader(const RmNode& rm_node, std::shared_ptr* reader) { + slash::MutexLock l(&reader_mu_); + if (occupied_.find(rm_node) != occupied_.end()) { + return Status::Corruption(rm_node.ToString() + " exist"); + } + if (vacant_.empty()) { + *reader = std::make_shared(); + } else { + *reader = *(vacant_.begin()); + vacant_.erase(vacant_.begin()); + } + occupied_[rm_node] = *reader; + return Status::OK(); +} + +Status BinlogReaderManager::ReleaseBinlogReader(const RmNode& rm_node) { + slash::MutexLock l(&reader_mu_); + if (occupied_.find(rm_node) == occupied_.end()) { + return Status::NotFound(rm_node.ToString()); + } + std::shared_ptr reader = occupied_[rm_node]; + occupied_.erase(rm_node); + vacant_.push_back(reader); + return Status::OK(); +} + +/* SlaveNode */ + +SlaveNode::SlaveNode(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id, int session_id) + : RmNode(ip, port, table_name, partition_id, session_id), + slave_state(kSlaveNotSync), + b_state(kNotSync), sent_offset(), acked_offset() { +} + +SlaveNode::~SlaveNode() { + if (b_state == kReadFromFile && binlog_reader != nullptr) { + RmNode rm_node(Ip(), Port(), TableName(), PartitionId()); + ReleaseBinlogFileReader(); + } +} + +Status SlaveNode::InitBinlogFileReader(const std::shared_ptr& binlog, + const BinlogOffset& offset) { + Status s = g_pika_rm->binlog_reader_mgr.FetchBinlogReader( + RmNode(Ip(), Port(), NodePartitionInfo()), &binlog_reader); + if (!s.ok()) { + return s; + } + int res = binlog_reader->Seek(binlog, offset.filenum, offset.offset); + if (res) { + g_pika_rm->binlog_reader_mgr.ReleaseBinlogReader( + RmNode(Ip(), Port(), NodePartitionInfo())); + return Status::Corruption(ToString() + " binlog reader init failed"); + } + return Status::OK(); +} + +void SlaveNode::ReleaseBinlogFileReader() { + g_pika_rm->binlog_reader_mgr.ReleaseBinlogReader( + RmNode(Ip(), Port(), NodePartitionInfo())); + binlog_reader = nullptr; +} + +std::string SlaveNode::ToStringStatus() { + std::stringstream tmp_stream; + tmp_stream << " Slave_state: " << SlaveStateMsg[slave_state] << "\r\n"; + tmp_stream << " Binlog_sync_state: " << BinlogSyncStateMsg[b_state] << "\r\n"; + tmp_stream << " Sync_window: " << "\r\n" << sync_win.ToStringStatus(); + tmp_stream << " Sent_offset: " << sent_offset.ToString() << "\r\n"; + tmp_stream << " Acked_offset: " << acked_offset.ToString() << "\r\n"; + tmp_stream << " Binlog_reader activated: " << (binlog_reader != nullptr) << "\r\n"; + return tmp_stream.str(); +} + +/* SyncPartition */ + +SyncPartition::SyncPartition(const std::string& table_name, uint32_t partition_id) + : partition_info_(table_name, partition_id) { +} + +/* SyncMasterPartition*/ + +SyncMasterPartition::SyncMasterPartition(const std::string& table_name, uint32_t partition_id) + : SyncPartition(table_name, partition_id), + session_id_(0) {} + +bool SyncMasterPartition::CheckReadBinlogFromCache() { + return false; +} + +int SyncMasterPartition::GetNumberOfSlaveNode() { + slash::MutexLock l(&partition_mu_); + return slaves_.size(); +} + +bool SyncMasterPartition::CheckSlaveNodeExist(const std::string& ip, int port) { + slash::MutexLock l(&partition_mu_); + for (auto& slave : slaves_) { + if (ip == slave->Ip() && port == slave->Port()) { + return true; + } + } + return false; +} + +Status SyncMasterPartition::GetSlaveNodeSession( + const std::string& ip, int port, int32_t* session) { + slash::MutexLock l(&partition_mu_); + for (auto& slave : slaves_) { + if (ip == slave->Ip() && port == slave->Port()) { + *session = slave->SessionId(); + return Status::OK(); + } + } + return Status::NotFound("slave " + ip + ":" + std::to_string(port) + " not found"); +} + +Status SyncMasterPartition::AddSlaveNode(const std::string& ip, int port, int session_id) { + slash::MutexLock l(&partition_mu_); + for (auto& slave : slaves_) { + if (ip == slave->Ip() && port == slave->Port()) { + slave->SetSessionId(session_id); + return Status::OK(); + } + } + std::shared_ptr slave_ptr = + std::make_shared(ip, port, SyncPartitionInfo().table_name_, SyncPartitionInfo().partition_id_, session_id); + slave_ptr->SetLastSendTime(slash::NowMicros()); + slave_ptr->SetLastRecvTime(slash::NowMicros()); + slaves_.push_back(slave_ptr); + LOG(INFO) << "Add Slave Node, partition: " << SyncPartitionInfo().ToString() << ", ip_port: "<< ip << ":" << port; + return Status::OK(); +} + +Status SyncMasterPartition::RemoveSlaveNode(const std::string& ip, int port) { + slash::MutexLock l(&partition_mu_); + for (size_t i = 0; i < slaves_.size(); ++i) { + std::shared_ptr slave = slaves_[i]; + if (ip == slave->Ip() && port == slave->Port()) { + slaves_.erase(slaves_.begin() + i); + LOG(INFO) << "Remove Slave Node, Partition: " << SyncPartitionInfo().ToString() + << ", ip_port: "<< ip << ":" << port; + return Status::OK(); + } + } + return Status::NotFound("RemoveSlaveNode" + ip + std::to_string(port)); +} + +Status SyncMasterPartition::ActivateSlaveBinlogSync(const std::string& ip, + int port, + const std::shared_ptr binlog, + const BinlogOffset& offset) { + { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + bool read_cache = CheckReadBinlogFromCache(); + + slave_ptr->Lock(); + slave_ptr->slave_state = kSlaveBinlogSync; + slave_ptr->sent_offset = offset; + slave_ptr->acked_offset = offset; + if (read_cache) { + slave_ptr->Unlock(); + // RegistToBinlogCacheWindow(ip, port, offset); + slave_ptr->Lock(); + slave_ptr->b_state = kReadFromCache; + } else { + // read binlog file from file + s = slave_ptr->InitBinlogFileReader(binlog, offset); + if (!s.ok()) { + slave_ptr->Unlock(); + return Status::Corruption("Init binlog file reader failed" + s.ToString()); + } + slave_ptr->b_state = kReadFromFile; + } + slave_ptr->Unlock(); + } + + Status s = SyncBinlogToWq(ip, port); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status SyncMasterPartition::SyncBinlogToWq(const std::string& ip, int port) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + if (slave_ptr->b_state == kReadFromFile) { + ReadBinlogFileToWq(slave_ptr); + } else if (slave_ptr->b_state == kReadFromCache) { + ReadCachedBinlogToWq(slave_ptr); + } + } + return Status::OK(); +} + +Status SyncMasterPartition::ActivateSlaveDbSync(const std::string& ip, int port) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + slave_ptr->slave_state = kSlaveDbSync; + // invoke db sync + } + return Status::OK(); +} + +Status SyncMasterPartition::ReadCachedBinlogToWq(const std::shared_ptr& slave_ptr) { + return Status::OK(); +} + +Status SyncMasterPartition::ReadBinlogFileToWq(const std::shared_ptr& slave_ptr) { + int cnt = slave_ptr->sync_win.Remainings(); + std::shared_ptr reader = slave_ptr->binlog_reader; + std::vector tasks; + for (int i = 0; i < cnt; ++i) { + std::string msg; + uint32_t filenum; + uint64_t offset; + Status s = reader->Get(&msg, &filenum, &offset); + if (s.IsEndFile()) { + break; + } else if (s.IsCorruption() || s.IsIOError()) { + LOG(WARNING) << SyncPartitionInfo().ToString() + << " Read Binlog error : " << s.ToString(); + return s; + } + slave_ptr->sync_win.Push(SyncWinItem(filenum, offset)); + + BinlogOffset sent_offset = BinlogOffset(filenum, offset); + slave_ptr->sent_offset = sent_offset; + slave_ptr->SetLastSendTime(slash::NowMicros()); + RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->TableName(), slave_ptr->PartitionId(), slave_ptr->SessionId()); + WriteTask task(rm_node, BinlogChip(sent_offset, msg)); + tasks.push_back(task); + } + + if (!tasks.empty()) { + g_pika_rm->ProduceWriteQueue(slave_ptr->Ip(), slave_ptr->Port(), tasks); + } + return Status::OK(); +} + +Status SyncMasterPartition::GetSlaveNode(const std::string& ip, int port, std::shared_ptr* slave_node) { + for (size_t i = 0; i < slaves_.size(); ++i) { + std::shared_ptr tmp_slave = slaves_[i]; + if (ip == tmp_slave->Ip() && port == tmp_slave->Port()) { + *slave_node = tmp_slave; + return Status::OK(); + } + } + return Status::NotFound("ip " + ip + " port " + std::to_string(port)); +} + +Status SyncMasterPartition::UpdateSlaveBinlogAckInfo(const std::string& ip, int port, const BinlogOffset& start, const BinlogOffset& end) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + if (slave_ptr->slave_state != kSlaveBinlogSync) { + return Status::Corruption(ip + std::to_string(port) + "state not BinlogSync"); + } + bool res = slave_ptr->sync_win.Update(SyncWinItem(start), SyncWinItem(end), &(slave_ptr->acked_offset)); + if (!res) { + return Status::Corruption("UpdateAckedInfo failed"); + } + } + return Status::OK(); +} + +Status SyncMasterPartition::GetSlaveSyncBinlogInfo(const std::string& ip, + int port, + BinlogOffset* sent_offset, + BinlogOffset* acked_offset) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + *sent_offset = slave_ptr->sent_offset; + *acked_offset = slave_ptr->acked_offset; + } + return Status::OK(); +} + +Status SyncMasterPartition::GetSlaveState(const std::string& ip, + int port, + SlaveState* const slave_state) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + *slave_state = slave_ptr->slave_state; + } + return Status::OK(); +} + +Status SyncMasterPartition::WakeUpSlaveBinlogSync() { + slash::MutexLock l(&partition_mu_); + for (auto& slave_ptr : slaves_) { + { + slash::MutexLock l(&slave_ptr->slave_mu); + if (slave_ptr->sent_offset == slave_ptr->acked_offset) { + if (slave_ptr->b_state == kReadFromFile) { + ReadBinlogFileToWq(slave_ptr); + } else if (slave_ptr->b_state == kReadFromCache) { + ReadCachedBinlogToWq(slave_ptr); + } + } + } + } + return Status::OK(); +} + +Status SyncMasterPartition::SetLastSendTime(const std::string& ip, int port, uint64_t time) { + slash::MutexLock l(&partition_mu_); + + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + slave_ptr->SetLastSendTime(time); + } + + return Status::OK(); +} + +Status SyncMasterPartition::GetLastSendTime(const std::string& ip, int port, uint64_t* time) { + slash::MutexLock l(&partition_mu_); + + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + *time = slave_ptr->LastSendTime(); + } + + return Status::OK(); +} + +Status SyncMasterPartition::SetLastRecvTime(const std::string& ip, int port, uint64_t time) { + slash::MutexLock l(&partition_mu_); + + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + slave_ptr->SetLastRecvTime(time); + } + + return Status::OK(); +} + +Status SyncMasterPartition::GetLastRecvTime(const std::string& ip, int port, uint64_t* time) { + slash::MutexLock l(&partition_mu_); + + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + return s; + } + + { + slash::MutexLock l(&slave_ptr->slave_mu); + *time = slave_ptr->LastRecvTime(); + } + + return Status::OK(); +} + +Status SyncMasterPartition::GetSafetyPurgeBinlog(std::string* safety_purge) { + BinlogOffset boffset; + std::string table_name = partition_info_.table_name_; + uint32_t partition_id = partition_info_.partition_id_; + std::shared_ptr partition = + g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition || !partition->GetBinlogOffset(&boffset)) { + return Status::NotFound("Partition NotFound"); + } else { + bool success = false; + uint32_t purge_max = boffset.filenum; + if (purge_max >= 10) { + success = true; + purge_max -= 10; + slash::MutexLock l(&partition_mu_); + for (const auto& slave : slaves_) { + if (slave->slave_state == SlaveState::kSlaveBinlogSync + && slave->acked_offset.filenum > 0) { + purge_max = std::min(slave->acked_offset.filenum - 1, purge_max); + } else { + success = false; + break; + } + } + } + *safety_purge = (success ? kBinlogPrefix + std::to_string(static_cast(purge_max)) : "none"); + } + return Status::OK(); +} + +bool SyncMasterPartition::BinlogCloudPurge(uint32_t index) { + BinlogOffset boffset; + std::string table_name = partition_info_.table_name_; + uint32_t partition_id = partition_info_.partition_id_; + std::shared_ptr partition = + g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition || !partition->GetBinlogOffset(&boffset)) { + return false; + } else { + if (index > boffset.filenum - 10) { // remain some more + return false; + } else { + slash::MutexLock l(&partition_mu_); + for (const auto& slave : slaves_) { + if (slave->slave_state == SlaveState::kSlaveDbSync) { + return false; + } else if (slave->slave_state == SlaveState::kSlaveBinlogSync) { + if (index >= slave->acked_offset.filenum) { + return false; + } + } + } + } + } + return true; +} + +Status SyncMasterPartition::CheckSyncTimeout(uint64_t now) { + slash::MutexLock l(&partition_mu_); + + std::shared_ptr slave_ptr = nullptr; + std::vector to_del; + + for (auto& slave_ptr : slaves_) { + slash::MutexLock l(&slave_ptr->slave_mu); + if (slave_ptr->LastRecvTime() + kRecvKeepAliveTimeout < now) { + to_del.push_back(Node(slave_ptr->Ip(), slave_ptr->Port())); + } else if (slave_ptr->LastSendTime() + kSendKeepAliveTimeout < now) { + std::vector task; + RmNode rm_node(slave_ptr->Ip(), slave_ptr->Port(), slave_ptr->TableName(), slave_ptr->PartitionId(), slave_ptr->SessionId()); + WriteTask empty_task(rm_node, BinlogChip(BinlogOffset(0, 0), "")); + task.push_back(empty_task); + Status s = g_pika_rm->SendSlaveBinlogChipsRequest(slave_ptr->Ip(), slave_ptr->Port(), task); + slave_ptr->SetLastSendTime(now); + if (!s.ok()) { + LOG(INFO)<< "Send ping failed: " << s.ToString(); + return Status::Corruption("Send ping failed: " + slave_ptr->Ip() + ":" + std::to_string(slave_ptr->Port())); + } + } + } + for (auto& node : to_del) { + for (size_t i = 0; i < slaves_.size(); ++i) { + if (node.Ip() == slaves_[i]->Ip() && node.Port() == slaves_[i]->Port()) { + slaves_.erase(slaves_.begin() + i); + LOG(WARNING) << SyncPartitionInfo().ToString() << " Master del Recv Timeout slave success " << node.ToString(); + break; + } + } + } + return Status::OK(); +} + +std::string SyncMasterPartition::ToStringStatus() { + std::stringstream tmp_stream; + tmp_stream << " Current Master Session: " << session_id_ << "\r\n"; + slash::MutexLock l(&partition_mu_); + for (size_t i = 0; i < slaves_.size(); ++i) { + std::shared_ptr slave_ptr = slaves_[i]; + slash::MutexLock l(&slave_ptr->slave_mu); + tmp_stream << " slave[" << i << "]: " << slave_ptr->ToString() << + "\r\n" << slave_ptr->ToStringStatus(); + } + return tmp_stream.str(); +} + +void SyncMasterPartition::GetValidSlaveNames(std::vector* slavenames) { + slash::MutexLock l(&partition_mu_); + for (auto ptr : slaves_) { + if (ptr->slave_state != kSlaveBinlogSync) { + continue; + } + std::string name = ptr->Ip() + ":" + std::to_string(ptr->Port()); + slavenames->push_back(name); + } +} + +Status SyncMasterPartition::GetInfo(std::string* info) { + std::stringstream tmp_stream; + slash::MutexLock l(&partition_mu_); + tmp_stream << " Role: Master" << "\r\n"; + tmp_stream << " connected_slaves: " << slaves_.size() << "\r\n"; + for (size_t i = 0; i < slaves_.size(); ++i) { + std::shared_ptr slave_ptr = slaves_[i]; + slash::MutexLock l(&slave_ptr->slave_mu); + tmp_stream << " slave[" << i << "]: " + << slave_ptr->Ip() << ":" << std::to_string(slave_ptr->Port()) << "\r\n"; + tmp_stream << " replication_status: " << SlaveStateMsg[slave_ptr->slave_state] << "\r\n"; + if (slave_ptr->slave_state == kSlaveBinlogSync) { + std::shared_ptr partition = g_pika_server->GetTablePartitionById(slave_ptr->TableName(), slave_ptr->PartitionId()); + BinlogOffset binlog_offset; + if (!partition || !partition->GetBinlogOffset(&binlog_offset)) { + return Status::Corruption("Get Info failed."); + } + uint64_t lag = (binlog_offset.filenum - slave_ptr->acked_offset.filenum) * + g_pika_conf->binlog_file_size() + + (binlog_offset.offset - slave_ptr->acked_offset.offset); + tmp_stream << " lag: " << lag << "\r\n"; + } + } + info->append(tmp_stream.str()); + return Status::OK(); +} + +int32_t SyncMasterPartition::GenSessionId() { + slash::MutexLock ml(&session_mu_); + return session_id_++; +} + +bool SyncMasterPartition::CheckSessionId(const std::string& ip, int port, + const std::string& table_name, + uint64_t partition_id, int session_id) { + slash::MutexLock l(&partition_mu_); + std::shared_ptr slave_ptr = nullptr; + Status s = GetSlaveNode(ip, port, &slave_ptr); + if (!s.ok()) { + LOG(WARNING)<< "Check SessionId Get Slave Node Error: " + << ip << ":" << port << "," << table_name << "_" << partition_id; + return false; + } + if (session_id != slave_ptr->SessionId()) { + LOG(WARNING)<< "Check SessionId Mismatch: " << ip << ":" << port << ", " + << table_name << "_" << partition_id << " expected_session: " << session_id + << ", actual_session:" << slave_ptr->SessionId(); + return false; + } + return true; +} + + +/* SyncSlavePartition */ +SyncSlavePartition::SyncSlavePartition(const std::string& table_name, + uint32_t partition_id) + : SyncPartition(table_name, partition_id), + m_info_(), + repl_state_(kNoConnect), + local_ip_("") { + m_info_.SetLastRecvTime(slash::NowMicros()); +} + +void SyncSlavePartition::SetReplState(const ReplState& repl_state) { + if (repl_state == ReplState::kNoConnect) { + // deactivate + Deactivate(); + return; + } + slash::MutexLock l(&partition_mu_); + repl_state_ = repl_state; +} + +ReplState SyncSlavePartition::State() { + slash::MutexLock l(&partition_mu_); + return repl_state_; +} + +void SyncSlavePartition::SetLastRecvTime(uint64_t time) { + slash::MutexLock l(&partition_mu_); + m_info_.SetLastRecvTime(time); +} + +uint64_t SyncSlavePartition::LastRecvTime() { + slash::MutexLock l(&partition_mu_); + return m_info_.LastRecvTime(); +} + +Status SyncSlavePartition::CheckSyncTimeout(uint64_t now) { + slash::MutexLock l(&partition_mu_); + // no need to do session keepalive return ok + if (repl_state_ != ReplState::kWaitDBSync && repl_state_ != ReplState::kConnected) { + return Status::OK(); + } + if (m_info_.LastRecvTime() + kRecvKeepAliveTimeout < now) { + m_info_ = RmNode(); + repl_state_ = ReplState::kTryConnect; + g_pika_server->SetLoopPartitionStateMachine(true); + } + return Status::OK(); +} + +Status SyncSlavePartition::GetInfo(std::string* info) { + std::string tmp_str = " Role: Slave\r\n"; + tmp_str += " master: " + MasterIp() + ":" + std::to_string(MasterPort()) + "\r\n"; + info->append(tmp_str); + return Status::OK(); +} + +void SyncSlavePartition::Activate(const RmNode& master, const ReplState& repl_state) { + slash::MutexLock l(&partition_mu_); + m_info_ = master; + repl_state_ = repl_state; + m_info_.SetLastRecvTime(slash::NowMicros()); +} + +void SyncSlavePartition::Deactivate() { + slash::MutexLock l(&partition_mu_); + m_info_ = RmNode(); + repl_state_ = ReplState::kNoConnect; +} + +std::string SyncSlavePartition::ToStringStatus() { + return " Master: " + MasterIp() + ":" + std::to_string(MasterPort()) + "\r\n" + + " SessionId: " + std::to_string(MasterSessionId()) + "\r\n" + + " SyncStatus " + ReplStateMsg[repl_state_] + "\r\n"; +} + +/* SyncWindow */ + +void SyncWindow::Push(const SyncWinItem& item) { + win_.push_back(item); +} + +bool SyncWindow::Update(const SyncWinItem& start_item, + const SyncWinItem& end_item, BinlogOffset* acked_offset) { + size_t start_pos = win_.size(), end_pos = win_.size(); + for (size_t i = 0; i < win_.size(); ++i) { + if (win_[i] == start_item) { + start_pos = i; + } + if (win_[i] == end_item) { + end_pos = i; + break; + } + } + if (start_pos == win_.size() || end_pos == win_.size()) { + LOG(WARNING) << "Ack offset Start: " << + start_item.ToString() << "End: " << end_item.ToString() << + " not found in binlog controller window." << + std::endl << "window status "<< std::endl << ToStringStatus(); + return false; + } + for (size_t i = start_pos; i <= end_pos; ++i) { + win_[i].acked_ = true; + } + while (!win_.empty()) { + if (win_[0].acked_) { + *acked_offset = win_[0].offset_; + win_.pop_front(); + } else { + break; + } + } + return true; +} + +int SyncWindow::Remainings() { + std::size_t remaining_size = g_pika_conf->sync_window_size() - win_.size(); + return remaining_size > 0? remaining_size:0 ; +} + +/* PikaReplicaManger */ + +PikaReplicaManager::PikaReplicaManager() + : last_meta_sync_timestamp_(0) { + std::set ips; + ips.insert("0.0.0.0"); + int port = g_pika_conf->port() + kPortShiftReplServer; + pika_repl_client_ = new PikaReplClient(3000, 60); + pika_repl_server_ = new PikaReplServer(ips, port, 3000); + InitPartition(); + pthread_rwlock_init(&partitions_rw_, NULL); +} + +PikaReplicaManager::~PikaReplicaManager() { + delete pika_repl_client_; + delete pika_repl_server_; + pthread_rwlock_destroy(&partitions_rw_); +} + +void PikaReplicaManager::Start() { + int ret = 0; + ret = pika_repl_client_->Start(); + if (ret != pink::kSuccess) { + LOG(FATAL) << "Start Repl Client Error: " << ret << (ret == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + + ret = pika_repl_server_->Start(); + if (ret != pink::kSuccess) { + LOG(FATAL) << "Start Repl Server Error: " << ret << (ret == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } +} + +void PikaReplicaManager::Stop() { + pika_repl_client_->Stop(); + pika_repl_server_->Stop(); +} + +void PikaReplicaManager::InitPartition() { + std::vector table_structs = g_pika_conf->table_structs(); + for (const auto& table : table_structs) { + const std::string& table_name = table.table_name; + for (const auto& partition_id : table.partition_ids) { + sync_master_partitions_[PartitionInfo(table_name, partition_id)] + = std::make_shared(table_name, partition_id); + sync_slave_partitions_[PartitionInfo(table_name, partition_id)] + = std::make_shared(table_name, partition_id); + } + } +} + +void PikaReplicaManager::ProduceWriteQueue(const std::string& ip, int port, const std::vector& tasks) { + slash::MutexLock l(&write_queue_mu_); + std::string index = ip + ":" + std::to_string(port); + for (auto& task : tasks) { + write_queues_[index].push(task); + } +} + +int PikaReplicaManager::ConsumeWriteQueue() { + std::vector to_delete; + std::unordered_map>> to_send_map; + int counter = 0; + { + slash::MutexLock l(&write_queue_mu_); + std::vector to_delete; + for (auto& iter : write_queues_) { + std::queue& queue = iter.second; + for (int i = 0; i < kBinlogSendPacketNum; ++i) { + if (queue.empty()) { + break; + } + size_t batch_index = queue.size() > kBinlogSendBatchNum ? kBinlogSendBatchNum : queue.size(); + std::vector to_send; + for (size_t i = 0; i < batch_index; ++i) { + to_send.push_back(queue.front()); + queue.pop(); + counter++; + } + to_send_map[iter.first].push_back(std::move(to_send)); + } + } + } + + for (auto& iter : to_send_map) { + std::string ip; + int port = 0; + if (!slash::ParseIpPortString(iter.first, ip, port)) { + LOG(WARNING) << "Parse ip_port error " << iter.first; + continue; + } + for (auto& to_send : iter.second) { + Status s = pika_repl_server_->SendSlaveBinlogChips(ip, port, to_send); + if (!s.ok()) { + LOG(WARNING) << "send binlog to " << ip << ":" << port << " failed, " << s.ToString(); + to_delete.push_back(iter.first); + continue; + } + } + } + + if (!to_delete.empty()) { + { + slash::MutexLock l(&write_queue_mu_); + for (auto& del_queue : to_delete) { + write_queues_.erase(del_queue); + } + } + } + return counter; +} + +void PikaReplicaManager::DropItemInWriteQueue(const std::string& ip, int port) { + slash::MutexLock l(&write_queue_mu_); + std::string index = ip + ":" + std::to_string(port); + write_queues_.erase(index); +} + +void PikaReplicaManager::ScheduleReplServerBGTask(pink::TaskFunc func, void* arg) { + pika_repl_server_->Schedule(func, arg); +} + +void PikaReplicaManager::ScheduleReplClientBGTask(pink::TaskFunc func, void* arg) { + pika_repl_client_->Schedule(func, arg); +} + +void PikaReplicaManager::ScheduleWriteBinlogTask(const std::string& table_partition, + const std::shared_ptr res, + std::shared_ptr conn, + void* res_private_data) { + pika_repl_client_->ScheduleWriteBinlogTask(table_partition, res, conn, res_private_data); +} + +void PikaReplicaManager::ScheduleWriteDBTask(const std::string& dispatch_key, + PikaCmdArgsType* argv, BinlogItem* binlog_item, + const std::string& table_name, uint32_t partition_id) { + pika_repl_client_->ScheduleWriteDBTask(dispatch_key, argv, binlog_item, table_name, partition_id); +} + +void PikaReplicaManager::ReplServerRemoveClientConn(int fd) { + pika_repl_server_->RemoveClientConn(fd); +} + +void PikaReplicaManager::ReplServerUpdateClientConnMap(const std::string& ip_port, + int fd) { + pika_repl_server_->UpdateClientConnMap(ip_port, fd); +} + +Status PikaReplicaManager::UpdateSyncBinlogStatus(const RmNode& slave, const BinlogOffset& range_start, const BinlogOffset& range_end) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s = partition->UpdateSlaveBinlogAckInfo(slave.Ip(), slave.Port(), range_start, range_end); + if (!s.ok()) { + return s; + } + s = partition->SyncBinlogToWq(slave.Ip(), slave.Port()); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::GetSyncBinlogStatus(const RmNode& slave, BinlogOffset* sent_offset, BinlogOffset* acked_offset) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s = partition->GetSlaveSyncBinlogInfo(slave.Ip(), slave.Port(), sent_offset, acked_offset); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::GetSyncMasterPartitionSlaveState(const RmNode& slave, + SlaveState* const slave_state) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s = partition->GetSlaveState(slave.Ip(), slave.Port(), slave_state); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +bool PikaReplicaManager::CheckPartitionSlaveExist(const RmNode& slave) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return false; + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + return partition->CheckSlaveNodeExist(slave.Ip(), slave.Port()); +} + +Status PikaReplicaManager::GetPartitionSlaveSession(const RmNode& slave, int32_t* session) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString(), + "not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + return partition->GetSlaveNodeSession(slave.Ip(), slave.Port(), session); +} + +Status PikaReplicaManager::AddPartitionSlave(const RmNode& slave) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s= partition->RemoveSlaveNode(slave.Ip(), slave.Port()); + if (!s.ok() && !s.IsNotFound()) { + return s; + } + s = partition->AddSlaveNode(slave.Ip(), slave.Port(), slave.SessionId()); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::RemovePartitionSlave(const RmNode& slave) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s = partition->RemoveSlaveNode(slave.Ip(), slave.Port()); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::LostConnection(const std::string& ip, int port) { + slash::RWLock l(&partitions_rw_, false); + for (auto& iter : sync_master_partitions_) { + std::shared_ptr partition = iter.second; + Status s = partition->RemoveSlaveNode(ip, port); + if (!s.ok() && !s.IsNotFound()) { + LOG(WARNING) << "Lost Connection failed " << s.ToString(); + } + } + + for (auto& iter : sync_slave_partitions_) { + std::shared_ptr partition = iter.second; + if (partition->MasterIp() == ip && partition->MasterPort() == port) { + partition->Deactivate(); + } + } + return Status::OK(); +} + +Status PikaReplicaManager::ActivateBinlogSync(const RmNode& slave, const BinlogOffset& offset) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr sync_partition = sync_master_partitions_[slave.NodePartitionInfo()]; + + std::shared_ptr partition = g_pika_server->GetTablePartitionById(slave.TableName(), slave.PartitionId()); + if (!partition) { + return Status::Corruption("Found Binlog faile"); + } + + Status s = sync_partition->ActivateSlaveBinlogSync(slave.Ip(), slave.Port(), partition->logger(), offset); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::ActivateDbSync(const RmNode& slave) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(slave.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(slave.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[slave.NodePartitionInfo()]; + Status s = partition->ActivateSlaveDbSync(slave.Ip(), slave.Port()); + if (!s.ok()) { + return s; + } + return Status::OK(); +} + +Status PikaReplicaManager::SetMasterLastRecvTime(const RmNode& node, uint64_t time) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(node.NodePartitionInfo()) == sync_master_partitions_.end()) { + return Status::NotFound(node.ToString() + " not found"); + } + std::shared_ptr partition = sync_master_partitions_[node.NodePartitionInfo()]; + partition->SetLastRecvTime(node.Ip(), node.Port(), time); + return Status::OK(); +} + +Status PikaReplicaManager::SetSlaveLastRecvTime(const RmNode& node, uint64_t time) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(node.NodePartitionInfo()) == sync_slave_partitions_.end()) { + return Status::NotFound(node.ToString() + " not found"); + } + std::shared_ptr partition = sync_slave_partitions_[node.NodePartitionInfo()]; + partition->SetLastRecvTime(time); + return Status::OK(); +} + +Status PikaReplicaManager::WakeUpBinlogSync() { + slash::RWLock l(&partitions_rw_, false); + for (auto& iter : sync_master_partitions_) { + std::shared_ptr partition = iter.second; + Status s = partition->WakeUpSlaveBinlogSync(); + if (!s.ok()) { + return s; + } + } + return Status::OK(); +} + +int32_t PikaReplicaManager::GenPartitionSessionId(const std::string& table_name, + uint32_t partition_id) { + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table_name, partition_id); + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end()) { + return -1; + } else { + std::shared_ptr sync_master_partition = sync_master_partitions_[p_info]; + return sync_master_partition->GenSessionId(); + } +} + +int32_t PikaReplicaManager::GetSlavePartitionSessionId(const std::string& table_name, + uint32_t partition_id) { + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table_name, partition_id); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return -1; + } else { + std::shared_ptr sync_slave_partition = sync_slave_partitions_[p_info]; + return sync_slave_partition->MasterSessionId(); + } +} + +bool PikaReplicaManager::CheckSlavePartitionSessionId(const std::string& table_name, + uint32_t partition_id, + int session_id) { + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table_name, partition_id); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + LOG(WARNING)<< "Slave Partition Not Found: " << p_info.ToString().data(); + return false; + } else { + std::shared_ptr sync_slave_partition = sync_slave_partitions_[p_info]; + if (sync_slave_partition->MasterSessionId() != session_id) { + LOG(WARNING)<< "Check SessionId Mismatch: " << sync_slave_partition->MasterIp() + << ":" << sync_slave_partition->MasterPort() << ", " + << sync_slave_partition->SyncPartitionInfo().ToString() + << " expected_session: " << session_id << ", actual_session:" + << sync_slave_partition->MasterSessionId(); + return false; + } + } + return true; +} + +bool PikaReplicaManager::CheckMasterPartitionSessionId(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id, int session_id) { + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table_name, partition_id); + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end()) { + return false; + } else { + std::shared_ptr sync_master_partition = sync_master_partitions_[p_info]; + return sync_master_partition->CheckSessionId(ip, port, table_name, partition_id, session_id); + } +} + +Status PikaReplicaManager::CheckSyncTimeout(uint64_t now) { + slash::RWLock l(&partitions_rw_, false); + + for (auto& iter : sync_master_partitions_) { + std::shared_ptr partition = iter.second; + Status s = partition->CheckSyncTimeout(now); + if (!s.ok()) { + LOG(WARNING) << "CheckSyncTimeout Failed " << s.ToString(); + } + } + for (auto& iter : sync_slave_partitions_) { + std::shared_ptr partition = iter.second; + Status s = partition->CheckSyncTimeout(now); + if (!s.ok()) { + LOG(WARNING) << "CheckSyncTimeout Failed " << s.ToString(); + } + } + return Status::OK(); +} + +Status PikaReplicaManager::CheckPartitionRole( + const std::string& table, uint32_t partition_id, int* role) { + slash::RWLock l(&partitions_rw_, false); + *role = 0; + PartitionInfo p_info(table, partition_id); + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end()) { + return Status::NotFound(table + std::to_string(partition_id) + " not found"); + } + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound(table + std::to_string(partition_id) + " not found"); + } + if (sync_master_partitions_[p_info]->GetNumberOfSlaveNode() != 0) { + *role |= PIKA_ROLE_MASTER; + } + if (sync_slave_partitions_[p_info]->State() == ReplState::kConnected) { + *role |= PIKA_ROLE_SLAVE; + } + // if role is not master or slave, the rest situations are all single + return Status::OK(); +} + +Status PikaReplicaManager::GetPartitionInfo( + const std::string& table, uint32_t partition_id, std::string* info) { + int role = 0; + std::string tmp_res; + Status s = CheckPartitionRole(table, partition_id, &role); + if (!s.ok()) { + return s; + } + + bool add_divider_line = ((role & PIKA_ROLE_MASTER) && (role & PIKA_ROLE_SLAVE)); + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table, partition_id); + if (role & PIKA_ROLE_MASTER) { + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end()) { + return Status::NotFound(table + std::to_string(partition_id) + " not found"); + } + Status s = sync_master_partitions_[p_info]->GetInfo(info); + if (!s.ok()) { + return s; + } + } + if (add_divider_line) { + info->append(" -----------\r\n"); + } + if (role & PIKA_ROLE_SLAVE) { + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound(table + std::to_string(partition_id) + " not found"); + } + Status s = sync_slave_partitions_[p_info]->GetInfo(info); + if (!s.ok()) { + return s; + } + } + info->append("\r\n"); + return Status::OK(); +} + +Status PikaReplicaManager::SelectLocalIp(const std::string& remote_ip, + const int remote_port, + std::string* const local_ip) { + pink::PinkCli* cli = pink::NewRedisCli(); + cli->set_connect_timeout(1500); + if ((cli->Connect(remote_ip, remote_port, "")).ok()) { + struct sockaddr_in laddr; + socklen_t llen = sizeof(laddr); + getsockname(cli->fd(), (struct sockaddr*) &laddr, &llen); + std::string tmp_ip(inet_ntoa(laddr.sin_addr)); + *local_ip = tmp_ip; + cli->Close(); + delete cli; + } else { + LOG(WARNING) << "Failed to connect remote node(" + << remote_ip << ":" << remote_port << ")"; + delete cli; + return Status::Corruption("connect remote node error"); + } + return Status::OK(); +} + +Status PikaReplicaManager::ActivateSyncSlavePartition(const RmNode& node, + const ReplState& repl_state) { + slash::RWLock l(&partitions_rw_, false); + const PartitionInfo& p_info = node.NodePartitionInfo(); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + node.ToString() + " not found"); + } + ReplState ssp_state = sync_slave_partitions_[p_info]->State(); + if (ssp_state != ReplState::kNoConnect) { + return Status::Corruption("Sync Slave partition in " + ReplStateMsg[ssp_state]); + } + std::string local_ip; + Status s = SelectLocalIp(node.Ip(), node.Port(), &local_ip); + if (s.ok()) { + sync_slave_partitions_[p_info]->SetLocalIp(local_ip); + sync_slave_partitions_[p_info]->Activate(node, repl_state); + } + return s; +} + +Status PikaReplicaManager::UpdateSyncSlavePartitionSessionId(const PartitionInfo& p_info, + int32_t session_id) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + p_info.ToString()); + } + sync_slave_partitions_[p_info]->SetMasterSessionId(session_id); + return Status::OK(); +} + +Status PikaReplicaManager::DeactivateSyncSlavePartition(const PartitionInfo& p_info) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + p_info.ToString()); + } + sync_slave_partitions_[p_info]->Deactivate(); + return Status::OK(); +} + +Status PikaReplicaManager::SetSlaveReplState(const PartitionInfo& p_info, + const ReplState& repl_state) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + p_info.ToString()); + } + sync_slave_partitions_[p_info]->SetReplState(repl_state); + return Status::OK(); +} + +Status PikaReplicaManager::GetSlaveReplState(const PartitionInfo& p_info, + ReplState* repl_state) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + p_info.ToString()); + } + *repl_state = sync_slave_partitions_[p_info]->State(); + return Status::OK(); +} + +Status PikaReplicaManager::SendMetaSyncRequest() { + Status s; + int now = time(NULL); + if (now - last_meta_sync_timestamp_ >= PIKA_META_SYNC_MAX_WAIT_TIME) { + s = pika_repl_client_->SendMetaSync(); + if (s.ok()) { + last_meta_sync_timestamp_ = now; + } + } + return s; +} + +Status PikaReplicaManager::SendRemoveSlaveNodeRequest(const std::string& table, + uint32_t partition_id) { + slash::Status s; + slash::RWLock l(&partitions_rw_, false); + PartitionInfo p_info(table, partition_id); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return Status::NotFound("Sync Slave partition " + p_info.ToString()); + } else { + std::shared_ptr s_partition = sync_slave_partitions_[p_info]; + s = pika_repl_client_->SendRemoveSlaveNode(s_partition->MasterIp(), + s_partition->MasterPort(), table, partition_id, s_partition->LocalIp()); + if (s.ok()) { + s_partition->Deactivate(); + } + } + + if (s.ok()) { + LOG(INFO) << "SlaveNode (" << table << ":" << partition_id + << "), stop sync success"; + } else { + LOG(WARNING) << "SlaveNode (" << table << ":" << partition_id + << "), stop sync faild, " << s.ToString(); + } + return s; +} + +Status PikaReplicaManager::SendPartitionTrySyncRequest( + const std::string& table_name, size_t partition_id) { + BinlogOffset boffset; + if (!g_pika_server->GetTablePartitionBinlogOffset( + table_name, partition_id, &boffset)) { + LOG(WARNING) << "Partition: " << table_name << ":" << partition_id + << ", Get partition binlog offset failed"; + return Status::Corruption("Partition get binlog offset error"); + } + + std::shared_ptr slave_partition = + GetSyncSlavePartitionByName(PartitionInfo(table_name, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << table_name << ":" << partition_id + << ", NotFound"; + return Status::Corruption("Slave Partition not found"); + } + + Status status = pika_repl_client_->SendPartitionTrySync(slave_partition->MasterIp(), + slave_partition->MasterPort(), + table_name, partition_id, boffset, + slave_partition->LocalIp()); + + Status s; + if (status.ok()) { + s = g_pika_rm->SetSlaveReplState(PartitionInfo(table_name, partition_id), ReplState::kWaitReply); + } else { + s = g_pika_rm->SetSlaveReplState(PartitionInfo(table_name, partition_id), ReplState::kError); + LOG(WARNING) << "SendPartitionTrySyncRequest failed " << status.ToString(); + } + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + return status; +} + +static bool already_dbsync = false; +Status PikaReplicaManager::SendPartitionDBSyncRequest( + const std::string& table_name, size_t partition_id) { + if (!already_dbsync) { + already_dbsync = true; + } else { + LOG(FATAL) << "we only allow one DBSync action to avoid passing duplicate commands to target Redis multiple times"; + } + + BinlogOffset boffset; + if (!g_pika_server->GetTablePartitionBinlogOffset( + table_name, partition_id, &boffset)) { + LOG(WARNING) << "Partition: " << table_name << ":" << partition_id + << ", Get partition binlog offset failed"; + return Status::Corruption("Partition get binlog offset error"); + } + + std::shared_ptr partition = + g_pika_server->GetTablePartitionById(table_name, partition_id); + if (!partition) { + LOG(WARNING) << "Partition: " << table_name << ":" << partition_id + << ", NotFound"; + return Status::Corruption("Partition not found"); + } + partition->PrepareRsync(); + + std::shared_ptr slave_partition = + GetSyncSlavePartitionByName(PartitionInfo(table_name, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << table_name << ":" << partition_id + << ", NotFound"; + return Status::Corruption("Slave Partition not found"); + } + + Status status = pika_repl_client_->SendPartitionDBSync(slave_partition->MasterIp(), + slave_partition->MasterPort(), + table_name, partition_id, boffset, + slave_partition->LocalIp()); + + Status s; + if (status.ok()) { + s = g_pika_rm->SetSlaveReplState(PartitionInfo(table_name, partition_id), ReplState::kWaitReply); + } else { + LOG(WARNING) << "SendPartitionDbSync failed " << status.ToString(); + s = g_pika_rm->SetSlaveReplState(PartitionInfo(table_name, partition_id), ReplState::kError); + } + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + return status; +} + +Status PikaReplicaManager::SendPartitionBinlogSyncAckRequest( + const std::string& table, uint32_t partition_id, + const BinlogOffset& ack_start, const BinlogOffset& ack_end, + bool is_first_send) { + std::shared_ptr slave_partition = + GetSyncSlavePartitionByName(PartitionInfo(table, partition_id)); + if (!slave_partition) { + LOG(WARNING) << "Slave Partition: " << table << ":" << partition_id + << ", NotFound"; + return Status::Corruption("Slave Partition not found"); + } + return pika_repl_client_->SendPartitionBinlogSync( + slave_partition->MasterIp(), slave_partition->MasterPort(), + table, partition_id, ack_start, ack_end, slave_partition->LocalIp(), + is_first_send); +} + +Status PikaReplicaManager::CloseReplClientConn(const std::string& ip, int32_t port) { + return pika_repl_client_->Close(ip, port); +} + +Status PikaReplicaManager::SendSlaveBinlogChipsRequest(const std::string& ip, + int port, + const std::vector& tasks) { + return pika_repl_server_->SendSlaveBinlogChips(ip, port, tasks); +} + +std::shared_ptr +PikaReplicaManager::GetSyncMasterPartitionByName(const PartitionInfo& p_info) { + slash::RWLock l(&partitions_rw_, false); + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end()) { + return nullptr; + } + return sync_master_partitions_[p_info]; +} + +Status PikaReplicaManager::GetSafetyPurgeBinlogFromSMP(const std::string& table_name, + uint32_t partition_id, + std::string* safety_purge) { + std::shared_ptr master_partition = + GetSyncMasterPartitionByName(PartitionInfo(table_name, partition_id)); + if (!master_partition) { + LOG(WARNING) << "Sync Master Partition: " << table_name << ":" << partition_id + << ", NotFound"; + return Status::NotFound("SyncMasterPartition NotFound"); + } else { + return master_partition->GetSafetyPurgeBinlog(safety_purge); + } +} + +bool PikaReplicaManager::BinlogCloudPurgeFromSMP(const std::string& table_name, + uint32_t partition_id, uint32_t index) { + std::shared_ptr master_partition = + GetSyncMasterPartitionByName(PartitionInfo(table_name, partition_id)); + if (!master_partition) { + LOG(WARNING) << "Sync Master Partition: " << table_name << ":" << partition_id + << ", NotFound"; + return false; + } else { + return master_partition->BinlogCloudPurge(index); + } +} + +std::shared_ptr +PikaReplicaManager::GetSyncSlavePartitionByName(const PartitionInfo& p_info) { + slash::RWLock l(&partitions_rw_, false); + if (sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + return nullptr; + } + return sync_slave_partitions_[p_info]; +} + +Status PikaReplicaManager::RunSyncSlavePartitionStateMachine() { + slash::RWLock l(&partitions_rw_, false); + for (const auto& item : sync_slave_partitions_) { + PartitionInfo p_info = item.first; + std::shared_ptr s_partition = item.second; + if (s_partition->State() == ReplState::kTryConnect) { + SendPartitionTrySyncRequest(p_info.table_name_, p_info.partition_id_); + } else if (s_partition->State() == ReplState::kTryDBSync) { + SendPartitionDBSyncRequest(p_info.table_name_, p_info.partition_id_); + } else if (s_partition->State() == ReplState::kWaitReply) { + continue; + } else if (s_partition->State() == ReplState::kWaitDBSync) { + std::shared_ptr partition = + g_pika_server->GetTablePartitionById( + p_info.table_name_, p_info.partition_id_); + if (partition) { + partition->TryUpdateMasterOffset(); + } else { + LOG(WARNING) << "Partition not found, Table Name: " + << p_info.table_name_ << " Partition Id: " << p_info.partition_id_; + } + } else if (s_partition->State() == ReplState::kConnected + || s_partition->State() == ReplState::kNoConnect) { + continue; + } + } + return Status::OK(); +} + +Status PikaReplicaManager::AddSyncPartitionSanityCheck(const std::set& p_infos) { + slash::RWLock l(&partitions_rw_, false); + for (const auto& p_info : p_infos) { + if (sync_master_partitions_.find(p_info) != sync_master_partitions_.end() + || sync_slave_partitions_.find(p_info) != sync_slave_partitions_.end()) { + LOG(WARNING) << "sync partition: " << p_info.ToString() << " exist"; + return Status::Corruption("sync partition " + p_info.ToString() + + " exist"); + } + } + return Status::OK(); +} + +Status PikaReplicaManager::AddSyncPartition( + const std::set& p_infos) { + Status s = AddSyncPartitionSanityCheck(p_infos); + if (!s.ok()) { + return s; + } + + slash::RWLock l(&partitions_rw_, true); + for (const auto& p_info : p_infos) { + sync_master_partitions_[p_info] = + std::make_shared(p_info.table_name_, + p_info.partition_id_); + sync_slave_partitions_[p_info] = + std::make_shared(p_info.table_name_, + p_info.partition_id_); + } + return Status::OK(); +} + +Status PikaReplicaManager::RemoveSyncPartitionSanityCheck( + const std::set& p_infos) { + slash::RWLock l(&partitions_rw_, false); + for (const auto& p_info : p_infos) { + if (sync_master_partitions_.find(p_info) == sync_master_partitions_.end() + || sync_slave_partitions_.find(p_info) == sync_slave_partitions_.end()) { + LOG(WARNING) << "sync partition: " << p_info.ToString() << " not found"; + return Status::Corruption("sync partition " + p_info.ToString() + + " not found"); + } + + if (sync_master_partitions_[p_info]->GetNumberOfSlaveNode() != 0) { + LOG(WARNING) << "sync master partition: " << p_info.ToString() + << " in syncing"; + return Status::Corruption("sync master partition " + p_info.ToString() + + " in syncing"); + } + + ReplState state = sync_slave_partitions_[p_info]->State(); + if (state != kNoConnect && state != kError) { + LOG(WARNING) << "sync slave partition: " << p_info.ToString() + << " in " << ReplStateMsg[state] + " state"; + return Status::Corruption("sync slave partition " + p_info.ToString() + + " in " + ReplStateMsg[state] + " state"); + } + } + return Status::OK(); +} + +Status PikaReplicaManager::RemoveSyncPartition( + const std::set& p_infos) { + Status s = RemoveSyncPartitionSanityCheck(p_infos); + if (!s.ok()) { + return s; + } + + slash::RWLock l(&partitions_rw_, true); + for (const auto& p_info : p_infos) { + sync_master_partitions_.erase(p_info); + sync_slave_partitions_.erase(p_info); + } + return Status::OK(); +} + +void PikaReplicaManager::FindCompleteReplica(std::vector* replica) { + std::unordered_map replica_slotnum; + slash::RWLock l(&partitions_rw_, false); + for (auto& iter : sync_master_partitions_) { + std::vector names; + iter.second->GetValidSlaveNames(&names); + for (auto& name : names) { + if (replica_slotnum.find(name) == replica_slotnum.end()) { + replica_slotnum[name] = 0; + } + replica_slotnum[name]++; + } + } + for (auto item : replica_slotnum) { + if (item.second == sync_master_partitions_.size()) { + replica->push_back(item.first); + } + } +} + +void PikaReplicaManager::FindCommonMaster(std::string* master) { + slash::RWLock l(&partitions_rw_, false); + std::string common_master_ip; + int common_master_port = 0; + for (auto& iter : sync_slave_partitions_) { + if (iter.second->State() != kConnected) { + return; + } + std::string tmp_ip = iter.second->MasterIp(); + int tmp_port = iter.second->MasterPort(); + if (common_master_ip.empty() && common_master_port == 0) { + common_master_ip = tmp_ip; + common_master_port = tmp_port; + } + if (tmp_ip != common_master_ip || tmp_port != common_master_port) { + return; + } + } + if (!common_master_ip.empty() && common_master_port != 0) { + *master = common_master_ip + ":" + std::to_string(common_master_port); + } +} + +void PikaReplicaManager::RmStatus(std::string* info) { + slash::RWLock l(&partitions_rw_, false); + std::stringstream tmp_stream; + tmp_stream << "Master partition(" << sync_master_partitions_.size() << "):" << "\r\n"; + for (auto& iter : sync_master_partitions_) { + tmp_stream << " Partition " << iter.second->SyncPartitionInfo().ToString() + << "\r\n" << iter.second->ToStringStatus() << "\r\n"; + } + tmp_stream << "Slave partition(" << sync_slave_partitions_.size() << "):" << "\r\n"; + for (auto& iter : sync_slave_partitions_) { + tmp_stream << " Partition " << iter.second->SyncPartitionInfo().ToString() + << "\r\n" << iter.second->ToStringStatus() << "\r\n"; + } + info->append(tmp_stream.str()); +} diff --git a/tools/pika_migrate/src/pika_rsync_service.cc b/tools/pika_migrate/src/pika_rsync_service.cc new file mode 100644 index 0000000000..00f3e70ee4 --- /dev/null +++ b/tools/pika_migrate/src/pika_rsync_service.cc @@ -0,0 +1,105 @@ +// Copyright (c) 2019-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_rsync_service.h" + +#include +#include + +#include "slash/include/env.h" +#include "slash/include/rsync.h" + +#include "include/pika_define.h" +#include "include/pika_conf.h" + +extern PikaConf *g_pika_conf; + +PikaRsyncService::PikaRsyncService(const std::string& raw_path, + const int port) + : raw_path_(raw_path), port_(port) { + if (raw_path_.back() != '/') { + raw_path_ += "/"; + } + rsync_path_ = raw_path_ + slash::kRsyncSubDir + "/"; + pid_path_ = rsync_path_ + slash::kRsyncPidFile; +} + +PikaRsyncService::~PikaRsyncService() { + if (!CheckRsyncAlive()) { + slash::DeleteDirIfExist(rsync_path_); + } else { + slash::StopRsync(raw_path_); + } + LOG(INFO) << "PikaRsyncService exit!!!"; +} + +int PikaRsyncService::StartRsync() { + int ret = 0; + std::string auth; + if (g_pika_conf->masterauth().empty()) { + auth = kDefaultRsyncAuth; + } else { + auth = g_pika_conf->masterauth(); + } + ret = slash::StartRsync(raw_path_, kDBSyncModule, "0.0.0.0", port_, auth); + if (ret != 0) { + LOG(WARNING) << "Failed to start rsync, path:" << raw_path_ << " error : " << ret; + return -1; + } + ret = CreateSecretFile(); + if (ret != 0) { + LOG(WARNING) << "Failed to create secret file"; + return -1; + } + // Make sure the listening addr of rsyncd is accessible, avoid the corner case + // that rsync --daemon process is started but not finished listening on the socket + sleep(1); + + if (!CheckRsyncAlive()) { + LOG(WARNING) << "Rsync service is no live, path:" << raw_path_; + return -1; + } + return 0; +} + +int PikaRsyncService::CreateSecretFile() { + std::string secret_file_path = g_pika_conf->db_sync_path(); + if (g_pika_conf->db_sync_path().back() != '/') { + secret_file_path += "/"; + } + secret_file_path += slash::kRsyncSubDir + "/"; + slash::CreatePath(secret_file_path); + secret_file_path += kPikaSecretFile; + + std::string auth; + if (g_pika_conf->requirepass().empty()) { + auth = kDefaultRsyncAuth; + } else { + auth = g_pika_conf->requirepass(); + } + + std::ofstream secret_stream(secret_file_path.c_str()); + if (!secret_stream) { + return -1; + } + secret_stream << auth; + secret_stream.close(); + + // secret file cant be other-accessible + std::string cmd = "chmod 600 " + secret_file_path; + int ret = system(cmd.c_str()); + if (ret == 0 || (WIFEXITED(ret) && !WEXITSTATUS(ret))) { + return 0; + } + return ret; +} + +bool PikaRsyncService::CheckRsyncAlive() { + return slash::FileExists(pid_path_); +} + +int PikaRsyncService::ListenPort() { + return port_; +} diff --git a/tools/pika_migrate/src/pika_sender.cc b/tools/pika_migrate/src/pika_sender.cc new file mode 100644 index 0000000000..a2109b22e8 --- /dev/null +++ b/tools/pika_migrate/src/pika_sender.cc @@ -0,0 +1,189 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_sender.h" + +#include + +#include "slash/include/xdebug.h" + +PikaSender::PikaSender(std::string ip, int64_t port, std::string password): + cli_(NULL), + signal_(&keys_mutex_), + ip_(ip), + port_(port), + password_(password), + should_exit_(false), + elements_(0) + { + } + +PikaSender::~PikaSender() { +} + +int PikaSender::QueueSize() { + slash::MutexLock l(&keys_mutex_); + return keys_queue_.size(); +} + +void PikaSender::Stop() { + should_exit_ = true; + keys_mutex_.Lock(); + signal_.Signal(); + keys_mutex_.Unlock(); +} + +void PikaSender::ConnectRedis() { + while (cli_ == NULL) { + // Connect to redis + cli_ = pink::NewRedisCli(); + cli_->set_connect_timeout(1000); + slash::Status s = cli_->Connect(ip_, port_); + if (!s.ok()) { + delete cli_; + cli_ = NULL; + LOG(WARNING) << "Can not connect to " << ip_ << ":" << port_ << ", status: " << s.ToString(); + continue; + } else { + // Connect success + + // Authentication + if (!password_.empty()) { + pink::RedisCmdArgsType argv, resp; + std::string cmd; + + argv.push_back("AUTH"); + argv.push_back(password_); + pink::SerializeRedisCommand(argv, &cmd); + slash::Status s = cli_->Send(&cmd); + + if (s.ok()) { + s = cli_->Recv(&resp); + if (resp[0] == "OK") { + } else { + LOG(FATAL) << "Connect to redis(" << ip_ << ":" << port_ << ") Invalid password"; + cli_->Close(); + delete cli_; + cli_ = NULL; + should_exit_ = true; + return; + } + } else { + LOG(WARNING) << "send auth failed: " << s.ToString(); + cli_->Close(); + delete cli_; + cli_ = NULL; + continue; + } + } else { + // If forget to input password + pink::RedisCmdArgsType argv, resp; + std::string cmd; + + argv.push_back("PING"); + pink::SerializeRedisCommand(argv, &cmd); + slash::Status s = cli_->Send(&cmd); + + if (s.ok()) { + s = cli_->Recv(&resp); + if (s.ok()) { + if (resp[0] == "NOAUTH Authentication required.") { + LOG(FATAL) << "Ping redis(" << ip_ << ":" << port_ << ") NOAUTH Authentication required"; + cli_->Close(); + delete cli_; + cli_ = NULL; + should_exit_ = true; + return; + } + } else { + LOG(WARNING) << "Recv failed: " << s.ToString(); + cli_->Close(); + delete cli_; + cli_ = NULL; + } + } + } + } + } +} + +void PikaSender::LoadKey(const std::string &key) { + keys_mutex_.Lock(); + if (keys_queue_.size() < 100000) { + keys_queue_.push(key); + signal_.Signal(); + keys_mutex_.Unlock(); + } else { + while (keys_queue_.size() > 100000 && !should_exit_) { + signal_.TimedWait(100); + } + keys_queue_.push(key); + signal_.Signal(); + keys_mutex_.Unlock(); + } +} + +void PikaSender::SendCommand(std::string &command, const std::string &key) { + // Send command + slash::Status s = cli_->Send(&command); + if (!s.ok()) { + elements_--; + LoadKey(key); + cli_->Close(); + log_info("%s", s.ToString().data()); + delete cli_; + cli_ = NULL; + ConnectRedis(); + } +} + +void *PikaSender::ThreadMain() { + log_info("Start sender thread..."); + int cnt = 0; + + if (cli_ == NULL) { + ConnectRedis(); + } + + while (!should_exit_ || QueueSize() != 0) { + std::string command; + + keys_mutex_.Lock(); + while (keys_queue_.size() == 0 && !should_exit_) { + signal_.TimedWait(200); + } + keys_mutex_.Unlock(); + if (QueueSize() == 0 && should_exit_) { + // if (should_exit_) { + return NULL; + } + + keys_mutex_.Lock(); + std::string key = keys_queue_.front(); + elements_++; + keys_queue_.pop(); + keys_mutex_.Unlock(); + + SendCommand(key, key); + cnt++; + if (cnt >= 200) { + for(; cnt > 0; cnt--) { + cli_->Recv(NULL); + } + } + } + for(; cnt > 0; cnt--) { + cli_->Recv(NULL); + } + + if (cli_) { + cli_->Close(); + delete cli_; + cli_ = NULL; + } + log_info("PikaSender thread complete"); + return NULL; +} + diff --git a/tools/pika_migrate/src/pika_server.cc b/tools/pika_migrate/src/pika_server.cc new file mode 100644 index 0000000000..c03b06a5ab --- /dev/null +++ b/tools/pika_migrate/src/pika_server.cc @@ -0,0 +1,1598 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_server.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "slash/include/env.h" +#include "slash/include/rsync.h" +#include "pink/include/pink_cli.h" +#include "pink/include/redis_cli.h" +#include "pink/include/bg_thread.h" + +#include "include/pika_rm.h" +#include "include/pika_server.h" +#include "include/pika_sender.h" +#include "include/migrator_thread.h" +#include "include/pika_dispatch_thread.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaServer* g_pika_server; +extern PikaReplicaManager* g_pika_rm; +extern PikaCmdTableManager* g_pika_cmd_table_manager; + +void DoPurgeDir(void* arg) { + std::string path = *(static_cast(arg)); + LOG(INFO) << "Delete dir: " << path << " start"; + slash::DeleteDir(path); + LOG(INFO) << "Delete dir: " << path << " done"; + delete static_cast(arg); +} + +void DoDBSync(void* arg) { + DBSyncArg* dbsa = reinterpret_cast(arg); + PikaServer* const ps = dbsa->p; + ps->DbSyncSendFile(dbsa->ip, dbsa->port, + dbsa->table_name, dbsa->partition_id); + delete dbsa; +} + +PikaServer::PikaServer() : + exit_(false), + slot_state_(INFREE), + have_scheduled_crontask_(false), + last_check_compact_time_({0, 0}), + master_ip_(""), + master_port_(0), + repl_state_(PIKA_REPL_NO_CONNECT), + role_(PIKA_ROLE_SINGLE), + loop_partition_state_machine_(false), + force_full_sync_(false), + slowlog_entry_id_(0) { + + //Init server ip host + if (!ServerInit()) { + LOG(FATAL) << "ServerInit iotcl error"; + } + + InitBlackwidowOptions(); + + pthread_rwlockattr_t tables_rw_attr; + pthread_rwlockattr_init(&tables_rw_attr); + pthread_rwlockattr_setkind_np(&tables_rw_attr, + PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP); + pthread_rwlock_init(&tables_rw_, &tables_rw_attr); + + // Create thread + worker_num_ = std::min(g_pika_conf->thread_num(), + PIKA_MAX_WORKER_THREAD_NUM); + + std::set ips; + if (g_pika_conf->network_interface().empty()) { + ips.insert("0.0.0.0"); + } else { + ips.insert("127.0.0.1"); + ips.insert(host_); + } + // We estimate the queue size + int worker_queue_limit = g_pika_conf->maxclients() / worker_num_ + 100; + LOG(INFO) << "Worker queue limit is " << worker_queue_limit; + pika_dispatch_thread_ = new PikaDispatchThread(ips, port_, worker_num_, 3000, + worker_queue_limit); + pika_monitor_thread_ = new PikaMonitorThread(); + pika_rsync_service_ = new PikaRsyncService(g_pika_conf->db_sync_path(), + g_pika_conf->port() + kPortShiftRSync); + pika_pubsub_thread_ = new pink::PubSubThread(); + pika_auxiliary_thread_ = new PikaAuxiliaryThread(); + pika_thread_pool_ = new pink::ThreadPool(g_pika_conf->thread_pool_size(), 100000); + + // Create redis sender + for (int i = 0; i < g_pika_conf->redis_sender_num(); i++) { + redis_senders_.emplace_back( + new RedisSender(int(i), + g_pika_conf->target_redis_host(), + g_pika_conf->target_redis_port(), + g_pika_conf->target_redis_pwd())); + } + + pthread_rwlock_init(&state_protector_, NULL); + pthread_rwlock_init(&slowlog_protector_, NULL); +} + +PikaServer::~PikaServer() { + + // DispatchThread will use queue of worker thread, + // so we need to delete dispatch before worker. + pika_thread_pool_->stop_thread_pool(); + delete pika_dispatch_thread_; + + { + slash::MutexLock l(&slave_mutex_); + std::vector::iterator iter = slaves_.begin(); + while (iter != slaves_.end()) { + iter = slaves_.erase(iter); + LOG(INFO) << "Delete slave success"; + } + } + + delete pika_pubsub_thread_; + delete pika_auxiliary_thread_; + delete pika_rsync_service_; + delete pika_thread_pool_; + delete pika_monitor_thread_; + + for (size_t i = 0; i < redis_senders_.size(); i++) { + redis_senders_[i]->Stop(); + } + // wait thread exit + sleep(1); + for (size_t i = 0; i < redis_senders_.size(); i++) { + delete redis_senders_[i]; + } + redis_senders_.clear(); + + bgsave_thread_.StopThread(); + key_scan_thread_.StopThread(); + + tables_.clear(); + + pthread_rwlock_destroy(&tables_rw_); + pthread_rwlock_destroy(&state_protector_); + pthread_rwlock_destroy(&slowlog_protector_); + + LOG(INFO) << "PikaServer " << pthread_self() << " exit!!!"; +} + +bool PikaServer::ServerInit() { + std::string network_interface = g_pika_conf->network_interface(); + + if (network_interface == "") { + + std::ifstream routeFile("/proc/net/route", std::ios_base::in); + if (!routeFile.good()) + { + return false; + } + + std::string line; + std::vector tokens; + while(std::getline(routeFile, line)) + { + std::istringstream stream(line); + std::copy(std::istream_iterator(stream), + std::istream_iterator(), + std::back_inserter >(tokens)); + + // the default interface is the one having the second + // field, Destination, set to "00000000" + if ((tokens.size() >= 2) && (tokens[1] == std::string("00000000"))) + { + network_interface = tokens[0]; + break; + } + + tokens.clear(); + } + routeFile.close(); + } + LOG(INFO) << "Using Networker Interface: " << network_interface; + + struct ifaddrs * ifAddrStruct = NULL; + struct ifaddrs * ifa = NULL; + void * tmpAddrPtr = NULL; + + if (getifaddrs(&ifAddrStruct) == -1) { + LOG(FATAL) << "getifaddrs failed: " << strerror(errno); + } + + for (ifa = ifAddrStruct; ifa != NULL; ifa = ifa->ifa_next) { + if (ifa->ifa_addr == NULL) { + continue; + } + if (ifa ->ifa_addr->sa_family==AF_INET) { // Check it is + // a valid IPv4 address + tmpAddrPtr = &((struct sockaddr_in *)ifa->ifa_addr)->sin_addr; + char addressBuffer[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, tmpAddrPtr, addressBuffer, INET_ADDRSTRLEN); + if (std::string(ifa->ifa_name) == network_interface) { + host_ = addressBuffer; + break; + } + } else if (ifa->ifa_addr->sa_family==AF_INET6) { // Check it is + // a valid IPv6 address + tmpAddrPtr = &((struct sockaddr_in6 *)ifa->ifa_addr)->sin6_addr; + char addressBuffer[INET6_ADDRSTRLEN]; + inet_ntop(AF_INET6, tmpAddrPtr, addressBuffer, INET6_ADDRSTRLEN); + if (std::string(ifa->ifa_name) == network_interface) { + host_ = addressBuffer; + break; + } + } + } + + if (ifAddrStruct != NULL) { + freeifaddrs(ifAddrStruct); + } + if (ifa == NULL) { + LOG(FATAL) << "error network interface: " << network_interface << ", please check!"; + } + + port_ = g_pika_conf->port(); + LOG(INFO) << "host: " << host_ << " port: " << port_; + return true; +} + +void PikaServer::Start() { + int ret = 0; + // start rsync first, rocksdb opened fd will not appear in this fork + ret = pika_rsync_service_->StartRsync(); + if (0 != ret) { + tables_.clear(); + LOG(FATAL) << "Start Rsync Error: bind port " +std::to_string(pika_rsync_service_->ListenPort()) + " failed" + << ", Listen on this port to receive Master FullSync Data"; + } + + // We Init Table Struct Before Start The following thread + InitTableStruct(); + + ret = pika_thread_pool_->start_thread_pool(); + if (ret != pink::kSuccess) { + tables_.clear(); + LOG(FATAL) << "Start ThreadPool Error: " << ret << (ret == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + ret = pika_dispatch_thread_->StartThread(); + if (ret != pink::kSuccess) { + tables_.clear(); + LOG(FATAL) << "Start Dispatch Error: " << ret << (ret == pink::kBindError ? ": bind port " + std::to_string(port_) + " conflict" + : ": other error") << ", Listen on this port to handle the connected redis client"; + } + ret = pika_pubsub_thread_->StartThread(); + if (ret != pink::kSuccess) { + tables_.clear(); + LOG(FATAL) << "Start Pubsub Error: " << ret << (ret == pink::kBindError ? ": bind port conflict" : ": other error"); + } + + ret = pika_auxiliary_thread_->StartThread(); + if (ret != pink::kSuccess) { + tables_.clear(); + LOG(FATAL) << "Start Auxiliary Thread Error: " << ret << (ret == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + for (size_t i = 0; i < redis_senders_.size(); i++) { + ret = redis_senders_[i]->StartThread(); + if (ret != pink::kSuccess) { + LOG(FATAL) << "Start Redis Sender Thread Error: " << ret << (ret == pink::kCreateThreadError ? ": create thread error " : ": other error"); + } + } + + time(&start_time_s_); + + std::string slaveof = g_pika_conf->slaveof(); + if (!slaveof.empty()) { + int32_t sep = slaveof.find(":"); + std::string master_ip = slaveof.substr(0, sep); + int32_t master_port = std::stoi(slaveof.substr(sep+1)); + if ((master_ip == "127.0.0.1" || master_ip == host_) && master_port == port_) { + LOG(FATAL) << "you will slaveof yourself as the config file, please check"; + } else { + SetMaster(master_ip, master_port); + } + } + + LOG(INFO) << "Pika Server going to start"; + while (!exit_) { + DoTimingTask(); + // wake up every 10 second + int try_num = 0; + while (!exit_ && try_num++ < 10) { + sleep(1); + } + } + LOG(INFO) << "Goodbye..."; +} + +void PikaServer::Exit() { + exit_ = true; +} + +std::string PikaServer::host() { + return host_; +} + +int PikaServer::port() { + return port_; +} + +time_t PikaServer::start_time_s() { + return start_time_s_; +} + +std::string PikaServer::master_ip() { + slash::RWLock(&state_protector_, false); + return master_ip_; +} + +int PikaServer::master_port() { + slash::RWLock(&state_protector_, false); + return master_port_; +} + +int PikaServer::role() { + slash::RWLock(&state_protector_, false); + return role_; +} + +bool PikaServer::readonly(const std::string& table_name, const std::string& key) { + slash::RWLock(&state_protector_, false); + if ((role_ & PIKA_ROLE_SLAVE) + && g_pika_conf->slave_read_only()) { + return true; + } + if (!g_pika_conf->classic_mode()) { + std::shared_ptr
table = GetTable(table_name); + if (table == nullptr) { + // swallow this error will process later + return false; + } + uint32_t index = g_pika_cmd_table_manager->DistributeKey( + key, table->PartitionNum()); + int role = 0; + Status s = g_pika_rm->CheckPartitionRole(table_name, index, &role); + if (!s.ok()) { + // swallow this error will process later + return false; + } + if (role & PIKA_ROLE_SLAVE) { + return true; + } + } + return false; +} + +int PikaServer::repl_state() { + slash::RWLock(&state_protector_, false); + return repl_state_; +} + +std::string PikaServer::repl_state_str() { + slash::RWLock(&state_protector_, false); + switch (repl_state_) { + case PIKA_REPL_NO_CONNECT: + return "no connect"; + case PIKA_REPL_SHOULD_META_SYNC: + return "should meta sync"; + case PIKA_REPL_META_SYNC_DONE: + return "meta sync done"; + case PIKA_REPL_ERROR: + return "error"; + default: + return ""; + } +} + +bool PikaServer::force_full_sync() { + return force_full_sync_; +} + +void PikaServer::SetForceFullSync(bool v) { + force_full_sync_ = v; +} + +void PikaServer::SetDispatchQueueLimit(int queue_limit) { + rlimit limit; + rlim_t maxfiles = g_pika_conf->maxclients() + PIKA_MIN_RESERVED_FDS; + if (getrlimit(RLIMIT_NOFILE, &limit) == -1) { + LOG(WARNING) << "getrlimit error: " << strerror(errno); + } else if (limit.rlim_cur < maxfiles) { + rlim_t old_limit = limit.rlim_cur; + limit.rlim_cur = maxfiles; + limit.rlim_max = maxfiles; + if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { + LOG(WARNING) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika have successfully reconfig it to " << limit.rlim_cur; + } else { + LOG(FATAL) << "your 'limit -n ' of " << old_limit << " is not enough for Redis to start. pika can not reconfig it(" << strerror(errno) << "), do it by yourself"; + } + } + pika_dispatch_thread_->SetQueueLimit(queue_limit); +} + +blackwidow::BlackwidowOptions PikaServer::bw_options() { + return bw_options_; +} + +void PikaServer::InitTableStruct() { + std::string db_path = g_pika_conf->db_path(); + std::string log_path = g_pika_conf->log_path(); + std::vector table_structs = g_pika_conf->table_structs(); + slash::RWLock rwl(&tables_rw_, true); + for (const auto& table : table_structs) { + std::string name = table.table_name; + uint32_t num = table.partition_num; + std::shared_ptr
table_ptr = std::make_shared
( + name, num, db_path, log_path); + table_ptr->AddPartitions(table.partition_ids); + tables_.emplace(name, table_ptr); + } +} + +std::shared_ptr
PikaServer::GetTable(const std::string &table_name) { + slash::RWLock l(&tables_rw_, false); + auto iter = tables_.find(table_name); + return (iter == tables_.end()) ? NULL : iter->second; +} + +std::set PikaServer::GetTablePartitionIds(const std::string& table_name) { + std::set empty; + slash::RWLock l(&tables_rw_, false); + auto iter = tables_.find(table_name); + return (iter == tables_.end()) ? empty : iter->second->GetPartitionIds(); +} + +bool PikaServer::IsBgSaving() { + slash::RWLock table_rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + slash::RWLock partition_rwl(&table_item.second->partitions_rw_, false); + for (const auto& patition_item : table_item.second->partitions_) { + if (patition_item.second->IsBgSaving()) { + return true; + } + } + } + return false; +} + +bool PikaServer::IsKeyScaning() { + slash::RWLock table_rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + if (table_item.second->IsKeyScaning()) { + return true; + } + } + return false; +} + +bool PikaServer::IsCompacting() { + slash::RWLock table_rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + slash::RWLock partition_rwl(&table_item.second->partitions_rw_, false); + for (const auto& partition_item : table_item.second->partitions_) { + partition_item.second->DbRWLockReader(); + std::string task_type = partition_item.second->db()->GetCurrentTaskType(); + partition_item.second->DbRWUnLock(); + if (strcasecmp(task_type.data(), "no")) { + return true; + } + } + } + return false; +} + +bool PikaServer::IsTableExist(const std::string& table_name) { + return GetTable(table_name) ? true : false; +} + +bool PikaServer::IsTablePartitionExist(const std::string& table_name, + uint32_t partition_id) { + std::shared_ptr
table_ptr = GetTable(table_name); + if (!table_ptr) { + return false; + } else { + return table_ptr->GetPartitionById(partition_id) ? true : false; + } +} + +bool PikaServer::IsCommandSupport(const std::string& command) { + if (g_pika_conf->classic_mode()) { + return true; + } else { + std::string cmd = command; + slash::StringToLower(cmd); + return !ShardingModeNotSupportCommands.count(cmd); + } +} + +bool PikaServer::IsTableBinlogIoError(const std::string& table_name) { + std::shared_ptr
table = GetTable(table_name); + return table ? table->IsBinlogIoError() : true; +} + +// If no collection of specified tables is given, we execute task in all tables +Status PikaServer::DoSameThingSpecificTable(const TaskType& type, const std::set& tables) { + slash::RWLock rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + if (!tables.empty() + && tables.find(table_item.first) == tables.end()) { + continue; + } else { + switch (type) { + case TaskType::kCompactAll: + table_item.second->Compact(blackwidow::DataType::kAll); + break; + case TaskType::kCompactStrings: + table_item.second->Compact(blackwidow::DataType::kStrings); + break; + case TaskType::kCompactHashes: + table_item.second->Compact(blackwidow::DataType::kHashes); + break; + case TaskType::kCompactSets: + table_item.second->Compact(blackwidow::DataType::kSets); + break; + case TaskType::kCompactZSets: + table_item.second->Compact(blackwidow::DataType::kZSets); + break; + case TaskType::kCompactList: + table_item.second->Compact(blackwidow::DataType::kLists); + break; + case TaskType::kStartKeyScan: + table_item.second->KeyScan(); + break; + case TaskType::kStopKeyScan: + table_item.second->StopKeyScan(); + break; + case TaskType::kBgSave: + table_item.second->BgSaveTable(); + break; + default: + break; + } + } + } + return Status::OK(); +} + +void PikaServer::PreparePartitionTrySync() { + slash::RWLock rwl(&tables_rw_, false); + ReplState state = force_full_sync_ ? + ReplState::kTryDBSync : ReplState::kTryConnect; + for (const auto& table_item : tables_) { + for (const auto& partition_item : table_item.second->partitions_) { + Status s = g_pika_rm->ActivateSyncSlavePartition( + RmNode(g_pika_server->master_ip(), + g_pika_server->master_port(), + table_item.second->GetTableName(), + partition_item.second->GetPartitionId()), state); + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + } + } + force_full_sync_ = false; + loop_partition_state_machine_ = true; + LOG(INFO) << "Mark try connect finish"; +} + +void PikaServer::PartitionSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys) { + slash::RWLock rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + for (const auto& partition_item : table_item.second->partitions_) { + partition_item.second->DbRWLockReader(); + partition_item.second->db()->SetMaxCacheStatisticKeys(max_cache_statistic_keys); + partition_item.second->DbRWUnLock(); + } + } +} + +void PikaServer::PartitionSetSmallCompactionThreshold(uint32_t small_compaction_threshold) { + slash::RWLock rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + for (const auto& partition_item : table_item.second->partitions_) { + partition_item.second->DbRWLockReader(); + partition_item.second->db()->SetSmallCompactionThreshold(small_compaction_threshold); + partition_item.second->DbRWUnLock(); + } + } +} + +bool PikaServer::GetTablePartitionBinlogOffset(const std::string& table_name, + uint32_t partition_id, + BinlogOffset* const boffset) { + std::shared_ptr partition = GetTablePartitionById(table_name, partition_id); + if (!partition) { + return false; + } else { + return partition->GetBinlogOffset(boffset); + } +} + +// Only use in classic mode +std::shared_ptr PikaServer::GetPartitionByDbName(const std::string& db_name) { + std::shared_ptr
table = GetTable(db_name); + return table ? table->GetPartitionById(0) : NULL; +} + +std::shared_ptr PikaServer::GetTablePartitionById( + const std::string& table_name, + uint32_t partition_id) { + std::shared_ptr
table = GetTable(table_name); + return table ? table->GetPartitionById(partition_id) : NULL; +} + +std::shared_ptr PikaServer::GetTablePartitionByKey( + const std::string& table_name, + const std::string& key) { + std::shared_ptr
table = GetTable(table_name); + return table ? table->GetPartitionByKey(key) : NULL; +} + +Status PikaServer::DoSameThingEveryPartition(const TaskType& type) { + slash::RWLock rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + for (const auto& partition_item : table_item.second->partitions_) { + switch (type) { + case TaskType::kResetReplState: + { + Status s = g_pika_rm->SetSlaveReplState( + PartitionInfo(table_item.second->GetTableName(), + partition_item.second->GetPartitionId()), + ReplState::kNoConnect); + if (!s.ok()) { + LOG(WARNING) << s.ToString(); + } + break; + } + case TaskType::kPurgeLog: + partition_item.second->PurgeLogs(); + break; + default: + break; + } + } + } + return Status::OK(); +} + +void PikaServer::BecomeMaster() { + slash::RWLock l(&state_protector_, true); + role_ |= PIKA_ROLE_MASTER; +} + +void PikaServer::DeleteSlave(int fd) { + std::string ip; + int port = -1; + bool is_find = false; + int slave_num = -1; + { + slash::MutexLock l(&slave_mutex_); + std::vector::iterator iter = slaves_.begin(); + while (iter != slaves_.end()) { + if (iter->conn_fd == fd) { + ip = iter->ip; + port = iter->port; + is_find = true; + g_pika_rm->LostConnection(iter->ip, iter->port); + g_pika_rm->DropItemInWriteQueue(iter->ip, iter->port); + LOG(INFO) << "Delete Slave Success, ip_port: " << iter->ip << ":" << iter->port; + slaves_.erase(iter); + break; + } + iter++; + } + slave_num = slaves_.size(); + } + + if (is_find) { + g_pika_rm->LostConnection(ip, port); + g_pika_rm->DropItemInWriteQueue(ip, port); + } + + if (slave_num == 0) { + slash::RWLock l(&state_protector_, true); + role_ &= ~PIKA_ROLE_MASTER; + } +} + +int32_t PikaServer::CountSyncSlaves() { + slash::MutexLock ldb(&db_sync_protector_); + return db_sync_slaves_.size(); +} + +int32_t PikaServer::GetShardingSlaveListString(std::string& slave_list_str) { + std::vector complete_replica; + g_pika_rm->FindCompleteReplica(&complete_replica); + std::stringstream tmp_stream; + size_t index = 0; + for (auto replica : complete_replica) { + std::string ip; + int port; + if(!slash::ParseIpPortString(replica, ip, port)) { + continue; + } + tmp_stream << "slave" << index++ << ":ip=" << ip << ",port=" << port << "\r\n"; + } + slave_list_str.assign(tmp_stream.str()); + return index; +} + +int32_t PikaServer::GetSlaveListString(std::string& slave_list_str) { + size_t index = 0; + SlaveState slave_state; + BinlogOffset master_boffset; + BinlogOffset sent_slave_boffset; + BinlogOffset acked_slave_boffset; + std::stringstream tmp_stream; + slash::MutexLock l(&slave_mutex_); + for (const auto& slave : slaves_) { + tmp_stream << "slave" << index++ << ":ip=" << slave.ip << ",port=" << slave.port << ",conn_fd=" << slave.conn_fd << ",lag="; + for (const auto& ts : slave.table_structs) { + for (size_t idx = 0; idx < ts.partition_num; ++idx) { + std::shared_ptr partition = GetTablePartitionById(ts.table_name, idx); + RmNode rm_node(slave.ip, slave.port, ts.table_name, idx); + Status s = g_pika_rm->GetSyncMasterPartitionSlaveState(rm_node, &slave_state); + if (s.ok() + && slave_state == SlaveState::kSlaveBinlogSync + && g_pika_rm->GetSyncBinlogStatus(rm_node, &sent_slave_boffset, &acked_slave_boffset).ok()) { + if (!partition || !partition->GetBinlogOffset(&master_boffset)) { + continue; + } else { + uint64_t lag = + (master_boffset.filenum - sent_slave_boffset.filenum) * g_pika_conf->binlog_file_size() + + (master_boffset.offset - sent_slave_boffset.offset); + tmp_stream << "(" << partition->GetPartitionName() << ":" << lag << ")"; + } + } else { + tmp_stream << "(" << partition->GetPartitionName() << ":not syncing)"; + } + } + } + tmp_stream << "\r\n"; + } + slave_list_str.assign(tmp_stream.str()); + return index; +} + +// Try add Slave, return true if success, +// return false when slave already exist +bool PikaServer::TryAddSlave(const std::string& ip, int64_t port, int fd, + const std::vector& table_structs) { + std::string ip_port = slash::IpPortString(ip, port); + + slash::MutexLock l(&slave_mutex_); + std::vector::iterator iter = slaves_.begin(); + while (iter != slaves_.end()) { + if (iter->ip_port == ip_port) { + LOG(WARNING) << "Slave Already Exist, ip_port: " << ip << ":" << port; + return false; + } + iter++; + } + + // Not exist, so add new + LOG(INFO) << "Add New Slave, " << ip << ":" << port; + SlaveItem s; + s.ip_port = ip_port; + s.ip = ip; + s.port = port; + s.conn_fd = fd; + s.stage = SLAVE_ITEM_STAGE_ONE; + s.table_structs = table_structs; + gettimeofday(&s.create_time, NULL); + slaves_.push_back(s); + return true; +} + +void PikaServer::SyncError() { + slash::RWLock l(&state_protector_, true); + repl_state_ = PIKA_REPL_ERROR; + LOG(WARNING) << "Sync error, set repl_state to PIKA_REPL_ERROR"; +} + +void PikaServer::RemoveMaster() { + { + slash::RWLock l(&state_protector_, true); + repl_state_ = PIKA_REPL_NO_CONNECT; + role_ &= ~PIKA_ROLE_SLAVE; + + if (master_ip_ != "" && master_port_ != -1) { + g_pika_rm->CloseReplClientConn(master_ip_, master_port_ + kPortShiftReplServer); + g_pika_rm->LostConnection(master_ip_, master_port_); + loop_partition_state_machine_ = false; + LOG(INFO) << "Remove Master Success, ip_port: " << master_ip_ << ":" << master_port_; + } + + master_ip_ = ""; + master_port_ = -1; + DoSameThingEveryPartition(TaskType::kResetReplState); + } +} + +bool PikaServer::SetMaster(std::string& master_ip, int master_port) { + if (master_ip == "127.0.0.1") { + master_ip = host_; + } + slash::RWLock l(&state_protector_, true); + if ((role_ ^ PIKA_ROLE_SLAVE) && repl_state_ == PIKA_REPL_NO_CONNECT) { + master_ip_ = master_ip; + master_port_ = master_port; + role_ |= PIKA_ROLE_SLAVE; + repl_state_ = PIKA_REPL_SHOULD_META_SYNC; + return true; + } + return false; +} + +bool PikaServer::ShouldMetaSync() { + slash::RWLock l(&state_protector_, false); + return repl_state_ == PIKA_REPL_SHOULD_META_SYNC; +} + +void PikaServer::FinishMetaSync() { + slash::RWLock l(&state_protector_, true); + assert(repl_state_ == PIKA_REPL_SHOULD_META_SYNC); + repl_state_ = PIKA_REPL_META_SYNC_DONE; +} + +bool PikaServer::MetaSyncDone() { + slash::RWLock l(&state_protector_, false); + return repl_state_ == PIKA_REPL_META_SYNC_DONE; +} + +void PikaServer::ResetMetaSyncStatus() { + slash::RWLock sp_l(&state_protector_, true); + if (role_ & PIKA_ROLE_SLAVE) { + // not change by slaveof no one, so set repl_state = PIKA_REPL_SHOULD_META_SYNC, + // continue to connect master + repl_state_ = PIKA_REPL_SHOULD_META_SYNC; + loop_partition_state_machine_ = false; + DoSameThingEveryPartition(TaskType::kResetReplState); + } +} + +bool PikaServer::AllPartitionConnectSuccess() { + bool all_partition_connect_success = true; + slash::RWLock rwl(&tables_rw_, false); + for (const auto& table_item : tables_) { + for (const auto& partition_item : table_item.second->partitions_) { + ReplState repl_state; + Status s = g_pika_rm->GetSlaveReplState( + PartitionInfo(table_item.second->GetTableName(), + partition_item.second->GetPartitionId()), &repl_state); + if (!s.ok()) { + return false; + } + if (repl_state != ReplState::kConnected) { + all_partition_connect_success = false; + break; + } + } + } + return all_partition_connect_success; +} + +bool PikaServer::LoopPartitionStateMachine() { + slash::RWLock sp_l(&state_protector_, false); + return loop_partition_state_machine_; +} + +void PikaServer::SetLoopPartitionStateMachine(bool need_loop) { + slash::RWLock sp_l(&state_protector_, true); + assert(repl_state_ == PIKA_REPL_META_SYNC_DONE); + loop_partition_state_machine_ = need_loop; +} + +void PikaServer::Schedule(pink::TaskFunc func, void* arg) { + pika_thread_pool_->Schedule(func, arg); +} + +void PikaServer::BGSaveTaskSchedule(pink::TaskFunc func, void* arg) { + bgsave_thread_.StartThread(); + bgsave_thread_.Schedule(func, arg); +} + +void PikaServer::PurgelogsTaskSchedule(pink::TaskFunc func, void* arg) { + purge_thread_.StartThread(); + purge_thread_.Schedule(func, arg); +} + +void PikaServer::PurgeDir(const std::string& path) { + std::string* dir_path = new std::string(path); + PurgeDirTaskSchedule(&DoPurgeDir, static_cast(dir_path)); +} + +void PikaServer::PurgeDirTaskSchedule(void (*function)(void*), void* arg) { + purge_thread_.StartThread(); + purge_thread_.Schedule(function, arg); +} + +void PikaServer::DBSync(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id) { + { + std::string task_index = + DbSyncTaskIndex(ip, port, table_name, partition_id); + slash::MutexLock ml(&db_sync_protector_); + if (db_sync_slaves_.find(task_index) != db_sync_slaves_.end()) { + return; + } + db_sync_slaves_.insert(task_index); + } + // Reuse the bgsave_thread_ + // Since we expect BgSave and DBSync execute serially + bgsave_thread_.StartThread(); + DBSyncArg* arg = new DBSyncArg(this, ip, port, table_name, partition_id); + bgsave_thread_.Schedule(&DoDBSync, reinterpret_cast(arg)); +} + +void PikaServer::TryDBSync(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id, int32_t top) { + std::shared_ptr partition = + GetTablePartitionById(table_name, partition_id); + if (!partition) { + LOG(WARNING) << "Partition: " << partition->GetPartitionName() + << " Not Found, TryDBSync Failed"; + } else { + BgSaveInfo bgsave_info = partition->bgsave_info(); + std::string logger_filename = partition->logger()->filename; + if (slash::IsDir(bgsave_info.path) != 0 + || !slash::FileExists(NewFileName(logger_filename, bgsave_info.filenum)) + || top - bgsave_info.filenum > kDBSyncMaxGap) { + // Need Bgsave first + partition->BgSavePartition(); + } + DBSync(ip, port, table_name, partition_id); + } +} + +void PikaServer::DbSyncSendFile(const std::string& ip, int port, + const std::string& table_name, + uint32_t partition_id) { + std::shared_ptr partition = GetTablePartitionById(table_name, partition_id); + if (!partition) { + LOG(WARNING) << "Partition: " << partition->GetPartitionName() + << " Not Found, DbSync send file Failed"; + return; + } + + BgSaveInfo bgsave_info = partition->bgsave_info(); + std::string bg_path = bgsave_info.path; + uint32_t binlog_filenum = bgsave_info.filenum; + uint64_t binlog_offset = bgsave_info.offset; + + // Get all files need to send + std::vector descendant; + int ret = 0; + LOG(INFO) << "Partition: " << partition->GetPartitionName() + << " Start Send files in " << bg_path << " to " << ip; + ret = slash::GetChildren(bg_path, descendant); + if (ret != 0) { + std::string ip_port = slash::IpPortString(ip, port); + slash::MutexLock ldb(&db_sync_protector_); + db_sync_slaves_.erase(ip_port); + LOG(WARNING) << "Partition: " << partition->GetPartitionName() + << " Get child directory when try to do sync failed, error: " << strerror(ret); + return; + } + + std::string local_path, target_path; + std::string remote_path = g_pika_conf->classic_mode() ? table_name : table_name + "/" + std::to_string(partition_id); + std::vector::const_iterator iter = descendant.begin(); + slash::RsyncRemote remote(ip, port, kDBSyncModule, g_pika_conf->db_sync_speed() * 1024); + std::string secret_file_path = g_pika_conf->db_sync_path(); + if (g_pika_conf->db_sync_path().back() != '/') { + secret_file_path += "/"; + } + secret_file_path += slash::kRsyncSubDir + "/" + kPikaSecretFile; + + for (; iter != descendant.end(); ++iter) { + local_path = bg_path + "/" + *iter; + target_path = remote_path + "/" + *iter; + + if (*iter == kBgsaveInfoFile) { + continue; + } + + if (slash::IsDir(local_path) == 0 && + local_path.back() != '/') { + local_path.push_back('/'); + target_path.push_back('/'); + } + + // We need specify the speed limit for every single file + ret = slash::RsyncSendFile(local_path, target_path, secret_file_path, remote); + if (0 != ret) { + LOG(WARNING) << "Partition: " << partition->GetPartitionName() + << " RSync send file failed! From: " << *iter + << ", To: " << target_path + << ", At: " << ip << ":" << port + << ", Error: " << ret; + break; + } + } + // Clear target path + slash::RsyncSendClearTarget(bg_path + "/strings", remote_path + "/strings", secret_file_path, remote); + slash::RsyncSendClearTarget(bg_path + "/hashes", remote_path + "/hashes", secret_file_path, remote); + slash::RsyncSendClearTarget(bg_path + "/lists", remote_path + "/lists", secret_file_path, remote); + slash::RsyncSendClearTarget(bg_path + "/sets", remote_path + "/sets", secret_file_path, remote); + slash::RsyncSendClearTarget(bg_path + "/zsets", remote_path + "/zsets", secret_file_path, remote); + + pink::PinkCli* cli = pink::NewRedisCli(); + std::string lip(host_); + if (cli->Connect(ip, port, "").ok()) { + struct sockaddr_in laddr; + socklen_t llen = sizeof(laddr); + getsockname(cli->fd(), (struct sockaddr*) &laddr, &llen); + lip = inet_ntoa(laddr.sin_addr); + cli->Close(); + delete cli; + } else { + LOG(WARNING) << "Rsync try connect slave rsync service error" + << ", slave rsync service(" << ip << ":" << port << ")"; + delete cli; + } + + // Send info file at last + if (0 == ret) { + // need to modify the IP addr in the info file + if (lip.compare(host_)) { + std::ofstream fix; + std::string fn = bg_path + "/" + kBgsaveInfoFile + "." + std::to_string(time(NULL)); + fix.open(fn, std::ios::in | std::ios::trunc); + if (fix.is_open()) { + fix << "0s\n" << lip << "\n" << port_ << "\n" << binlog_filenum << "\n" << binlog_offset << "\n"; + fix.close(); + } + ret = slash::RsyncSendFile(fn, remote_path + "/" + kBgsaveInfoFile, secret_file_path, remote); + slash::DeleteFile(fn); + if (ret != 0) { + LOG(WARNING) << "Partition: " << partition->GetPartitionName() << " Send Modified Info File Failed"; + } + } else if (0 != (ret = slash::RsyncSendFile(bg_path + "/" + kBgsaveInfoFile, remote_path + "/" + kBgsaveInfoFile, secret_file_path, remote))) { + LOG(WARNING) << "Partition: " << partition->GetPartitionName() << " Send Info File Failed"; + } + } + // remove slave + { + std::string task_index = + DbSyncTaskIndex(ip, port, table_name, partition_id); + slash::MutexLock ml(&db_sync_protector_); + db_sync_slaves_.erase(task_index); + } + + if (0 == ret) { + LOG(INFO) << "Partition: " << partition->GetPartitionName() << " RSync Send Files Success"; + } +} + +std::string PikaServer::DbSyncTaskIndex(const std::string& ip, + int port, + const std::string& table_name, + uint32_t partition_id) { + char buf[256]; + snprintf(buf, sizeof(buf), "%s:%d_%s:%d", + ip.data(), port, table_name.data(), partition_id); + return buf; +} + +void PikaServer::KeyScanTaskSchedule(pink::TaskFunc func, void* arg) { + key_scan_thread_.StartThread(); + key_scan_thread_.Schedule(func, arg); +} + +void PikaServer::ClientKillAll() { + pika_dispatch_thread_->ClientKillAll(); + pika_monitor_thread_->ThreadClientKill(); +} + +int PikaServer::ClientKill(const std::string &ip_port) { + if (pika_dispatch_thread_->ClientKill(ip_port) + || pika_monitor_thread_->ThreadClientKill(ip_port)) { + return 1; + } + return 0; +} + +int64_t PikaServer::ClientList(std::vector *clients) { + int64_t clients_num = 0; + clients_num += pika_dispatch_thread_->ThreadClientList(clients); + clients_num += pika_monitor_thread_->ThreadClientList(clients); + return clients_num; +} + +bool PikaServer::HasMonitorClients() { + return pika_monitor_thread_->HasMonitorClients(); +} + +void PikaServer::AddMonitorMessage(const std::string& monitor_message) { + pika_monitor_thread_->AddMonitorMessage(monitor_message); +} + +void PikaServer::AddMonitorClient(std::shared_ptr client_ptr) { + pika_monitor_thread_->AddMonitorClient(client_ptr); +} + +void PikaServer::SlowlogTrim() { + pthread_rwlock_wrlock(&slowlog_protector_); + while (slowlog_list_.size() > static_cast(g_pika_conf->slowlog_max_len())) { + slowlog_list_.pop_back(); + } + pthread_rwlock_unlock(&slowlog_protector_); +} + +void PikaServer::SlowlogReset() { + pthread_rwlock_wrlock(&slowlog_protector_); + slowlog_list_.clear(); + pthread_rwlock_unlock(&slowlog_protector_); +} + +uint32_t PikaServer::SlowlogLen() { + RWLock l(&slowlog_protector_, false); + return slowlog_list_.size(); +} + +void PikaServer::SlowlogObtain(int64_t number, std::vector* slowlogs) { + pthread_rwlock_rdlock(&slowlog_protector_); + slowlogs->clear(); + std::list::const_iterator iter = slowlog_list_.begin(); + while (number-- && iter != slowlog_list_.end()) { + slowlogs->push_back(*iter); + iter++; + } + pthread_rwlock_unlock(&slowlog_protector_); +} + +void PikaServer::SlowlogPushEntry(const PikaCmdArgsType& argv, int32_t time, int64_t duration) { + SlowlogEntry entry; + uint32_t slargc = (argv.size() < SLOWLOG_ENTRY_MAX_ARGC) + ? argv.size() : SLOWLOG_ENTRY_MAX_ARGC; + + for (uint32_t idx = 0; idx < slargc; ++idx) { + if (slargc != argv.size() && idx == slargc - 1) { + char buffer[32]; + sprintf(buffer, "... (%lu more arguments)", argv.size() - slargc + 1); + entry.argv.push_back(std::string(buffer)); + } else { + if (argv[idx].size() > SLOWLOG_ENTRY_MAX_STRING) { + char buffer[32]; + sprintf(buffer, "... (%lu more bytes)", argv[idx].size() - SLOWLOG_ENTRY_MAX_STRING); + std::string suffix(buffer); + std::string brief = argv[idx].substr(0, SLOWLOG_ENTRY_MAX_STRING); + entry.argv.push_back(brief + suffix); + } else { + entry.argv.push_back(argv[idx]); + } + } + } + + pthread_rwlock_wrlock(&slowlog_protector_); + entry.id = slowlog_entry_id_++; + entry.start_time = time; + entry.duration = duration; + slowlog_list_.push_front(entry); + pthread_rwlock_unlock(&slowlog_protector_); + + SlowlogTrim(); +} + +void PikaServer::ResetStat() { + statistic_data_.accumulative_connections.store(0); + statistic_data_.thread_querynum.store(0); + statistic_data_.last_thread_querynum.store(0); +} + +uint64_t PikaServer::ServerQueryNum() { + return statistic_data_.thread_querynum.load(); +} + +uint64_t PikaServer::ServerCurrentQps() { + return statistic_data_.last_sec_thread_querynum.load(); +} + +uint64_t PikaServer::accumulative_connections() { + return statistic_data_.accumulative_connections.load(); +} + +void PikaServer::incr_accumulative_connections() { + ++statistic_data_.accumulative_connections; +} + +// only one thread invoke this right now +void PikaServer::ResetLastSecQuerynum() { + uint64_t last_query = statistic_data_.last_thread_querynum.load(); + uint64_t cur_query = statistic_data_.thread_querynum.load(); + uint64_t last_time_us = statistic_data_.last_time_us.load(); + if (cur_query < last_query) { + cur_query = last_query; + } + uint64_t delta_query = cur_query - last_query; + uint64_t cur_time_us = slash::NowMicros(); + if (cur_time_us <= last_time_us) { + cur_time_us = last_time_us + 1; + } + uint64_t delta_time_us = cur_time_us - last_time_us; + statistic_data_.last_sec_thread_querynum.store(delta_query + * 1000000 / (delta_time_us)); + statistic_data_.last_thread_querynum.store(cur_query); + statistic_data_.last_time_us.store(cur_time_us); +} + +void PikaServer::UpdateQueryNumAndExecCountTable(const std::string& command) { + std::string cmd(command); + statistic_data_.thread_querynum++; + statistic_data_.exec_count_table[slash::StringToUpper(cmd)]++; +} + +std::unordered_map PikaServer::ServerExecCountTable() { + std::unordered_map res; + for (auto& cmd : statistic_data_.exec_count_table) { + res[cmd.first] = cmd.second.load(); + } + return res; +} + +int PikaServer::SendToPeer() { + return g_pika_rm->ConsumeWriteQueue(); +} + +void PikaServer::SignalAuxiliary() { + pika_auxiliary_thread_->mu_.Lock(); + pika_auxiliary_thread_->cv_.Signal(); + pika_auxiliary_thread_->mu_.Unlock(); +} + +Status PikaServer::TriggerSendBinlogSync() { + return g_pika_rm->WakeUpBinlogSync(); +} + +int PikaServer::PubSubNumPat() { + return pika_pubsub_thread_->PubSubNumPat(); +} + +int PikaServer::Publish(const std::string& channel, const std::string& msg) { + int receivers = pika_pubsub_thread_->Publish(channel, msg); + return receivers; +} + +int PikaServer::UnSubscribe(std::shared_ptr conn, + const std::vector& channels, + bool pattern, + std::vector>* result) { + int subscribed = pika_pubsub_thread_->UnSubscribe(conn, channels, pattern, result); + return subscribed; +} + +void PikaServer::Subscribe(std::shared_ptr conn, + const std::vector& channels, + bool pattern, + std::vector>* result) { + pika_pubsub_thread_->Subscribe(conn, channels, pattern, result); +} + +void PikaServer::PubSubChannels(const std::string& pattern, + std::vector* result) { + pika_pubsub_thread_->PubSubChannels(pattern, result); +} + +void PikaServer::PubSubNumSub(const std::vector& channels, + std::vector>* result) { + pika_pubsub_thread_->PubSubNumSub(channels, result); +} + +int PikaServer::SendRedisCommand(const std::string& command, const std::string& key) { + // Send command + size_t idx = std::hash()(key) % redis_senders_.size(); + redis_senders_[idx]->SendRedisCommand(command); + return 0; +} + +void PikaServer::RetransmitData(const std::string& path) { + + blackwidow::BlackWidow *db = new blackwidow::BlackWidow(); + rocksdb::Status s = db->Open(g_pika_server->bw_options(), path); + + if (!s.ok()) { + LOG(FATAL) << "open received database error: " << s.ToString(); + return; + } + + // Init SenderThread + int thread_num = g_pika_conf->redis_sender_num(); + std::string target_host = g_pika_conf->target_redis_host(); + int target_port = g_pika_conf->target_redis_port(); + std::string target_pwd = g_pika_conf->target_redis_pwd(); + + LOG(INFO) << "open received database success, start retransmit data to redis(" + << target_host << ":" << target_port << ")"; + + std::vector pika_senders; + std::vector migrators; + + for (int i = 0; i < thread_num; i++) { + pika_senders.emplace_back(new PikaSender(target_host, target_port, target_pwd)); + } + migrators.emplace_back(new MigratorThread(db, &pika_senders, blackwidow::kStrings, thread_num)); + migrators.emplace_back(new MigratorThread(db, &pika_senders, blackwidow::kLists, thread_num)); + migrators.emplace_back(new MigratorThread(db, &pika_senders, blackwidow::kHashes, thread_num)); + migrators.emplace_back(new MigratorThread(db, &pika_senders, blackwidow::kSets, thread_num)); + migrators.emplace_back(new MigratorThread(db, &pika_senders, blackwidow::kZSets, thread_num)); + + for (size_t i = 0; i < pika_senders.size(); i++) { + pika_senders[i]->StartThread(); + } + for (size_t i = 0; i < migrators.size(); i++) { + migrators[i]->StartThread(); + } + + for (size_t i = 0; i < migrators.size(); i++) { + migrators[i]->JoinThread(); + } + for (size_t i = 0; i < pika_senders.size(); i++) { + pika_senders[i]->Stop(); + } + for (size_t i = 0; i < pika_senders.size(); i++) { + pika_senders[i]->JoinThread(); + } + + int64_t replies = 0, records = 0; + for (size_t i = 0; i < migrators.size(); i++) { + records += migrators[i]->num(); + delete migrators[i]; + } + migrators.clear(); + for (size_t i = 0; i < pika_senders.size(); i++) { + replies += pika_senders[i]->elements(); + delete pika_senders[i]; + } + pika_senders.clear(); + + LOG(INFO) << "=============== Retransmit Finish ====================="; + LOG(INFO) << "Total records : " << records << " have been Scaned"; + LOG(INFO) << "Total replies : " << replies << " received from redis server"; + LOG(INFO) << "======================================================="; +} + +/******************************* PRIVATE *******************************/ + +void PikaServer::DoTimingTask() { + // Maybe schedule compactrange + AutoCompactRange(); + // Purge log + AutoPurge(); + // Delete expired dump + AutoDeleteExpiredDump(); + // Cheek Rsync Status + AutoKeepAliveRSync(); +} + +void PikaServer::AutoCompactRange() { + struct statfs disk_info; + int ret = statfs(g_pika_conf->db_path().c_str(), &disk_info); + if (ret == -1) { + LOG(WARNING) << "statfs error: " << strerror(errno); + return; + } + + uint64_t total_size = disk_info.f_bsize * disk_info.f_blocks; + uint64_t free_size = disk_info.f_bsize * disk_info.f_bfree; + std::string ci = g_pika_conf->compact_interval(); + std::string cc = g_pika_conf->compact_cron(); + + if (ci != "") { + std::string::size_type slash = ci.find("/"); + int interval = std::atoi(ci.substr(0, slash).c_str()); + int usage = std::atoi(ci.substr(slash+1).c_str()); + struct timeval now; + gettimeofday(&now, NULL); + if (last_check_compact_time_.tv_sec == 0 || + now.tv_sec - last_check_compact_time_.tv_sec >= interval * 3600) { + gettimeofday(&last_check_compact_time_, NULL); + if (((double)free_size / total_size) * 100 >= usage) { + Status s = DoSameThingSpecificTable(TaskType::kCompactAll); + if (s.ok()) { + LOG(INFO) << "[Interval]schedule compactRange, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB"; + } else { + LOG(INFO) << "[Interval]schedule compactRange Failed, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 + << "MB, error: " << s.ToString(); + } + } else { + LOG(WARNING) << "compact-interval failed, because there is not enough disk space left, freesize" + << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB"; + } + } + return; + } + + if (cc != "") { + bool have_week = false; + std::string compact_cron, week_str; + int slash_num = count(cc.begin(), cc.end(), '/'); + if (slash_num == 2) { + have_week = true; + std::string::size_type first_slash = cc.find("/"); + week_str = cc.substr(0, first_slash); + compact_cron = cc.substr(first_slash + 1); + } else { + compact_cron = cc; + } + + std::string::size_type colon = compact_cron.find("-"); + std::string::size_type underline = compact_cron.find("/"); + int week = have_week ? (std::atoi(week_str.c_str()) % 7) : 0; + int start = std::atoi(compact_cron.substr(0, colon).c_str()); + int end = std::atoi(compact_cron.substr(colon+1, underline).c_str()); + int usage = std::atoi(compact_cron.substr(underline+1).c_str()); + std::time_t t = std::time(nullptr); + std::tm* t_m = std::localtime(&t); + + bool in_window = false; + if (start < end && (t_m->tm_hour >= start && t_m->tm_hour < end)) { + in_window = have_week ? (week == t_m->tm_wday) : true; + } else if (start > end && ((t_m->tm_hour >= start && t_m->tm_hour < 24) || + (t_m->tm_hour >= 0 && t_m->tm_hour < end))) { + in_window = have_week ? false : true; + } else { + have_scheduled_crontask_ = false; + } + + if (!have_scheduled_crontask_ && in_window) { + if (((double)free_size / total_size) * 100 >= usage) { + Status s = DoSameThingEveryPartition(TaskType::kCompactAll); + if (s.ok()) { + LOG(INFO) << "[Cron]schedule compactRange, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB"; + } else { + LOG(INFO) << "[Cron]schedule compactRange Failed, freesize: " << free_size/1048576 << "MB, disksize: " << total_size/1048576 + << "MB, error: " << s.ToString(); + } + have_scheduled_crontask_ = true; + } else { + LOG(WARNING) << "compact-cron failed, because there is not enough disk space left, freesize" + << free_size/1048576 << "MB, disksize: " << total_size/1048576 << "MB"; + } + } + } +} + +void PikaServer::AutoPurge() { + DoSameThingEveryPartition(TaskType::kPurgeLog); +} + +void PikaServer::AutoDeleteExpiredDump() { + std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); + std::string db_sync_path = g_pika_conf->bgsave_path(); + int expiry_days = g_pika_conf->expire_dump_days(); + std::vector dump_dir; + + // Never expire + if (expiry_days <= 0) { + return; + } + + // Dump is not exist + if (!slash::FileExists(db_sync_path)) { + return; + } + + // Directory traversal + if (slash::GetChildren(db_sync_path, dump_dir) != 0) { + return; + } + // Handle dump directory + for (size_t i = 0; i < dump_dir.size(); i++) { + if (dump_dir[i].substr(0, db_sync_prefix.size()) != db_sync_prefix || dump_dir[i].size() != (db_sync_prefix.size() + 8)) { + continue; + } + + std::string str_date = dump_dir[i].substr(db_sync_prefix.size(), (dump_dir[i].size() - db_sync_prefix.size())); + char *end = NULL; + std::strtol(str_date.c_str(), &end, 10); + if (*end != 0) { + continue; + } + + // Parse filename + int dump_year = std::atoi(str_date.substr(0, 4).c_str()); + int dump_month = std::atoi(str_date.substr(4, 2).c_str()); + int dump_day = std::atoi(str_date.substr(6, 2).c_str()); + + time_t t = time(NULL); + struct tm *now = localtime(&t); + int now_year = now->tm_year + 1900; + int now_month = now->tm_mon + 1; + int now_day = now->tm_mday; + + struct tm dump_time, now_time; + + dump_time.tm_year = dump_year; + dump_time.tm_mon = dump_month; + dump_time.tm_mday = dump_day; + dump_time.tm_hour = 0; + dump_time.tm_min = 0; + dump_time.tm_sec = 0; + + now_time.tm_year = now_year; + now_time.tm_mon = now_month; + now_time.tm_mday = now_day; + now_time.tm_hour = 0; + now_time.tm_min = 0; + now_time.tm_sec = 0; + + long dump_timestamp = mktime(&dump_time); + long now_timestamp = mktime(&now_time); + // How many days, 1 day = 86400s + int interval_days = (now_timestamp - dump_timestamp) / 86400; + + if (interval_days >= expiry_days) { + std::string dump_file = db_sync_path + dump_dir[i]; + if (CountSyncSlaves() == 0) { + LOG(INFO) << "Not syncing, delete dump file: " << dump_file; + slash::DeleteDirIfExist(dump_file); + } else { + LOG(INFO) << "Syncing, can not delete " << dump_file << " dump file"; + } + } + } +} + +void PikaServer::AutoKeepAliveRSync() { + if (!pika_rsync_service_->CheckRsyncAlive()) { + LOG(WARNING) << "The Rsync service is down, Try to restart"; + pika_rsync_service_->StartRsync(); + } +} + +void PikaServer::InitBlackwidowOptions() { + + // For rocksdb::Options + bw_options_.options.create_if_missing = true; + bw_options_.options.keep_log_file_num = 10; + bw_options_.options.max_manifest_file_size = 64 * 1024 * 1024; + bw_options_.options.max_log_file_size = 512 * 1024 * 1024; + + bw_options_.options.write_buffer_size = + g_pika_conf->write_buffer_size(); + bw_options_.options.write_buffer_manager.reset( + new rocksdb::WriteBufferManager(g_pika_conf->max_write_buffer_size())); + bw_options_.options.target_file_size_base = + g_pika_conf->target_file_size_base(); + bw_options_.options.max_background_flushes = + g_pika_conf->max_background_flushes(); + bw_options_.options.max_background_compactions = + g_pika_conf->max_background_compactions(); + bw_options_.options.max_open_files = + g_pika_conf->max_cache_files(); + bw_options_.options.max_bytes_for_level_multiplier = + g_pika_conf->max_bytes_for_level_multiplier(); + bw_options_.options.optimize_filters_for_hits = + g_pika_conf->optimize_filters_for_hits(); + bw_options_.options.level_compaction_dynamic_level_bytes = + g_pika_conf->level_compaction_dynamic_level_bytes(); + + + if (g_pika_conf->compression() == "none") { + bw_options_.options.compression = + rocksdb::CompressionType::kNoCompression; + } else if (g_pika_conf->compression() == "snappy") { + bw_options_.options.compression = + rocksdb::CompressionType::kSnappyCompression; + } else if (g_pika_conf->compression() == "zlib") { + bw_options_.options.compression = + rocksdb::CompressionType::kZlibCompression; + } + + // For rocksdb::BlockBasedTableOptions + bw_options_.table_options.block_size = g_pika_conf->block_size(); + bw_options_.table_options.cache_index_and_filter_blocks = + g_pika_conf->cache_index_and_filter_blocks(); + bw_options_.block_cache_size = g_pika_conf->block_cache(); + bw_options_.share_block_cache = g_pika_conf->share_block_cache(); + + if (bw_options_.block_cache_size == 0) { + bw_options_.table_options.no_block_cache = true; + } else if (bw_options_.share_block_cache) { + bw_options_.table_options.block_cache = + rocksdb::NewLRUCache(bw_options_.block_cache_size); + } + + // For Blackwidow small compaction + bw_options_.statistics_max_size = g_pika_conf->max_cache_statistic_keys(); + bw_options_.small_compaction_threshold = + g_pika_conf->small_compaction_threshold(); +} diff --git a/tools/pika_migrate/src/pika_set.cc b/tools/pika_migrate/src/pika_set.cc new file mode 100644 index 0000000000..c78784487a --- /dev/null +++ b/tools/pika_migrate/src/pika_set.cc @@ -0,0 +1,391 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_set.h" + +#include "slash/include/slash_string.h" + +void SAddCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSAdd); + return; + } + key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + iter++; + members_.assign(iter, argv_.end()); + return; +} + +void SAddCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->SAdd(key_, members_, &count); + if (!s.ok()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res_.AppendInteger(count); + return; +} + +void SPopCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSPop); + return; + } + key_ = argv_[1]; + return; +} + +void SPopCmd::Do(std::shared_ptr partition) { + std::string member; + rocksdb::Status s = partition->db()->SPop(key_, &member); + if (s.ok()) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } else if (s.IsNotFound()) { + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SCardCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSCard); + return; + } + key_ = argv_[1]; + return; +} + +void SCardCmd::Do(std::shared_ptr partition) { + int32_t card = 0; + rocksdb::Status s = partition->db()->SCard(key_, &card); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(card); + } else { + res_.SetRes(CmdRes::kErrOther, "scard error"); + } + return; +} + +void SMembersCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSMembers); + return; + } + key_ = argv_[1]; + return; +} + +void SMembersCmd::Do(std::shared_ptr partition) { + std::vector members; + rocksdb::Status s = partition->db()->SMembers(key_, &members); + if (s.ok() || s.IsNotFound()) { + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSScan); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &cursor_)) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSScan); + return; + } + size_t argc = argv_.size(), index = 3; + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + if (count_ < 0) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + return; +} + +void SScanCmd::Do(std::shared_ptr partition) { + int64_t next_cursor = 0; + std::vector members; + rocksdb::Status s = partition->db()->SScan(key_, cursor_, pattern_, count_, &members, &next_cursor); + + if (s.ok() || s.IsNotFound()) { + res_.AppendContent("*2"); + char buf[32]; + int64_t len = slash::ll2string(buf, sizeof(buf), next_cursor); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendString(member); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SRemCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSMembers); + return; + } + key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + members_.assign(++iter, argv_.end()); + return; +} + +void SRemCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->SRem(key_, members_, &count); + res_.AppendInteger(count); + return; +} + +void SUnionCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSUnion); + return; + } + PikaCmdArgsType::iterator iter = argv_.begin(); + keys_.assign(++iter, argv_.end()); + return; +} + +void SUnionCmd::Do(std::shared_ptr partition) { + std::vector members; + partition->db()->SUnion(keys_, &members); + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } + return; +} + +void SUnionstoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSUnionstore); + return; + } + dest_key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + keys_.assign(++iter, argv_.end()); + return; +} + +void SUnionstoreCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->SUnionstore(dest_key_, keys_, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SInterCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSInter); + return; + } + PikaCmdArgsType::iterator iter = argv_.begin(); + keys_.assign(++iter, argv_.end()); + return; +} + +void SInterCmd::Do(std::shared_ptr partition) { + std::vector members; + partition->db()->SInter(keys_, &members); + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } + return; +} + +void SInterstoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSInterstore); + return; + } + dest_key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + keys_.assign(++iter, argv_.end()); + return; +} + +void SInterstoreCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->SInterstore(dest_key_, keys_, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SIsmemberCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSIsmember); + return; + } + key_ = argv_[1]; + member_ = argv_[2]; + return; +} + +void SIsmemberCmd::Do(std::shared_ptr partition) { + int32_t is_member = 0; + partition->db()->SIsmember(key_, member_, &is_member); + if (is_member) { + res_.AppendContent(":1"); + } else { + res_.AppendContent(":0"); + } +} + +void SDiffCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSDiff); + return; + } + PikaCmdArgsType::iterator iter = argv_.begin(); + keys_.assign(++iter, argv_.end()); + return; +} + +void SDiffCmd::Do(std::shared_ptr partition) { + std::vector members; + partition->db()->SDiff(keys_, &members); + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } + return; +} + +void SDiffstoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSDiffstore); + return; + } + dest_key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin(); + iter++; + keys_.assign(++iter, argv_.end()); + return; +} + +void SDiffstoreCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->SDiffstore(dest_key_, keys_, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void SMoveCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSMove); + return; + } + src_key_ = argv_[1]; + dest_key_ = argv_[2]; + member_ = argv_[3]; + return; +} + +void SMoveCmd::Do(std::shared_ptr partition) { + int32_t res = 0; + rocksdb::Status s = partition->db()->SMove(src_key_, dest_key_, member_, &res); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(res); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void SRandmemberCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSRandmember); + return; + } + key_ = argv_[1]; + if (argv_.size() > 3) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSRandmember); + return; + } else if (argv_.size() == 3) { + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + } else { + reply_arr = true;; + } + } + return; +} + +void SRandmemberCmd::Do(std::shared_ptr partition) { + std::vector members; + rocksdb::Status s = partition->db()->SRandmember(key_, count_, &members); + if (s.ok() || s.IsNotFound()) { + if (!reply_arr && members.size()) { + res_.AppendStringLen(members[0].size()); + res_.AppendContent(members[0]); + } else { + res_.AppendArrayLen(members.size()); + for (const auto& member : members) { + res_.AppendStringLen(member.size()); + res_.AppendContent(member); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} diff --git a/tools/pika_migrate/src/pika_slot.cc b/tools/pika_migrate/src/pika_slot.cc new file mode 100644 index 0000000000..adeecf8bb7 --- /dev/null +++ b/tools/pika_migrate/src/pika_slot.cc @@ -0,0 +1,436 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_rm.h" +#include "include/pika_slot.h" +#include "include/pika_table.h" +#include "include/pika_server.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaCmdTableManager* g_pika_cmd_table_manager; +extern PikaReplicaManager* g_pika_rm; +extern PikaServer* g_pika_server; +extern PikaConf* g_pika_conf; + +// SLOTSINFO +void SlotsInfoCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsInfo); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSINFO only support on sharding mode"); + return; + } + + return; +} + +void SlotsInfoCmd::Do(std::shared_ptr partition) { + std::shared_ptr
table_ptr = g_pika_server->GetTable(g_pika_conf->default_table()); + if (!table_ptr) { + res_.SetRes(CmdRes::kNotFound, kCmdNameSlotsInfo); + return; + } + table_ptr->KeyScan(); + // this get will get last time scan info + KeyScanInfo key_scan_info = table_ptr->GetKeyScanInfo(); + + std::map infos; + Status s = table_ptr->GetPartitionsKeyScanInfo(&infos); + if (!s.ok()) { + res_.SetRes(CmdRes::kInvalidParameter, kCmdNameSlotsInfo); + return; + } + res_.AppendArrayLen(infos.size()); + for (auto& key_info : infos) { + uint64_t total_key_size = 0; + for (size_t idx = 0; idx < key_info.second.key_infos.size(); ++idx) { + total_key_size += key_info.second.key_infos[idx].keys; + } + res_.AppendArrayLen(2); + res_.AppendInteger(key_info.first); + res_.AppendInteger(total_key_size); + } + return; +} + +// SLOTSHASHKEY key1 [key2 …] +void SlotsHashKeyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsHashKey); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSHASHKEY only support on sharding mode"); + return; + } + + return; +} + +void SlotsHashKeyCmd::Do(std::shared_ptr partition) { + res_.AppendArrayLen(argv_.size() - 1); + std::shared_ptr
table_ptr = g_pika_server->GetTable(g_pika_conf->default_table()); + uint32_t partition_num = table_ptr->PartitionNum(); + if (!table_ptr) { + res_.SetRes(CmdRes::kInvalidParameter, kCmdNameSlotsHashKey); + } + // iter starts from real key, first item in argv_ is command name + std::vector::const_iterator iter = argv_.begin() + 1; + for (; iter != argv_.end(); iter++) { + res_.AppendInteger(g_pika_cmd_table_manager->DistributeKey(*iter, partition_num)); + } + return; +} + +// slotsmgrtslot-async host port timeout maxbulks maxbytes slot numkeys +void SlotsMgrtSlotAsyncCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtSlotAsync); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSMGRTTAGSLOT-ASYNC only support on sharding mode"); + return; + } + + return; +} + +void SlotsMgrtSlotAsyncCmd::Do(std::shared_ptr partition) { + int64_t moved = 0; + int64_t remained = 0; + res_.AppendArrayLen(2); + res_.AppendInteger(moved); + res_.AppendInteger(remained); +} + +// SLOTSMGRTTAGSLOT-ASYNC host port timeout maxbulks maxbytes slot numkeys +void SlotsMgrtTagSlotAsyncCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtTagSlotAsync); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSMGRTTAGSLOT-ASYNC only support on sharding mode"); + return; + } + + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; //Remember the first args is the opt name + dest_ip_ = *it++; + slash::StringToLower(dest_ip_); + + std::string str_dest_port = *it++; + if (!slash::string2l(str_dest_port.data(), str_dest_port.size(), &dest_port_) || dest_port_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameSlotsMgrtTagSlotAsync); + return; + } + + if ((dest_ip_ == "127.0.0.1" || dest_ip_ == g_pika_server->host()) && dest_port_ == g_pika_server->port()) { + res_.SetRes(CmdRes::kErrOther, "destination address error"); + return; + } + + std::string str_timeout_ms = *it++; + + std::string str_max_bulks = *it++; + + std::string str_max_bytes_ = *it++; + + std::string str_slot_num = *it++; + if (!slash::string2l(str_slot_num.data(), str_slot_num.size(), &slot_num_) + || slot_num_ < 0 || slot_num_ >= g_pika_conf->default_slot_num()) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameSlotsMgrtTagSlotAsync); + return; + } + + std::string str_keys_num = *it++; + return; +} + +void SlotsMgrtTagSlotAsyncCmd::Do(std::shared_ptr partition) { + int64_t moved = 0; + int64_t remained = 0; + // check if this slave node exist. + // if exist, dont mark migrate done + // cache coming request in codis proxy and keep retrying + // Until sync done, new node slaveof no one. + // mark this migrate done + // proxy retry cached request in new node + bool is_exist = g_pika_rm->CheckPartitionSlaveExist( + RmNode(dest_ip_, dest_port_, g_pika_conf->default_table(), slot_num_)); + if (is_exist) { + remained = 1; + } else { + remained = 0; + } + res_.AppendArrayLen(2); + res_.AppendInteger(moved); + res_.AppendInteger(remained); +} + +// SLOTSSCAN slotnum cursor [COUNT count] +void SlotsScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsScan); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSSCAN only support on sharding mode"); + return; + } + + int64_t slotnum; + if (!slash::string2l(argv_[1].data(), argv_[1].size(), &slotnum)) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameSlotsScan); + return; + } + slotnum_ = static_cast(slotnum); + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &cursor_)) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameSlotsScan); + return; + } + size_t argc = argv_.size(), index = 3; + + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_) || count_ <= 0) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + return; +} + +void SlotsScanCmd::Do(std::shared_ptr partition) { + std::shared_ptr
table_ptr = g_pika_server->GetTable(g_pika_conf->default_table()); + if (!table_ptr) { + res_.SetRes(CmdRes::kNotFound, kCmdNameSlotsScan); + return; + } + std::shared_ptr cur_partition = table_ptr->GetPartitionById(slotnum_); + if (!cur_partition) { + res_.SetRes(CmdRes::kNotFound, kCmdNameSlotsScan); + return; + } + std::vector keys; + int64_t cursor_ret = cur_partition->db()->Scan(blackwidow::DataType::kAll, + cursor_, pattern_, count_, &keys); + + res_.AppendArrayLen(2); + + char buf[32]; + int len = slash::ll2string(buf, sizeof(buf), cursor_ret); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(keys.size()); + std::vector::iterator iter; + for (iter = keys.begin(); iter != keys.end(); iter++) { + res_.AppendStringLen(iter->size()); + res_.AppendContent(*iter); + } + return; +} + +// SLOTSDEL slot1 [slot2 …] +void SlotsDelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsDel); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSDEL only support on sharding mode"); + return; + } + + // iter starts from real key, first item in argv_ is command name + std::vector::const_iterator iter = argv_.begin() + 1; + for (; iter != argv_.end(); iter++) { + int64_t slotnum; + if (!slash::string2l(iter->data(), iter->size(), &slotnum)) { + res_.SetRes(CmdRes::kInvalidInt, kCmdNameSlotsDel); + return; + } + slots_.push_back(static_cast(slotnum)); + } + return; +} + +void SlotsDelCmd::Do(std::shared_ptr partition) { + std::shared_ptr
table_ptr = g_pika_server->GetTable(g_pika_conf->default_table()); + if (!table_ptr) { + res_.SetRes(CmdRes::kNotFound, kCmdNameSlotsDel); + return; + } + if (table_ptr->IsKeyScaning()) { + res_.SetRes(CmdRes::kErrOther, "The keyscan operation is executing, Try again later"); + return; + } + std::vector successed_slots; + for (auto& slotnum : slots_) { + std::shared_ptr cur_partition = table_ptr->GetPartitionById(slotnum); + if (!cur_partition) { + continue; + } + cur_partition->FlushDB(); + successed_slots.push_back(slotnum); + } + res_.AppendArrayLen(successed_slots.size()); + for (auto& slotnum : successed_slots) { + res_.AppendArrayLen(2); + res_.AppendInteger(slotnum); + res_.AppendInteger(0); + } + return; +} + +// SLOTSMGRT-EXEC-WRAPPER $hashkey $command [$arg1 ...] +void SlotsMgrtExecWrapperCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtExecWrapper); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSMGRT-EXEC-WRAPPER only support on sharding mode"); + return; + } + + PikaCmdArgsType::const_iterator it = argv_.begin() + 1; + key_ = *it++; + //slash::StringToLower(key_); + return; +} + +void SlotsMgrtExecWrapperCmd::Do(std::shared_ptr partition) { + // return 0 means proxy will request to new slot server + // return 1 means proxy will keey trying + // return 2 means return this key directly + res_.AppendArrayLen(2); + res_.AppendInteger(1); + res_.AppendInteger(1); + return; +} + +// slotsmgrt-async-status +void SlotsMgrtAsyncStatusCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtAsyncStatus); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSMGRT-ASYNC-STATUS only support on sharding mode"); + return; + } + + return; +} + +void SlotsMgrtAsyncStatusCmd::Do(std::shared_ptr partition) { + std::string status; + std::string ip = "none"; + int64_t port = -1, slot = -1, moved = -1, remained = -1; + std::string mstatus = "no"; + res_.AppendArrayLen(5); + status = "dest server: " + ip + ":" + std::to_string(port); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "slot number: " + std::to_string(slot); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "migrating : " + mstatus; + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "moved keys : " + std::to_string(moved); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + status = "remain keys: " + std::to_string(remained); + res_.AppendStringLen(status.size()); + res_.AppendContent(status); + return; +} + +// slotsmgrt-async-cancel +void SlotsMgrtAsyncCancelCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameSlotsMgrtAsyncCancel); + return; + } + + if (g_pika_conf->classic_mode()) { + res_.SetRes(CmdRes::kErrOther, "SLOTSMGRT-ASYNC-CANCEL only support on sharding mode"); + return; + } + + return; +} + +void SlotsMgrtAsyncCancelCmd::Do(std::shared_ptr partition) { + res_.SetRes(CmdRes::kOk); + return; +} + +// slotsmgrtslot host port timeout slot +void SlotsMgrtSlotCmd::DoInitial() { + res_.SetRes(CmdRes::kErrOther, kCmdNameSlotsMgrtSlot + " NOT supported"); + return; +} + +void SlotsMgrtSlotCmd::Do(std::shared_ptr partition) { + return; +} + +// slotsmgrttagslot host port timeout slot +void SlotsMgrtTagSlotCmd::DoInitial() { + res_.SetRes(CmdRes::kErrOther, kCmdNameSlotsMgrtTagSlot + " NOT supported"); + return; +} + +void SlotsMgrtTagSlotCmd::Do(std::shared_ptr partition) { + return; +} + +// slotsmgrtone host port timeout key +void SlotsMgrtOneCmd::DoInitial() { + res_.SetRes(CmdRes::kErrOther, kCmdNameSlotsMgrtOne + " NOT supported"); + return; +} + +void SlotsMgrtOneCmd::Do(std::shared_ptr partition) { + return; +} + +// slotsmgrttagone host port timeout key +void SlotsMgrtTagOneCmd::DoInitial() { + res_.SetRes(CmdRes::kErrOther, kCmdNameSlotsMgrtTagOne + " NOT supported"); + return; +} + +void SlotsMgrtTagOneCmd::Do(std::shared_ptr partition) { + return; +} diff --git a/tools/pika_migrate/src/pika_table.cc b/tools/pika_migrate/src/pika_table.cc new file mode 100644 index 0000000000..20d2c9be4f --- /dev/null +++ b/tools/pika_migrate/src/pika_table.cc @@ -0,0 +1,261 @@ +// Copyright (c) 2018-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_table.h" + +#include "include/pika_server.h" +#include "include/pika_cmd_table_manager.h" + +extern PikaServer* g_pika_server; +extern PikaCmdTableManager* g_pika_cmd_table_manager; + +std::string TablePath(const std::string& path, + const std::string& table_name) { + char buf[100]; + snprintf(buf, sizeof(buf), "%s/", table_name.data()); + return path + buf; +} + +Table::Table(const std::string& table_name, + uint32_t partition_num, + const std::string& db_path, + const std::string& log_path) : + table_name_(table_name), + partition_num_(partition_num) { + + db_path_ = TablePath(db_path, table_name_); + log_path_ = TablePath(log_path, "log_" + table_name_); + + slash::CreatePath(db_path_); + slash::CreatePath(log_path_); + + pthread_rwlock_init(&partitions_rw_, NULL); +} + +Table::~Table() { + StopKeyScan(); + pthread_rwlock_destroy(&partitions_rw_); + partitions_.clear(); +} + +std::string Table::GetTableName() { + return table_name_; +} + +void Table::BgSaveTable() { + slash::RWLock l(&partitions_rw_, false); + for (const auto& item : partitions_) { + item.second->BgSavePartition(); + } +} + +void Table::CompactTable(const blackwidow::DataType& type) { + slash::RWLock l(&partitions_rw_, false); + for (const auto& item : partitions_) { + item.second->Compact(type); + } +} + +bool Table::FlushPartitionDB() { + slash::RWLock rwl(&partitions_rw_, false); + slash::MutexLock ml(&key_scan_protector_); + if (key_scan_info_.key_scaning_) { + return false; + } + for (const auto& item : partitions_) { + item.second->FlushDB(); + } + return true; +} + +bool Table::FlushPartitionSubDB(const std::string& db_name) { + slash::RWLock rwl(&partitions_rw_, false); + slash::MutexLock ml(&key_scan_protector_); + if (key_scan_info_.key_scaning_) { + return false; + } + for (const auto& item : partitions_) { + item.second->FlushSubDB(db_name); + } + return true; +} + +bool Table::IsBinlogIoError() { + slash::RWLock l(&partitions_rw_, false); + for (const auto& item : partitions_) { + if (item.second->IsBinlogIoError()) { + return true; + } + } + return false; +} + +uint32_t Table::PartitionNum() { + return partition_num_; +} + +Status Table::AddPartitions(const std::set& partition_ids) { + slash::RWLock l(&partitions_rw_, true); + for (const uint32_t& id : partition_ids) { + if (id >= partition_num_) { + return Status::Corruption("partition index out of range[0, " + + std::to_string(partition_num_ - 1) + "]"); + } else if (partitions_.find(id) != partitions_.end()) { + return Status::Corruption("partition " + + std::to_string(id) + " already exist"); + } + } + + for (const uint32_t& id : partition_ids) { + partitions_.emplace(id, std::make_shared( + table_name_, id, db_path_, log_path_)); + } + return Status::OK(); +} + +Status Table::RemovePartitions(const std::set& partition_ids) { + slash::RWLock l(&partitions_rw_, true); + for (const uint32_t& id : partition_ids) { + if (partitions_.find(id) == partitions_.end()) { + return Status::Corruption("partition " + std::to_string(id) + " not found"); + } + } + + for (const uint32_t& id : partition_ids) { + partitions_[id]->Leave(); + partitions_.erase(id); + } + return Status::OK(); +} + +void Table::KeyScan() { + slash::MutexLock ml(&key_scan_protector_); + if (key_scan_info_.key_scaning_) { + return; + } + + key_scan_info_.key_scaning_ = true; + key_scan_info_.duration = -2; // duration -2 mean the task in waiting status, + // has not been scheduled for exec + BgTaskArg* bg_task_arg = new BgTaskArg(); + bg_task_arg->table = shared_from_this(); + g_pika_server->KeyScanTaskSchedule(&DoKeyScan, reinterpret_cast(bg_task_arg)); +} + +bool Table::IsKeyScaning() { + slash::MutexLock ml(&key_scan_protector_); + return key_scan_info_.key_scaning_; +} + +void Table::RunKeyScan() { + Status s; + std::vector new_key_infos(5); + + InitKeyScan(); + slash::RWLock rwl(&partitions_rw_, false); + for (const auto& item : partitions_) { + std::vector tmp_key_infos; + s = item.second->GetKeyNum(&tmp_key_infos); + if (s.ok()) { + for (size_t idx = 0; idx < tmp_key_infos.size(); ++idx) { + new_key_infos[idx].keys += tmp_key_infos[idx].keys; + new_key_infos[idx].expires += tmp_key_infos[idx].expires; + new_key_infos[idx].avg_ttl += tmp_key_infos[idx].avg_ttl; + new_key_infos[idx].invaild_keys += tmp_key_infos[idx].invaild_keys; + } + } else { + break; + } + } + key_scan_info_.duration = time(NULL) - key_scan_info_.start_time; + + slash::MutexLock lm(&key_scan_protector_); + if (s.ok()) { + key_scan_info_.key_infos = new_key_infos; + } + key_scan_info_.key_scaning_ = false; +} + +void Table::StopKeyScan() { + slash::RWLock rwl(&partitions_rw_, false); + slash::MutexLock ml(&key_scan_protector_); + for (const auto& item : partitions_) { + item.second->db()->StopScanKeyNum(); + } + key_scan_info_.key_scaning_ = false; +} + +void Table::ScanDatabase(const blackwidow::DataType& type) { + slash::RWLock rwl(&partitions_rw_, false); + for (const auto& item : partitions_) { + printf("\n\npartition name : %s\n", item.second->GetPartitionName().c_str()); + item.second->db()->ScanDatabase(type); + } +} + +Status Table::GetPartitionsKeyScanInfo(std::map* infos) { + slash::RWLock rwl(&partitions_rw_, false); + for (const auto& item : partitions_) { + (*infos)[item.first] = item.second->GetKeyScanInfo(); + } + return Status::OK(); +} + +KeyScanInfo Table::GetKeyScanInfo() { + slash::MutexLock lm(&key_scan_protector_); + return key_scan_info_; +} + +void Table::Compact(const blackwidow::DataType& type) { + slash::RWLock rwl(&partitions_rw_, true); + for (const auto& item : partitions_) { + item.second->Compact(type); + } +} + +void Table::DoKeyScan(void *arg) { + BgTaskArg* bg_task_arg = reinterpret_cast(arg); + bg_task_arg->table->RunKeyScan(); + delete bg_task_arg; +} + +void Table::InitKeyScan() { + key_scan_info_.start_time = time(NULL); + char s_time[32]; + int len = strftime(s_time, sizeof(s_time), "%Y-%m-%d %H:%M:%S", localtime(&key_scan_info_.start_time)); + key_scan_info_.s_start_time.assign(s_time, len); + key_scan_info_.duration = -1; // duration -1 mean the task in processing +} + +void Table::LeaveAllPartition() { + slash::RWLock rwl(&partitions_rw_, true); + for (const auto& item : partitions_) { + item.second->Leave(); + } + partitions_.clear(); +} + +std::set Table::GetPartitionIds() { + std::set ids; + slash::RWLock l(&partitions_rw_, false); + for (const auto& item : partitions_) { + ids.insert(item.first); + } + return ids; +} + +std::shared_ptr Table::GetPartitionById(uint32_t partition_id) { + slash::RWLock rwl(&partitions_rw_, false); + auto iter = partitions_.find(partition_id); + return (iter == partitions_.end()) ? NULL : iter->second; +} + +std::shared_ptr Table::GetPartitionByKey(const std::string& key) { + assert(partition_num_ != 0); + uint32_t index = g_pika_cmd_table_manager->DistributeKey(key, partition_num_); + slash::RWLock rwl(&partitions_rw_, false); + auto iter = partitions_.find(index); + return (iter == partitions_.end()) ? NULL : iter->second; +} diff --git a/tools/pika_migrate/src/pika_zset.cc b/tools/pika_migrate/src/pika_zset.cc new file mode 100644 index 0000000000..23e144567e --- /dev/null +++ b/tools/pika_migrate/src/pika_zset.cc @@ -0,0 +1,946 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "include/pika_zset.h" + +#include "slash/include/slash_string.h" + +void ZAddCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZAdd); + return; + } + size_t argc = argv_.size(); + if (argc % 2 == 1) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + key_ = argv_[1]; + score_members.clear(); + double score; + size_t index = 2; + for (; index < argc; index += 2) { + if (!slash::string2d(argv_[index].data(), argv_[index].size(), &score)) { + res_.SetRes(CmdRes::kInvalidFloat); + return; + } + score_members.push_back({score, argv_[index + 1]}); + } + return; +} + +void ZAddCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->ZAdd(key_, score_members, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZCardCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZCard); + return; + } + key_ = argv_[1]; + return; +} + +void ZCardCmd::Do(std::shared_ptr partition) { + int32_t card = 0; + rocksdb::Status s = partition->db()->ZCard(key_, &card); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(card); + } else { + res_.SetRes(CmdRes::kErrOther, "zcard error"); + } + return; +} + +void ZScanCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZScan); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &cursor_)) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZScan); + return; + } + size_t argc = argv_.size(), index = 3; + while (index < argc) { + std::string opt = argv_[index]; + if (!strcasecmp(opt.data(), "match") + || !strcasecmp(opt.data(), "count")) { + index++; + if (index >= argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(opt.data(), "match")) { + pattern_ = argv_[index]; + } else if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } + if (count_ < 0) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + return; +} + +void ZScanCmd::Do(std::shared_ptr partition) { + int64_t next_cursor = 0; + std::vector score_members; + rocksdb::Status s = partition->db()->ZScan(key_, cursor_, pattern_, count_, &score_members, &next_cursor); + if (s.ok() || s.IsNotFound()) { + res_.AppendContent("*2"); + char buf[32]; + int64_t len = slash::ll2string(buf, sizeof(buf), next_cursor); + res_.AppendStringLen(len); + res_.AppendContent(buf); + + res_.AppendArrayLen(score_members.size() * 2); + for (const auto& score_member : score_members) { + res_.AppendString(score_member.member); + + len = slash::d2string(buf, sizeof(buf), score_member.score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZIncrbyCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZIncrby); + return; + } + key_ = argv_[1]; + if (!slash::string2d(argv_[2].data(), argv_[2].size(), &by_)) { + res_.SetRes(CmdRes::kInvalidFloat); + return; + } + member_ = argv_[3]; + return; +} + +void ZIncrbyCmd::Do(std::shared_ptr partition) { + double score = 0; + rocksdb::Status s = partition->db()->ZIncrby(key_, member_, by_, &score); + if (s.ok()) { + char buf[32]; + int64_t len = slash::d2string(buf, sizeof(buf), score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZsetRangeParentCmd::DoInitial() { + if (argv_.size() == 5 && !strcasecmp(argv_[4].data(), "withscores")) { + is_ws_ = true; + } else if (argv_.size() != 4) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &start_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &stop_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + return; +} + +void ZRangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRange); + return; + } + ZsetRangeParentCmd::DoInitial(); +} + +void ZRangeCmd::Do(std::shared_ptr partition) { + std::vector score_members; + rocksdb::Status s = partition->db()->ZRange(key_, start_, stop_, &score_members); + if (s.ok() || s.IsNotFound()) { + if (is_ws_) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(score_members.size() * 2); + for (const auto& sm : score_members) { + res_.AppendStringLen(sm.member.size()); + res_.AppendContent(sm.member); + len = slash::d2string(buf, sizeof(buf), sm.score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.AppendArrayLen(score_members.size()); + for (const auto& sm : score_members) { + res_.AppendStringLen(sm.member.size()); + res_.AppendContent(sm.member); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZRevrangeCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRevrange); + return; + } + ZsetRangeParentCmd::DoInitial(); +} + +void ZRevrangeCmd::Do(std::shared_ptr partition) { + std::vector score_members; + rocksdb::Status s = partition->db()->ZRevrange(key_, start_, stop_, &score_members); + if (s.ok() || s.IsNotFound()) { + if (is_ws_) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(score_members.size() * 2); + for (const auto& sm : score_members) { + res_.AppendStringLen(sm.member.size()); + res_.AppendContent(sm.member); + len = slash::d2string(buf, sizeof(buf), sm.score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.AppendArrayLen(score_members.size()); + for (const auto& sm : score_members) { + res_.AppendStringLen(sm.member.size()); + res_.AppendContent(sm.member); + } + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +int32_t DoScoreStrRange(std::string begin_score, std::string end_score, bool *left_close, bool *right_close, double *min_score, double *max_score) { + if (begin_score.size() > 0 && begin_score.at(0) == '(') { + *left_close = false; + begin_score.erase(begin_score.begin()); + } + if (begin_score == "-inf") { + *min_score = blackwidow::ZSET_SCORE_MIN; + } else if (begin_score == "inf" || begin_score == "+inf") { + *min_score = blackwidow::ZSET_SCORE_MAX; + } else if (!slash::string2d(begin_score.data(), begin_score.size(), min_score)) { + return -1; + } + + if (end_score.size() > 0 && end_score.at(0) == '(') { + *right_close = false; + end_score.erase(end_score.begin()); + } + if (end_score == "+inf" || end_score == "inf") { + *max_score = blackwidow::ZSET_SCORE_MAX; + } else if (end_score == "-inf") { + *max_score = blackwidow::ZSET_SCORE_MIN; + } else if (!slash::string2d(end_score.data(), end_score.size(), max_score)) { + return -1; + } + return 0; +} + +static void FitLimit(int64_t &count, int64_t &offset, const int64_t size) { + count = count >= 0 ? count : size; + offset = (offset >= 0 && offset < size) ? offset : size; + count = (offset + count < size) ? count : size - offset; +} + +void ZsetRangebyscoreParentCmd::DoInitial() { + key_ = argv_[1]; + int32_t ret = DoScoreStrRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_score_, &max_score_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max is not a float"); + return; + } + size_t argc = argv_.size(); + if (argc < 5) { + return; + } + size_t index = 4; + while (index < argc) { + if (!strcasecmp(argv_[index].data(), "withscores")) { + with_scores_ = true; + } else if (!strcasecmp(argv_[index].data(), "limit")) { + if (index + 3 > argc) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + if (!slash::string2l(argv_[index].data(), argv_[index].size(), &offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + index++; + if (!slash::string2l(argv_[index].data(), argv_[index].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } +} + +void ZRangebyscoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRangebyscore); + return; + } + ZsetRangebyscoreParentCmd::DoInitial(); +} + +void ZRangebyscoreCmd::Do(std::shared_ptr partition) { + if (min_score_ == blackwidow::ZSET_SCORE_MAX || max_score_ == blackwidow::ZSET_SCORE_MIN) { + res_.AppendContent("*0"); + return; + } + std::vector score_members; + rocksdb::Status s = partition->db()->ZRangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + FitLimit(count_, offset_, score_members.size()); + size_t index = offset_, end = offset_ + count_; + if (with_scores_) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(count_ * 2); + for (; index < end; index++) { + res_.AppendStringLen(score_members[index].member.size()); + res_.AppendContent(score_members[index].member); + len = slash::d2string(buf, sizeof(buf), score_members[index].score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.AppendArrayLen(count_); + for (; index < end; index++) { + res_.AppendStringLen(score_members[index].member.size()); + res_.AppendContent(score_members[index].member); + } + } + return; +} + +void ZRevrangebyscoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRevrangebyscore); + return; + } + ZsetRangebyscoreParentCmd::DoInitial(); + double tmp_score; + tmp_score = min_score_; + min_score_ = max_score_; + max_score_ = tmp_score; + + bool tmp_close; + tmp_close = left_close_; + left_close_ = right_close_; + right_close_ = tmp_close; +} + +void ZRevrangebyscoreCmd::Do(std::shared_ptr partition) { + if (min_score_ == blackwidow::ZSET_SCORE_MAX || max_score_ == blackwidow::ZSET_SCORE_MIN) { + res_.AppendContent("*0"); + return; + } + std::vector score_members; + rocksdb::Status s = partition->db()->ZRevrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &score_members); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + FitLimit(count_, offset_, score_members.size()); + int64_t index = offset_, end = offset_ + count_; + if (with_scores_) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(count_ * 2); + for (; index < end; index++) { + res_.AppendStringLen(score_members[index].member.size()); + res_.AppendContent(score_members[index].member); + len = slash::d2string(buf, sizeof(buf), score_members[index].score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.AppendArrayLen(count_); + for (; index < end; index++) { + res_.AppendStringLen(score_members[index].member.size()); + res_.AppendContent(score_members[index].member); + } + } + return; +} + +void ZCountCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZCount); + return; + } + key_ = argv_[1]; + int32_t ret = DoScoreStrRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_score_, &max_score_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max is not a float"); + return; + } + return; +} + +void ZCountCmd::Do(std::shared_ptr partition) { + if (min_score_ == blackwidow::ZSET_SCORE_MAX || max_score_ == blackwidow::ZSET_SCORE_MIN) { + res_.AppendContent("*0"); + return; + } + + int32_t count = 0; + rocksdb::Status s = partition->db()->ZCount(key_, min_score_, max_score_, left_close_, right_close_, &count); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZRemCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRem); + return; + } + key_ = argv_[1]; + PikaCmdArgsType::iterator iter = argv_.begin() + 2; + members_.assign(iter, argv_.end()); + return; +} + +void ZRemCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->ZRem(key_, members_, &count); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZsetUIstoreParentCmd::DoInitial() { + dest_key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &num_keys_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (num_keys_ < 1) { + res_.SetRes(CmdRes::kErrOther, "at least 1 input key is needed for ZUNIONSTORE/ZINTERSTORE"); + return; + } + int argc = argv_.size(); + if (argc < num_keys_ + 3) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + keys_.assign(argv_.begin() + 3, argv_.begin() + 3 + num_keys_); + weights_.assign(num_keys_, 1); + int index = num_keys_ + 3; + while (index < argc) { + if (!strcasecmp(argv_[index].data(), "weights")) { + index++; + if (argc < index + num_keys_) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + double weight; + int base = index; + for (; index < base + num_keys_; index++) { + if (!slash::string2d(argv_[index].data(), argv_[index].size(), &weight)) { + res_.SetRes(CmdRes::kErrOther, "weight value is not a float"); + return; + } + weights_[index-base] = weight; + } + } else if (!strcasecmp(argv_[index].data(), "aggregate")) { + index++; + if (argc < index + 1) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!strcasecmp(argv_[index].data(), "sum")) { + aggregate_ = blackwidow::SUM; + } else if (!strcasecmp(argv_[index].data(), "min")) { + aggregate_ = blackwidow::MIN; + } else if (!strcasecmp(argv_[index].data(), "max")) { + aggregate_ = blackwidow::MAX; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + index++; + } else { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + } + return; +} + +void ZUnionstoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZUnionstore); + return; + } + ZsetUIstoreParentCmd::DoInitial(); +} + +void ZUnionstoreCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->ZUnionstore(dest_key_, keys_, weights_, aggregate_, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZInterstoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZInterstore); + return; + } + ZsetUIstoreParentCmd::DoInitial(); + return; +} + +void ZInterstoreCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->ZInterstore(dest_key_, keys_, weights_, aggregate_, &count); + if (s.ok()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZsetRankParentCmd::DoInitial() { + key_ = argv_[1]; + member_ = argv_[2]; + return; +} + +void ZRankCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRank); + return; + } + ZsetRankParentCmd::DoInitial(); +} + +void ZRankCmd::Do(std::shared_ptr partition) { + int32_t rank = 0; + rocksdb::Status s = partition->db()->ZRank(key_, member_, &rank); + if (s.ok()) { + res_.AppendInteger(rank); + } else if (s.IsNotFound()){ + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void ZRevrankCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRevrank); + return; + } + ZsetRankParentCmd::DoInitial(); +} + +void ZRevrankCmd::Do(std::shared_ptr partition) { + int32_t revrank = 0; + rocksdb::Status s = partition->db()->ZRevrank(key_, member_, &revrank); + if (s.ok()) { + res_.AppendInteger(revrank); + } else if (s.IsNotFound()){ + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + +void ZScoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZScore); + return; + } + key_ = argv_[1]; + member_ = argv_[2]; +} + +void ZScoreCmd::Do(std::shared_ptr partition) { + double score = 0; + rocksdb::Status s = partition->db()->ZScore(key_, member_, &score); + if (s.ok()) { + char buf[32]; + int64_t len = slash::d2string(buf, sizeof(buf), score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } else if (s.IsNotFound()) { + res_.AppendContent("$-1"); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +static int32_t DoMemberRange(const std::string &raw_min_member, + const std::string &raw_max_member, + bool *left_close, + bool *right_close, + std::string* min_member, + std::string* max_member) { + if (raw_min_member == "-") { + *min_member = "-"; + } else if (raw_min_member == "+") { + *min_member = "+"; + } else { + if (raw_min_member.size() > 0 && raw_min_member.at(0) == '(') { + *left_close = false; + } else if (raw_min_member.size() > 0 && raw_min_member.at(0) == '[') { + *left_close = true; + } else { + return -1; + } + min_member->assign(raw_min_member.begin() + 1, raw_min_member.end()); + } + + if (raw_max_member == "+") { + *max_member = "+"; + } else if (raw_max_member == "-") { + *max_member = "-"; + } else { + if (raw_max_member.size() > 0 && raw_max_member.at(0) == '(') { + *right_close = false; + } else if (raw_max_member.size() > 0 && raw_max_member.at(0) == '[') { + *right_close = true; + } else { + return -1; + } + max_member->assign(raw_max_member.begin() + 1, raw_max_member.end()); + } + return 0; +} + +void ZsetRangebylexParentCmd::DoInitial() { + key_ = argv_[1]; + int32_t ret = DoMemberRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_member_, &max_member_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max not valid string range item"); + return; + } + size_t argc = argv_.size(); + if (argc == 4) { + return; + } else if (argc != 7 || strcasecmp(argv_[4].data(), "limit")) { + res_.SetRes(CmdRes::kSyntaxErr); + return; + } + if (!slash::string2l(argv_[5].data(), argv_[5].size(), &offset_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[6].data(), argv_[6].size(), &count_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } +} + +void ZRangebylexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRangebylex); + return; + } + ZsetRangebylexParentCmd::DoInitial(); +} + +void ZRangebylexCmd::Do(std::shared_ptr partition) { + if (min_member_ == "+" || max_member_ == "-") { + res_.AppendContent("*0"); + return; + } + std::vector members; + rocksdb::Status s = partition->db()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + FitLimit(count_, offset_, members.size()); + + res_.AppendArrayLen(count_); + size_t index = offset_, end = offset_ + count_; + for (; index < end; index++) { + res_.AppendStringLen(members[index].size()); + res_.AppendContent(members[index]); + } + return; +} + +void ZRevrangebylexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRevrangebylex); + return; + } + ZsetRangebylexParentCmd::DoInitial(); + + std::string tmp_s; + tmp_s = min_member_; + min_member_ = max_member_; + max_member_ = tmp_s; + + bool tmp_b; + tmp_b = left_close_; + left_close_ = right_close_; + right_close_ = tmp_b; +} + +void ZRevrangebylexCmd::Do(std::shared_ptr partition) { + if (min_member_ == "+" || max_member_ == "-") { + res_.AppendContent("*0"); + return; + } + std::vector members; + rocksdb::Status s = partition->db()->ZRangebylex(key_, min_member_, max_member_, left_close_, right_close_, &members); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + FitLimit(count_, offset_, members.size()); + + res_.AppendArrayLen(count_); + int64_t index = members.size() - 1 - offset_, end = index - count_; + for (; index > end; index--) { + res_.AppendStringLen(members[index].size()); + res_.AppendContent(members[index]); + } + return; +} + +void ZLexcountCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZLexcount); + return; + } + key_ = argv_[1]; + int32_t ret = DoMemberRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_member_, &max_member_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max not valid string range item"); + return; + } +} + +void ZLexcountCmd::Do(std::shared_ptr partition) { + if (min_member_ == "+" || max_member_ == "-") { + res_.AppendContent(":0"); + return; + } + int32_t count = 0; + rocksdb::Status s = partition->db()->ZLexcount(key_, min_member_, max_member_, left_close_, right_close_, &count); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res_.AppendInteger(count); + return; +} + +void ZRemrangebyrankCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRemrangebyrank); + return; + } + key_ = argv_[1]; + if (!slash::string2l(argv_[2].data(), argv_[2].size(), &start_rank_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } + if (!slash::string2l(argv_[3].data(), argv_[3].size(), &stop_rank_)) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } +} + +void ZRemrangebyrankCmd::Do(std::shared_ptr partition) { + int32_t count = 0; + rocksdb::Status s = partition->db()->ZRemrangebyrank(key_, start_rank_, stop_rank_, &count); + if (s.ok() || s.IsNotFound()) { + res_.AppendInteger(count); + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } + return; +} + +void ZRemrangebyscoreCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRemrangebyscore); + return; + } + key_ = argv_[1]; + int32_t ret = DoScoreStrRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_score_, &max_score_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max is not a float"); + return; + } + return; +} + +void ZRemrangebyscoreCmd::Do(std::shared_ptr partition) { + if (min_score_ == blackwidow::ZSET_SCORE_MAX || max_score_ == blackwidow::ZSET_SCORE_MIN) { + res_.AppendContent(":0"); + return; + } + int32_t count = 0; + rocksdb::Status s = partition->db()->ZRemrangebyscore(key_, min_score_, max_score_, left_close_, right_close_, &count); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res_.AppendInteger(count); + return; +} + +void ZRemrangebylexCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZRemrangebylex); + return; + } + key_ = argv_[1]; + int32_t ret = DoMemberRange(argv_[2], argv_[3], &left_close_, &right_close_, &min_member_, &max_member_); + if (ret == -1) { + res_.SetRes(CmdRes::kErrOther, "min or max not valid string range item"); + return; + } + return; +} + +void ZRemrangebylexCmd::Do(std::shared_ptr partition) { + if (min_member_ == "+" || max_member_ == "-") { + res_.AppendContent("*0"); + return; + } + int32_t count = 0; + rocksdb::Status s = partition->db()->ZRemrangebylex(key_, min_member_, max_member_, left_close_, right_close_, &count); + if (!s.ok() && !s.IsNotFound()) { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + return; + } + res_.AppendInteger(count); + return; +} + + +void ZPopmaxCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZPopmax); + return; + } + key_ = argv_[1]; + if (argv_.size() == 2) { + count_ = 1; + return; + } + if (!slash::string2ll(argv_[2].data(), argv_[2].size(), (long long*)(&count_))) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } +} + +void ZPopmaxCmd::Do(std::shared_ptr partition) { + std::vector score_members; + rocksdb::Status s = partition->db()->ZPopMax(key_, count_, &score_members); + if (s.ok() || s.IsNotFound()) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(score_members.size() * 2); + for (const auto& sm : score_members) { + res_.AppendString(sm.member); + len = slash::d2string(buf, sizeof(buf), sm.score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} + + +void ZPopminCmd::DoInitial() { + if (!CheckArg(argv_.size())) { + res_.SetRes(CmdRes::kWrongNum, kCmdNameZPopmin); + return; + } + key_ = argv_[1]; + if (argv_.size() == 2) { + count_ = 1; + return; + } + if (!slash::string2ll(argv_[2].data(), argv_[2].size(), (long long*)(&count_))) { + res_.SetRes(CmdRes::kInvalidInt); + return; + } +} + +void ZPopminCmd::Do(std::shared_ptr partition) { + std::vector score_members; + rocksdb::Status s = partition->db()->ZPopMin(key_, count_, &score_members); + if (s.ok() || s.IsNotFound()) { + char buf[32]; + int64_t len; + res_.AppendArrayLen(score_members.size() * 2); + for (const auto& sm : score_members) { + res_.AppendString(sm.member); + len = slash::d2string(buf, sizeof(buf), sm.score); + res_.AppendStringLen(len); + res_.AppendContent(buf); + } + } else { + res_.SetRes(CmdRes::kErrOther, s.ToString()); + } +} diff --git a/tools/pika_migrate/src/redis_sender.cc b/tools/pika_migrate/src/redis_sender.cc new file mode 100644 index 0000000000..74c41eabbd --- /dev/null +++ b/tools/pika_migrate/src/redis_sender.cc @@ -0,0 +1,222 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + + +#include "include/redis_sender.h" + +#include +#include + +#include + +#include "slash/include/xdebug.h" + +static time_t kCheckDiff = 1; + +RedisSender::RedisSender(int id, std::string ip, int64_t port, std::string password): + id_(id), + cli_(NULL), + rsignal_(&commands_mutex_), + wsignal_(&commands_mutex_), + ip_(ip), + port_(port), + password_(password), + should_exit_(false), + cnt_(0), + elements_(0) { + + last_write_time_ = ::time(NULL); +} + +RedisSender::~RedisSender() { + LOG(INFO) << "RedisSender thread " << id_ << " exit!!!"; +} + +void RedisSender::ConnectRedis() { + while (cli_ == NULL) { + // Connect to redis + cli_ = pink::NewRedisCli(); + cli_->set_connect_timeout(1000); + cli_->set_recv_timeout(10000); + cli_->set_send_timeout(10000); + slash::Status s = cli_->Connect(ip_, port_); + if (!s.ok()) { + LOG(WARNING) << "Can not connect to " << ip_ << ":" << port_ << ", status: " << s.ToString(); + delete cli_; + cli_ = NULL; + sleep(3); + continue; + } else { + // Connect success + + // Authentication + if (!password_.empty()) { + pink::RedisCmdArgsType argv, resp; + std::string cmd; + + argv.push_back("AUTH"); + argv.push_back(password_); + pink::SerializeRedisCommand(argv, &cmd); + slash::Status s = cli_->Send(&cmd); + + if (s.ok()) { + s = cli_->Recv(&resp); + if (resp[0] == "OK") { + } else { + LOG(FATAL) << "Connect to redis(" << ip_ << ":" << port_ << ") Invalid password"; + cli_->Close(); + delete cli_; + cli_ = NULL; + should_exit_ = true; + return; + } + } else { + LOG(WARNING) << "send auth failed: " << s.ToString(); + cli_->Close(); + delete cli_; + cli_ = NULL; + continue; + } + } else { + // If forget to input password + pink::RedisCmdArgsType argv, resp; + std::string cmd; + + argv.push_back("PING"); + pink::SerializeRedisCommand(argv, &cmd); + slash::Status s = cli_->Send(&cmd); + + if (s.ok()) { + s = cli_->Recv(&resp); + if (s.ok()) { + if (resp[0] == "NOAUTH Authentication required.") { + LOG(FATAL) << "Ping redis(" << ip_ << ":" << port_ << ") NOAUTH Authentication required"; + cli_->Close(); + delete cli_; + cli_ = NULL; + should_exit_ = true; + return; + } + } else { + LOG(WARNING) << s.ToString(); + cli_->Close(); + delete cli_; + cli_ = NULL; + } + } + } + } + } +} + +void RedisSender::Stop() { + set_should_stop(); + should_exit_ = true; + commands_mutex_.Lock(); + rsignal_.Signal(); + commands_mutex_.Unlock(); +} + +void RedisSender::SendRedisCommand(const std::string &command) { + commands_mutex_.Lock(); + if (commands_queue_.size() < 100000) { + commands_queue_.push(command); + rsignal_.Signal(); + commands_mutex_.Unlock(); + return; + } + + while (commands_queue_.size() > 100000) { + wsignal_.Wait(); + } + commands_queue_.push(command); + rsignal_.Signal(); + commands_mutex_.Unlock(); +} + +int RedisSender::SendCommand(std::string &command) { + time_t now = ::time(NULL); + if (kCheckDiff < now - last_write_time_) { + int ret = cli_->CheckAliveness(); + if (ret < 0) { + ConnectRedis(); + } + last_write_time_ = now; + } + + // Send command + int idx = 0; + do { + slash::Status s = cli_->Send(&command); + if (s.ok()) { + return 0; + } + + LOG(WARNING) << "RedisSender " << id_ << "fails to send redis command " << command << ", times: " << idx + 1 << ", error: " << s.ToString(); + + cli_->Close(); + delete cli_; + cli_ = NULL; + ConnectRedis(); + } while(++idx < 3); + + return -1; +} + +void *RedisSender::ThreadMain() { + LOG(INFO) << "Start redis sender " << id_ << " thread..."; + // sleep(15); + int ret = 0; + + ConnectRedis(); + + while (!should_exit_) { + commands_mutex_.Lock(); + while (commands_queue_.size() == 0 && !should_exit_) { + rsignal_.TimedWait(100); + // rsignal_.Wait(); + } + // if (commands_queue_.size() == 0 && should_exit_) { + if (should_exit_) { + commands_mutex_.Unlock(); + break; + } + + if (commands_queue_.size() == 0) { + commands_mutex_.Unlock(); + continue; + } + commands_mutex_.Unlock(); + + // get redis command + std::string command; + commands_mutex_.Lock(); + command = commands_queue_.front(); + // printf("%d, command %s\n", id_, command.c_str()); + elements_++; + commands_queue_.pop(); + wsignal_.Signal(); + commands_mutex_.Unlock(); + ret = SendCommand(command); + if (ret == 0) { + cnt_++; + } + + if (cnt_ >= 200) { + for(; cnt_ > 0; cnt_--) { + cli_->Recv(NULL); + } + } + } + for(; cnt_ > 0; cnt_--) { + cli_->Recv(NULL); + } + + LOG(INFO) << "RedisSender thread " << id_ << " complete"; + delete cli_; + cli_ = NULL; + return NULL; +} + diff --git a/tools/pika_migrate/tests/README.md b/tools/pika_migrate/tests/README.md new file mode 100644 index 0000000000..47b371236f --- /dev/null +++ b/tools/pika_migrate/tests/README.md @@ -0,0 +1,4 @@ +### Pika test + + * 在Pika目录下执行 `./pikatests.sh geo` 测试Pika GEO命令 + * 如果是`unit/type`接口, 例如 SET, 执行 `./pikatests.sh type/set` 测试Pika SET命令 diff --git a/tools/pika_migrate/tests/assets/default.conf b/tools/pika_migrate/tests/assets/default.conf new file mode 100644 index 0000000000..c9cb8183f7 --- /dev/null +++ b/tools/pika_migrate/tests/assets/default.conf @@ -0,0 +1,79 @@ +# Pika port +port : 9221 +# Thread Number +thread-num : 1 +# Sync Thread Number +sync-thread-num : 6 +# Item count of sync thread queue +sync-buffer-size : 10 +# Pika log path +log-path : ./log/ +# Pika glog level: only INFO and ERROR +loglevel : info +# Pika db path +db-path : ./db/ +# Pika write-buffer-size +write-buffer-size : 268435456 +# Pika timeout +timeout : 60 +# Requirepass +requirepass : +# Masterauth +masterauth : +# Userpass +userpass : +# User Blacklist +userblacklist : +# Dump Prefix +dump-prefix : +# daemonize [yes | no] +#daemonize : yes +# slotmigrate [yes | no] +#slotmigrate : no +# Dump Path +dump-path : ./dump/ +# Expire-dump-days +dump-expire : 0 +# pidfile Path +pidfile : ./pika.pid +# Max Connection +maxclients : 20000 +# the per file size of sst to compact, defalut is 2M +target-file-size-base : 20971520 +# Expire-logs-days +expire-logs-days : 7 +# Expire-logs-nums +expire-logs-nums : 10 +# Root-connection-num +root-connection-num : 2 +# Slowlog-log-slower-than +slowlog-log-slower-than : 10000 +# slave-read-only(yes/no, 1/0) +slave-read-only : 0 +# Pika db sync path +db-sync-path : ./dbsync/ +# db sync speed(MB) max is set to 125MB, min is set to 0, and if below 0 or above 125, the value will be adjust to 125 +db-sync-speed : -1 +# network interface +# network-interface : eth1 +# replication +# slaveof : master-ip:master-port +# CronTask, format: start:end-ratio, like 02-04/60, pika will check to schedule compaction between 2 to 4 o'clock everyday +# if the freesize/disksize > 60% +# compact-cron : + +################### +## Critical Settings +################### +# binlog file size: default is 100M, limited in [1K, 2G] +binlog-file-size : 104857600 +# Compression +compression : snappy +# max-background-flushes: default is 1, limited in [1, 4] +max-background-flushes : 1 +# max-background-compactions: default is 1, limited in [1, 4] +max-background-compactions : 2 +# max-cache-files default is 5000 +max-cache-files : 5000 +# max_bytes_for_level_multiplier: default is 10, you can change it to 5 +max-bytes-for-level-multiplier : 10 diff --git a/tools/pika_migrate/tests/assets/encodings.rdb b/tools/pika_migrate/tests/assets/encodings.rdb new file mode 100644 index 0000000000..9fd9b705d1 Binary files /dev/null and b/tools/pika_migrate/tests/assets/encodings.rdb differ diff --git a/tools/pika_migrate/tests/assets/hash-zipmap.rdb b/tools/pika_migrate/tests/assets/hash-zipmap.rdb new file mode 100644 index 0000000000..27a42ed4bb Binary files /dev/null and b/tools/pika_migrate/tests/assets/hash-zipmap.rdb differ diff --git a/tools/pika_migrate/tests/helpers/bg_complex_data.tcl b/tools/pika_migrate/tests/helpers/bg_complex_data.tcl new file mode 100644 index 0000000000..dffd7c6688 --- /dev/null +++ b/tools/pika_migrate/tests/helpers/bg_complex_data.tcl @@ -0,0 +1,10 @@ +source tests/support/redis.tcl +source tests/support/util.tcl + +proc bg_complex_data {host port db ops} { + set r [redis $host $port] + $r select $db + createComplexDataset $r $ops +} + +bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] diff --git a/tools/pika_migrate/tests/helpers/gen_write_load.tcl b/tools/pika_migrate/tests/helpers/gen_write_load.tcl new file mode 100644 index 0000000000..6d1a345166 --- /dev/null +++ b/tools/pika_migrate/tests/helpers/gen_write_load.tcl @@ -0,0 +1,15 @@ +source tests/support/redis.tcl + +proc gen_write_load {host port seconds} { + set start_time [clock seconds] + set r [redis $host $port 1] + $r select 9 + while 1 { + $r set [expr rand()] [expr rand()] + if {[clock seconds]-$start_time > $seconds} { + exit 0 + } + } +} + +gen_write_load [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] diff --git a/tools/pika_migrate/tests/instances.tcl b/tools/pika_migrate/tests/instances.tcl new file mode 100644 index 0000000000..426508f33a --- /dev/null +++ b/tools/pika_migrate/tests/instances.tcl @@ -0,0 +1,407 @@ +# Multi-instance test framework. +# This is used in order to test Sentinel and Redis Cluster, and provides +# basic capabilities for spawning and handling N parallel Redis / Sentinel +# instances. +# +# Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com +# This software is released under the BSD License. See the COPYING file for +# more information. + +package require Tcl 8.5 + +set tcl_precision 17 +source ../support/redis.tcl +source ../support/util.tcl +source ../support/server.tcl +source ../support/test.tcl + +set ::verbose 0 +set ::pause_on_error 0 +set ::simulate_error 0 +set ::sentinel_instances {} +set ::redis_instances {} +set ::sentinel_base_port 20000 +set ::redis_base_port 30000 +set ::pids {} ; # We kill everything at exit +set ::dirs {} ; # We remove all the temp dirs at exit +set ::run_matching {} ; # If non empty, only tests matching pattern are run. + +if {[catch {cd tmp}]} { + puts "tmp directory not found." + puts "Please run this test from the Redis source root." + exit 1 +} + +# Spawn a redis or sentinel instance, depending on 'type'. +proc spawn_instance {type base_port count {conf {}}} { + for {set j 0} {$j < $count} {incr j} { + set port [find_available_port $base_port] + incr base_port + puts "Starting $type #$j at port $port" + + # Create a directory for this instance. + set dirname "${type}_${j}" + lappend ::dirs $dirname + catch {exec rm -rf $dirname} + file mkdir $dirname + + # Write the instance config file. + set cfgfile [file join $dirname $type.conf] + set cfg [open $cfgfile w] + puts $cfg "port $port" + puts $cfg "dir ./$dirname" + puts $cfg "logfile log.txt" + # Add additional config files + foreach directive $conf { + puts $cfg $directive + } + close $cfg + + # Finally exec it and remember the pid for later cleanup. + if {$type eq "redis"} { + set prgname redis-server + } elseif {$type eq "sentinel"} { + set prgname redis-sentinel + } else { + error "Unknown instance type." + } + set pid [exec ../../../src/${prgname} $cfgfile &] + lappend ::pids $pid + + # Check availability + if {[server_is_up 127.0.0.1 $port 100] == 0} { + abort_sentinel_test "Problems starting $type #$j: ping timeout" + } + + # Push the instance into the right list + set link [redis 127.0.0.1 $port] + $link reconnect 1 + lappend ::${type}_instances [list \ + pid $pid \ + host 127.0.0.1 \ + port $port \ + link $link \ + ] + } +} + +proc cleanup {} { + puts "Cleaning up..." + foreach pid $::pids { + catch {exec kill -9 $pid} + } + foreach dir $::dirs { + catch {exec rm -rf $dir} + } +} + +proc abort_sentinel_test msg { + puts "WARNING: Aborting the test." + puts ">>>>>>>> $msg" + cleanup + exit 1 +} + +proc parse_options {} { + for {set j 0} {$j < [llength $::argv]} {incr j} { + set opt [lindex $::argv $j] + set val [lindex $::argv [expr $j+1]] + if {$opt eq "--single"} { + incr j + set ::run_matching "*${val}*" + } elseif {$opt eq "--pause-on-error"} { + set ::pause_on_error 1 + } elseif {$opt eq "--fail"} { + set ::simulate_error 1 + } elseif {$opt eq "--help"} { + puts "Hello, I'm sentinel.tcl and I run Sentinel unit tests." + puts "\nOptions:" + puts "--single Only runs tests specified by pattern." + puts "--pause-on-error Pause for manual inspection on error." + puts "--fail Simulate a test failure." + puts "--help Shows this help." + exit 0 + } else { + puts "Unknown option $opt" + exit 1 + } + } +} + +# If --pause-on-error option was passed at startup this function is called +# on error in order to give the developer a chance to understand more about +# the error condition while the instances are still running. +proc pause_on_error {} { + puts "" + puts [colorstr yellow "*** Please inspect the error now ***"] + puts "\nType \"continue\" to resume the test, \"help\" for help screen.\n" + while 1 { + puts -nonewline "> " + flush stdout + set line [gets stdin] + set argv [split $line " "] + set cmd [lindex $argv 0] + if {$cmd eq {continue}} { + break + } elseif {$cmd eq {show-redis-logs}} { + set count 10 + if {[lindex $argv 1] ne {}} {set count [lindex $argv 1]} + foreach_redis_id id { + puts "=== REDIS $id ====" + puts [exec tail -$count redis_$id/log.txt] + puts "---------------------\n" + } + } elseif {$cmd eq {show-sentinel-logs}} { + set count 10 + if {[lindex $argv 1] ne {}} {set count [lindex $argv 1]} + foreach_sentinel_id id { + puts "=== SENTINEL $id ====" + puts [exec tail -$count sentinel_$id/log.txt] + puts "---------------------\n" + } + } elseif {$cmd eq {ls}} { + foreach_redis_id id { + puts -nonewline "Redis $id" + set errcode [catch { + set str {} + append str "@[RI $id tcp_port]: " + append str "[RI $id role] " + if {[RI $id role] eq {slave}} { + append str "[RI $id master_host]:[RI $id master_port]" + } + set str + } retval] + if {$errcode} { + puts " -- $retval" + } else { + puts $retval + } + } + foreach_sentinel_id id { + puts -nonewline "Sentinel $id" + set errcode [catch { + set str {} + append str "@[SI $id tcp_port]: " + append str "[join [S $id sentinel get-master-addr-by-name mymaster]]" + set str + } retval] + if {$errcode} { + puts " -- $retval" + } else { + puts $retval + } + } + } elseif {$cmd eq {help}} { + puts "ls List Sentinel and Redis instances." + puts "show-sentinel-logs \[N\] Show latest N lines of logs." + puts "show-redis-logs \[N\] Show latest N lines of logs." + puts "S cmd ... arg Call command in Sentinel ." + puts "R cmd ... arg Call command in Redis ." + puts "SI Show Sentinel INFO ." + puts "RI Show Sentinel INFO ." + puts "continue Resume test." + } else { + set errcode [catch {eval $line} retval] + if {$retval ne {}} {puts "$retval"} + } + } +} + +# We redefine 'test' as for Sentinel we don't use the server-client +# architecture for the test, everything is sequential. +proc test {descr code} { + set ts [clock format [clock seconds] -format %H:%M:%S] + puts -nonewline "$ts> $descr: " + flush stdout + + if {[catch {set retval [uplevel 1 $code]} error]} { + if {[string match "assertion:*" $error]} { + set msg [string range $error 10 end] + puts [colorstr red $msg] + if {$::pause_on_error} pause_on_error + puts "(Jumping to next unit after error)" + return -code continue + } else { + # Re-raise, let handler up the stack take care of this. + error $error $::errorInfo + } + } else { + puts [colorstr green OK] + } +} + +proc run_tests {} { + set tests [lsort [glob ../tests/*]] + foreach test $tests { + if {$::run_matching ne {} && [string match $::run_matching $test] == 0} { + continue + } + if {[file isdirectory $test]} continue + puts [colorstr yellow "Testing unit: [lindex [file split $test] end]"] + source $test + } +} + +# The "S" command is used to interact with the N-th Sentinel. +# The general form is: +# +# S command arg arg arg ... +# +# Example to ping the Sentinel 0 (first instance): S 0 PING +proc S {n args} { + set s [lindex $::sentinel_instances $n] + [dict get $s link] {*}$args +} + +# Like R but to chat with Redis instances. +proc R {n args} { + set r [lindex $::redis_instances $n] + [dict get $r link] {*}$args +} + +proc get_info_field {info field} { + set fl [string length $field] + append field : + foreach line [split $info "\n"] { + set line [string trim $line "\r\n "] + if {[string range $line 0 $fl] eq $field} { + return [string range $line [expr {$fl+1}] end] + } + } + return {} +} + +proc SI {n field} { + get_info_field [S $n info] $field +} + +proc RI {n field} { + get_info_field [R $n info] $field +} + +# Iterate over IDs of sentinel or redis instances. +proc foreach_instance_id {instances idvar code} { + upvar 1 $idvar id + for {set id 0} {$id < [llength $instances]} {incr id} { + set errcode [catch {uplevel 1 $code} result] + if {$errcode == 1} { + error $result $::errorInfo $::errorCode + } elseif {$errcode == 4} { + continue + } elseif {$errcode == 3} { + break + } elseif {$errcode != 0} { + return -code $errcode $result + } + } +} + +proc foreach_sentinel_id {idvar code} { + set errcode [catch {uplevel 1 [list foreach_instance_id $::sentinel_instances $idvar $code]} result] + return -code $errcode $result +} + +proc foreach_redis_id {idvar code} { + set errcode [catch {uplevel 1 [list foreach_instance_id $::redis_instances $idvar $code]} result] + return -code $errcode $result +} + +# Get the specific attribute of the specified instance type, id. +proc get_instance_attrib {type id attrib} { + dict get [lindex [set ::${type}_instances] $id] $attrib +} + +# Set the specific attribute of the specified instance type, id. +proc set_instance_attrib {type id attrib newval} { + set d [lindex [set ::${type}_instances] $id] + dict set d $attrib $newval + lset ::${type}_instances $id $d +} + +# Create a master-slave cluster of the given number of total instances. +# The first instance "0" is the master, all others are configured as +# slaves. +proc create_redis_master_slave_cluster n { + foreach_redis_id id { + if {$id == 0} { + # Our master. + R $id slaveof no one + R $id flushall + } elseif {$id < $n} { + R $id slaveof [get_instance_attrib redis 0 host] \ + [get_instance_attrib redis 0 port] + } else { + # Instances not part of the cluster. + R $id slaveof no one + } + } + # Wait for all the slaves to sync. + wait_for_condition 1000 50 { + [RI 0 connected_slaves] == ($n-1) + } else { + fail "Unable to create a master-slaves cluster." + } +} + +proc get_instance_id_by_port {type port} { + foreach_${type}_id id { + if {[get_instance_attrib $type $id port] == $port} { + return $id + } + } + fail "Instance $type port $port not found." +} + +# Kill an instance of the specified type/id with SIGKILL. +# This function will mark the instance PID as -1 to remember that this instance +# is no longer running and will remove its PID from the list of pids that +# we kill at cleanup. +# +# The instance can be restarted with restart-instance. +proc kill_instance {type id} { + set pid [get_instance_attrib $type $id pid] + if {$pid == -1} { + error "You tried to kill $type $id twice." + } + exec kill -9 $pid + set_instance_attrib $type $id pid -1 + set_instance_attrib $type $id link you_tried_to_talk_with_killed_instance + + # Remove the PID from the list of pids to kill at exit. + set ::pids [lsearch -all -inline -not -exact $::pids $pid] +} + +# Return true of the instance of the specified type/id is killed. +proc instance_is_killed {type id} { + set pid [get_instance_attrib $type $id pid] + expr {$pid == -1} +} + +# Restart an instance previously killed by kill_instance +proc restart_instance {type id} { + set dirname "${type}_${id}" + set cfgfile [file join $dirname $type.conf] + set port [get_instance_attrib $type $id port] + + # Execute the instance with its old setup and append the new pid + # file for cleanup. + if {$type eq "redis"} { + set prgname redis-server + } else { + set prgname redis-sentinel + } + set pid [exec ../../../src/${prgname} $cfgfile &] + set_instance_attrib $type $id pid $pid + lappend ::pids $pid + + # Check that the instance is running + if {[server_is_up 127.0.0.1 $port 100] == 0} { + abort_sentinel_test "Problems starting $type #$id: ping timeout" + } + + # Connect with it with a fresh link + set link [redis 127.0.0.1 $port] + $link reconnect 1 + set_instance_attrib $type $id link $link +} + diff --git a/tools/pika_migrate/tests/integration/aof-race.tcl b/tools/pika_migrate/tests/integration/aof-race.tcl new file mode 100644 index 0000000000..207f207393 --- /dev/null +++ b/tools/pika_migrate/tests/integration/aof-race.tcl @@ -0,0 +1,35 @@ +set defaults { appendonly {yes} appendfilename {appendonly.aof} } +set server_path [tmpdir server.aof] +set aof_path "$server_path/appendonly.aof" + +proc start_server_aof {overrides code} { + upvar defaults defaults srv srv server_path server_path + set config [concat $defaults $overrides] + start_server [list overrides $config] $code +} + +tags {"aof"} { + # Specific test for a regression where internal buffers were not properly + # cleaned after a child responsible for an AOF rewrite exited. This buffer + # was subsequently appended to the new AOF, resulting in duplicate commands. + start_server_aof [list dir $server_path] { + set client [redis [srv host] [srv port]] + set bench [open "|src/redis-benchmark -q -p [srv port] -c 20 -n 20000 incr foo" "r+"] + after 100 + + # Benchmark should be running by now: start background rewrite + $client bgrewriteaof + + # Read until benchmark pipe reaches EOF + while {[string length [read $bench]] > 0} {} + + # Check contents of foo + assert_equal 20000 [$client get foo] + } + + # Restart server to replay AOF + start_server_aof [list dir $server_path] { + set client [redis [srv host] [srv port]] + assert_equal 20000 [$client get foo] + } +} diff --git a/tools/pika_migrate/tests/integration/aof.tcl b/tools/pika_migrate/tests/integration/aof.tcl new file mode 100644 index 0000000000..7ea70943c6 --- /dev/null +++ b/tools/pika_migrate/tests/integration/aof.tcl @@ -0,0 +1,236 @@ +set defaults { appendonly {yes} appendfilename {appendonly.aof} } +set server_path [tmpdir server.aof] +set aof_path "$server_path/appendonly.aof" + +proc append_to_aof {str} { + upvar fp fp + puts -nonewline $fp $str +} + +proc create_aof {code} { + upvar fp fp aof_path aof_path + set fp [open $aof_path w+] + uplevel 1 $code + close $fp +} + +proc start_server_aof {overrides code} { + upvar defaults defaults srv srv server_path server_path + set config [concat $defaults $overrides] + set srv [start_server [list overrides $config]] + uplevel 1 $code + kill_server $srv +} + +tags {"aof"} { + ## Server can start when aof-load-truncated is set to yes and AOF + ## is truncated, with an incomplete MULTI block. + create_aof { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand multi] + append_to_aof [formatCommand set bar world] + } + + start_server_aof [list dir $server_path aof-load-truncated yes] { + test "Unfinished MULTI: Server should start if load-truncated is yes" { + assert_equal 1 [is_alive $srv] + } + } + + ## Should also start with truncated AOF without incomplete MULTI block. + create_aof { + append_to_aof [formatCommand incr foo] + append_to_aof [formatCommand incr foo] + append_to_aof [formatCommand incr foo] + append_to_aof [formatCommand incr foo] + append_to_aof [formatCommand incr foo] + append_to_aof [string range [formatCommand incr foo] 0 end-1] + } + + start_server_aof [list dir $server_path aof-load-truncated yes] { + test "Short read: Server should start if load-truncated is yes" { + assert_equal 1 [is_alive $srv] + } + + set client [redis [dict get $srv host] [dict get $srv port]] + + test "Truncated AOF loaded: we expect foo to be equal to 5" { + assert {[$client get foo] eq "5"} + } + + test "Append a new command after loading an incomplete AOF" { + $client incr foo + } + } + + # Now the AOF file is expected to be correct + start_server_aof [list dir $server_path aof-load-truncated yes] { + test "Short read + command: Server should start" { + assert_equal 1 [is_alive $srv] + } + + set client [redis [dict get $srv host] [dict get $srv port]] + + test "Truncated AOF loaded: we expect foo to be equal to 6 now" { + assert {[$client get foo] eq "6"} + } + } + + ## Test that the server exits when the AOF contains a format error + create_aof { + append_to_aof [formatCommand set foo hello] + append_to_aof "!!!" + append_to_aof [formatCommand set foo hello] + } + + start_server_aof [list dir $server_path aof-load-truncated yes] { + test "Bad format: Server should have logged an error" { + set pattern "*Bad file format reading the append only file*" + set retry 10 + while {$retry} { + set result [exec tail -n1 < [dict get $srv stdout]] + if {[string match $pattern $result]} { + break + } + incr retry -1 + after 1000 + } + if {$retry == 0} { + error "assertion:expected error not found on config file" + } + } + } + + ## Test the server doesn't start when the AOF contains an unfinished MULTI + create_aof { + append_to_aof [formatCommand set foo hello] + append_to_aof [formatCommand multi] + append_to_aof [formatCommand set bar world] + } + + start_server_aof [list dir $server_path aof-load-truncated no] { + test "Unfinished MULTI: Server should have logged an error" { + set pattern "*Unexpected end of file reading the append only file*" + set retry 10 + while {$retry} { + set result [exec tail -n1 < [dict get $srv stdout]] + if {[string match $pattern $result]} { + break + } + incr retry -1 + after 1000 + } + if {$retry == 0} { + error "assertion:expected error not found on config file" + } + } + } + + ## Test that the server exits when the AOF contains a short read + create_aof { + append_to_aof [formatCommand set foo hello] + append_to_aof [string range [formatCommand set bar world] 0 end-1] + } + + start_server_aof [list dir $server_path aof-load-truncated no] { + test "Short read: Server should have logged an error" { + set pattern "*Unexpected end of file reading the append only file*" + set retry 10 + while {$retry} { + set result [exec tail -n1 < [dict get $srv stdout]] + if {[string match $pattern $result]} { + break + } + incr retry -1 + after 1000 + } + if {$retry == 0} { + error "assertion:expected error not found on config file" + } + } + } + + ## Test that redis-check-aof indeed sees this AOF is not valid + test "Short read: Utility should confirm the AOF is not valid" { + catch { + exec src/redis-check-aof $aof_path + } result + assert_match "*not valid*" $result + } + + test "Short read: Utility should be able to fix the AOF" { + set result [exec src/redis-check-aof --fix $aof_path << "y\n"] + assert_match "*Successfully truncated AOF*" $result + } + + ## Test that the server can be started using the truncated AOF + start_server_aof [list dir $server_path aof-load-truncated no] { + test "Fixed AOF: Server should have been started" { + assert_equal 1 [is_alive $srv] + } + + test "Fixed AOF: Keyspace should contain values that were parseable" { + set client [redis [dict get $srv host] [dict get $srv port]] + wait_for_condition 50 100 { + [catch {$client ping} e] == 0 + } else { + fail "Loading DB is taking too much time." + } + assert_equal "hello" [$client get foo] + assert_equal "" [$client get bar] + } + } + + ## Test that SPOP (that modifies the client's argc/argv) is correctly free'd + create_aof { + append_to_aof [formatCommand sadd set foo] + append_to_aof [formatCommand sadd set bar] + append_to_aof [formatCommand spop set] + } + + start_server_aof [list dir $server_path aof-load-truncated no] { + test "AOF+SPOP: Server should have been started" { + assert_equal 1 [is_alive $srv] + } + + test "AOF+SPOP: Set should have 1 member" { + set client [redis [dict get $srv host] [dict get $srv port]] + wait_for_condition 50 100 { + [catch {$client ping} e] == 0 + } else { + fail "Loading DB is taking too much time." + } + assert_equal 1 [$client scard set] + } + } + + ## Test that EXPIREAT is loaded correctly + create_aof { + append_to_aof [formatCommand rpush list foo] + append_to_aof [formatCommand expireat list 1000] + append_to_aof [formatCommand rpush list bar] + } + + start_server_aof [list dir $server_path aof-load-truncated no] { + test "AOF+EXPIRE: Server should have been started" { + assert_equal 1 [is_alive $srv] + } + + test "AOF+EXPIRE: List should be empty" { + set client [redis [dict get $srv host] [dict get $srv port]] + wait_for_condition 50 100 { + [catch {$client ping} e] == 0 + } else { + fail "Loading DB is taking too much time." + } + assert_equal 0 [$client llen list] + } + } + + start_server {overrides {appendonly {yes} appendfilename {appendonly.aof}}} { + test {Redis should not try to convert DEL into EXPIREAT for EXPIRE -1} { + r set x 10 + r expire x -1 + } + } +} diff --git a/tools/pika_migrate/tests/integration/convert-zipmap-hash-on-load.tcl b/tools/pika_migrate/tests/integration/convert-zipmap-hash-on-load.tcl new file mode 100644 index 0000000000..cf3577f284 --- /dev/null +++ b/tools/pika_migrate/tests/integration/convert-zipmap-hash-on-load.tcl @@ -0,0 +1,35 @@ +# Copy RDB with zipmap encoded hash to server path +set server_path [tmpdir "server.convert-zipmap-hash-on-load"] + +exec cp -f tests/assets/hash-zipmap.rdb $server_path +start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { + test "RDB load zipmap hash: converts to ziplist" { + r select 0 + + assert_match "*ziplist*" [r debug object hash] + assert_equal 2 [r hlen hash] + assert_match {v1 v2} [r hmget hash f1 f2] + } +} + +exec cp -f tests/assets/hash-zipmap.rdb $server_path +start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-entries" 1]] { + test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-entries is exceeded" { + r select 0 + + assert_match "*hashtable*" [r debug object hash] + assert_equal 2 [r hlen hash] + assert_match {v1 v2} [r hmget hash f1 f2] + } +} + +exec cp -f tests/assets/hash-zipmap.rdb $server_path +start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-value" 1]] { + test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-value is exceeded" { + r select 0 + + assert_match "*hashtable*" [r debug object hash] + assert_equal 2 [r hlen hash] + assert_match {v1 v2} [r hmget hash f1 f2] + } +} diff --git a/tools/pika_migrate/tests/integration/rdb.tcl b/tools/pika_migrate/tests/integration/rdb.tcl new file mode 100644 index 0000000000..71876a6edc --- /dev/null +++ b/tools/pika_migrate/tests/integration/rdb.tcl @@ -0,0 +1,98 @@ +set server_path [tmpdir "server.rdb-encoding-test"] + +# Copy RDB with different encodings in server path +exec cp tests/assets/encodings.rdb $server_path + +start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { + test "RDB encoding loading test" { + r select 0 + csvdump r + } {"compressible","string","aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +"hash","hash","a","1","aa","10","aaa","100","b","2","bb","20","bbb","200","c","3","cc","30","ccc","300","ddd","400","eee","5000000000", +"hash_zipped","hash","a","1","b","2","c","3", +"list","list","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000","1","2","3","a","b","c","100000","6000000000", +"list_zipped","list","1","2","3","a","b","c","100000","6000000000", +"number","string","10" +"set","set","1","100000","2","3","6000000000","a","b","c", +"set_zipped_1","set","1","2","3","4", +"set_zipped_2","set","100000","200000","300000","400000", +"set_zipped_3","set","1000000000","2000000000","3000000000","4000000000","5000000000","6000000000", +"string","string","Hello World" +"zset","zset","a","1","b","2","c","3","aa","10","bb","20","cc","30","aaa","100","bbb","200","ccc","300","aaaa","1000","cccc","123456789","bbbb","5000000000", +"zset_zipped","zset","a","1","b","2","c","3", +} +} + +set server_path [tmpdir "server.rdb-startup-test"] + +start_server [list overrides [list "dir" $server_path]] { + test {Server started empty with non-existing RDB file} { + r debug digest + } {0000000000000000000000000000000000000000} + # Save an RDB file, needed for the next test. + r save +} + +start_server [list overrides [list "dir" $server_path]] { + test {Server started empty with empty RDB file} { + r debug digest + } {0000000000000000000000000000000000000000} +} + +# Helper function to start a server and kill it, just to check the error +# logged. +set defaults {} +proc start_server_and_kill_it {overrides code} { + upvar defaults defaults srv srv server_path server_path + set config [concat $defaults $overrides] + set srv [start_server [list overrides $config]] + uplevel 1 $code + kill_server $srv +} + +# Make the RDB file unreadable +file attributes [file join $server_path dump.rdb] -permissions 0222 + +# Detect root account (it is able to read the file even with 002 perm) +set isroot 0 +catch { + open [file join $server_path dump.rdb] + set isroot 1 +} + +# Now make sure the server aborted with an error +if {!$isroot} { + start_server_and_kill_it [list "dir" $server_path] { + test {Server should not start if RDB file can't be open} { + wait_for_condition 50 100 { + [string match {*Fatal error loading*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was unreadable!" + } + } + } +} + +# Fix permissions of the RDB file. +file attributes [file join $server_path dump.rdb] -permissions 0666 + +# Corrupt its CRC64 checksum. +set filesize [file size [file join $server_path dump.rdb]] +set fd [open [file join $server_path dump.rdb] r+] +fconfigure $fd -translation binary +seek $fd -8 end +puts -nonewline $fd "foobar00"; # Corrupt the checksum +close $fd + +# Now make sure the server aborted with an error +start_server_and_kill_it [list "dir" $server_path] { + test {Server should not start if RDB is corrupted} { + wait_for_condition 50 100 { + [string match {*RDB checksum*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was corrupted!" + } + } +} diff --git a/tools/pika_migrate/tests/integration/redis-cli.tcl b/tools/pika_migrate/tests/integration/redis-cli.tcl new file mode 100644 index 0000000000..40e4222e3e --- /dev/null +++ b/tools/pika_migrate/tests/integration/redis-cli.tcl @@ -0,0 +1,208 @@ +start_server {tags {"cli"}} { + proc open_cli {} { + set ::env(TERM) dumb + set fd [open [format "|src/redis-cli -p %d -n 9" [srv port]] "r+"] + fconfigure $fd -buffering none + fconfigure $fd -blocking false + fconfigure $fd -translation binary + assert_equal "redis> " [read_cli $fd] + set _ $fd + } + + proc close_cli {fd} { + close $fd + } + + proc read_cli {fd} { + set buf [read $fd] + while {[string length $buf] == 0} { + # wait some time and try again + after 10 + set buf [read $fd] + } + set _ $buf + } + + proc write_cli {fd buf} { + puts $fd $buf + flush $fd + } + + # Helpers to run tests in interactive mode + proc run_command {fd cmd} { + write_cli $fd $cmd + set lines [split [read_cli $fd] "\n"] + assert_equal "redis> " [lindex $lines end] + join [lrange $lines 0 end-1] "\n" + } + + proc test_interactive_cli {name code} { + set ::env(FAKETTY) 1 + set fd [open_cli] + test "Interactive CLI: $name" $code + close_cli $fd + unset ::env(FAKETTY) + } + + # Helpers to run tests where stdout is not a tty + proc write_tmpfile {contents} { + set tmp [tmpfile "cli"] + set tmpfd [open $tmp "w"] + puts -nonewline $tmpfd $contents + close $tmpfd + set _ $tmp + } + + proc _run_cli {opts args} { + set cmd [format "src/redis-cli -p %d -n 9 $args" [srv port]] + foreach {key value} $opts { + if {$key eq "pipe"} { + set cmd "sh -c \"$value | $cmd\"" + } + if {$key eq "path"} { + set cmd "$cmd < $value" + } + } + + set fd [open "|$cmd" "r"] + fconfigure $fd -buffering none + fconfigure $fd -translation binary + set resp [read $fd 1048576] + close $fd + set _ $resp + } + + proc run_cli {args} { + _run_cli {} {*}$args + } + + proc run_cli_with_input_pipe {cmd args} { + _run_cli [list pipe $cmd] {*}$args + } + + proc run_cli_with_input_file {path args} { + _run_cli [list path $path] {*}$args + } + + proc test_nontty_cli {name code} { + test "Non-interactive non-TTY CLI: $name" $code + } + + # Helpers to run tests where stdout is a tty (fake it) + proc test_tty_cli {name code} { + set ::env(FAKETTY) 1 + test "Non-interactive TTY CLI: $name" $code + unset ::env(FAKETTY) + } + + test_interactive_cli "INFO response should be printed raw" { + set lines [split [run_command $fd info] "\n"] + foreach line $lines { + assert [regexp {^[a-z0-9_]+:[a-z0-9_]+} $line] + } + } + + test_interactive_cli "Status reply" { + assert_equal "OK" [run_command $fd "set key foo"] + } + + test_interactive_cli "Integer reply" { + assert_equal "(integer) 1" [run_command $fd "incr counter"] + } + + test_interactive_cli "Bulk reply" { + r set key foo + assert_equal "\"foo\"" [run_command $fd "get key"] + } + + test_interactive_cli "Multi-bulk reply" { + r rpush list foo + r rpush list bar + assert_equal "1. \"foo\"\n2. \"bar\"" [run_command $fd "lrange list 0 -1"] + } + + test_interactive_cli "Parsing quotes" { + assert_equal "OK" [run_command $fd "set key \"bar\""] + assert_equal "bar" [r get key] + assert_equal "OK" [run_command $fd "set key \" bar \""] + assert_equal " bar " [r get key] + assert_equal "OK" [run_command $fd "set key \"\\\"bar\\\"\""] + assert_equal "\"bar\"" [r get key] + assert_equal "OK" [run_command $fd "set key \"\tbar\t\""] + assert_equal "\tbar\t" [r get key] + + # invalid quotation + assert_equal "Invalid argument(s)" [run_command $fd "get \"\"key"] + assert_equal "Invalid argument(s)" [run_command $fd "get \"key\"x"] + + # quotes after the argument are weird, but should be allowed + assert_equal "OK" [run_command $fd "set key\"\" bar"] + assert_equal "bar" [r get key] + } + + test_tty_cli "Status reply" { + assert_equal "OK\n" [run_cli set key bar] + assert_equal "bar" [r get key] + } + + test_tty_cli "Integer reply" { + r del counter + assert_equal "(integer) 1\n" [run_cli incr counter] + } + + test_tty_cli "Bulk reply" { + r set key "tab\tnewline\n" + assert_equal "\"tab\\tnewline\\n\"\n" [run_cli get key] + } + + test_tty_cli "Multi-bulk reply" { + r del list + r rpush list foo + r rpush list bar + assert_equal "1. \"foo\"\n2. \"bar\"\n" [run_cli lrange list 0 -1] + } + + test_tty_cli "Read last argument from pipe" { + assert_equal "OK\n" [run_cli_with_input_pipe "echo foo" set key] + assert_equal "foo\n" [r get key] + } + + test_tty_cli "Read last argument from file" { + set tmpfile [write_tmpfile "from file"] + assert_equal "OK\n" [run_cli_with_input_file $tmpfile set key] + assert_equal "from file" [r get key] + } + + test_nontty_cli "Status reply" { + assert_equal "OK" [run_cli set key bar] + assert_equal "bar" [r get key] + } + + test_nontty_cli "Integer reply" { + r del counter + assert_equal "1" [run_cli incr counter] + } + + test_nontty_cli "Bulk reply" { + r set key "tab\tnewline\n" + assert_equal "tab\tnewline\n" [run_cli get key] + } + + test_nontty_cli "Multi-bulk reply" { + r del list + r rpush list foo + r rpush list bar + assert_equal "foo\nbar" [run_cli lrange list 0 -1] + } + + test_nontty_cli "Read last argument from pipe" { + assert_equal "OK" [run_cli_with_input_pipe "echo foo" set key] + assert_equal "foo\n" [r get key] + } + + test_nontty_cli "Read last argument from file" { + set tmpfile [write_tmpfile "from file"] + assert_equal "OK" [run_cli_with_input_file $tmpfile set key] + assert_equal "from file" [r get key] + } +} diff --git a/tools/pika_migrate/tests/integration/replication-2.tcl b/tools/pika_migrate/tests/integration/replication-2.tcl new file mode 100644 index 0000000000..9446e5cd91 --- /dev/null +++ b/tools/pika_migrate/tests/integration/replication-2.tcl @@ -0,0 +1,87 @@ +start_server {tags {"repl"}} { + start_server {} { + test {First server should have role slave after SLAVEOF} { + r -1 slaveof [srv 0 host] [srv 0 port] + after 1000 + s -1 role + } {slave} + + test {If min-slaves-to-write is honored, write is accepted} { + r config set min-slaves-to-write 1 + r config set min-slaves-max-lag 10 + r set foo 12345 + wait_for_condition 50 100 { + [r -1 get foo] eq {12345} + } else { + fail "Write did not reached slave" + } + } + + test {No write if min-slaves-to-write is < attached slaves} { + r config set min-slaves-to-write 2 + r config set min-slaves-max-lag 10 + catch {r set foo 12345} err + set err + } {NOREPLICAS*} + + test {If min-slaves-to-write is honored, write is accepted (again)} { + r config set min-slaves-to-write 1 + r config set min-slaves-max-lag 10 + r set foo 12345 + wait_for_condition 50 100 { + [r -1 get foo] eq {12345} + } else { + fail "Write did not reached slave" + } + } + + test {No write if min-slaves-max-lag is > of the slave lag} { + r -1 deferred 1 + r config set min-slaves-to-write 1 + r config set min-slaves-max-lag 2 + r -1 debug sleep 6 + assert {[r set foo 12345] eq {OK}} + after 4000 + catch {r set foo 12345} err + assert {[r -1 read] eq {OK}} + r -1 deferred 0 + set err + } {NOREPLICAS*} + + test {min-slaves-to-write is ignored by slaves} { + r config set min-slaves-to-write 1 + r config set min-slaves-max-lag 10 + r -1 config set min-slaves-to-write 1 + r -1 config set min-slaves-max-lag 10 + r set foo aaabbb + wait_for_condition 50 100 { + [r -1 get foo] eq {aaabbb} + } else { + fail "Write did not reached slave" + } + } + + # Fix parameters for the next test to work + r config set min-slaves-to-write 0 + r -1 config set min-slaves-to-write 0 + r flushall + + test {MASTER and SLAVE dataset should be identical after complex ops} { + createComplexDataset r 10000 + after 500 + if {[r debug digest] ne [r -1 debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + } + } +} diff --git a/tools/pika_migrate/tests/integration/replication-3.tcl b/tools/pika_migrate/tests/integration/replication-3.tcl new file mode 100644 index 0000000000..0fcbad45b0 --- /dev/null +++ b/tools/pika_migrate/tests/integration/replication-3.tcl @@ -0,0 +1,101 @@ +start_server {tags {"repl"}} { + start_server {} { + test {First server should have role slave after SLAVEOF} { + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + if {$::accurate} {set numops 50000} else {set numops 5000} + + test {MASTER and SLAVE consistency with expire} { + createComplexDataset r $numops useexpire + after 4000 ;# Make sure everything expired before taking the digest + r keys * ;# Force DEL syntesizing to slave + after 1000 ;# Wait another second. Now everything should be fine. + if {[r debug digest] ne [r -1 debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + } + } +} + +start_server {tags {"repl"}} { + start_server {} { + test {First server should have role slave after SLAVEOF} { + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + set numops 20000 ;# Enough to trigger the Script Cache LRU eviction. + + # While we are at it, enable AOF to test it will be consistent as well + # after the test. + r config set appendonly yes + + test {MASTER and SLAVE consistency with EVALSHA replication} { + array set oldsha {} + for {set j 0} {$j < $numops} {incr j} { + set key "key:$j" + # Make sure to create scripts that have different SHA1s + set script "return redis.call('incr','$key')" + set sha1 [r eval "return redis.sha1hex(\"$script\")" 0] + set oldsha($j) $sha1 + r eval $script 0 + set res [r evalsha $sha1 0] + assert {$res == 2} + # Additionally call one of the old scripts as well, at random. + set res [r evalsha $oldsha([randomInt $j]) 0] + assert {$res > 2} + + # Trigger an AOF rewrite while we are half-way, this also + # forces the flush of the script cache, and we will cover + # more code as a result. + if {$j == $numops / 2} { + catch {r bgrewriteaof} + } + } + + wait_for_condition 50 100 { + [r dbsize] == $numops && + [r -1 dbsize] == $numops && + [r debug digest] eq [r -1 debug digest] + } else { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + + } + + set old_digest [r debug digest] + r config set appendonly no + r debug loadaof + set new_digest [r debug digest] + assert {$old_digest eq $new_digest} + } + } +} diff --git a/tools/pika_migrate/tests/integration/replication-4.tcl b/tools/pika_migrate/tests/integration/replication-4.tcl new file mode 100644 index 0000000000..6db9ffe2bc --- /dev/null +++ b/tools/pika_migrate/tests/integration/replication-4.tcl @@ -0,0 +1,136 @@ +proc start_bg_complex_data {host port db ops} { + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops & +} + +proc stop_bg_complex_data {handle} { + catch {exec /bin/kill -9 $handle} +} + +start_server {tags {"repl"}} { + start_server {} { + + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] + set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] + set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + after 1000 + s 0 role + } {slave} + + test {Test replication with parallel clients writing in differnet DBs} { + after 5000 + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + + if {[$master debug digest] ne [$slave debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + } + } +} + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + test {With min-slaves-to-write (1,3): master should be writable} { + $master config set min-slaves-max-lag 3 + $master config set min-slaves-to-write 1 + $master set foo bar + } {OK} + + test {With min-slaves-to-write (2,3): master should not be writable} { + $master config set min-slaves-max-lag 3 + $master config set min-slaves-to-write 2 + catch {$master set foo bar} e + set e + } {NOREPLICAS*} + + test {With min-slaves-to-write: master not writable with lagged slave} { + $master config set min-slaves-max-lag 2 + $master config set min-slaves-to-write 1 + assert {[$master set foo bar] eq {OK}} + $slave deferred 1 + $slave debug sleep 6 + after 4000 + catch {$master set foo bar} e + set e + } {NOREPLICAS*} + } +} + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 role] eq {slave} + } else { + fail "Replication not started." + } + } + + test {Replication: commands with many arguments (issue #1221)} { + # We now issue large MSET commands, that may trigger a specific + # class of bugs, see issue #1221. + for {set j 0} {$j < 100} {incr j} { + set cmd [list mset] + for {set x 0} {$x < 1000} {incr x} { + lappend cmd [randomKey] [randomValue] + } + $master {*}$cmd + } + + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + } + } +} diff --git a/tools/pika_migrate/tests/integration/replication-psync.tcl b/tools/pika_migrate/tests/integration/replication-psync.tcl new file mode 100644 index 0000000000..f131dafe31 --- /dev/null +++ b/tools/pika_migrate/tests/integration/replication-psync.tcl @@ -0,0 +1,115 @@ +proc start_bg_complex_data {host port db ops} { + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops & +} + +proc stop_bg_complex_data {handle} { + catch {exec /bin/kill -9 $handle} +} + +# Creates a master-slave pair and breaks the link continuously to force +# partial resyncs attempts, all this while flooding the master with +# write queries. +# +# You can specifiy backlog size, ttl, delay before reconnection, test duration +# in seconds, and an additional condition to verify at the end. +proc test_psync {descr duration backlog_size backlog_ttl delay cond} { + start_server {tags {"repl"}} { + start_server {} { + + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + $master config set repl-backlog-size $backlog_size + $master config set repl-backlog-ttl $backlog_ttl + + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] + set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] + set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] + + test {Slave should be able to synchronize with the master} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [lindex [r role] 0] eq {slave} && + [lindex [r role] 3] eq {connected} + } else { + fail "Replication not started." + } + } + + # Check that the background clients are actually writing. + test {Detect write load to master} { + wait_for_condition 50 100 { + [$master dbsize] > 100 + } else { + fail "Can't detect write load from background clients." + } + } + + test "Test replication partial resync: $descr" { + # Now while the clients are writing data, break the maste-slave + # link multiple times. + for {set j 0} {$j < $duration*10} {incr j} { + after 100 + # catch {puts "MASTER [$master dbsize] keys, SLAVE [$slave dbsize] keys"} + + if {($j % 20) == 0} { + catch { + if {$delay} { + $slave multi + $slave client kill $master_host:$master_port + $slave debug sleep $delay + $slave exec + } else { + $slave client kill $master_host:$master_port + } + } + } + } + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + + if {[$master debug digest] ne [$slave debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + eval $cond + } + } + } +} + +test_psync {ok psync} 6 1000000 3600 0 { + assert {[s -1 sync_partial_ok] > 0} +} + +test_psync {no backlog} 6 100 3600 0.5 { + assert {[s -1 sync_partial_err] > 0} +} + +test_psync {ok after delay} 3 100000000 3600 3 { + assert {[s -1 sync_partial_ok] > 0} +} + +test_psync {backlog expired} 3 100000000 1 3 { + assert {[s -1 sync_partial_err] > 0} +} diff --git a/tools/pika_migrate/tests/integration/replication.tcl b/tools/pika_migrate/tests/integration/replication.tcl new file mode 100644 index 0000000000..bb907eba8e --- /dev/null +++ b/tools/pika_migrate/tests/integration/replication.tcl @@ -0,0 +1,215 @@ +start_server {tags {"repl"}} { + set A [srv 0 client] + set A_host [srv 0 host] + set A_port [srv 0 port] + start_server {} { + set B [srv 0 client] + set B_host [srv 0 host] + set B_port [srv 0 port] + + test {Set instance A as slave of B} { + $A slaveof $B_host $B_port + wait_for_condition 50 100 { + [lindex [$A role] 0] eq {slave} && + [string match {*master_link_status:up*} [$A info replication]] + } else { + fail "Can't turn the instance into a slave" + } + } + + test {BRPOPLPUSH replication, when blocking against empty list} { + set rd [redis_deferring_client] + $rd brpoplpush a b 5 + r lpush a foo + wait_for_condition 50 100 { + [$A debug digest] eq [$B debug digest] + } else { + fail "Master and slave have different digest: [$A debug digest] VS [$B debug digest]" + } + } + + test {BRPOPLPUSH replication, list exists} { + set rd [redis_deferring_client] + r lpush c 1 + r lpush c 2 + r lpush c 3 + $rd brpoplpush c d 5 + after 1000 + assert_equal [$A debug digest] [$B debug digest] + } + + test {BLPOP followed by role change, issue #2473} { + set rd [redis_deferring_client] + $rd blpop foo 0 ; # Block while B is a master + + # Turn B into master of A + $A slaveof no one + $B slaveof $A_host $A_port + wait_for_condition 50 100 { + [lindex [$B role] 0] eq {slave} && + [string match {*master_link_status:up*} [$B info replication]] + } else { + fail "Can't turn the instance into a slave" + } + + # Push elements into the "foo" list of the new slave. + # If the client is still attached to the instance, we'll get + # a desync between the two instances. + $A rpush foo a b c + after 100 + + wait_for_condition 50 100 { + [$A debug digest] eq [$B debug digest] && + [$A lrange foo 0 -1] eq {a b c} && + [$B lrange foo 0 -1] eq {a b c} + } else { + fail "Master and slave have different digest: [$A debug digest] VS [$B debug digest]" + } + } + } +} + +start_server {tags {"repl"}} { + r set mykey foo + + start_server {} { + test {Second server should have role master at first} { + s role + } {master} + + test {SLAVEOF should start with link status "down"} { + r slaveof [srv -1 host] [srv -1 port] + s master_link_status + } {down} + + test {The role should immediately be changed to "slave"} { + s role + } {slave} + + wait_for_sync r + test {Sync should have transferred keys from master} { + r get mykey + } {foo} + + test {The link status should be up} { + s master_link_status + } {up} + + test {SET on the master should immediately propagate} { + r -1 set mykey bar + + wait_for_condition 500 100 { + [r 0 get mykey] eq {bar} + } else { + fail "SET on master did not propagated on slave" + } + } + + test {FLUSHALL should replicate} { + r -1 flushall + if {$::valgrind} {after 2000} + list [r -1 dbsize] [r 0 dbsize] + } {0 0} + + test {ROLE in master reports master with a slave} { + set res [r -1 role] + lassign $res role offset slaves + assert {$role eq {master}} + assert {$offset > 0} + assert {[llength $slaves] == 1} + lassign [lindex $slaves 0] master_host master_port slave_offset + assert {$slave_offset <= $offset} + } + + test {ROLE in slave reports slave in connected state} { + set res [r role] + lassign $res role master_host master_port slave_state slave_offset + assert {$role eq {slave}} + assert {$slave_state eq {connected}} + } + } +} + +foreach dl {no yes} { + start_server {tags {"repl"}} { + set master [srv 0 client] + $master config set repl-diskless-sync $dl + set master_host [srv 0 host] + set master_port [srv 0 port] + set slaves {} + set load_handle0 [start_write_load $master_host $master_port 3] + set load_handle1 [start_write_load $master_host $master_port 5] + set load_handle2 [start_write_load $master_host $master_port 20] + set load_handle3 [start_write_load $master_host $master_port 8] + set load_handle4 [start_write_load $master_host $master_port 4] + start_server {} { + lappend slaves [srv 0 client] + start_server {} { + lappend slaves [srv 0 client] + start_server {} { + lappend slaves [srv 0 client] + test "Connect multiple slaves at the same time (issue #141), diskless=$dl" { + # Send SLAVEOF commands to slaves + [lindex $slaves 0] slaveof $master_host $master_port + [lindex $slaves 1] slaveof $master_host $master_port + [lindex $slaves 2] slaveof $master_host $master_port + + # Wait for all the three slaves to reach the "online" + # state from the POV of the master. + set retry 500 + while {$retry} { + set info [r -3 info] + if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} { + break + } else { + incr retry -1 + after 100 + } + } + if {$retry == 0} { + error "assertion:Slaves not correctly synchronized" + } + + # Wait that slaves acknowledge they are online so + # we are sure that DBSIZE and DEBUG DIGEST will not + # fail because of timing issues. + wait_for_condition 500 100 { + [lindex [[lindex $slaves 0] role] 3] eq {connected} && + [lindex [[lindex $slaves 1] role] 3] eq {connected} && + [lindex [[lindex $slaves 2] role] 3] eq {connected} + } else { + fail "Slaves still not connected after some time" + } + + # Stop the write load + stop_write_load $load_handle0 + stop_write_load $load_handle1 + stop_write_load $load_handle2 + stop_write_load $load_handle3 + stop_write_load $load_handle4 + + # Make sure that slaves and master have same + # number of keys + wait_for_condition 500 100 { + [$master dbsize] == [[lindex $slaves 0] dbsize] && + [$master dbsize] == [[lindex $slaves 1] dbsize] && + [$master dbsize] == [[lindex $slaves 2] dbsize] + } else { + fail "Different number of keys between masted and slave after too long time." + } + + # Check digests + set digest [$master debug digest] + set digest0 [[lindex $slaves 0] debug digest] + set digest1 [[lindex $slaves 1] debug digest] + set digest2 [[lindex $slaves 2] debug digest] + assert {$digest ne 0000000000000000000000000000000000000000} + assert {$digest eq $digest0} + assert {$digest eq $digest1} + assert {$digest eq $digest2} + } + } + } + } + } +} diff --git a/tools/pika_migrate/tests/sentinel/run.tcl b/tools/pika_migrate/tests/sentinel/run.tcl new file mode 100644 index 0000000000..f330299599 --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/run.tcl @@ -0,0 +1,22 @@ +# Sentinel test suite. Copyright (C) 2014 Salvatore Sanfilippo antirez@gmail.com +# This software is released under the BSD License. See the COPYING file for +# more information. + +cd tests/sentinel +source ../instances.tcl + +set ::instances_count 5 ; # How many instances we use at max. + +proc main {} { + parse_options + spawn_instance sentinel $::sentinel_base_port $::instances_count + spawn_instance redis $::redis_base_port $::instances_count + run_tests + cleanup +} + +if {[catch main e]} { + puts $::errorInfo + cleanup + exit 1 +} diff --git a/tools/pika_migrate/tests/sentinel/tests/00-base.tcl b/tools/pika_migrate/tests/sentinel/tests/00-base.tcl new file mode 100644 index 0000000000..a79d0c371c --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/00-base.tcl @@ -0,0 +1,126 @@ +# Check the basic monitoring and failover capabilities. + +source "../tests/includes/init-tests.tcl" + +if {$::simulate_error} { + test "This test will fail" { + fail "Simulated error" + } +} + +test "Basic failover works if the master is down" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} + +test "All the other slaves now point to the new master" { + foreach_redis_id id { + if {$id != $master_id && $id != 0} { + wait_for_condition 1000 50 { + [RI $id master_port] == [lindex $addr 1] + } else { + fail "Redis ID $id not configured to replicate with new master" + } + } + } +} + +test "The old master eventually gets reconfigured as a slave" { + wait_for_condition 1000 50 { + [RI 0 master_port] == [lindex $addr 1] + } else { + fail "Old master not reconfigured as slave of new master" + } +} + +test "ODOWN is not possible without N (quorum) Sentinels reports" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum [expr $sentinels+1] + } + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + + # Make sure failover did not happened. + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + restart_instance redis $master_id +} + +test "Failover is not possible without majority agreement" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $quorum + } + + # Crash majority of sentinels + for {set id 0} {$id < $quorum} {incr id} { + kill_instance sentinel $id + } + + # Kill the current master + kill_instance redis $master_id + + # Make sure failover did not happened. + set addr [S $quorum SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + restart_instance redis $master_id + + # Cleanup: restart Sentinels to monitor the master. + for {set id 0} {$id < $quorum} {incr id} { + restart_instance sentinel $id + } +} + +test "Failover works if we configure for absolute agreement" { + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $sentinels + } + + # Wait for Sentinels to monitor the master again + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [dict get [S $id SENTINEL MASTER mymaster] info-refresh] < 100000 + } else { + fail "At least one Sentinel is not monitoring the master" + } + } + + kill_instance redis $master_id + + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] + + # Set the min ODOWN agreement back to strict majority. + foreach_sentinel_id id { + S $id SENTINEL SET mymaster quorum $quorum + } +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} diff --git a/tools/pika_migrate/tests/sentinel/tests/01-conf-update.tcl b/tools/pika_migrate/tests/sentinel/tests/01-conf-update.tcl new file mode 100644 index 0000000000..4998104d2f --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/01-conf-update.tcl @@ -0,0 +1,39 @@ +# Test Sentinel configuration consistency after partitions heal. + +source "../tests/includes/init-tests.tcl" + +test "We can failover with Sentinel 1 crashed" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + + # Crash Sentinel 1 + kill_instance sentinel 1 + + kill_instance redis $master_id + foreach_sentinel_id id { + if {$id != 1} { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "Sentinel $id did not received failover info" + } + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "After Sentinel 1 is restarted, its config gets updated" { + restart_instance sentinel 1 + wait_for_condition 1000 50 { + [lindex [S 1 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "Restarted Sentinel did not received failover info" + } +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} diff --git a/tools/pika_migrate/tests/sentinel/tests/02-slaves-reconf.tcl b/tools/pika_migrate/tests/sentinel/tests/02-slaves-reconf.tcl new file mode 100644 index 0000000000..fa15d2efde --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/02-slaves-reconf.tcl @@ -0,0 +1,84 @@ +# Check that slaves are reconfigured at a latter time if they are partitioned. +# +# Here we should test: +# 1) That slaves point to the new master after failover. +# 2) That partitioned slaves point to new master when they are partitioned +# away during failover and return at a latter time. + +source "../tests/includes/init-tests.tcl" + +proc 02_test_slaves_replication {} { + uplevel 1 { + test "Check that slaves replicate from current master" { + set master_port [RI $master_id tcp_port] + foreach_redis_id id { + if {$id == $master_id} continue + if {[instance_is_killed redis $id]} continue + wait_for_condition 1000 50 { + ([RI $id master_port] == $master_port) && + ([RI $id master_link_status] eq {up}) + } else { + fail "Redis slave $id is replicating from wrong master" + } + } + } + } +} + +proc 02_crash_and_failover {} { + uplevel 1 { + test "Crash the master and force a failover" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + kill_instance redis $master_id + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + restart_instance redis $master_id + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] + } + } +} + +02_test_slaves_replication +02_crash_and_failover +02_test_slaves_replication + +test "Kill a slave instance" { + foreach_redis_id id { + if {$id == $master_id} continue + set killed_slave_id $id + kill_instance redis $id + break + } +} + +02_crash_and_failover +02_test_slaves_replication + +test "Wait for failover to end" { + set inprogress 1 + while {$inprogress} { + set inprogress 0 + foreach_sentinel_id id { + if {[dict exists [S $id SENTINEL MASTER mymaster] failover-state]} { + incr inprogress + } + } + if {$inprogress} {after 100} + } +} + +test "Restart killed slave and test replication of slaves again..." { + restart_instance redis $killed_slave_id +} + +# Now we check if the slave rejoining the partition is reconfigured even +# if the failover finished. +02_test_slaves_replication diff --git a/tools/pika_migrate/tests/sentinel/tests/03-runtime-reconf.tcl b/tools/pika_migrate/tests/sentinel/tests/03-runtime-reconf.tcl new file mode 100644 index 0000000000..426596c37e --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/03-runtime-reconf.tcl @@ -0,0 +1 @@ +# Test runtime reconfiguration command SENTINEL SET. diff --git a/tools/pika_migrate/tests/sentinel/tests/04-slave-selection.tcl b/tools/pika_migrate/tests/sentinel/tests/04-slave-selection.tcl new file mode 100644 index 0000000000..3d2ca64845 --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/04-slave-selection.tcl @@ -0,0 +1,5 @@ +# Test slave selection algorithm. +# +# This unit should test: +# 1) That when there are no suitable slaves no failover is performed. +# 2) That among the available slaves, the one with better offset is picked. diff --git a/tools/pika_migrate/tests/sentinel/tests/05-manual.tcl b/tools/pika_migrate/tests/sentinel/tests/05-manual.tcl new file mode 100644 index 0000000000..1a60d814b3 --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/05-manual.tcl @@ -0,0 +1,44 @@ +# Test manual failover + +source "../tests/includes/init-tests.tcl" + +test "Manual failover works" { + set old_port [RI $master_id tcp_port] + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + assert {[lindex $addr 1] == $old_port} + S 0 SENTINEL FAILOVER mymaster + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [lindex [S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] 1] != $old_port + } else { + fail "At least one Sentinel did not received failover info" + } + } + set addr [S 0 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster] + set master_id [get_instance_id_by_port redis [lindex $addr 1]] +} + +test "New master [join $addr {:}] role matches" { + assert {[RI $master_id role] eq {master}} +} + +test "All the other slaves now point to the new master" { + foreach_redis_id id { + if {$id != $master_id && $id != 0} { + wait_for_condition 1000 50 { + [RI $id master_port] == [lindex $addr 1] + } else { + fail "Redis ID $id not configured to replicate with new master" + } + } + } +} + +test "The old master eventually gets reconfigured as a slave" { + wait_for_condition 1000 50 { + [RI 0 master_port] == [lindex $addr 1] + } else { + fail "Old master not reconfigured as slave of new master" + } +} + diff --git a/tools/pika_migrate/tests/sentinel/tests/includes/init-tests.tcl b/tools/pika_migrate/tests/sentinel/tests/includes/init-tests.tcl new file mode 100644 index 0000000000..c8165dcfa9 --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tests/includes/init-tests.tcl @@ -0,0 +1,72 @@ +# Initialization tests -- most units will start including this. + +test "(init) Restart killed instances" { + foreach type {redis sentinel} { + foreach_${type}_id id { + if {[get_instance_attrib $type $id pid] == -1} { + puts -nonewline "$type/$id " + flush stdout + restart_instance $type $id + } + } + } +} + +test "(init) Remove old master entry from sentinels" { + foreach_sentinel_id id { + catch {S $id SENTINEL REMOVE mymaster} + } +} + +set redis_slaves 4 +test "(init) Create a master-slaves cluster of [expr $redis_slaves+1] instances" { + create_redis_master_slave_cluster [expr {$redis_slaves+1}] +} +set master_id 0 + +test "(init) Sentinels can start monitoring a master" { + set sentinels [llength $::sentinel_instances] + set quorum [expr {$sentinels/2+1}] + foreach_sentinel_id id { + S $id SENTINEL MONITOR mymaster \ + [get_instance_attrib redis $master_id host] \ + [get_instance_attrib redis $master_id port] $quorum + } + foreach_sentinel_id id { + assert {[S $id sentinel master mymaster] ne {}} + S $id SENTINEL SET mymaster down-after-milliseconds 2000 + S $id SENTINEL SET mymaster failover-timeout 20000 + S $id SENTINEL SET mymaster parallel-syncs 10 + } +} + +test "(init) Sentinels can talk with the master" { + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [catch {S $id SENTINEL GET-MASTER-ADDR-BY-NAME mymaster}] == 0 + } else { + fail "Sentinel $id can't talk with the master." + } + } +} + +test "(init) Sentinels are able to auto-discover other sentinels" { + set sentinels [llength $::sentinel_instances] + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [dict get [S $id SENTINEL MASTER mymaster] num-other-sentinels] == ($sentinels-1) + } else { + fail "At least some sentinel can't detect some other sentinel" + } + } +} + +test "(init) Sentinels are able to auto-discover slaves" { + foreach_sentinel_id id { + wait_for_condition 1000 50 { + [dict get [S $id SENTINEL MASTER mymaster] num-slaves] == $redis_slaves + } else { + fail "At least some sentinel can't detect some slave" + } + } +} diff --git a/tools/pika_migrate/tests/sentinel/tmp/.gitignore b/tools/pika_migrate/tests/sentinel/tmp/.gitignore new file mode 100644 index 0000000000..f581f73e2d --- /dev/null +++ b/tools/pika_migrate/tests/sentinel/tmp/.gitignore @@ -0,0 +1,2 @@ +redis_* +sentinel_* diff --git a/tools/pika_migrate/tests/support/redis.tcl b/tools/pika_migrate/tests/support/redis.tcl new file mode 100644 index 0000000000..7c78360812 --- /dev/null +++ b/tools/pika_migrate/tests/support/redis.tcl @@ -0,0 +1,294 @@ +# Tcl clinet library - used by test-redis.tcl script for now +# Copyright (C) 2009 Salvatore Sanfilippo +# Released under the BSD license like Redis itself +# +# Example usage: +# +# set r [redis 127.0.0.1 6379] +# $r lpush mylist foo +# $r lpush mylist bar +# $r lrange mylist 0 -1 +# $r close +# +# Non blocking usage example: +# +# proc handlePong {r type reply} { +# puts "PONG $type '$reply'" +# if {$reply ne "PONG"} { +# $r ping [list handlePong] +# } +# } +# +# set r [redis] +# $r blocking 0 +# $r get fo [list handlePong] +# +# vwait forever + +package require Tcl 8.5 +package provide redis 0.1 + +namespace eval redis {} +set ::redis::id 0 +array set ::redis::fd {} +array set ::redis::addr {} +array set ::redis::blocking {} +array set ::redis::deferred {} +array set ::redis::reconnect {} +array set ::redis::callback {} +array set ::redis::state {} ;# State in non-blocking reply reading +array set ::redis::statestack {} ;# Stack of states, for nested mbulks + +proc redis {{server 127.0.0.1} {port 6379} {defer 0}} { + set fd [socket $server $port] + fconfigure $fd -translation binary + set id [incr ::redis::id] + set ::redis::fd($id) $fd + set ::redis::addr($id) [list $server $port] + set ::redis::blocking($id) 1 + set ::redis::deferred($id) $defer + set ::redis::reconnect($id) 0 + ::redis::redis_reset_state $id + interp alias {} ::redis::redisHandle$id {} ::redis::__dispatch__ $id +} + +# This is a wrapper to the actual dispatching procedure that handles +# reconnection if needed. +proc ::redis::__dispatch__ {id method args} { + set errorcode [catch {::redis::__dispatch__raw__ $id $method $args} retval] + if {$errorcode && $::redis::reconnect($id) && $::redis::fd($id) eq {}} { + # Try again if the connection was lost. + # FIXME: we don't re-select the previously selected DB, nor we check + # if we are inside a transaction that needs to be re-issued from + # scratch. + set errorcode [catch {::redis::__dispatch__raw__ $id $method $args} retval] + } + return -code $errorcode $retval +} + +proc ::redis::__dispatch__raw__ {id method argv} { + set fd $::redis::fd($id) + + # Reconnect the link if needed. + if {$fd eq {}} { + lassign $::redis::addr($id) host port + set ::redis::fd($id) [socket $host $port] + fconfigure $::redis::fd($id) -translation binary + set fd $::redis::fd($id) + } + + set blocking $::redis::blocking($id) + set deferred $::redis::deferred($id) + if {$blocking == 0} { + if {[llength $argv] == 0} { + error "Please provide a callback in non-blocking mode" + } + set callback [lindex $argv end] + set argv [lrange $argv 0 end-1] + } + if {[info command ::redis::__method__$method] eq {}} { + set cmd "*[expr {[llength $argv]+1}]\r\n" + append cmd "$[string length $method]\r\n$method\r\n" + foreach a $argv { + append cmd "$[string length $a]\r\n$a\r\n" + } + ::redis::redis_write $fd $cmd + if {[catch {flush $fd}]} { + set ::redis::fd($id) {} + return -code error "I/O error reading reply" + } + + if {!$deferred} { + if {$blocking} { + ::redis::redis_read_reply $id $fd + } else { + # Every well formed reply read will pop an element from this + # list and use it as a callback. So pipelining is supported + # in non blocking mode. + lappend ::redis::callback($id) $callback + fileevent $fd readable [list ::redis::redis_readable $fd $id] + } + } + } else { + uplevel 1 [list ::redis::__method__$method $id $fd] $argv + } +} + +proc ::redis::__method__blocking {id fd val} { + set ::redis::blocking($id) $val + fconfigure $fd -blocking $val +} + +proc ::redis::__method__reconnect {id fd val} { + set ::redis::reconnect($id) $val +} + +proc ::redis::__method__read {id fd} { + ::redis::redis_read_reply $id $fd +} + +proc ::redis::__method__write {id fd buf} { + ::redis::redis_write $fd $buf +} + +proc ::redis::__method__flush {id fd} { + flush $fd +} + +proc ::redis::__method__close {id fd} { + catch {close $fd} + catch {unset ::redis::fd($id)} + catch {unset ::redis::addr($id)} + catch {unset ::redis::blocking($id)} + catch {unset ::redis::deferred($id)} + catch {unset ::redis::reconnect($id)} + catch {unset ::redis::state($id)} + catch {unset ::redis::statestack($id)} + catch {unset ::redis::callback($id)} + catch {interp alias {} ::redis::redisHandle$id {}} +} + +proc ::redis::__method__channel {id fd} { + return $fd +} + +proc ::redis::__method__deferred {id fd val} { + set ::redis::deferred($id) $val +} + +proc ::redis::redis_write {fd buf} { + puts -nonewline $fd $buf +} + +proc ::redis::redis_writenl {fd buf} { + redis_write $fd $buf + redis_write $fd "\r\n" + flush $fd +} + +proc ::redis::redis_readnl {fd len} { + set buf [read $fd $len] + read $fd 2 ; # discard CR LF + return $buf +} + +proc ::redis::redis_bulk_read {fd} { + set count [redis_read_line $fd] + if {$count == -1} return {} + set buf [redis_readnl $fd $count] + return $buf +} + +proc ::redis::redis_multi_bulk_read {id fd} { + set count [redis_read_line $fd] + if {$count == -1} return {} + set l {} + set err {} + for {set i 0} {$i < $count} {incr i} { + if {[catch { + lappend l [redis_read_reply $id $fd] + } e] && $err eq {}} { + set err $e + } + } + if {$err ne {}} {return -code error $err} + return $l +} + +proc ::redis::redis_read_line fd { + string trim [gets $fd] +} + +proc ::redis::redis_read_reply {id fd} { + set type [read $fd 1] + switch -exact -- $type { + : - + + {redis_read_line $fd} + - {return -code error [redis_read_line $fd]} + $ {redis_bulk_read $fd} + * {redis_multi_bulk_read $id $fd} + default { + if {$type eq {}} { + set ::redis::fd($id) {} + return -code error "I/O error reading reply" + } + return -code error "Bad protocol, '$type' as reply type byte" + } + } +} + +proc ::redis::redis_reset_state id { + set ::redis::state($id) [dict create buf {} mbulk -1 bulk -1 reply {}] + set ::redis::statestack($id) {} +} + +proc ::redis::redis_call_callback {id type reply} { + set cb [lindex $::redis::callback($id) 0] + set ::redis::callback($id) [lrange $::redis::callback($id) 1 end] + uplevel #0 $cb [list ::redis::redisHandle$id $type $reply] + ::redis::redis_reset_state $id +} + +# Read a reply in non-blocking mode. +proc ::redis::redis_readable {fd id} { + if {[eof $fd]} { + redis_call_callback $id eof {} + ::redis::__method__close $id $fd + return + } + if {[dict get $::redis::state($id) bulk] == -1} { + set line [gets $fd] + if {$line eq {}} return ;# No complete line available, return + switch -exact -- [string index $line 0] { + : - + + {redis_call_callback $id reply [string range $line 1 end-1]} + - {redis_call_callback $id err [string range $line 1 end-1]} + $ { + dict set ::redis::state($id) bulk \ + [expr [string range $line 1 end-1]+2] + if {[dict get $::redis::state($id) bulk] == 1} { + # We got a $-1, hack the state to play well with this. + dict set ::redis::state($id) bulk 2 + dict set ::redis::state($id) buf "\r\n" + ::redis::redis_readable $fd $id + } + } + * { + dict set ::redis::state($id) mbulk [string range $line 1 end-1] + # Handle *-1 + if {[dict get $::redis::state($id) mbulk] == -1} { + redis_call_callback $id reply {} + } + } + default { + redis_call_callback $id err \ + "Bad protocol, $type as reply type byte" + } + } + } else { + set totlen [dict get $::redis::state($id) bulk] + set buflen [string length [dict get $::redis::state($id) buf]] + set toread [expr {$totlen-$buflen}] + set data [read $fd $toread] + set nread [string length $data] + dict append ::redis::state($id) buf $data + # Check if we read a complete bulk reply + if {[string length [dict get $::redis::state($id) buf]] == + [dict get $::redis::state($id) bulk]} { + if {[dict get $::redis::state($id) mbulk] == -1} { + redis_call_callback $id reply \ + [string range [dict get $::redis::state($id) buf] 0 end-2] + } else { + dict with ::redis::state($id) { + lappend reply [string range $buf 0 end-2] + incr mbulk -1 + set bulk -1 + } + if {[dict get $::redis::state($id) mbulk] == 0} { + redis_call_callback $id reply \ + [dict get $::redis::state($id) reply] + } + } + } + } +} diff --git a/tools/pika_migrate/tests/support/server.tcl b/tools/pika_migrate/tests/support/server.tcl new file mode 100644 index 0000000000..c7777fe5d3 --- /dev/null +++ b/tools/pika_migrate/tests/support/server.tcl @@ -0,0 +1,337 @@ +set ::global_overrides {} +set ::tags {} +set ::valgrind_errors {} + +proc start_server_error {config_file error} { + set err {} + append err "Cant' start the Redis server\n" + append err "CONFIGURATION:" + append err [exec cat $config_file] + append err "\nERROR:" + append err [string trim $error] + send_data_packet $::test_server_fd err $err +} + +proc check_valgrind_errors stderr { + set fd [open $stderr] + set buf [read $fd] + close $fd + + if {[regexp -- { at 0x} $buf] || + (![regexp -- {definitely lost: 0 bytes} $buf] && + ![regexp -- {no leaks are possible} $buf])} { + send_data_packet $::test_server_fd err "Valgrind error: $buf\n" + } +} + +proc kill_server config { + # nothing to kill when running against external server + if {$::external} return + + # nevermind if its already dead + if {![is_alive $config]} { return } + set pid [dict get $config pid] + + # check for leaks + if {![dict exists $config "skipleaks"]} { + catch { + if {[string match {*Darwin*} [exec uname -a]]} { + tags {"leaks"} { + test "Check for memory leaks (pid $pid)" { + set output {0 leaks} + catch {exec leaks $pid} output + if {[string match {*process does not exist*} $output] || + [string match {*cannot examine*} $output]} { + # In a few tests we kill the server process. + set output "0 leaks" + } + set output + } {*0 leaks*} + } + } + } + } + + # kill server and wait for the process to be totally exited + catch {exec kill $pid} + while {[is_alive $config]} { + incr wait 10 + + if {$wait >= 5000} { + puts "Forcing process $pid to exit..." + catch {exec kill -KILL $pid} + } elseif {$wait % 1000 == 0} { + puts "Waiting for process $pid to exit..." + } + after 10 + } + + # Check valgrind errors if needed + if {$::valgrind} { + check_valgrind_errors [dict get $config stderr] + } + + # Remove this pid from the set of active pids in the test server. + send_data_packet $::test_server_fd server-killed $pid +} + +proc is_alive config { + set pid [dict get $config pid] + if {[catch {exec ps -p $pid} err]} { + return 0 + } else { + return 1 + } +} + +proc ping_server {host port} { + set retval 0 + if {[catch { + set fd [socket $host $port] + fconfigure $fd -translation binary + puts $fd "PING\r\n" + flush $fd + set reply [gets $fd] + if {[string range $reply 0 0] eq {+} || + [string range $reply 0 0] eq {-}} { + set retval 1 + } + close $fd + } e]} { + if {$::verbose} { + puts -nonewline "." + } + } else { + if {$::verbose} { + puts -nonewline "ok" + } + } + return $retval +} + +# Return 1 if the server at the specified addr is reachable by PING, otherwise +# returns 0. Performs a try every 50 milliseconds for the specified number +# of retries. +proc server_is_up {host port retrynum} { + after 10 ;# Use a small delay to make likely a first-try success. + set retval 0 + while {[incr retrynum -1]} { + if {[catch {ping_server $host $port} ping]} { + set ping 0 + } + if {$ping} {return 1} + after 50 + } + return 0 +} + +# doesn't really belong here, but highly coupled to code in start_server +proc tags {tags code} { + set ::tags [concat $::tags $tags] + uplevel 1 $code + set ::tags [lrange $::tags 0 end-[llength $tags]] +} + +proc start_server {options {code undefined}} { + # If we are running against an external server, we just push the + # host/port pair in the stack the first time + if {$::external} { + if {[llength $::servers] == 0} { + set srv {} + dict set srv "host" $::host + dict set srv "port" $::port + set client [redis $::host $::port] + dict set srv "client" $client + $client select 9 + + # append the server to the stack + lappend ::servers $srv + } + uplevel 1 $code + return + } + + # setup defaults + set baseconfig "default.conf" + set overrides {} + set tags {} + + # parse options + foreach {option value} $options { + switch $option { + "config" { + set baseconfig $value } + "overrides" { + set overrides $value } + "tags" { + set tags $value + set ::tags [concat $::tags $value] } + default { + error "Unknown option $option" } + } + } + + set data [split [exec cat "tests/assets/$baseconfig"] "\n"] + set config {} + foreach line $data { + if {[string length $line] > 0 && [string index $line 0] ne "#"} { + set elements [split $line " "] + set directive [lrange $elements 0 0] + set arguments [lrange $elements 1 end] + dict set config $directive $arguments + } + } + + # use a different directory every time a server is started + dict set config dir [tmpdir server] + + # start every server on a different port + set ::port [find_available_port [expr {$::port+1}]] + dict set config port $::port + + # apply overrides from global space and arguments + foreach {directive arguments} [concat $::global_overrides $overrides] { + dict set config $directive $arguments + } + + # write new configuration to temporary file + set config_file [tmpfile redis.conf] + set fp [open $config_file w+] + foreach directive [dict keys $config] { + if {$directive == "port"} { + puts -nonewline $fp "$directive : " + puts $fp [dict get $config $directive] + } elseif {$directive == "requirepass"} { + puts $fp "$directive :" + } elseif {$directive == "dump_prefix"} { + puts $fp "$directive :" + } else { + puts -nonewline $fp "$directive " + puts $fp [dict get $config $directive] + } + } + close $fp + + set stdout [format "%s/%s" [dict get $config "dir"] "stdout"] + set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] + + if {$::valgrind} { + set pid [exec valgrind --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr &] + } else { + set pid [exec src/redis-server -c $config_file > $stdout 2> $stderr &] + #set pid [exec src/redis-server $config_file > $stdout 2> $stderr &] + } + + puts "Starting ---- " + + # Tell the test server about this new instance. + send_data_packet $::test_server_fd server-spawned $pid + + # check that the server actually started + # ugly but tries to be as fast as possible... + if {$::valgrind} {set retrynum 1000} else {set retrynum 100} + + if {$::verbose} { + puts -nonewline "=== ($tags) Starting server ${::host}:${::port} " + } + + if {$code ne "undefined"} { + set serverisup [server_is_up $::host $::port $retrynum] + } else { + set serverisup 1 + } + + if {$::verbose} { + puts "" + } + + if {!$serverisup} { + set err {} + append err [exec cat $stdout] "\n" [exec cat $stderr] + start_server_error $config_file $err + return + } + + puts "Before Wait" + # Wait for actual startup + #while {![info exists _pid]} { + # regexp {PID:\s(\d+)} [exec cat $stdout] _ _pid + # after 100 + #} + puts "After Wait" + + # setup properties to be able to initialize a client object + set host $::host + set port $::port + if {[dict exists $config bind]} { set host [dict get $config bind] } + if {[dict exists $config port]} { set port [dict get $config port] } + + # setup config dict + dict set srv "config_file" $config_file + dict set srv "config" $config + dict set srv "pid" $pid + dict set srv "host" $host + dict set srv "port" $port + dict set srv "stdout" $stdout + dict set srv "stderr" $stderr + + # if a block of code is supplied, we wait for the server to become + # available, create a client object and kill the server afterwards + if {$code ne "undefined"} { + set line [exec head -n1 $stdout] + if {[string match {*already in use*} $line]} { + error_and_quit $config_file $line + } + + while 1 { + # check that the server actually started and is ready for connections + if {[exec grep "going to start" | wc -l < $stderr] > 0} { + break + } + puts "Fuck YYB" + after 10 + } + + # append the server to the stack + lappend ::servers $srv + + # connect client (after server dict is put on the stack) + reconnect + + # execute provided block + set num_tests $::num_tests + if {[catch { uplevel 1 $code } error]} { + set backtrace $::errorInfo + + # Kill the server without checking for leaks + dict set srv "skipleaks" 1 + kill_server $srv + + # Print warnings from log + puts [format "\nLogged warnings (pid %d):" [dict get $srv "pid"]] + set warnings [warnings_from_file [dict get $srv "stdout"]] + if {[string length $warnings] > 0} { + puts "$warnings" + } else { + puts "(none)" + } + puts "" + + error $error $backtrace + } + + # Don't do the leak check when no tests were run + if {$num_tests == $::num_tests} { + dict set srv "skipleaks" 1 + } + + # pop the server object + set ::servers [lrange $::servers 0 end-1] + + set ::tags [lrange $::tags 0 end-[llength $tags]] + kill_server $srv + } else { + set ::tags [lrange $::tags 0 end-[llength $tags]] + set _ $srv + } +} diff --git a/tools/pika_migrate/tests/support/test.tcl b/tools/pika_migrate/tests/support/test.tcl new file mode 100644 index 0000000000..7d390cc47a --- /dev/null +++ b/tools/pika_migrate/tests/support/test.tcl @@ -0,0 +1,130 @@ +set ::num_tests 0 +set ::num_passed 0 +set ::num_failed 0 +set ::tests_failed {} + +proc fail {msg} { + error "assertion:$msg" +} + +proc assert {condition} { + if {![uplevel 1 [list expr $condition]]} { + error "assertion:Expected condition '$condition' to be true ([uplevel 1 [list subst -nocommands $condition]])" + } +} + +proc assert_match {pattern value} { + if {![string match $pattern $value]} { + error "assertion:Expected '$value' to match '$pattern'" + } +} + +proc assert_equal {expected value} { + if {$expected ne $value} { + error "assertion:Expected '$value' to be equal to '$expected'" + } +} + +proc assert_error {pattern code} { + if {[catch {uplevel 1 $code} error]} { + assert_match $pattern $error + } else { + error "assertion:Expected an error but nothing was caught" + } +} + +proc assert_encoding {enc key} { + # Swapped out values don't have an encoding, so make sure that + # the value is swapped in before checking the encoding. + set dbg [r debug object $key] + while {[string match "* swapped at:*" $dbg]} { + r debug swapin $key + set dbg [r debug object $key] + } + assert_match "* encoding:$enc *" $dbg +} + +proc assert_type {type key} { + assert_equal $type [r type $key] +} + +# Wait for the specified condition to be true, with the specified number of +# max retries and delay between retries. Otherwise the 'elsescript' is +# executed. +proc wait_for_condition {maxtries delay e _else_ elsescript} { + while {[incr maxtries -1] >= 0} { + set errcode [catch {uplevel 1 [list expr $e]} result] + if {$errcode == 0} { + if {$result} break + } else { + return -code $errcode $result + } + after $delay + } + if {$maxtries == -1} { + set errcode [catch [uplevel 1 $elsescript] result] + return -code $errcode $result + } +} + +proc test {name code {okpattern undefined}} { + # abort if tagged with a tag to deny + foreach tag $::denytags { + if {[lsearch $::tags $tag] >= 0} { + return + } + } + + # check if tagged with at least 1 tag to allow when there *is* a list + # of tags to allow, because default policy is to run everything + if {[llength $::allowtags] > 0} { + set matched 0 + foreach tag $::allowtags { + if {[lsearch $::tags $tag] >= 0} { + incr matched + } + } + if {$matched < 1} { + return + } + } + + incr ::num_tests + set details {} + lappend details "$name in $::curfile" + + send_data_packet $::test_server_fd testing $name + + if {[catch {set retval [uplevel 1 $code]} error]} { + if {[string match "assertion:*" $error]} { + set msg [string range $error 10 end] + lappend details $msg + lappend ::tests_failed $details + + incr ::num_failed + send_data_packet $::test_server_fd err [join $details "\n"] + } else { + # Re-raise, let handler up the stack take care of this. + error $error $::errorInfo + } + } else { + if {$okpattern eq "undefined" || $okpattern eq $retval || [string match $okpattern $retval]} { + incr ::num_passed + send_data_packet $::test_server_fd ok $name + } else { + set msg "Expected '$okpattern' to equal or match '$retval'" + lappend details $msg + lappend ::tests_failed $details + + incr ::num_failed + send_data_packet $::test_server_fd err [join $details "\n"] + } + } + + if {$::traceleaks} { + set output [exec leaks redis-server] + if {![string match {*0 leaks*} $output]} { + send_data_packet $::test_server_fd err "Detected a memory leak in test '$name': $output" + } + } +} diff --git a/tools/pika_migrate/tests/support/tmpfile.tcl b/tools/pika_migrate/tests/support/tmpfile.tcl new file mode 100644 index 0000000000..809f587306 --- /dev/null +++ b/tools/pika_migrate/tests/support/tmpfile.tcl @@ -0,0 +1,15 @@ +set ::tmpcounter 0 +set ::tmproot "./tests/tmp" +file mkdir $::tmproot + +# returns a dirname unique to this process to write to +proc tmpdir {basename} { + set dir [file join $::tmproot $basename.[pid].[incr ::tmpcounter]] + file mkdir $dir + set _ $dir +} + +# return a filename unique to this process to write to +proc tmpfile {basename} { + file join $::tmproot $basename.[pid].[incr ::tmpcounter] +} diff --git a/tools/pika_migrate/tests/support/util.tcl b/tools/pika_migrate/tests/support/util.tcl new file mode 100644 index 0000000000..cd5b9b511f --- /dev/null +++ b/tools/pika_migrate/tests/support/util.tcl @@ -0,0 +1,371 @@ +proc randstring {min max {type binary}} { + set len [expr {$min+int(rand()*($max-$min+1))}] + set output {} + if {$type eq {binary}} { + set minval 0 + set maxval 255 + } elseif {$type eq {alpha}} { + set minval 48 + set maxval 122 + } elseif {$type eq {compr}} { + set minval 48 + set maxval 52 + } + while {$len} { + append output [format "%c" [expr {$minval+int(rand()*($maxval-$minval+1))}]] + incr len -1 + } + return $output +} + +# Useful for some test +proc zlistAlikeSort {a b} { + if {[lindex $a 0] > [lindex $b 0]} {return 1} + if {[lindex $a 0] < [lindex $b 0]} {return -1} + string compare [lindex $a 1] [lindex $b 1] +} + +# Return all log lines starting with the first line that contains a warning. +# Generally, this will be an assertion error with a stack trace. +proc warnings_from_file {filename} { + set lines [split [exec cat $filename] "\n"] + set matched 0 + set logall 0 + set result {} + foreach line $lines { + if {[string match {*REDIS BUG REPORT START*} $line]} { + set logall 1 + } + if {[regexp {^\[\d+\]\s+\d+\s+\w+\s+\d{2}:\d{2}:\d{2} \#} $line]} { + set matched 1 + } + if {$logall || $matched} { + lappend result $line + } + } + join $result "\n" +} + +# Return value for INFO property +proc status {r property} { + if {[regexp "\r\n$property:(.*?)\r\n" [{*}$r info] _ value]} { + set _ $value + } +} + +proc waitForBgsave r { + while 1 { + if {[status r rdb_bgsave_in_progress] eq 1} { + if {$::verbose} { + puts -nonewline "\nWaiting for background save to finish... " + flush stdout + } + after 1000 + } else { + break + } + } +} + +proc waitForBgrewriteaof r { + while 1 { + if {[status r aof_rewrite_in_progress] eq 1} { + if {$::verbose} { + puts -nonewline "\nWaiting for background AOF rewrite to finish... " + flush stdout + } + after 1000 + } else { + break + } + } +} + +proc wait_for_sync r { + while 1 { + if {[status $r master_link_status] eq "down"} { + after 10 + } else { + break + } + } +} + +# Random integer between 0 and max (excluded). +proc randomInt {max} { + expr {int(rand()*$max)} +} + +# Random signed integer between -max and max (both extremes excluded). +proc randomSignedInt {max} { + set i [randomInt $max] + if {rand() > 0.5} { + set i -$i + } + return $i +} + +proc randpath args { + set path [expr {int(rand()*[llength $args])}] + uplevel 1 [lindex $args $path] +} + +proc randomValue {} { + randpath { + # Small enough to likely collide + randomSignedInt 1000 + } { + # 32 bit compressible signed/unsigned + randpath {randomSignedInt 2000000000} {randomSignedInt 4000000000} + } { + # 64 bit + randpath {randomSignedInt 1000000000000} + } { + # Random string + randpath {randstring 0 256 alpha} \ + {randstring 0 256 compr} \ + {randstring 0 256 binary} + } +} + +proc randomKey {} { + randpath { + # Small enough to likely collide + randomInt 1000 + } { + # 32 bit compressible signed/unsigned + randpath {randomInt 2000000000} {randomInt 4000000000} + } { + # 64 bit + randpath {randomInt 1000000000000} + } { + # Random string + randpath {randstring 1 256 alpha} \ + {randstring 1 256 compr} + } +} + +proc findKeyWithType {r type} { + for {set j 0} {$j < 20} {incr j} { + set k [{*}$r randomkey] + if {$k eq {}} { + return {} + } + if {[{*}$r type $k] eq $type} { + return $k + } + } + return {} +} + +proc createComplexDataset {r ops {opt {}}} { + for {set j 0} {$j < $ops} {incr j} { + set k [randomKey] + set k2 [randomKey] + set f [randomValue] + set v [randomValue] + + if {[lsearch -exact $opt useexpire] != -1} { + if {rand() < 0.1} { + {*}$r expire [randomKey] [randomInt 2] + } + } + + randpath { + set d [expr {rand()}] + } { + set d [expr {rand()}] + } { + set d [expr {rand()}] + } { + set d [expr {rand()}] + } { + set d [expr {rand()}] + } { + randpath {set d +inf} {set d -inf} + } + set t [{*}$r type $k] + + if {$t eq {none}} { + randpath { + {*}$r set $k $v + } { + {*}$r lpush $k $v + } { + {*}$r sadd $k $v + } { + {*}$r zadd $k $d $v + } { + {*}$r hset $k $f $v + } { + {*}$r del $k + } + set t [{*}$r type $k] + } + + switch $t { + {string} { + # Nothing to do + } + {list} { + randpath {{*}$r lpush $k $v} \ + {{*}$r rpush $k $v} \ + {{*}$r lrem $k 0 $v} \ + {{*}$r rpop $k} \ + {{*}$r lpop $k} + } + {set} { + randpath {{*}$r sadd $k $v} \ + {{*}$r srem $k $v} \ + { + set otherset [findKeyWithType {*}$r set] + if {$otherset ne {}} { + randpath { + {*}$r sunionstore $k2 $k $otherset + } { + {*}$r sinterstore $k2 $k $otherset + } { + {*}$r sdiffstore $k2 $k $otherset + } + } + } + } + {zset} { + randpath {{*}$r zadd $k $d $v} \ + {{*}$r zrem $k $v} \ + { + set otherzset [findKeyWithType {*}$r zset] + if {$otherzset ne {}} { + randpath { + {*}$r zunionstore $k2 2 $k $otherzset + } { + {*}$r zinterstore $k2 2 $k $otherzset + } + } + } + } + {hash} { + randpath {{*}$r hset $k $f $v} \ + {{*}$r hdel $k $f} + } + } + } +} + +proc formatCommand {args} { + set cmd "*[llength $args]\r\n" + foreach a $args { + append cmd "$[string length $a]\r\n$a\r\n" + } + set _ $cmd +} + +proc csvdump r { + set o {} + foreach k [lsort [{*}$r keys *]] { + set type [{*}$r type $k] + append o [csvstring $k] , [csvstring $type] , + switch $type { + string { + append o [csvstring [{*}$r get $k]] "\n" + } + list { + foreach e [{*}$r lrange $k 0 -1] { + append o [csvstring $e] , + } + append o "\n" + } + set { + foreach e [lsort [{*}$r smembers $k]] { + append o [csvstring $e] , + } + append o "\n" + } + zset { + foreach e [{*}$r zrange $k 0 -1 withscores] { + append o [csvstring $e] , + } + append o "\n" + } + hash { + set fields [{*}$r hgetall $k] + set newfields {} + foreach {k v} $fields { + lappend newfields [list $k $v] + } + set fields [lsort -index 0 $newfields] + foreach kv $fields { + append o [csvstring [lindex $kv 0]] , + append o [csvstring [lindex $kv 1]] , + } + append o "\n" + } + } + } + return $o +} + +proc csvstring s { + return "\"$s\"" +} + +proc roundFloat f { + format "%.10g" $f +} + +proc find_available_port start { + for {set j $start} {$j < $start+1024} {incr j} { + if {[catch { + set fd [socket 127.0.0.1 $j] + }]} { + return $j + } else { + close $fd + } + } + if {$j == $start+1024} { + error "Can't find a non busy port in the $start-[expr {$start+1023}] range." + } +} + +# Test if TERM looks like to support colors +proc color_term {} { + expr {[info exists ::env(TERM)] && [string match *xterm* $::env(TERM)]} +} + +proc colorstr {color str} { + if {[color_term]} { + set b 0 + if {[string range $color 0 4] eq {bold-}} { + set b 1 + set color [string range $color 5 end] + } + switch $color { + red {set colorcode {31}} + green {set colorcode {32}} + yellow {set colorcode {33}} + blue {set colorcode {34}} + magenta {set colorcode {35}} + cyan {set colorcode {36}} + white {set colorcode {37}} + default {set colorcode {37}} + } + if {$colorcode ne {}} { + return "\033\[$b;${colorcode};49m$str\033\[0m" + } + } else { + return $str + } +} + +# Execute a background process writing random data for the specified number +# of seconds to the specified Redis instance. +proc start_write_load {host port seconds} { + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds & +} + +# Stop a process generating write load executed with start_write_load. +proc stop_write_load {handle} { + catch {exec /bin/kill -9 $handle} +} diff --git a/tools/pika_migrate/tests/test_helper.tcl b/tools/pika_migrate/tests/test_helper.tcl new file mode 100644 index 0000000000..d1ebde1c48 --- /dev/null +++ b/tools/pika_migrate/tests/test_helper.tcl @@ -0,0 +1,545 @@ +# Redis test suite. Copyright (C) 2009 Salvatore Sanfilippo antirez@gmail.com +# This software is released under the BSD License. See the COPYING file for +# more information. + +package require Tcl 8.5 + +set tcl_precision 17 +source tests/support/redis.tcl +source tests/support/server.tcl +source tests/support/tmpfile.tcl +source tests/support/test.tcl +source tests/support/util.tcl + +set ::all_tests { + unit/printver + unit/auth + unit/protocol + unit/basic + unit/scan + unit/type/list + unit/type/list-2 + unit/type/list-3 + unit/type/set + unit/type/zset + unit/type/hash + unit/sort + unit/expire + unit/other + unit/multi + unit/quit + unit/aofrw + integration/replication + integration/replication-2 + integration/replication-3 + integration/replication-4 + integration/replication-psync + integration/aof + integration/rdb + integration/convert-zipmap-hash-on-load + unit/pubsub + unit/slowlog + unit/scripting + unit/maxmemory + unit/introspection + unit/limits + unit/obuf-limits + unit/dump + unit/bitops + unit/memefficiency + unit/hyperloglog +} +# Index to the next test to run in the ::all_tests list. +set ::next_test 0 + +set ::host 127.0.0.1 +set ::port 21111 +set ::traceleaks 0 +set ::valgrind 0 +set ::verbose 0 +set ::quiet 0 +set ::denytags {} +set ::allowtags {} +set ::external 0; # If "1" this means, we are running against external instance +set ::file ""; # If set, runs only the tests in this comma separated list +set ::curfile ""; # Hold the filename of the current suite +set ::accurate 0; # If true runs fuzz tests with more iterations +set ::force_failure 0 +set ::timeout 600; # 10 minutes without progresses will quit the test. +set ::last_progress [clock seconds] +set ::active_servers {} ; # Pids of active Redis instances. + +# Set to 1 when we are running in client mode. The Redis test uses a +# server-client model to run tests simultaneously. The server instance +# runs the specified number of client instances that will actually run tests. +# The server is responsible of showing the result to the user, and exit with +# the appropriate exit code depending on the test outcome. +set ::client 0 +set ::numclients 16 + +proc execute_tests name { + set path "tests/$name.tcl" + set ::curfile $path + source $path + send_data_packet $::test_server_fd done "$name" +} + +# Setup a list to hold a stack of server configs. When calls to start_server +# are nested, use "srv 0 pid" to get the pid of the inner server. To access +# outer servers, use "srv -1 pid" etcetera. +set ::servers {} +proc srv {args} { + set level 0 + if {[string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set property [lindex $args 1] + } else { + set property [lindex $args 0] + } + set srv [lindex $::servers end+$level] + dict get $srv $property +} + +# Provide easy access to the client for the inner server. It's possible to +# prepend the argument list with a negative level to access clients for +# servers running in outer blocks. +proc r {args} { + set level 0 + if {[string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set args [lrange $args 1 end] + } + [srv $level "client"] {*}$args +} + +proc reconnect {args} { + set level [lindex $args 0] + if {[string length $level] == 0 || ![string is integer $level]} { + set level 0 + } + + set srv [lindex $::servers end+$level] + set host [dict get $srv "host"] + set port [dict get $srv "port"] + set config [dict get $srv "config"] + set client [redis $host $port] + dict set srv "client" $client + + # select the right db when we don't have to authenticate + if {![dict exists $config "requirepass"]} { + $client select 9 + } + + # re-set $srv in the servers list + lset ::servers end+$level $srv +} + +proc redis_deferring_client {args} { + set level 0 + if {[llength $args] > 0 && [string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set args [lrange $args 1 end] + } + + # create client that defers reading reply + set client [redis [srv $level "host"] [srv $level "port"] 1] + + # select the right db and read the response (OK) + $client select 9 + $client read + return $client +} + +# Provide easy access to INFO properties. Same semantic as "proc r". +proc s {args} { + set level 0 + if {[string is integer [lindex $args 0]]} { + set level [lindex $args 0] + set args [lrange $args 1 end] + } + status [srv $level "client"] [lindex $args 0] +} + +proc cleanup {} { + if {!$::quiet} {puts -nonewline "Cleanup: may take some time... "} + flush stdout + catch {exec rm -rf {*}[glob tests/tmp/redis.conf.*]} + catch {exec rm -rf {*}[glob tests/tmp/server.*]} + if {!$::quiet} {puts "OK"} +} + +proc test_server_main {} { + cleanup + set tclsh [info nameofexecutable] + # Open a listening socket, trying different ports in order to find a + # non busy one. + set port [find_available_port 11111] + if {!$::quiet} { + puts "Starting test server at port $port" + } + socket -server accept_test_clients -myaddr 127.0.0.1 $port + + # Start the client instances + set ::clients_pids {} + set start_port [expr {$::port+100}] + for {set j 0} {$j < $::numclients} {incr j} { + set start_port [find_available_port $start_port] + set p [exec $tclsh [info script] {*}$::argv \ + --client $port --port $start_port &] + lappend ::clients_pids $p + incr start_port 10 + } + + # Setup global state for the test server + set ::idle_clients {} + set ::active_clients {} + array set ::active_clients_task {} + array set ::clients_start_time {} + set ::clients_time_history {} + set ::failed_tests {} + + # Enter the event loop to handle clients I/O + after 100 test_server_cron + vwait forever +} + +# This function gets called 10 times per second. +proc test_server_cron {} { + set elapsed [expr {[clock seconds]-$::last_progress}] + + if {$elapsed > $::timeout} { + set err "\[[colorstr red TIMEOUT]\]: clients state report follows." + puts $err + show_clients_state + kill_clients + force_kill_all_servers + the_end + } + + after 100 test_server_cron +} + +proc accept_test_clients {fd addr port} { + fconfigure $fd -encoding binary + fileevent $fd readable [list read_from_test_client $fd] +} + +# This is the readable handler of our test server. Clients send us messages +# in the form of a status code such and additional data. Supported +# status types are: +# +# ready: the client is ready to execute the command. Only sent at client +# startup. The server will queue the client FD in the list of idle +# clients. +# testing: just used to signal that a given test started. +# ok: a test was executed with success. +# err: a test was executed with an error. +# exception: there was a runtime exception while executing the test. +# done: all the specified test file was processed, this test client is +# ready to accept a new task. +proc read_from_test_client fd { + set bytes [gets $fd] + set payload [read $fd $bytes] + foreach {status data} $payload break + set ::last_progress [clock seconds] + + if {$status eq {ready}} { + if {!$::quiet} { + puts "\[$status\]: $data" + } + signal_idle_client $fd + } elseif {$status eq {done}} { + set elapsed [expr {[clock seconds]-$::clients_start_time($fd)}] + set all_tests_count [llength $::all_tests] + set running_tests_count [expr {[llength $::active_clients]-1}] + set completed_tests_count [expr {$::next_test-$running_tests_count}] + puts "\[$completed_tests_count/$all_tests_count [colorstr yellow $status]\]: $data ($elapsed seconds)" + lappend ::clients_time_history $elapsed $data + signal_idle_client $fd + set ::active_clients_task($fd) DONE + } elseif {$status eq {ok}} { + if {!$::quiet} { + puts "\[[colorstr green $status]\]: $data" + } + set ::active_clients_task($fd) "(OK) $data" + } elseif {$status eq {err}} { + set err "\[[colorstr red $status]\]: $data" + puts $err + lappend ::failed_tests $err + set ::active_clients_task($fd) "(ERR) $data" + } elseif {$status eq {exception}} { + puts "\[[colorstr red $status]\]: $data" + kill_clients + force_kill_all_servers + exit 1 + } elseif {$status eq {testing}} { + set ::active_clients_task($fd) "(IN PROGRESS) $data" + } elseif {$status eq {server-spawned}} { + lappend ::active_servers $data + } elseif {$status eq {server-killed}} { + set ::active_servers [lsearch -all -inline -not -exact $::active_servers $data] + } else { + if {!$::quiet} { + puts "\[$status\]: $data" + } + } +} + +proc show_clients_state {} { + # The following loop is only useful for debugging tests that may + # enter an infinite loop. Commented out normally. + foreach x $::active_clients { + if {[info exist ::active_clients_task($x)]} { + puts "$x => $::active_clients_task($x)" + } else { + puts "$x => ???" + } + } +} + +proc kill_clients {} { + foreach p $::clients_pids { + catch {exec kill $p} + } +} + +proc force_kill_all_servers {} { + foreach p $::active_servers { + puts "Killing still running Redis server $p" + catch {exec kill -9 $p} + } +} + +# A new client is idle. Remove it from the list of active clients and +# if there are still test units to run, launch them. +proc signal_idle_client fd { + # Remove this fd from the list of active clients. + set ::active_clients \ + [lsearch -all -inline -not -exact $::active_clients $fd] + + if 0 {show_clients_state} + + # New unit to process? + if {$::next_test != [llength $::all_tests]} { + if {!$::quiet} { + puts [colorstr bold-white "Testing [lindex $::all_tests $::next_test]"] + set ::active_clients_task($fd) "ASSIGNED: $fd ([lindex $::all_tests $::next_test])" + } + set ::clients_start_time($fd) [clock seconds] + send_data_packet $fd run [lindex $::all_tests $::next_test] + lappend ::active_clients $fd + incr ::next_test + } else { + lappend ::idle_clients $fd + if {[llength $::active_clients] == 0} { + the_end + } + } +} + +# The the_end function gets called when all the test units were already +# executed, so the test finished. +proc the_end {} { + # TODO: print the status, exit with the rigth exit code. + puts "\n The End\n" + puts "Execution time of different units:" + foreach {time name} $::clients_time_history { + puts " $time seconds - $name" + } + if {[llength $::failed_tests]} { + puts "\n[colorstr bold-red {!!! WARNING}] The following tests failed:\n" + foreach failed $::failed_tests { + puts "*** $failed" + } + cleanup + exit 1 + } else { + puts "\n[colorstr bold-white {\o/}] [colorstr bold-green {All tests passed without errors!}]\n" + cleanup + exit 0 + } +} + +# The client is not even driven (the test server is instead) as we just need +# to read the command, execute, reply... all this in a loop. +proc test_client_main server_port { + set ::test_server_fd [socket localhost $server_port] + fconfigure $::test_server_fd -encoding binary + send_data_packet $::test_server_fd ready [pid] + while 1 { + set bytes [gets $::test_server_fd] + set payload [read $::test_server_fd $bytes] + foreach {cmd data} $payload break + if {$cmd eq {run}} { + execute_tests $data + } else { + error "Unknown test client command: $cmd" + } + } +} + +proc send_data_packet {fd status data} { + set payload [list $status $data] + puts $fd [string length $payload] + puts -nonewline $fd $payload + flush $fd +} + +proc print_help_screen {} { + puts [join { + "--valgrind Run the test over valgrind." + "--accurate Run slow randomized tests for more iterations." + "--quiet Don't show individual tests." + "--single Just execute the specified unit (see next option)." + "--list-tests List all the available test units." + "--clients Number of test clients (default 16)." + "--timeout Test timeout in seconds (default 10 min)." + "--force-failure Force the execution of a test that always fails." + "--help Print this help screen." + } "\n"] +} + +# parse arguments +for {set j 0} {$j < [llength $argv]} {incr j} { + set opt [lindex $argv $j] + set arg [lindex $argv [expr $j+1]] + if {$opt eq {--tags}} { + foreach tag $arg { + if {[string index $tag 0] eq "-"} { + lappend ::denytags [string range $tag 1 end] + } else { + lappend ::allowtags $tag + } + } + incr j + } elseif {$opt eq {--valgrind}} { + set ::valgrind 1 + } elseif {$opt eq {--quiet}} { + set ::quiet 1 + } elseif {$opt eq {--host}} { + set ::external 1 + set ::host $arg + incr j + } elseif {$opt eq {--port}} { + set ::port $arg + incr j + } elseif {$opt eq {--accurate}} { + set ::accurate 1 + } elseif {$opt eq {--force-failure}} { + set ::force_failure 1 + } elseif {$opt eq {--single}} { + set ::all_tests $arg + incr j + } elseif {$opt eq {--list-tests}} { + foreach t $::all_tests { + puts $t + } + exit 0 + } elseif {$opt eq {--client}} { + set ::client 1 + set ::test_server_port $arg + incr j + } elseif {$opt eq {--clients}} { + set ::numclients $arg + incr j + } elseif {$opt eq {--timeout}} { + set ::timeout $arg + incr j + } elseif {$opt eq {--help}} { + print_help_screen + exit 0 + } else { + puts "Wrong argument: $opt" + exit 1 + } +} + +proc attach_to_replication_stream {} { + set s [socket [srv 0 "host"] [srv 0 "port"]] + fconfigure $s -translation binary + puts -nonewline $s "SYNC\r\n" + flush $s + + # Get the count + set count [gets $s] + set prefix [string range $count 0 0] + if {$prefix ne {$}} { + error "attach_to_replication_stream error. Received '$count' as count." + } + set count [string range $count 1 end] + + # Consume the bulk payload + while {$count} { + set buf [read $s $count] + set count [expr {$count-[string length $buf]}] + } + return $s +} + +proc read_from_replication_stream {s} { + fconfigure $s -blocking 0 + set attempt 0 + while {[gets $s count] == -1} { + if {[incr attempt] == 10} return "" + after 100 + } + fconfigure $s -blocking 1 + set count [string range $count 1 end] + + # Return a list of arguments for the command. + set res {} + for {set j 0} {$j < $count} {incr j} { + read $s 1 + set arg [::redis::redis_bulk_read $s] + if {$j == 0} {set arg [string tolower $arg]} + lappend res $arg + } + return $res +} + +proc assert_replication_stream {s patterns} { + for {set j 0} {$j < [llength $patterns]} {incr j} { + assert_match [lindex $patterns $j] [read_from_replication_stream $s] + } +} + +proc close_replication_stream {s} { + close $s +} + +# With the parallel test running multiple Redis instances at the same time +# we need a fast enough computer, otherwise a lot of tests may generate +# false positives. +# If the computer is too slow we revert the sequential test without any +# parallelism, that is, clients == 1. +proc is_a_slow_computer {} { + set start [clock milliseconds] + for {set j 0} {$j < 1000000} {incr j} {} + set elapsed [expr [clock milliseconds]-$start] + expr {$elapsed > 200} +} + +if {$::client} { + if {[catch { test_client_main $::test_server_port } err]} { + set estr "Executing test client: $err.\n$::errorInfo" + if {[catch {send_data_packet $::test_server_fd exception $estr}]} { + puts $estr + } + exit 1 + } +} else { + if {[is_a_slow_computer]} { + puts "** SLOW COMPUTER ** Using a single client to avoid false positives." + set ::numclients 1 + } + + if {[catch { test_server_main } err]} { + if {[string length $err] > 0} { + # only display error when not generated by the test suite + if {$err ne "exception"} { + puts $::errorInfo + } + exit 1 + } + } +} diff --git a/tools/pika_migrate/tests/unit/aofrw.tcl b/tools/pika_migrate/tests/unit/aofrw.tcl new file mode 100644 index 0000000000..a2d74168f3 --- /dev/null +++ b/tools/pika_migrate/tests/unit/aofrw.tcl @@ -0,0 +1,210 @@ +start_server {tags {"aofrw"}} { + # Enable the AOF + r config set appendonly yes + r config set auto-aof-rewrite-percentage 0 ; # Disable auto-rewrite. + waitForBgrewriteaof r + + test {AOF rewrite during write load} { + # Start a write load for 10 seconds + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set load_handle0 [start_write_load $master_host $master_port 10] + set load_handle1 [start_write_load $master_host $master_port 10] + set load_handle2 [start_write_load $master_host $master_port 10] + set load_handle3 [start_write_load $master_host $master_port 10] + set load_handle4 [start_write_load $master_host $master_port 10] + + # Make sure the instance is really receiving data + wait_for_condition 50 100 { + [r dbsize] > 0 + } else { + fail "No write load detected." + } + + # After 3 seconds, start a rewrite, while the write load is still + # active. + after 3000 + r bgrewriteaof + waitForBgrewriteaof r + + # Let it run a bit more so that we'll append some data to the new + # AOF. + after 1000 + + # Stop the processes generating the load if they are still active + stop_write_load $load_handle0 + stop_write_load $load_handle1 + stop_write_load $load_handle2 + stop_write_load $load_handle3 + stop_write_load $load_handle4 + + # Make sure that we remain the only connected client. + # This step is needed to make sure there are no pending writes + # that will be processed between the two "debug digest" calls. + wait_for_condition 50 100 { + [llength [split [string trim [r client list]] "\n"]] == 1 + } else { + puts [r client list] + fail "Clients generating loads are not disconnecting" + } + + # Get the data set digest + set d1 [r debug digest] + + # Load the AOF + r debug loadaof + set d2 [r debug digest] + + # Make sure they are the same + assert {$d1 eq $d2} + } +} + +start_server {tags {"aofrw"}} { + test {Turning off AOF kills the background writing child if any} { + r config set appendonly yes + waitForBgrewriteaof r + r multi + r bgrewriteaof + r config set appendonly no + r exec + wait_for_condition 50 100 { + [string match {*Killing*AOF*child*} [exec tail -n5 < [srv 0 stdout]]] + } else { + fail "Can't find 'Killing AOF child' into recent logs" + } + } + + foreach d {string int} { + foreach e {ziplist linkedlist} { + test "AOF rewrite of list with $e encoding, $d data" { + r flushall + if {$e eq {ziplist}} {set len 10} else {set len 1000} + for {set j 0} {$j < $len} {incr j} { + if {$d eq {string}} { + set data [randstring 0 16 alpha] + } else { + set data [randomInt 4000000000] + } + r lpush key $data + } + assert_equal [r object encoding key] $e + set d1 [r debug digest] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set d2 [r debug digest] + if {$d1 ne $d2} { + error "assertion:$d1 is not equal to $d2" + } + } + } + } + + foreach d {string int} { + foreach e {intset hashtable} { + test "AOF rewrite of set with $e encoding, $d data" { + r flushall + if {$e eq {intset}} {set len 10} else {set len 1000} + for {set j 0} {$j < $len} {incr j} { + if {$d eq {string}} { + set data [randstring 0 16 alpha] + } else { + set data [randomInt 4000000000] + } + r sadd key $data + } + if {$d ne {string}} { + assert_equal [r object encoding key] $e + } + set d1 [r debug digest] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set d2 [r debug digest] + if {$d1 ne $d2} { + error "assertion:$d1 is not equal to $d2" + } + } + } + } + + foreach d {string int} { + foreach e {ziplist hashtable} { + test "AOF rewrite of hash with $e encoding, $d data" { + r flushall + if {$e eq {ziplist}} {set len 10} else {set len 1000} + for {set j 0} {$j < $len} {incr j} { + if {$d eq {string}} { + set data [randstring 0 16 alpha] + } else { + set data [randomInt 4000000000] + } + r hset key $data $data + } + assert_equal [r object encoding key] $e + set d1 [r debug digest] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set d2 [r debug digest] + if {$d1 ne $d2} { + error "assertion:$d1 is not equal to $d2" + } + } + } + } + + foreach d {string int} { + foreach e {ziplist skiplist} { + test "AOF rewrite of zset with $e encoding, $d data" { + r flushall + if {$e eq {ziplist}} {set len 10} else {set len 1000} + for {set j 0} {$j < $len} {incr j} { + if {$d eq {string}} { + set data [randstring 0 16 alpha] + } else { + set data [randomInt 4000000000] + } + r zadd key [expr rand()] $data + } + assert_equal [r object encoding key] $e + set d1 [r debug digest] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set d2 [r debug digest] + if {$d1 ne $d2} { + error "assertion:$d1 is not equal to $d2" + } + } + } + } + + test {BGREWRITEAOF is delayed if BGSAVE is in progress} { + r multi + r bgsave + r bgrewriteaof + r info persistence + set res [r exec] + assert_match {*scheduled*} [lindex $res 1] + assert_match {*aof_rewrite_scheduled:1*} [lindex $res 2] + while {[string match {*aof_rewrite_scheduled:1*} [r info persistence]]} { + after 100 + } + } + + test {BGREWRITEAOF is refused if already in progress} { + catch { + r multi + r bgrewriteaof + r bgrewriteaof + r exec + } e + assert_match {*ERR*already*} $e + while {[string match {*aof_rewrite_scheduled:1*} [r info persistence]]} { + after 100 + } + } +} diff --git a/tools/pika_migrate/tests/unit/auth.tcl b/tools/pika_migrate/tests/unit/auth.tcl new file mode 100644 index 0000000000..633cda95c9 --- /dev/null +++ b/tools/pika_migrate/tests/unit/auth.tcl @@ -0,0 +1,27 @@ +start_server {tags {"auth"}} { + test {AUTH fails if there is no password configured server side} { + catch {r auth foo} err + set _ $err + } {ERR*no password*} +} + +start_server {tags {"auth"} overrides {requirepass foobar}} { + test {AUTH fails when a wrong password is given} { + catch {r auth wrong!} err + set _ $err + } {ERR*invalid password} + + test {Arbitrary command gives an error when AUTH is required} { + catch {r set foo bar} err + set _ $err + } {NOAUTH*} + + test {AUTH succeeds when the right password is given} { + r auth foobar + } {OK} + + test {Once AUTH succeeded we can actually send commands to the server} { + r set foo 100 + r incr foo + } {101} +} diff --git a/tools/pika_migrate/tests/unit/basic.tcl b/tools/pika_migrate/tests/unit/basic.tcl new file mode 100644 index 0000000000..6f725d299b --- /dev/null +++ b/tools/pika_migrate/tests/unit/basic.tcl @@ -0,0 +1,783 @@ +start_server {tags {"basic"}} { + test {DEL all keys to start with a clean DB} { + foreach key [r keys *] {r del $key} + r dbsize + } {0} + + test {SET and GET an item} { + r set x foobar + r get x + } {foobar} + + test {SET and GET an empty item} { + r set x {} + r get x + } {} + + test {DEL against a single item} { + r del x + r get x + } {} + + test {Vararg DEL} { + r set foo1 a + r set foo2 b + r set foo3 c + list [r del foo1 foo2 foo3 foo4] [r mget foo1 foo2 foo3] + } {3 {{} {} {}}} + + test {KEYS with pattern} { + foreach key {key_x key_y key_z foo_a foo_b foo_c} { + r set $key hello + } + lsort [r keys foo*] + } {foo_a foo_b foo_c} + + test {KEYS to get all keys} { + lsort [r keys *] + } {foo_a foo_b foo_c key_x key_y key_z} + + test {DBSIZE} { + r dbsize + } {6} + + test {DEL all keys} { + foreach key [r keys *] {r del $key} + r dbsize + } {0} + + test {Very big payload in GET/SET} { + set buf [string repeat "abcd" 1000000] + r set foo $buf + r get foo + } [string repeat "abcd" 1000000] + + tags {"slow"} { + test {Very big payload random access} { + set err {} + array set payload {} + for {set j 0} {$j < 100} {incr j} { + set size [expr 1+[randomInt 100000]] + set buf [string repeat "pl-$j" $size] + set payload($j) $buf + r set bigpayload_$j $buf + } + for {set j 0} {$j < 1000} {incr j} { + set index [randomInt 100] + set buf [r get bigpayload_$index] + if {$buf != $payload($index)} { + set err "Values differ: I set '$payload($index)' but I read back '$buf'" + break + } + } + unset payload + set _ $err + } {} + + test {SET 10000 numeric keys and access all them in reverse order} { + set err {} + for {set x 0} {$x < 10000} {incr x} { + r set $x $x + } + set sum 0 + for {set x 9999} {$x >= 0} {incr x -1} { + set val [r get $x] + if {$val ne $x} { + set err "Element at position $x is $val instead of $x" + break + } + } + set _ $err + } {} + + test {DBSIZE should be 10101 now} { + r dbsize + } {10101} + } + + test {INCR against non existing key} { + set res {} + append res [r incr novar] + append res [r get novar] + } {11} + + test {INCR against key created by incr itself} { + r incr novar + } {2} + + test {INCR against key originally set with SET} { + r set novar 100 + r incr novar + } {101} + + test {INCR over 32bit value} { + r set novar 17179869184 + r incr novar + } {17179869185} + + test {INCRBY over 32bit value with over 32bit increment} { + r set novar 17179869184 + r incrby novar 17179869184 + } {34359738368} + + test {INCR fails against key with spaces (left)} { + r set novar " 11" + catch {r incr novar} err + format $err + } {ERR*} + + test {INCR fails against key with spaces (right)} { + r set novar "11 " + catch {r incr novar} err + format $err + } {ERR*} + + test {INCR fails against key with spaces (both)} { + r set novar " 11 " + catch {r incr novar} err + format $err + } {ERR*} + + test {INCR fails against a key holding a list} { + r rpush mylist 1 + catch {r incr mylist} err + r rpop mylist + format $err + } {WRONGTYPE*} + + test {DECRBY over 32bit value with over 32bit increment, negative res} { + r set novar 17179869184 + r decrby novar 17179869185 + } {-1} + + test {INCRBYFLOAT against non existing key} { + r del novar + list [roundFloat [r incrbyfloat novar 1]] \ + [roundFloat [r get novar]] \ + [roundFloat [r incrbyfloat novar 0.25]] \ + [roundFloat [r get novar]] + } {1 1 1.25 1.25} + + test {INCRBYFLOAT against key originally set with SET} { + r set novar 1.5 + roundFloat [r incrbyfloat novar 1.5] + } {3} + + test {INCRBYFLOAT over 32bit value} { + r set novar 17179869184 + r incrbyfloat novar 1.5 + } {17179869185.5} + + test {INCRBYFLOAT over 32bit value with over 32bit increment} { + r set novar 17179869184 + r incrbyfloat novar 17179869184 + } {34359738368} + + test {INCRBYFLOAT fails against key with spaces (left)} { + set err {} + r set novar " 11" + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against key with spaces (right)} { + set err {} + r set novar "11 " + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against key with spaces (both)} { + set err {} + r set novar " 11 " + catch {r incrbyfloat novar 1.0} err + format $err + } {ERR*valid*} + + test {INCRBYFLOAT fails against a key holding a list} { + r del mylist + set err {} + r rpush mylist 1 + catch {r incrbyfloat mylist 1.0} err + r del mylist + format $err + } {WRONGTYPE*} + + test {INCRBYFLOAT does not allow NaN or Infinity} { + r set foo 0 + set err {} + catch {r incrbyfloat foo +inf} err + set err + # p.s. no way I can force NaN to test it from the API because + # there is no way to increment / decrement by infinity nor to + # perform divisions. + } {ERR*would produce*} + + test {INCRBYFLOAT decrement} { + r set foo 1 + roundFloat [r incrbyfloat foo -1.1] + } {-0.1} + + test "SETNX target key missing" { + r del novar + assert_equal 1 [r setnx novar foobared] + assert_equal "foobared" [r get novar] + } + + test "SETNX target key exists" { + r set novar foobared + assert_equal 0 [r setnx novar blabla] + assert_equal "foobared" [r get novar] + } + + test "SETNX against not-expired volatile key" { + r set x 10 + r expire x 10000 + assert_equal 0 [r setnx x 20] + assert_equal 10 [r get x] + } + + test "SETNX against expired volatile key" { + # Make it very unlikely for the key this test uses to be expired by the + # active expiry cycle. This is tightly coupled to the implementation of + # active expiry and dbAdd() but currently the only way to test that + # SETNX expires a key when it should have been. + for {set x 0} {$x < 9999} {incr x} { + r setex key-$x 3600 value + } + + # This will be one of 10000 expiring keys. A cycle is executed every + # 100ms, sampling 10 keys for being expired or not. This key will be + # expired for at most 1s when we wait 2s, resulting in a total sample + # of 100 keys. The probability of the success of this test being a + # false positive is therefore approx. 1%. + r set x 10 + r expire x 1 + + # Wait for the key to expire + after 2000 + + assert_equal 1 [r setnx x 20] + assert_equal 20 [r get x] + } + + test "DEL against expired key" { + r debug set-active-expire 0 + r setex keyExpire 1 valExpire + after 1100 + assert_equal 0 [r del keyExpire] + r debug set-active-expire 1 + } + + test {EXISTS} { + set res {} + r set newkey test + append res [r exists newkey] + r del newkey + append res [r exists newkey] + } {10} + + test {Zero length value in key. SET/GET/EXISTS} { + r set emptykey {} + set res [r get emptykey] + append res [r exists emptykey] + r del emptykey + append res [r exists emptykey] + } {10} + + test {Commands pipelining} { + set fd [r channel] + puts -nonewline $fd "SET k1 xyzk\r\nGET k1\r\nPING\r\n" + flush $fd + set res {} + append res [string match OK* [r read]] + append res [r read] + append res [string match PONG* [r read]] + format $res + } {1xyzk1} + + test {Non existing command} { + catch {r foobaredcommand} err + string match ERR* $err + } {1} + + test {RENAME basic usage} { + r set mykey hello + r rename mykey mykey1 + r rename mykey1 mykey2 + r get mykey2 + } {hello} + + test {RENAME source key should no longer exist} { + r exists mykey + } {0} + + test {RENAME against already existing key} { + r set mykey a + r set mykey2 b + r rename mykey2 mykey + set res [r get mykey] + append res [r exists mykey2] + } {b0} + + test {RENAMENX basic usage} { + r del mykey + r del mykey2 + r set mykey foobar + r renamenx mykey mykey2 + set res [r get mykey2] + append res [r exists mykey] + } {foobar0} + + test {RENAMENX against already existing key} { + r set mykey foo + r set mykey2 bar + r renamenx mykey mykey2 + } {0} + + test {RENAMENX against already existing key (2)} { + set res [r get mykey] + append res [r get mykey2] + } {foobar} + + test {RENAME against non existing source key} { + catch {r rename nokey foobar} err + format $err + } {ERR*} + + test {RENAME where source and dest key is the same} { + catch {r rename mykey mykey} err + format $err + } {ERR*} + + test {RENAME with volatile key, should move the TTL as well} { + r del mykey mykey2 + r set mykey foo + r expire mykey 100 + assert {[r ttl mykey] > 95 && [r ttl mykey] <= 100} + r rename mykey mykey2 + assert {[r ttl mykey2] > 95 && [r ttl mykey2] <= 100} + } + + test {RENAME with volatile key, should not inherit TTL of target key} { + r del mykey mykey2 + r set mykey foo + r set mykey2 bar + r expire mykey2 100 + assert {[r ttl mykey] == -1 && [r ttl mykey2] > 0} + r rename mykey mykey2 + r ttl mykey2 + } {-1} + + test {DEL all keys again (DB 0)} { + foreach key [r keys *] { + r del $key + } + r dbsize + } {0} + + test {DEL all keys again (DB 1)} { + r select 10 + foreach key [r keys *] { + r del $key + } + set res [r dbsize] + r select 9 + format $res + } {0} + + test {MOVE basic usage} { + r set mykey foobar + r move mykey 10 + set res {} + lappend res [r exists mykey] + lappend res [r dbsize] + r select 10 + lappend res [r get mykey] + lappend res [r dbsize] + r select 9 + format $res + } [list 0 0 foobar 1] + + test {MOVE against key existing in the target DB} { + r set mykey hello + r move mykey 10 + } {0} + + test {MOVE against non-integer DB (#1428)} { + r set mykey hello + catch {r move mykey notanumber} e + set e + } {*ERR*index out of range} + + test {SET/GET keys in different DBs} { + r set a hello + r set b world + r select 10 + r set a foo + r set b bared + r select 9 + set res {} + lappend res [r get a] + lappend res [r get b] + r select 10 + lappend res [r get a] + lappend res [r get b] + r select 9 + format $res + } {hello world foo bared} + + test {MGET} { + r flushdb + r set foo BAR + r set bar FOO + r mget foo bar + } {BAR FOO} + + test {MGET against non existing key} { + r mget foo baazz bar + } {BAR {} FOO} + + test {MGET against non-string key} { + r sadd myset ciao + r sadd myset bau + r mget foo baazz bar myset + } {BAR {} FOO {}} + + test {RANDOMKEY} { + r flushdb + r set foo x + r set bar y + set foo_seen 0 + set bar_seen 0 + for {set i 0} {$i < 100} {incr i} { + set rkey [r randomkey] + if {$rkey eq {foo}} { + set foo_seen 1 + } + if {$rkey eq {bar}} { + set bar_seen 1 + } + } + list $foo_seen $bar_seen + } {1 1} + + test {RANDOMKEY against empty DB} { + r flushdb + r randomkey + } {} + + test {RANDOMKEY regression 1} { + r flushdb + r set x 10 + r del x + r randomkey + } {} + + test {GETSET (set new value)} { + list [r getset foo xyz] [r get foo] + } {{} xyz} + + test {GETSET (replace old value)} { + r set foo bar + list [r getset foo xyz] [r get foo] + } {bar xyz} + + test {MSET base case} { + r mset x 10 y "foo bar" z "x x x x x x x\n\n\r\n" + r mget x y z + } [list 10 {foo bar} "x x x x x x x\n\n\r\n"] + + test {MSET wrong number of args} { + catch {r mset x 10 y "foo bar" z} err + format $err + } {*wrong number*} + + test {MSETNX with already existent key} { + list [r msetnx x1 xxx y2 yyy x 20] [r exists x1] [r exists y2] + } {0 0 0} + + test {MSETNX with not existing keys} { + list [r msetnx x1 xxx y2 yyy] [r get x1] [r get y2] + } {1 xxx yyy} + + test "STRLEN against non-existing key" { + assert_equal 0 [r strlen notakey] + } + + test "STRLEN against integer-encoded value" { + r set myinteger -555 + assert_equal 4 [r strlen myinteger] + } + + test "STRLEN against plain string" { + r set mystring "foozzz0123456789 baz" + assert_equal 20 [r strlen mystring] + } + + test "SETBIT against non-existing key" { + r del mykey + assert_equal 0 [r setbit mykey 1 1] + assert_equal [binary format B* 01000000] [r get mykey] + } + + test "SETBIT against string-encoded key" { + # Ascii "@" is integer 64 = 01 00 00 00 + r set mykey "@" + + assert_equal 0 [r setbit mykey 2 1] + assert_equal [binary format B* 01100000] [r get mykey] + assert_equal 1 [r setbit mykey 1 0] + assert_equal [binary format B* 00100000] [r get mykey] + } + + test "SETBIT against integer-encoded key" { + # Ascii "1" is integer 49 = 00 11 00 01 + r set mykey 1 + assert_encoding int mykey + + assert_equal 0 [r setbit mykey 6 1] + assert_equal [binary format B* 00110011] [r get mykey] + assert_equal 1 [r setbit mykey 2 0] + assert_equal [binary format B* 00010011] [r get mykey] + } + + test "SETBIT against key with wrong type" { + r del mykey + r lpush mykey "foo" + assert_error "WRONGTYPE*" {r setbit mykey 0 1} + } + + test "SETBIT with out of range bit offset" { + r del mykey + assert_error "*out of range*" {r setbit mykey [expr 4*1024*1024*1024] 1} + assert_error "*out of range*" {r setbit mykey -1 1} + } + + test "SETBIT with non-bit argument" { + r del mykey + assert_error "*out of range*" {r setbit mykey 0 -1} + assert_error "*out of range*" {r setbit mykey 0 2} + assert_error "*out of range*" {r setbit mykey 0 10} + assert_error "*out of range*" {r setbit mykey 0 20} + } + + test "SETBIT fuzzing" { + set str "" + set len [expr 256*8] + r del mykey + + for {set i 0} {$i < 2000} {incr i} { + set bitnum [randomInt $len] + set bitval [randomInt 2] + set fmt [format "%%-%ds%%d%%-s" $bitnum] + set head [string range $str 0 $bitnum-1] + set tail [string range $str $bitnum+1 end] + set str [string map {" " 0} [format $fmt $head $bitval $tail]] + + r setbit mykey $bitnum $bitval + assert_equal [binary format B* $str] [r get mykey] + } + } + + test "GETBIT against non-existing key" { + r del mykey + assert_equal 0 [r getbit mykey 0] + } + + test "GETBIT against string-encoded key" { + # Single byte with 2nd and 3rd bit set + r set mykey "`" + + # In-range + assert_equal 0 [r getbit mykey 0] + assert_equal 1 [r getbit mykey 1] + assert_equal 1 [r getbit mykey 2] + assert_equal 0 [r getbit mykey 3] + + # Out-range + assert_equal 0 [r getbit mykey 8] + assert_equal 0 [r getbit mykey 100] + assert_equal 0 [r getbit mykey 10000] + } + + test "GETBIT against integer-encoded key" { + r set mykey 1 + assert_encoding int mykey + + # Ascii "1" is integer 49 = 00 11 00 01 + assert_equal 0 [r getbit mykey 0] + assert_equal 0 [r getbit mykey 1] + assert_equal 1 [r getbit mykey 2] + assert_equal 1 [r getbit mykey 3] + + # Out-range + assert_equal 0 [r getbit mykey 8] + assert_equal 0 [r getbit mykey 100] + assert_equal 0 [r getbit mykey 10000] + } + + test "SETRANGE against non-existing key" { + r del mykey + assert_equal 3 [r setrange mykey 0 foo] + assert_equal "foo" [r get mykey] + + r del mykey + assert_equal 0 [r setrange mykey 0 ""] + assert_equal 0 [r exists mykey] + + r del mykey + assert_equal 4 [r setrange mykey 1 foo] + assert_equal "\000foo" [r get mykey] + } + + test "SETRANGE against string-encoded key" { + r set mykey "foo" + assert_equal 3 [r setrange mykey 0 b] + assert_equal "boo" [r get mykey] + + r set mykey "foo" + assert_equal 3 [r setrange mykey 0 ""] + assert_equal "foo" [r get mykey] + + r set mykey "foo" + assert_equal 3 [r setrange mykey 1 b] + assert_equal "fbo" [r get mykey] + + r set mykey "foo" + assert_equal 7 [r setrange mykey 4 bar] + assert_equal "foo\000bar" [r get mykey] + } + + test "SETRANGE against integer-encoded key" { + r set mykey 1234 + assert_encoding int mykey + assert_equal 4 [r setrange mykey 0 2] + assert_encoding raw mykey + assert_equal 2234 [r get mykey] + + # Shouldn't change encoding when nothing is set + r set mykey 1234 + assert_encoding int mykey + assert_equal 4 [r setrange mykey 0 ""] + assert_encoding int mykey + assert_equal 1234 [r get mykey] + + r set mykey 1234 + assert_encoding int mykey + assert_equal 4 [r setrange mykey 1 3] + assert_encoding raw mykey + assert_equal 1334 [r get mykey] + + r set mykey 1234 + assert_encoding int mykey + assert_equal 6 [r setrange mykey 5 2] + assert_encoding raw mykey + assert_equal "1234\0002" [r get mykey] + } + + test "SETRANGE against key with wrong type" { + r del mykey + r lpush mykey "foo" + assert_error "WRONGTYPE*" {r setrange mykey 0 bar} + } + + test "SETRANGE with out of range offset" { + r del mykey + assert_error "*maximum allowed size*" {r setrange mykey [expr 512*1024*1024-4] world} + + r set mykey "hello" + assert_error "*out of range*" {r setrange mykey -1 world} + assert_error "*maximum allowed size*" {r setrange mykey [expr 512*1024*1024-4] world} + } + + test "GETRANGE against non-existing key" { + r del mykey + assert_equal "" [r getrange mykey 0 -1] + } + + test "GETRANGE against string value" { + r set mykey "Hello World" + assert_equal "Hell" [r getrange mykey 0 3] + assert_equal "Hello World" [r getrange mykey 0 -1] + assert_equal "orld" [r getrange mykey -4 -1] + assert_equal "" [r getrange mykey 5 3] + assert_equal " World" [r getrange mykey 5 5000] + assert_equal "Hello World" [r getrange mykey -5000 10000] + } + + test "GETRANGE against integer-encoded value" { + r set mykey 1234 + assert_equal "123" [r getrange mykey 0 2] + assert_equal "1234" [r getrange mykey 0 -1] + assert_equal "234" [r getrange mykey -3 -1] + assert_equal "" [r getrange mykey 5 3] + assert_equal "4" [r getrange mykey 3 5000] + assert_equal "1234" [r getrange mykey -5000 10000] + } + + test "GETRANGE fuzzing" { + for {set i 0} {$i < 1000} {incr i} { + r set bin [set bin [randstring 0 1024 binary]] + set _start [set start [randomInt 1500]] + set _end [set end [randomInt 1500]] + if {$_start < 0} {set _start "end-[abs($_start)-1]"} + if {$_end < 0} {set _end "end-[abs($_end)-1]"} + assert_equal [string range $bin $_start $_end] [r getrange bin $start $end] + } + } + + test {Extended SET can detect syntax errors} { + set e {} + catch {r set foo bar non-existing-option} e + set e + } {*syntax*} + + test {Extended SET NX option} { + r del foo + set v1 [r set foo 1 nx] + set v2 [r set foo 2 nx] + list $v1 $v2 [r get foo] + } {OK {} 1} + + test {Extended SET XX option} { + r del foo + set v1 [r set foo 1 xx] + r set foo bar + set v2 [r set foo 2 xx] + list $v1 $v2 [r get foo] + } {{} OK 2} + + test {Extended SET EX option} { + r del foo + r set foo bar ex 10 + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } + + test {Extended SET PX option} { + r del foo + r set foo bar px 10000 + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } + + test {Extended SET using multiple options at once} { + r set foo val + assert {[r set foo bar xx px 10000] eq {OK}} + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } + + test {KEYS * two times with long key, Github issue #1208} { + r flushdb + r set dlskeriewrioeuwqoirueioqwrueoqwrueqw test + r keys * + r keys * + } {dlskeriewrioeuwqoirueioqwrueoqwrueqw} + + test {GETRANGE with huge ranges, Github issue #1844} { + r set foo bar + r getrange foo 0 4294967297 + } {bar} +} diff --git a/tools/pika_migrate/tests/unit/bitops.tcl b/tools/pika_migrate/tests/unit/bitops.tcl new file mode 100644 index 0000000000..9751850ad4 --- /dev/null +++ b/tools/pika_migrate/tests/unit/bitops.tcl @@ -0,0 +1,341 @@ +# Compare Redis commadns against Tcl implementations of the same commands. +proc count_bits s { + binary scan $s b* bits + string length [regsub -all {0} $bits {}] +} + +proc simulate_bit_op {op args} { + set maxlen 0 + set j 0 + set count [llength $args] + foreach a $args { + binary scan $a b* bits + set b($j) $bits + if {[string length $bits] > $maxlen} { + set maxlen [string length $bits] + } + incr j + } + for {set j 0} {$j < $count} {incr j} { + if {[string length $b($j)] < $maxlen} { + append b($j) [string repeat 0 [expr $maxlen-[string length $b($j)]]] + } + } + set out {} + for {set x 0} {$x < $maxlen} {incr x} { + set bit [string range $b(0) $x $x] + if {$op eq {not}} {set bit [expr {!$bit}]} + for {set j 1} {$j < $count} {incr j} { + set bit2 [string range $b($j) $x $x] + switch $op { + and {set bit [expr {$bit & $bit2}]} + or {set bit [expr {$bit | $bit2}]} + xor {set bit [expr {$bit ^ $bit2}]} + } + } + append out $bit + } + binary format b* $out +} + +start_server {tags {"bitops"}} { + test {BITCOUNT returns 0 against non existing key} { + r bitcount no-key + } 0 + + catch {unset num} + foreach vec [list "" "\xaa" "\x00\x00\xff" "foobar" "123"] { + incr num + test "BITCOUNT against test vector #$num" { + r set str $vec + assert {[r bitcount str] == [count_bits $vec]} + } + } + + test {BITCOUNT fuzzing without start/end} { + for {set j 0} {$j < 100} {incr j} { + set str [randstring 0 3000] + r set str $str + assert {[r bitcount str] == [count_bits $str]} + } + } + + test {BITCOUNT fuzzing with start/end} { + for {set j 0} {$j < 100} {incr j} { + set str [randstring 0 3000] + r set str $str + set l [string length $str] + set start [randomInt $l] + set end [randomInt $l] + if {$start > $end} { + lassign [list $end $start] start end + } + assert {[r bitcount str $start $end] == [count_bits [string range $str $start $end]]} + } + } + + test {BITCOUNT with start, end} { + r set s "foobar" + assert_equal [r bitcount s 0 -1] [count_bits "foobar"] + assert_equal [r bitcount s 1 -2] [count_bits "ooba"] + assert_equal [r bitcount s -2 1] [count_bits ""] + assert_equal [r bitcount s 0 1000] [count_bits "foobar"] + } + + test {BITCOUNT syntax error #1} { + catch {r bitcount s 0} e + set e + } {ERR*syntax*} + + test {BITCOUNT regression test for github issue #582} { + r del str + r setbit foo 0 1 + if {[catch {r bitcount foo 0 4294967296} e]} { + assert_match {*ERR*out of range*} $e + set _ 1 + } else { + set e + } + } {1} + + test {BITCOUNT misaligned prefix} { + r del str + r set str ab + r bitcount str 1 -1 + } {3} + + test {BITCOUNT misaligned prefix + full words + remainder} { + r del str + r set str __PPxxxxxxxxxxxxxxxxRR__ + r bitcount str 2 -3 + } {74} + + test {BITOP NOT (empty string)} { + r set s "" + r bitop not dest s + r get dest + } {} + + test {BITOP NOT (known string)} { + r set s "\xaa\x00\xff\x55" + r bitop not dest s + r get dest + } "\x55\xff\x00\xaa" + + test {BITOP where dest and target are the same key} { + r set s "\xaa\x00\xff\x55" + r bitop not s s + r get s + } "\x55\xff\x00\xaa" + + test {BITOP AND|OR|XOR don't change the string with single input key} { + r set a "\x01\x02\xff" + r bitop and res1 a + r bitop or res2 a + r bitop xor res3 a + list [r get res1] [r get res2] [r get res3] + } [list "\x01\x02\xff" "\x01\x02\xff" "\x01\x02\xff"] + + test {BITOP missing key is considered a stream of zero} { + r set a "\x01\x02\xff" + r bitop and res1 no-suck-key a + r bitop or res2 no-suck-key a no-such-key + r bitop xor res3 no-such-key a + list [r get res1] [r get res2] [r get res3] + } [list "\x00\x00\x00" "\x01\x02\xff" "\x01\x02\xff"] + + test {BITOP shorter keys are zero-padded to the key with max length} { + r set a "\x01\x02\xff\xff" + r set b "\x01\x02\xff" + r bitop and res1 a b + r bitop or res2 a b + r bitop xor res3 a b + list [r get res1] [r get res2] [r get res3] + } [list "\x01\x02\xff\x00" "\x01\x02\xff\xff" "\x00\x00\x00\xff"] + + foreach op {and or xor} { + test "BITOP $op fuzzing" { + for {set i 0} {$i < 10} {incr i} { + r flushall + set vec {} + set veckeys {} + set numvec [expr {[randomInt 10]+1}] + for {set j 0} {$j < $numvec} {incr j} { + set str [randstring 0 1000] + lappend vec $str + lappend veckeys vector_$j + r set vector_$j $str + } + r bitop $op target {*}$veckeys + assert_equal [r get target] [simulate_bit_op $op {*}$vec] + } + } + } + + test {BITOP NOT fuzzing} { + for {set i 0} {$i < 10} {incr i} { + r flushall + set str [randstring 0 1000] + r set str $str + r bitop not target str + assert_equal [r get target] [simulate_bit_op not $str] + } + } + + test {BITOP with integer encoded source objects} { + r set a 1 + r set b 2 + r bitop xor dest a b a + r get dest + } {2} + + test {BITOP with non string source key} { + r del c + r set a 1 + r set b 2 + r lpush c foo + catch {r bitop xor dest a b c d} e + set e + } {WRONGTYPE*} + + test {BITOP with empty string after non empty string (issue #529)} { + r flushdb + r set a "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + r bitop or x a b + } {32} + + test {BITPOS bit=0 with empty key returns 0} { + r del str + r bitpos str 0 + } {0} + + test {BITPOS bit=1 with empty key returns -1} { + r del str + r bitpos str 1 + } {-1} + + test {BITPOS bit=0 with string less than 1 word works} { + r set str "\xff\xf0\x00" + r bitpos str 0 + } {12} + + test {BITPOS bit=1 with string less than 1 word works} { + r set str "\x00\x0f\x00" + r bitpos str 1 + } {12} + + test {BITPOS bit=0 starting at unaligned address} { + r set str "\xff\xf0\x00" + r bitpos str 0 1 + } {12} + + test {BITPOS bit=1 starting at unaligned address} { + r set str "\x00\x0f\xff" + r bitpos str 1 1 + } {12} + + test {BITPOS bit=0 unaligned+full word+reminder} { + r del str + r set str "\xff\xff\xff" ; # Prefix + # Followed by two (or four in 32 bit systems) full words + r append str "\xff\xff\xff\xff\xff\xff\xff\xff" + r append str "\xff\xff\xff\xff\xff\xff\xff\xff" + r append str "\xff\xff\xff\xff\xff\xff\xff\xff" + # First zero bit. + r append str "\x0f" + assert {[r bitpos str 0] == 216} + assert {[r bitpos str 0 1] == 216} + assert {[r bitpos str 0 2] == 216} + assert {[r bitpos str 0 3] == 216} + assert {[r bitpos str 0 4] == 216} + assert {[r bitpos str 0 5] == 216} + assert {[r bitpos str 0 6] == 216} + assert {[r bitpos str 0 7] == 216} + assert {[r bitpos str 0 8] == 216} + } + + test {BITPOS bit=1 unaligned+full word+reminder} { + r del str + r set str "\x00\x00\x00" ; # Prefix + # Followed by two (or four in 32 bit systems) full words + r append str "\x00\x00\x00\x00\x00\x00\x00\x00" + r append str "\x00\x00\x00\x00\x00\x00\x00\x00" + r append str "\x00\x00\x00\x00\x00\x00\x00\x00" + # First zero bit. + r append str "\xf0" + assert {[r bitpos str 1] == 216} + assert {[r bitpos str 1 1] == 216} + assert {[r bitpos str 1 2] == 216} + assert {[r bitpos str 1 3] == 216} + assert {[r bitpos str 1 4] == 216} + assert {[r bitpos str 1 5] == 216} + assert {[r bitpos str 1 6] == 216} + assert {[r bitpos str 1 7] == 216} + assert {[r bitpos str 1 8] == 216} + } + + test {BITPOS bit=1 returns -1 if string is all 0 bits} { + r set str "" + for {set j 0} {$j < 20} {incr j} { + assert {[r bitpos str 1] == -1} + r append str "\x00" + } + } + + test {BITPOS bit=0 works with intervals} { + r set str "\x00\xff\x00" + assert {[r bitpos str 0 0 -1] == 0} + assert {[r bitpos str 0 1 -1] == 16} + assert {[r bitpos str 0 2 -1] == 16} + assert {[r bitpos str 0 2 200] == 16} + assert {[r bitpos str 0 1 1] == -1} + } + + test {BITPOS bit=1 works with intervals} { + r set str "\x00\xff\x00" + assert {[r bitpos str 1 0 -1] == 8} + assert {[r bitpos str 1 1 -1] == 8} + assert {[r bitpos str 1 2 -1] == -1} + assert {[r bitpos str 1 2 200] == -1} + assert {[r bitpos str 1 1 1] == 8} + } + + test {BITPOS bit=0 changes behavior if end is given} { + r set str "\xff\xff\xff" + assert {[r bitpos str 0] == 24} + assert {[r bitpos str 0 0] == 24} + assert {[r bitpos str 0 0 -1] == -1} + } + + test {BITPOS bit=1 fuzzy testing using SETBIT} { + r del str + set max 524288; # 64k + set first_one_pos -1 + for {set j 0} {$j < 1000} {incr j} { + assert {[r bitpos str 1] == $first_one_pos} + set pos [randomInt $max] + r setbit str $pos 1 + if {$first_one_pos == -1 || $first_one_pos > $pos} { + # Update the position of the first 1 bit in the array + # if the bit we set is on the left of the previous one. + set first_one_pos $pos + } + } + } + + test {BITPOS bit=0 fuzzy testing using SETBIT} { + set max 524288; # 64k + set first_zero_pos $max + r set str [string repeat "\xff" [expr $max/8]] + for {set j 0} {$j < 1000} {incr j} { + assert {[r bitpos str 0] == $first_zero_pos} + set pos [randomInt $max] + r setbit str $pos 0 + if {$first_zero_pos > $pos} { + # Update the position of the first 0 bit in the array + # if the bit we clear is on the left of the previous one. + set first_zero_pos $pos + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/dump.tcl b/tools/pika_migrate/tests/unit/dump.tcl new file mode 100644 index 0000000000..b79c3ba9d0 --- /dev/null +++ b/tools/pika_migrate/tests/unit/dump.tcl @@ -0,0 +1,142 @@ +start_server {tags {"dump"}} { + test {DUMP / RESTORE are able to serialize / unserialize a simple key} { + r set foo bar + set encoded [r dump foo] + r del foo + list [r exists foo] [r restore foo 0 $encoded] [r ttl foo] [r get foo] + } {0 OK -1 bar} + + test {RESTORE can set an arbitrary expire to the materialized key} { + r set foo bar + set encoded [r dump foo] + r del foo + r restore foo 5000 $encoded + set ttl [r pttl foo] + assert {$ttl >= 3000 && $ttl <= 5000} + r get foo + } {bar} + + test {RESTORE can set an expire that overflows a 32 bit integer} { + r set foo bar + set encoded [r dump foo] + r del foo + r restore foo 2569591501 $encoded + set ttl [r pttl foo] + assert {$ttl >= (2569591501-3000) && $ttl <= 2569591501} + r get foo + } {bar} + + test {RESTORE returns an error of the key already exists} { + r set foo bar + set e {} + catch {r restore foo 0 "..."} e + set e + } {*is busy*} + + test {DUMP of non existing key returns nil} { + r dump nonexisting_key + } {} + + test {MIGRATE is able to migrate a key between two instances} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 5000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second get key] eq {Some Value}} + assert {[$second ttl key] == -1} + } + } + + test {MIGRATE propagates TTL correctly} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + $first expire key 10 + set ret [r -1 migrate $second_host $second_port key 9 5000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second get key] eq {Some Value}} + assert {[$second ttl key] >= 7 && [$second ttl key] <= 10} + } + } + + test {MIGRATE can correctly transfer large values} { + set first [srv 0 client] + r del key + for {set j 0} {$j < 5000} {incr j} { + r rpush key 1 2 3 4 5 6 7 8 9 10 + r rpush key "item 1" "item 2" "item 3" "item 4" "item 5" \ + "item 6" "item 7" "item 8" "item 9" "item 10" + } + assert {[string length [r dump key]] > (1024*64)} + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 10000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second ttl key] == -1} + assert {[$second llen key] == 5000*20} + } + } + + test {MIGRATE can correctly transfer hashes} { + set first [srv 0 client] + r del key + r hmset key field1 "item 1" field2 "item 2" field3 "item 3" \ + field4 "item 4" field5 "item 5" field6 "item 6" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 10000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second ttl key] == -1} + } + } + + test {MIGRATE timeout actually works} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + + set rd [redis_deferring_client] + $rd debug sleep 5.0 ; # Make second server unable to reply. + set e {} + catch {r -1 migrate $second_host $second_port key 9 1000} e + assert_match {IOERR*} $e + } + } +} diff --git a/tools/pika_migrate/tests/unit/expire.tcl b/tools/pika_migrate/tests/unit/expire.tcl new file mode 100644 index 0000000000..ff3dacb337 --- /dev/null +++ b/tools/pika_migrate/tests/unit/expire.tcl @@ -0,0 +1,201 @@ +start_server {tags {"expire"}} { + test {EXPIRE - set timeouts multiple times} { + r set x foobar + set v1 [r expire x 5] + set v2 [r ttl x] + set v3 [r expire x 10] + set v4 [r ttl x] + r expire x 2 + list $v1 $v2 $v3 $v4 + } {1 [45] 1 10} + + test {EXPIRE - It should be still possible to read 'x'} { + r get x + } {foobar} + + tags {"slow"} { + test {EXPIRE - After 2.1 seconds the key should no longer be here} { + after 2100 + list [r get x] [r exists x] + } {{} 0} + } + + test {EXPIRE - write on expire should work} { + r del x + r lpush x foo + r expire x 1000 + r lpush x bar + r lrange x 0 -1 + } {bar foo} + + test {EXPIREAT - Check for EXPIRE alike behavior} { + r del x + r set x foo + r expireat x [expr [clock seconds]+15] + r ttl x + } {1[345]} + + test {SETEX - Set + Expire combo operation. Check for TTL} { + r setex x 12 test + r ttl x + } {1[012]} + + test {SETEX - Check value} { + r get x + } {test} + + test {SETEX - Overwrite old key} { + r setex y 1 foo + r get y + } {foo} + + tags {"slow"} { + test {SETEX - Wait for the key to expire} { + after 1100 + r get y + } {} + } + + test {SETEX - Wrong time parameter} { + catch {r setex z -10 foo} e + set _ $e + } {*invalid expire*} + + test {PERSIST can undo an EXPIRE} { + r set x foo + r expire x 50 + list [r ttl x] [r persist x] [r ttl x] [r get x] + } {50 1 -1 foo} + + test {PERSIST returns 0 against non existing or non volatile keys} { + r set x foo + list [r persist foo] [r persist nokeyatall] + } {0 0} + + test {EXPIRE pricision is now the millisecond} { + # This test is very likely to do a false positive if the + # server is under pressure, so if it does not work give it a few more + # chances. + for {set j 0} {$j < 3} {incr j} { + r del x + r setex x 1 somevalue + after 900 + set a [r get x] + after 1100 + set b [r get x] + if {$a eq {somevalue} && $b eq {}} break + } + list $a $b + } {somevalue {}} + + test {PEXPIRE/PSETEX/PEXPIREAT can set sub-second expires} { + # This test is very likely to do a false positive if the + # server is under pressure, so if it does not work give it a few more + # chances. + for {set j 0} {$j < 3} {incr j} { + r del x y z + r psetex x 100 somevalue + after 80 + set a [r get x] + after 120 + set b [r get x] + + r set x somevalue + r pexpire x 100 + after 80 + set c [r get x] + after 120 + set d [r get x] + + r set x somevalue + r pexpireat x [expr ([clock seconds]*1000)+100] + after 80 + set e [r get x] + after 120 + set f [r get x] + + if {$a eq {somevalue} && $b eq {} && + $c eq {somevalue} && $d eq {} && + $e eq {somevalue} && $f eq {}} break + } + list $a $b + } {somevalue {}} + + test {TTL returns tiem to live in seconds} { + r del x + r setex x 10 somevalue + set ttl [r ttl x] + assert {$ttl > 8 && $ttl <= 10} + } + + test {PTTL returns time to live in milliseconds} { + r del x + r setex x 1 somevalue + set ttl [r pttl x] + assert {$ttl > 900 && $ttl <= 1000} + } + + test {TTL / PTTL return -1 if key has no expire} { + r del x + r set x hello + list [r ttl x] [r pttl x] + } {-1 -1} + + test {TTL / PTTL return -2 if key does not exit} { + r del x + list [r ttl x] [r pttl x] + } {-2 -2} + + test {Redis should actively expire keys incrementally} { + r flushdb + r psetex key1 500 a + r psetex key2 500 a + r psetex key3 500 a + set size1 [r dbsize] + # Redis expires random keys ten times every second so we are + # fairly sure that all the three keys should be evicted after + # one second. + after 1000 + set size2 [r dbsize] + list $size1 $size2 + } {3 0} + + test {Redis should lazy expire keys} { + r flushdb + r debug set-active-expire 0 + r psetex key1 500 a + r psetex key2 500 a + r psetex key3 500 a + set size1 [r dbsize] + # Redis expires random keys ten times every second so we are + # fairly sure that all the three keys should be evicted after + # one second. + after 1000 + set size2 [r dbsize] + r mget key1 key2 key3 + set size3 [r dbsize] + r debug set-active-expire 1 + list $size1 $size2 $size3 + } {3 3 0} + + test {EXPIRE should not resurrect keys (issue #1026)} { + r debug set-active-expire 0 + r set foo bar + r pexpire foo 500 + after 1000 + r expire foo 10 + r debug set-active-expire 1 + r exists foo + } {0} + + test {5 keys in, 5 keys out} { + r flushdb + r set a c + r expire a 5 + r set t c + r set e c + r set s c + r set foo b + lsort [r keys *] + } {a e foo s t} +} diff --git a/tools/pika_migrate/tests/unit/geo.tcl b/tools/pika_migrate/tests/unit/geo.tcl new file mode 100644 index 0000000000..7ed8710980 --- /dev/null +++ b/tools/pika_migrate/tests/unit/geo.tcl @@ -0,0 +1,311 @@ +# Helper functions to simulate search-in-radius in the Tcl side in order to +# verify the Redis implementation with a fuzzy test. +proc geo_degrad deg {expr {$deg*atan(1)*8/360}} + +proc geo_distance {lon1d lat1d lon2d lat2d} { + set lon1r [geo_degrad $lon1d] + set lat1r [geo_degrad $lat1d] + set lon2r [geo_degrad $lon2d] + set lat2r [geo_degrad $lat2d] + set v [expr {sin(($lon2r - $lon1r) / 2)}] + set u [expr {sin(($lat2r - $lat1r) / 2)}] + expr {2.0 * 6372797.560856 * \ + asin(sqrt($u * $u + cos($lat1r) * cos($lat2r) * $v * $v))} +} + +proc geo_random_point {lonvar latvar} { + upvar 1 $lonvar lon + upvar 1 $latvar lat + # Note that the actual latitude limit should be -85 to +85, we restrict + # the test to -70 to +70 since in this range the algorithm is more precise + # while outside this range occasionally some element may be missing. + set lon [expr {-180 + rand()*360}] + set lat [expr {-70 + rand()*140}] +} + +# Return elements non common to both the lists. +# This code is from http://wiki.tcl.tk/15489 +proc compare_lists {List1 List2} { + set DiffList {} + foreach Item $List1 { + if {[lsearch -exact $List2 $Item] == -1} { + lappend DiffList $Item + } + } + foreach Item $List2 { + if {[lsearch -exact $List1 $Item] == -1} { + if {[lsearch -exact $DiffList $Item] == -1} { + lappend DiffList $Item + } + } + } + return $DiffList +} + +# The following list represents sets of random seed, search position +# and radius that caused bugs in the past. It is used by the randomized +# test later as a starting point. When the regression vectors are scanned +# the code reverts to using random data. +# +# The format is: seed km lon lat +set regression_vectors { + {1482225976969 7083 81.634948934258375 30.561509253718668} + {1482340074151 5416 -70.863281847379767 -46.347003465679947} + {1499014685896 6064 -89.818768962202014 -40.463868561416803} + {1412 156 149.29737817929004 15.95807862745508} + {441574 143 59.235461856813856 66.269555127373678} + {160645 187 -101.88575239939883 49.061997951502917} + {750269 154 -90.187939661642517 66.615930412251487} + {342880 145 163.03472387745728 64.012747720821181} + {729955 143 137.86663517256579 63.986745399416776} + {939895 151 59.149620271823181 65.204186651485145} + {1412 156 149.29737817929004 15.95807862745508} + {564862 149 84.062063109158544 -65.685403922426232} +} +set rv_idx 0 + +start_server {tags {"geo"}} { + test {GEOADD create} { + r geoadd nyc -73.9454966 40.747533 "lic market" + } {1} + + test {GEOADD update} { + r geoadd nyc -73.9454966 40.747533 "lic market" + } {0} + + test {GEOADD invalid coordinates} { + catch { + r geoadd nyc -73.9454966 40.747533 "lic market" \ + foo bar "luck market" + } err + set err + } {*valid*} + + test {GEOADD multi add} { + r geoadd nyc -73.9733487 40.7648057 "central park n/q/r" -73.9903085 40.7362513 "union square" -74.0131604 40.7126674 "wtc one" -73.7858139 40.6428986 "jfk" -73.9375699 40.7498929 "q4" -73.9564142 40.7480973 4545 + } {6} + + test {Check geoset values} { + r zrange nyc 0 -1 withscores + } {{wtc one} 1791873972053020 {union square} 1791875485187452 {central park n/q/r} 1791875761332224 4545 1791875796750882 {lic market} 1791875804419201 q4 1791875830079666 jfk 1791895905559723} + + test {GEORADIUS simple (sorted)} { + r georadius nyc -73.9798091 40.7598464 3 km asc + } {{central park n/q/r} 4545 {union square}} + + test {GEORADIUS withdist (sorted)} { + r georadius nyc -73.9798091 40.7598464 3 km withdist asc + } {{{central park n/q/r} 0.7750} {4545 2.3651} {{union square} 2.7697}} + + test {GEORADIUS with COUNT} { + r georadius nyc -73.9798091 40.7598464 10 km COUNT 3 + } {{wtc one} {union square} {central park n/q/r}} + + test {GEORADIUS with COUNT but missing integer argument} { + catch {r georadius nyc -73.9798091 40.7598464 10 km COUNT} e + set e + } {ERR*syntax*} + + test {GEORADIUS with COUNT DESC} { + r georadius nyc -73.9798091 40.7598464 10 km COUNT 2 DESC + } {{wtc one} q4} + + test {GEORADIUS HUGE, issue #2767} { + r geoadd users -47.271613776683807 -54.534504198047678 user_000000 + llength [r GEORADIUS users 0 0 50000 km WITHCOORD] + } {1} + + test {GEORADIUSBYMEMBER simple (sorted)} { + r georadiusbymember nyc "wtc one" 7 km + } {{wtc one} {union square} {central park n/q/r} 4545 {lic market}} + + test {GEORADIUSBYMEMBER withdist (sorted)} { + r georadiusbymember nyc "wtc one" 7 km withdist + } {{{wtc one} 0.0000} {{union square} 3.2544} {{central park n/q/r} 6.7000} {4545 6.1975} {{lic market} 6.8969}} + + test {GEOHASH is able to return geohash strings} { + # Example from Wikipedia. + r del points + r geoadd points -5.6 42.6 test + lindex [r geohash points test] 0 + } {ezs42e44yx0} + + test {GEOPOS simple} { + r del points + r geoadd points 10 20 a 30 40 b + lassign [lindex [r geopos points a b] 0] x1 y1 + lassign [lindex [r geopos points a b] 1] x2 y2 + assert {abs($x1 - 10) < 0.001} + assert {abs($y1 - 20) < 0.001} + assert {abs($x2 - 30) < 0.001} + assert {abs($y2 - 40) < 0.001} + } + + test {GEOPOS missing element} { + r del points + r geoadd points 10 20 a 30 40 b + lindex [r geopos points a x b] 1 + } {} + + test {GEODIST simple & unit} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + set m [r geodist points Palermo Catania] + assert {$m > 166274 && $m < 166275} + set km [r geodist points Palermo Catania km] + assert {$km > 166.2 && $km < 166.3} + } + + test {GEODIST missing elements} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + set m [r geodist points Palermo Agrigento] + assert {$m eq {}} + set m [r geodist points Ragusa Agrigento] + assert {$m eq {}} + set m [r geodist empty_key Palermo Catania] + assert {$m eq {}} + } + + test {GEORADIUS STORE option: syntax error} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + catch {r georadius points 13.361389 38.115556 50 km store} e + set e + } {*ERR*syntax*} + + test {GEORANGE STORE option: incompatible options} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + catch {r georadius points 13.361389 38.115556 50 km store points2 withdist} e + assert_match {*ERR*} $e + catch {r georadius points 13.361389 38.115556 50 km store points2 withhash} e + assert_match {*ERR*} $e + catch {r georadius points 13.361389 38.115556 50 km store points2 withcoords} e + assert_match {*ERR*} $e + } + + test {GEORANGE STORE option: plain usage} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + r georadius points 13.361389 38.115556 500 km store points2 + assert_equal [r zrange points 0 -1] [r zrange points2 0 -1] + } + + test {GEORANGE STOREDIST option: plain usage} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + r georadius points 13.361389 38.115556 500 km storedist points2 + set res [r zrange points2 0 -1 withscores] + assert {[lindex $res 1] < 1} + assert {[lindex $res 3] > 166} + assert {[lindex $res 3] < 167} + } + + test {GEORANGE STOREDIST option: COUNT ASC and DESC} { + r del points + r geoadd points 13.361389 38.115556 "Palermo" \ + 15.087269 37.502669 "Catania" + r georadius points 13.361389 38.115556 500 km storedist points2 asc count 1 + assert {[r zcard points2] == 1} + set res [r zrange points2 0 -1 withscores] + assert {[lindex $res 0] eq "Palermo"} + + r georadius points 13.361389 38.115556 500 km storedist points2 desc count 1 + assert {[r zcard points2] == 1} + set res [r zrange points2 0 -1 withscores] + assert {[lindex $res 0] eq "Catania"} + } + + test {GEOADD + GEORANGE randomized test} { + set attempt 30 + while {[incr attempt -1]} { + set rv [lindex $regression_vectors $rv_idx] + incr rv_idx + + unset -nocomplain debuginfo + set srand_seed [clock milliseconds] + if {$rv ne {}} {set srand_seed [lindex $rv 0]} + lappend debuginfo "srand_seed is $srand_seed" + expr {srand($srand_seed)} ; # If you need a reproducible run + r del mypoints + + if {[randomInt 10] == 0} { + # From time to time use very big radiuses + set radius_km [expr {[randomInt 50000]+10}] + } else { + # Normally use a few - ~200km radiuses to stress + # test the code the most in edge cases. + set radius_km [expr {[randomInt 200]+10}] + } + if {$rv ne {}} {set radius_km [lindex $rv 1]} + set radius_m [expr {$radius_km*1000}] + geo_random_point search_lon search_lat + if {$rv ne {}} { + set search_lon [lindex $rv 2] + set search_lat [lindex $rv 3] + } + lappend debuginfo "Search area: $search_lon,$search_lat $radius_km km" + set tcl_result {} + set argv {} + for {set j 0} {$j < 20000} {incr j} { + geo_random_point lon lat + lappend argv $lon $lat "place:$j" + set distance [geo_distance $lon $lat $search_lon $search_lat] + if {$distance < $radius_m} { + lappend tcl_result "place:$j" + } + lappend debuginfo "place:$j $lon $lat [expr {$distance/1000}] km" + } + r geoadd mypoints {*}$argv + set res [lsort [r georadius mypoints $search_lon $search_lat $radius_km km]] + set res2 [lsort $tcl_result] + set test_result OK + + if {$res != $res2} { + set rounding_errors 0 + set diff [compare_lists $res $res2] + foreach place $diff { + set mydist [geo_distance $lon $lat $search_lon $search_lat] + set mydist [expr $mydist/1000] + if {($mydist / $radius_km) > 0.999} {incr rounding_errors} + } + # Make sure this is a real error and not a rounidng issue. + if {[llength $diff] == $rounding_errors} { + set res $res2; # Error silenced + } + } + + if {$res != $res2} { + set diff [compare_lists $res $res2] + puts "*** Possible problem in GEO radius query ***" + puts "Redis: $res" + puts "Tcl : $res2" + puts "Diff : $diff" + puts [join $debuginfo "\n"] + foreach place $diff { + if {[lsearch -exact $res2 $place] != -1} { + set where "(only in Tcl)" + } else { + set where "(only in Redis)" + } + lassign [lindex [r geopos mypoints $place] 0] lon lat + set mydist [geo_distance $lon $lat $search_lon $search_lat] + set mydist [expr $mydist/1000] + puts "$place -> [r geopos mypoints $place] $mydist $where" + if {($mydist / $radius_km) > 0.999} {incr rounding_errors} + } + set test_result FAIL + } + unset -nocomplain debuginfo + if {$test_result ne {OK}} break + } + set test_result + } {OK} +} diff --git a/tools/pika_migrate/tests/unit/hyperloglog.tcl b/tools/pika_migrate/tests/unit/hyperloglog.tcl new file mode 100755 index 0000000000..6d614bb156 --- /dev/null +++ b/tools/pika_migrate/tests/unit/hyperloglog.tcl @@ -0,0 +1,250 @@ +start_server {tags {"hll"}} { +# test {HyperLogLog self test passes} { +# catch {r pfselftest} e +# set e +# } {OK} + + test {PFADD without arguments creates an HLL value} { + r pfadd hll + r exists hll + } {1} + + test {Approximated cardinality after creation is zero} { + r pfcount hll + } {0} + + test {PFADD returns 1 when at least 1 reg was modified} { + r pfadd hll a b c + } {1} + + test {PFADD returns 0 when no reg was modified} { + r pfadd hll a b c + } {0} + + test {PFADD works with empty string (regression)} { + r pfadd hll "" + } + + # Note that the self test stresses much better the + # cardinality estimation error. We are testing just the + # command implementation itself here. + test {PFCOUNT returns approximated cardinality of set} { + r del hll + set res {} + r pfadd hll 1 2 3 4 5 + lappend res [r pfcount hll] + # Call it again to test cached value invalidation. + r pfadd hll 6 7 8 8 9 10 + lappend res [r pfcount hll] + set res + } {5 10} + +# test {HyperLogLogs are promote from sparse to dense} { +# r del hll +# r config set hll-sparse-max-bytes 3000 +# set n 0 +# while {$n < 100000} { +# set elements {} +# for {set j 0} {$j < 100} {incr j} {lappend elements [expr rand()]} +# incr n 100 +# r pfadd hll {*}$elements +# set card [r pfcount hll] +# set err [expr {abs($card-$n)}] +# assert {$err < (double($card)/100)*5} +# if {$n < 1000} { +# assert {[r pfdebug encoding hll] eq {sparse}} +# } elseif {$n > 10000} { +# assert {[r pfdebug encoding hll] eq {dense}} +# } +# } +# } + +# test {HyperLogLog sparse encoding stress test} { +# for {set x 0} {$x < 1000} {incr x} { +# r del hll1 hll2 +# set numele [randomInt 100] +# set elements {} +# for {set j 0} {$j < $numele} {incr j} { +# lappend elements [expr rand()] +# } + # Force dense representation of hll2 +# r pfadd hll2 +# r pfdebug todense hll2 +# r pfadd hll1 {*}$elements +# r pfadd hll2 {*}$elements +# assert {[r pfdebug encoding hll1] eq {sparse}} +# assert {[r pfdebug encoding hll2] eq {dense}} + # Cardinality estimated should match exactly. +# assert {[r pfcount hll1] eq [r pfcount hll2]} +# } +# } + +# test {Corrupted sparse HyperLogLogs are detected: Additionl at tail} { +# r del hll +# r pfadd hll a b c +# r append hll "hello" +# set e {} +# catch {r pfcount hll} e +# set e +# } {*INVALIDOBJ*} + +# test {Corrupted sparse HyperLogLogs are detected: Broken magic} { +# r del hll +# r pfadd hll a b c +# r setrange hll 0 "0123" +# set e {} +# catch {r pfcount hll} e +# set e +# } {*WRONGTYPE*} + +# test {Corrupted sparse HyperLogLogs are detected: Invalid encoding} { +# r del hll +# r pfadd hll a b c +# r setrange hll 4 "x" +# set e {} +# catch {r pfcount hll} e +# set e +# } {*WRONGTYPE*} + +# test {Corrupted dense HyperLogLogs are detected: Wrong length} { +# r del hll +# r pfadd hll a b c +# r setrange hll 4 "\x00" +# set e {} +# catch {r pfcount hll} e +# set e +# } {*WRONGTYPE*} + +# test {PFADD, PFCOUNT, PFMERGE type checking works} { +# r set foo bar +# catch {r pfadd foo 1} e +# assert_match {*WRONGTYPE*} $e +# catch {r pfcount foo} e +# assert_match {*WRONGTYPE*} $e +# catch {r pfmerge bar foo} e +# assert_match {*WRONGTYPE*} $e +# catch {r pfmerge foo bar} e +# assert_match {*WRONGTYPE*} $e +# } + + test {PFMERGE results on the cardinality of union of sets} { + r del hll hll1 hll2 hll3 + r pfadd hll1 a b c + r pfadd hll2 b c d + r pfadd hll3 c d e + r pfmerge hll hll1 hll2 hll3 + r pfcount hll + } {5} + + test {PFCOUNT multiple-keys merge returns cardinality of union} { + r del hll1 hll2 hll3 + for {set x 1} {$x < 100000} {incr x} { + # Force dense representation of hll2 + r pfadd hll1 "foo-$x" + r pfadd hll2 "bar-$x" + r pfadd hll3 "zap-$x" + + set card [r pfcount hll1 hll2 hll3] + set realcard [expr {$x*3}] + set err [expr {abs($card-$realcard)}] + assert {$err < (double($card)/100)*5} + } + } + + test {HYPERLOGLOG press test: 5w, 10w, 15w, 20w, 30w, 50w, 100w} { + r del hll1 + for {set x 1} {$x <= 1000000} {incr x} { + r pfadd hll1 "foo-$x" + if {$x == 50000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.01} + } + if {$x == 100000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.01} + } + if {$x == 150000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.01} + } + if {$x == 300000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.01} + } + if {$x == 500000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.01} + } + if {$x == 1000000} { + set card [r pfcount hll1] + set realcard [expr {$x*1}] + set err [expr {abs($card-$realcard)}] + + set d_err [expr {$err * 1.0}] + set d_realcard [expr {$realcard * 1.0}] + set err_precentage [expr {double($d_err / $d_realcard)}] + puts "$x error rate: $err_precentage" + assert {$err < $realcard * 0.03} + } + } + } + +# test {PFDEBUG GETREG returns the HyperLogLog raw registers} { +# r del hll +# r pfadd hll 1 2 3 +# llength [r pfdebug getreg hll] +# } {16384} + + +# test {PFDEBUG GETREG returns the HyperLogLog raw registers} { +# r del hll +# r pfadd hll 1 2 3 +# llength [r pfdebug getreg hll] +# } {16384} + +# test {PFADD / PFCOUNT cache invalidation works} { +# r del hll +# r pfadd hll a b c +# r pfcount hll +# assert {[r getrange hll 15 15] eq "\x00"} +# r pfadd hll a b c +# assert {[r getrange hll 15 15] eq "\x00"} +# r pfadd hll 1 2 3 +# assert {[r getrange hll 15 15] eq "\x80"} +# } +} diff --git a/tools/pika_migrate/tests/unit/introspection.tcl b/tools/pika_migrate/tests/unit/introspection.tcl new file mode 100644 index 0000000000..342bb939a8 --- /dev/null +++ b/tools/pika_migrate/tests/unit/introspection.tcl @@ -0,0 +1,59 @@ +start_server {tags {"introspection"}} { + test {CLIENT LIST} { + r client list + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + + test {MONITOR can log executed commands} { + set rd [redis_deferring_client] + $rd monitor + r set foo bar + r get foo + list [$rd read] [$rd read] [$rd read] + } {*OK*"set" "foo"*"get" "foo"*} + + test {MONITOR can log commands issued by the scripting engine} { + set rd [redis_deferring_client] + $rd monitor + r eval {redis.call('set',KEYS[1],ARGV[1])} 1 foo bar + $rd read ;# Discard the OK + assert_match {*eval*} [$rd read] + assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] + } + + test {CLIENT GETNAME should return NIL if name is not assigned} { + r client getname + } {} + + test {CLIENT LIST shows empty fields for unassigned names} { + r client list + } {*name= *} + + test {CLIENT SETNAME does not accept spaces} { + catch {r client setname "foo bar"} e + set e + } {ERR*} + + test {CLIENT SETNAME can assign a name to this connection} { + assert_equal [r client setname myname] {OK} + r client list + } {*name=myname*} + + test {CLIENT SETNAME can change the name of an existing connection} { + assert_equal [r client setname someothername] {OK} + r client list + } {*name=someothername*} + + test {After CLIENT SETNAME, connection can still be closed} { + set rd [redis_deferring_client] + $rd client setname foobar + assert_equal [$rd read] "OK" + assert_match {*foobar*} [r client list] + $rd close + # Now the client should no longer be listed + wait_for_condition 50 100 { + [string match {*foobar*} [r client list]] == 0 + } else { + fail "Client still listed in CLIENT LIST after SETNAME." + } + } +} diff --git a/tools/pika_migrate/tests/unit/keys.tcl b/tools/pika_migrate/tests/unit/keys.tcl new file mode 100644 index 0000000000..cb62444f3f --- /dev/null +++ b/tools/pika_migrate/tests/unit/keys.tcl @@ -0,0 +1,54 @@ +start_server {tags {"keys"}} { + test {KEYS with pattern} { + foreach key {key_x key_y key_z foo_a foo_b foo_c} { + r set $key hello + } + assert_equal {foo_a foo_b foo_c} [r keys foo*] + assert_equal {foo_a foo_b foo_c} [r keys f*] + assert_equal {foo_a foo_b foo_c} [r keys f*o*] + } + + test {KEYS to get all keys} { + lsort [r keys *] + } {foo_a foo_b foo_c key_x key_y key_z} + + test {KEYS select by type} { + foreach key {key_x key_y key_z foo_a foo_b foo_c} { + r del $key + } + r set kv_1 value + r set kv_2 value + r hset hash_1 hash_field 1 + r hset hash_2 hash_field 1 + r lpush list_1 value + r lpush list_2 value + r zadd zset_1 1 "a" + r zadd zset_2 1 "a" + r sadd set_1 "a" + r sadd set_2 "a" + assert_equal {kv_1 kv_2} [r keys * string] + assert_equal {hash_1 hash_2} [r keys * hash] + assert_equal {list_1 list_2} [r keys * list] + assert_equal {zset_1 zset_2} [r keys * zset] + assert_equal {set_1 set_2} [r keys * set] + assert_equal {kv_1 kv_2 hash_1 hash_2 zset_1 zset_2 set_1 set_2 list_1 list_2} [r keys *] + assert_equal {kv_1 kv_2} [r keys * STRING] + assert_equal {hash_1 hash_2} [r keys * HASH] + assert_equal {list_1 list_2} [r keys * LIST] + assert_equal {zset_1 zset_2} [r keys * ZSET] + assert_equal {set_1 set_2} [r keys * SET] + } + + test {KEYS syntax error} { + catch {r keys * a} e1 + catch {r keys * strings} e2 + catch {r keys * c d} e3 + catch {r keys} e4 + catch {r keys * set zset} e5 + assert_equal {ERR syntax error} [set e1] + assert_equal {ERR syntax error} [set e2] + assert_equal {ERR syntax error} [set e3] + assert_equal {ERR wrong number of arguments for 'keys' command} [set e4] + assert_equal {ERR syntax error} [set e5] + } +} diff --git a/tools/pika_migrate/tests/unit/latency-monitor.tcl b/tools/pika_migrate/tests/unit/latency-monitor.tcl new file mode 100644 index 0000000000..b736cad98b --- /dev/null +++ b/tools/pika_migrate/tests/unit/latency-monitor.tcl @@ -0,0 +1,50 @@ +start_server {tags {"latency-monitor"}} { + # Set a threshold high enough to avoid spurious latency events. + r config set latency-monitor-threshold 200 + r latency reset + + test {Test latency events logging} { + r debug sleep 0.3 + after 1100 + r debug sleep 0.4 + after 1100 + r debug sleep 0.5 + assert {[r latency history command] >= 3} + } + + test {LATENCY HISTORY output is ok} { + set min 250 + set max 450 + foreach event [r latency history command] { + lassign $event time latency + assert {$latency >= $min && $latency <= $max} + incr min 100 + incr max 100 + set last_time $time ; # Used in the next test + } + } + + test {LATENCY LATEST output is ok} { + foreach event [r latency latest] { + lassign $event eventname time latency max + assert {$eventname eq "command"} + assert {$max >= 450 & $max <= 650} + assert {$time == $last_time} + break + } + } + + test {LATENCY HISTORY / RESET with wrong event name is fine} { + assert {[llength [r latency history blabla]] == 0} + assert {[r latency reset blabla] == 0} + } + + test {LATENCY DOCTOR produces some output} { + assert {[string length [r latency doctor]] > 0} + } + + test {LATENCY RESET is able to reset events} { + assert {[r latency reset] > 0} + assert {[r latency latest] eq {}} + } +} diff --git a/tools/pika_migrate/tests/unit/limits.tcl b/tools/pika_migrate/tests/unit/limits.tcl new file mode 100644 index 0000000000..b37ea9b0f5 --- /dev/null +++ b/tools/pika_migrate/tests/unit/limits.tcl @@ -0,0 +1,16 @@ +start_server {tags {"limits"} overrides {maxclients 10}} { + test {Check if maxclients works refusing connections} { + set c 0 + catch { + while {$c < 50} { + incr c + set rd [redis_deferring_client] + $rd ping + $rd read + after 100 + } + } e + assert {$c > 8 && $c <= 10} + set e + } {*ERR max*reached*} +} diff --git a/tools/pika_migrate/tests/unit/maxmemory.tcl b/tools/pika_migrate/tests/unit/maxmemory.tcl new file mode 100644 index 0000000000..e6bf7860cb --- /dev/null +++ b/tools/pika_migrate/tests/unit/maxmemory.tcl @@ -0,0 +1,144 @@ +start_server {tags {"maxmemory"}} { + test "Without maxmemory small integers are shared" { + r config set maxmemory 0 + r set a 1 + assert {[r object refcount a] > 1} + } + + test "With maxmemory and non-LRU policy integers are still shared" { + r config set maxmemory 1073741824 + r config set maxmemory-policy allkeys-random + r set a 1 + assert {[r object refcount a] > 1} + } + + test "With maxmemory and LRU policy integers are not shared" { + r config set maxmemory 1073741824 + r config set maxmemory-policy allkeys-lru + r set a 1 + r config set maxmemory-policy volatile-lru + r set b 1 + assert {[r object refcount a] == 1} + assert {[r object refcount b] == 1} + r config set maxmemory 0 + } + + foreach policy { + allkeys-random allkeys-lru volatile-lru volatile-random volatile-ttl + } { + test "maxmemory - is the memory limit honoured? (policy $policy)" { + # make sure to start with a blank instance + r flushall + # Get the current memory limit and calculate a new limit. + # We just add 100k to the current memory size so that it is + # fast for us to reach that limit. + set used [s used_memory] + set limit [expr {$used+100*1024}] + r config set maxmemory $limit + r config set maxmemory-policy $policy + # Now add keys until the limit is almost reached. + set numkeys 0 + while 1 { + r setex [randomKey] 10000 x + incr numkeys + if {[s used_memory]+4096 > $limit} { + assert {$numkeys > 10} + break + } + } + # If we add the same number of keys already added again, we + # should still be under the limit. + for {set j 0} {$j < $numkeys} {incr j} { + r setex [randomKey] 10000 x + } + assert {[s used_memory] < ($limit+4096)} + } + } + + foreach policy { + allkeys-random allkeys-lru volatile-lru volatile-random volatile-ttl + } { + test "maxmemory - only allkeys-* should remove non-volatile keys ($policy)" { + # make sure to start with a blank instance + r flushall + # Get the current memory limit and calculate a new limit. + # We just add 100k to the current memory size so that it is + # fast for us to reach that limit. + set used [s used_memory] + set limit [expr {$used+100*1024}] + r config set maxmemory $limit + r config set maxmemory-policy $policy + # Now add keys until the limit is almost reached. + set numkeys 0 + while 1 { + r set [randomKey] x + incr numkeys + if {[s used_memory]+4096 > $limit} { + assert {$numkeys > 10} + break + } + } + # If we add the same number of keys already added again and + # the policy is allkeys-* we should still be under the limit. + # Otherwise we should see an error reported by Redis. + set err 0 + for {set j 0} {$j < $numkeys} {incr j} { + if {[catch {r set [randomKey] x} e]} { + if {[string match {*used memory*} $e]} { + set err 1 + } + } + } + if {[string match allkeys-* $policy]} { + assert {[s used_memory] < ($limit+4096)} + } else { + assert {$err == 1} + } + } + } + + foreach policy { + volatile-lru volatile-random volatile-ttl + } { + test "maxmemory - policy $policy should only remove volatile keys." { + # make sure to start with a blank instance + r flushall + # Get the current memory limit and calculate a new limit. + # We just add 100k to the current memory size so that it is + # fast for us to reach that limit. + set used [s used_memory] + set limit [expr {$used+100*1024}] + r config set maxmemory $limit + r config set maxmemory-policy $policy + # Now add keys until the limit is almost reached. + set numkeys 0 + while 1 { + # Odd keys are volatile + # Even keys are non volatile + if {$numkeys % 2} { + r setex "key:$numkeys" 10000 x + } else { + r set "key:$numkeys" x + } + if {[s used_memory]+4096 > $limit} { + assert {$numkeys > 10} + break + } + incr numkeys + } + # Now we add the same number of volatile keys already added. + # We expect Redis to evict only volatile keys in order to make + # space. + set err 0 + for {set j 0} {$j < $numkeys} {incr j} { + catch {r setex "foo:$j" 10000 x} + } + # We should still be under the limit. + assert {[s used_memory] < ($limit+4096)} + # However all our non volatile keys should be here. + for {set j 0} {$j < $numkeys} {incr j 2} { + assert {[r exists "key:$j"]} + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/memefficiency.tcl b/tools/pika_migrate/tests/unit/memefficiency.tcl new file mode 100644 index 0000000000..7ca9a705bb --- /dev/null +++ b/tools/pika_migrate/tests/unit/memefficiency.tcl @@ -0,0 +1,37 @@ +proc test_memory_efficiency {range} { + r flushall + set rd [redis_deferring_client] + set base_mem [s used_memory] + set written 0 + for {set j 0} {$j < 10000} {incr j} { + set key key:$j + set val [string repeat A [expr {int(rand()*$range)}]] + $rd set $key $val + incr written [string length $key] + incr written [string length $val] + incr written 2 ;# A separator is the minimum to store key-value data. + } + for {set j 0} {$j < 10000} {incr j} { + $rd read ; # Discard replies + } + + set current_mem [s used_memory] + set used [expr {$current_mem-$base_mem}] + set efficiency [expr {double($written)/$used}] + return $efficiency +} + +start_server {tags {"memefficiency"}} { + foreach {size_range expected_min_efficiency} { + 32 0.15 + 64 0.25 + 128 0.35 + 1024 0.75 + 16384 0.82 + } { + test "Memory efficiency with values in range $size_range" { + set efficiency [test_memory_efficiency $size_range] + assert {$efficiency >= $expected_min_efficiency} + } + } +} diff --git a/tools/pika_migrate/tests/unit/multi.tcl b/tools/pika_migrate/tests/unit/multi.tcl new file mode 100644 index 0000000000..6655bf62c2 --- /dev/null +++ b/tools/pika_migrate/tests/unit/multi.tcl @@ -0,0 +1,309 @@ +start_server {tags {"multi"}} { + test {MUTLI / EXEC basics} { + r del mylist + r rpush mylist a + r rpush mylist b + r rpush mylist c + r multi + set v1 [r lrange mylist 0 -1] + set v2 [r ping] + set v3 [r exec] + list $v1 $v2 $v3 + } {QUEUED QUEUED {{a b c} PONG}} + + test {DISCARD} { + r del mylist + r rpush mylist a + r rpush mylist b + r rpush mylist c + r multi + set v1 [r del mylist] + set v2 [r discard] + set v3 [r lrange mylist 0 -1] + list $v1 $v2 $v3 + } {QUEUED OK {a b c}} + + test {Nested MULTI are not allowed} { + set err {} + r multi + catch {[r multi]} err + r exec + set _ $err + } {*ERR MULTI*} + + test {MULTI where commands alter argc/argv} { + r sadd myset a + r multi + r spop myset + list [r exec] [r exists myset] + } {a 0} + + test {WATCH inside MULTI is not allowed} { + set err {} + r multi + catch {[r watch x]} err + r exec + set _ $err + } {*ERR WATCH*} + + test {EXEC fails if there are errors while queueing commands #1} { + r del foo1 foo2 + r multi + r set foo1 bar1 + catch {r non-existing-command} + r set foo2 bar2 + catch {r exec} e + assert_match {EXECABORT*} $e + list [r exists foo1] [r exists foo2] + } {0 0} + + test {EXEC fails if there are errors while queueing commands #2} { + set rd [redis_deferring_client] + r del foo1 foo2 + r multi + r set foo1 bar1 + $rd config set maxmemory 1 + assert {[$rd read] eq {OK}} + catch {r lpush mylist myvalue} + $rd config set maxmemory 0 + assert {[$rd read] eq {OK}} + r set foo2 bar2 + catch {r exec} e + assert_match {EXECABORT*} $e + $rd close + list [r exists foo1] [r exists foo2] + } {0 0} + + test {If EXEC aborts, the client MULTI state is cleared} { + r del foo1 foo2 + r multi + r set foo1 bar1 + catch {r non-existing-command} + r set foo2 bar2 + catch {r exec} e + assert_match {EXECABORT*} $e + r ping + } {PONG} + + test {EXEC works on WATCHed key not modified} { + r watch x y z + r watch k + r multi + r ping + r exec + } {PONG} + + test {EXEC fail on WATCHed key modified (1 key of 1 watched)} { + r set x 30 + r watch x + r set x 40 + r multi + r ping + r exec + } {} + + test {EXEC fail on WATCHed key modified (1 key of 5 watched)} { + r set x 30 + r watch a b x k z + r set x 40 + r multi + r ping + r exec + } {} + + test {EXEC fail on WATCHed key modified by SORT with STORE even if the result is empty} { + r flushdb + r lpush foo bar + r watch foo + r sort emptylist store foo + r multi + r ping + r exec + } {} + + test {After successful EXEC key is no longer watched} { + r set x 30 + r watch x + r multi + r ping + r exec + r set x 40 + r multi + r ping + r exec + } {PONG} + + test {After failed EXEC key is no longer watched} { + r set x 30 + r watch x + r set x 40 + r multi + r ping + r exec + r set x 40 + r multi + r ping + r exec + } {PONG} + + test {It is possible to UNWATCH} { + r set x 30 + r watch x + r set x 40 + r unwatch + r multi + r ping + r exec + } {PONG} + + test {UNWATCH when there is nothing watched works as expected} { + r unwatch + } {OK} + + test {FLUSHALL is able to touch the watched keys} { + r set x 30 + r watch x + r flushall + r multi + r ping + r exec + } {} + + test {FLUSHALL does not touch non affected keys} { + r del x + r watch x + r flushall + r multi + r ping + r exec + } {PONG} + + test {FLUSHDB is able to touch the watched keys} { + r set x 30 + r watch x + r flushdb + r multi + r ping + r exec + } {} + + test {FLUSHDB does not touch non affected keys} { + r del x + r watch x + r flushdb + r multi + r ping + r exec + } {PONG} + + test {WATCH is able to remember the DB a key belongs to} { + r select 5 + r set x 30 + r watch x + r select 1 + r set x 10 + r select 5 + r multi + r ping + set res [r exec] + # Restore original DB + r select 9 + set res + } {PONG} + + test {WATCH will consider touched keys target of EXPIRE} { + r del x + r set x foo + r watch x + r expire x 10 + r multi + r ping + r exec + } {} + + test {WATCH will not consider touched expired keys} { + r del x + r set x foo + r expire x 1 + r watch x + after 1100 + r multi + r ping + r exec + } {PONG} + + test {DISCARD should clear the WATCH dirty flag on the client} { + r watch x + r set x 10 + r multi + r discard + r multi + r incr x + r exec + } {11} + + test {DISCARD should UNWATCH all the keys} { + r watch x + r set x 10 + r multi + r discard + r set x 10 + r multi + r incr x + r exec + } {11} + + test {MULTI / EXEC is propagated correctly (single write command)} { + set repl [attach_to_replication_stream] + r multi + r set foo bar + r exec + assert_replication_stream $repl { + {select *} + {multi} + {set foo bar} + {exec} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (empty transaction)} { + set repl [attach_to_replication_stream] + r multi + r exec + r set foo bar + assert_replication_stream $repl { + {select *} + {set foo bar} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (read-only commands)} { + r set foo value1 + set repl [attach_to_replication_stream] + r multi + r get foo + r exec + r set foo value2 + assert_replication_stream $repl { + {select *} + {set foo value2} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (write command, no effect)} { + r del bar foo bar + set repl [attach_to_replication_stream] + r multi + r del foo + r exec + assert_replication_stream $repl { + {select *} + {multi} + {exec} + } + close_replication_stream $repl + } +} diff --git a/tools/pika_migrate/tests/unit/obuf-limits.tcl b/tools/pika_migrate/tests/unit/obuf-limits.tcl new file mode 100644 index 0000000000..5d625cf453 --- /dev/null +++ b/tools/pika_migrate/tests/unit/obuf-limits.tcl @@ -0,0 +1,73 @@ +start_server {tags {"obuf-limits"}} { + test {Client output buffer hard limit is enforced} { + r config set client-output-buffer-limit {pubsub 100000 0 0} + set rd1 [redis_deferring_client] + + $rd1 subscribe foo + set reply [$rd1 read] + assert {$reply eq "subscribe foo 1"} + + set omem 0 + while 1 { + r publish foo bar + set clients [split [r client list] "\r\n"] + set c [split [lindex $clients 1] " "] + if {![regexp {omem=([0-9]+)} $c - omem]} break + if {$omem > 200000} break + } + assert {$omem >= 90000 && $omem < 200000} + $rd1 close + } + + test {Client output buffer soft limit is not enforced if time is not overreached} { + r config set client-output-buffer-limit {pubsub 0 100000 10} + set rd1 [redis_deferring_client] + + $rd1 subscribe foo + set reply [$rd1 read] + assert {$reply eq "subscribe foo 1"} + + set omem 0 + set start_time 0 + set time_elapsed 0 + while 1 { + r publish foo bar + set clients [split [r client list] "\r\n"] + set c [split [lindex $clients 1] " "] + if {![regexp {omem=([0-9]+)} $c - omem]} break + if {$omem > 100000} { + if {$start_time == 0} {set start_time [clock seconds]} + set time_elapsed [expr {[clock seconds]-$start_time}] + if {$time_elapsed >= 5} break + } + } + assert {$omem >= 100000 && $time_elapsed >= 5 && $time_elapsed <= 10} + $rd1 close + } + + test {Client output buffer soft limit is enforced if time is overreached} { + r config set client-output-buffer-limit {pubsub 0 100000 3} + set rd1 [redis_deferring_client] + + $rd1 subscribe foo + set reply [$rd1 read] + assert {$reply eq "subscribe foo 1"} + + set omem 0 + set start_time 0 + set time_elapsed 0 + while 1 { + r publish foo bar + set clients [split [r client list] "\r\n"] + set c [split [lindex $clients 1] " "] + if {![regexp {omem=([0-9]+)} $c - omem]} break + if {$omem > 100000} { + if {$start_time == 0} {set start_time [clock seconds]} + set time_elapsed [expr {[clock seconds]-$start_time}] + if {$time_elapsed >= 10} break + } + } + assert {$omem >= 100000 && $time_elapsed < 6} + $rd1 close + } +} diff --git a/tools/pika_migrate/tests/unit/other.tcl b/tools/pika_migrate/tests/unit/other.tcl new file mode 100644 index 0000000000..a53f3f5c81 --- /dev/null +++ b/tools/pika_migrate/tests/unit/other.tcl @@ -0,0 +1,245 @@ +start_server {tags {"other"}} { + if {$::force_failure} { + # This is used just for test suite development purposes. + test {Failing test} { + format err + } {ok} + } + + test {SAVE - make sure there are all the types as values} { + # Wait for a background saving in progress to terminate + waitForBgsave r + r lpush mysavelist hello + r lpush mysavelist world + r set myemptykey {} + r set mynormalkey {blablablba} + r zadd mytestzset 10 a + r zadd mytestzset 20 b + r zadd mytestzset 30 c + r save + } {OK} + + tags {slow} { + if {$::accurate} {set iterations 10000} else {set iterations 1000} + foreach fuzztype {binary alpha compr} { + test "FUZZ stresser with data model $fuzztype" { + set err 0 + for {set i 0} {$i < $iterations} {incr i} { + set fuzz [randstring 0 512 $fuzztype] + r set foo $fuzz + set got [r get foo] + if {$got ne $fuzz} { + set err [list $fuzz $got] + break + } + } + set _ $err + } {0} + } + } + + test {BGSAVE} { + waitForBgsave r + r flushdb + r save + r set x 10 + r bgsave + waitForBgsave r + r debug reload + r get x + } {10} + + test {SELECT an out of range DB} { + catch {r select 1000000} err + set _ $err + } {*invalid*} + + tags {consistency} { + if {![catch {package require sha1}]} { + if {$::accurate} {set numops 10000} else {set numops 1000} + test {Check consistency of different data types after a reload} { + r flushdb + createComplexDataset r $numops + set dump [csvdump r] + set sha1 [r debug digest] + r debug reload + set sha1_after [r debug digest] + if {$sha1 eq $sha1_after} { + set _ 1 + } else { + set newdump [csvdump r] + puts "Consistency test failed!" + puts "You can inspect the two dumps in /tmp/repldump*.txt" + + set fd [open /tmp/repldump1.txt w] + puts $fd $dump + close $fd + set fd [open /tmp/repldump2.txt w] + puts $fd $newdump + close $fd + + set _ 0 + } + } {1} + + test {Same dataset digest if saving/reloading as AOF?} { + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set sha1_after [r debug digest] + if {$sha1 eq $sha1_after} { + set _ 1 + } else { + set newdump [csvdump r] + puts "Consistency test failed!" + puts "You can inspect the two dumps in /tmp/aofdump*.txt" + + set fd [open /tmp/aofdump1.txt w] + puts $fd $dump + close $fd + set fd [open /tmp/aofdump2.txt w] + puts $fd $newdump + close $fd + + set _ 0 + } + } {1} + } + } + + test {EXPIRES after a reload (snapshot + append only file rewrite)} { + r flushdb + r set x 10 + r expire x 1000 + r save + r debug reload + set ttl [r ttl x] + set e1 [expr {$ttl > 900 && $ttl <= 1000}] + r bgrewriteaof + waitForBgrewriteaof r + r debug loadaof + set ttl [r ttl x] + set e2 [expr {$ttl > 900 && $ttl <= 1000}] + list $e1 $e2 + } {1 1} + + test {EXPIRES after AOF reload (without rewrite)} { + r flushdb + r config set appendonly yes + r set x somevalue + r expire x 1000 + r setex y 2000 somevalue + r set z somevalue + r expireat z [expr {[clock seconds]+3000}] + + # Milliseconds variants + r set px somevalue + r pexpire px 1000000 + r psetex py 2000000 somevalue + r set pz somevalue + r pexpireat pz [expr {([clock seconds]+3000)*1000}] + + # Reload and check + waitForBgrewriteaof r + # We need to wait two seconds to avoid false positives here, otherwise + # the DEBUG LOADAOF command may read a partial file. + # Another solution would be to set the fsync policy to no, since this + # prevents write() to be delayed by the completion of fsync(). + after 2000 + r debug loadaof + set ttl [r ttl x] + assert {$ttl > 900 && $ttl <= 1000} + set ttl [r ttl y] + assert {$ttl > 1900 && $ttl <= 2000} + set ttl [r ttl z] + assert {$ttl > 2900 && $ttl <= 3000} + set ttl [r ttl px] + assert {$ttl > 900 && $ttl <= 1000} + set ttl [r ttl py] + assert {$ttl > 1900 && $ttl <= 2000} + set ttl [r ttl pz] + assert {$ttl > 2900 && $ttl <= 3000} + r config set appendonly no + } + + tags {protocol} { + test {PIPELINING stresser (also a regression for the old epoll bug)} { + set fd2 [socket $::host $::port] + fconfigure $fd2 -encoding binary -translation binary + puts -nonewline $fd2 "SELECT 9\r\n" + flush $fd2 + gets $fd2 + + for {set i 0} {$i < 100000} {incr i} { + set q {} + set val "0000${i}0000" + append q "SET key:$i $val\r\n" + puts -nonewline $fd2 $q + set q {} + append q "GET key:$i\r\n" + puts -nonewline $fd2 $q + } + flush $fd2 + + for {set i 0} {$i < 100000} {incr i} { + gets $fd2 line + gets $fd2 count + set count [string range $count 1 end] + set val [read $fd2 $count] + read $fd2 2 + } + close $fd2 + set _ 1 + } {1} + } + + test {APPEND basics} { + list [r append foo bar] [r get foo] \ + [r append foo 100] [r get foo] + } {3 bar 6 bar100} + + test {APPEND basics, integer encoded values} { + set res {} + r del foo + r append foo 1 + r append foo 2 + lappend res [r get foo] + r set foo 1 + r append foo 2 + lappend res [r get foo] + } {12 12} + + test {APPEND fuzzing} { + set err {} + foreach type {binary alpha compr} { + set buf {} + r del x + for {set i 0} {$i < 1000} {incr i} { + set bin [randstring 0 10 $type] + append buf $bin + r append x $bin + } + if {$buf != [r get x]} { + set err "Expected '$buf' found '[r get x]'" + break + } + } + set _ $err + } {} + + # Leave the user with a clean DB before to exit + test {FLUSHDB} { + set aux {} + r select 9 + r flushdb + lappend aux [r dbsize] + r select 10 + r flushdb + lappend aux [r dbsize] + } {0 0} + + test {Perform a final SAVE to leave a clean DB on disk} { + waitForBgsave r + r save + } {OK} +} diff --git a/tools/pika_migrate/tests/unit/printver.tcl b/tools/pika_migrate/tests/unit/printver.tcl new file mode 100644 index 0000000000..c80f45144d --- /dev/null +++ b/tools/pika_migrate/tests/unit/printver.tcl @@ -0,0 +1,6 @@ +start_server {} { + set i [r info] + regexp {redis_version:(.*?)\r\n} $i - version + regexp {redis_git_sha1:(.*?)\r\n} $i - sha1 + puts "Testing Redis version $version ($sha1)" +} diff --git a/tools/pika_migrate/tests/unit/protocol.tcl b/tools/pika_migrate/tests/unit/protocol.tcl new file mode 100644 index 0000000000..ac99c3abb4 --- /dev/null +++ b/tools/pika_migrate/tests/unit/protocol.tcl @@ -0,0 +1,117 @@ +start_server {tags {"protocol"}} { + test "Handle an empty query" { + reconnect + r write "\r\n" + r flush + assert_equal "PONG" [r ping] + } + + test "Negative multibulk length" { + reconnect + r write "*-10\r\n" + r flush + assert_equal PONG [r ping] + } + + test "Out of range multibulk length" { + reconnect + r write "*20000000\r\n" + r flush + assert_error "*invalid multibulk length*" {r read} + } + + test "Wrong multibulk payload header" { + reconnect + r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\nfooz\r\n" + r flush + assert_error "*expected '$', got 'f'*" {r read} + } + + test "Negative multibulk payload length" { + reconnect + r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$-10\r\n" + r flush + assert_error "*invalid bulk length*" {r read} + } + + test "Out of range multibulk payload length" { + reconnect + r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$2000000000\r\n" + r flush + assert_error "*invalid bulk length*" {r read} + } + + test "Non-number multibulk payload length" { + reconnect + r write "*3\r\n\$3\r\nSET\r\n\$1\r\nx\r\n\$blabla\r\n" + r flush + assert_error "*invalid bulk length*" {r read} + } + + test "Multi bulk request not followed by bulk arguments" { + reconnect + r write "*1\r\nfoo\r\n" + r flush + assert_error "*expected '$', got 'f'*" {r read} + } + + test "Generic wrong number of args" { + reconnect + assert_error "*wrong*arguments*ping*" {r ping x y z} + } + + test "Unbalanced number of quotes" { + reconnect + r write "set \"\"\"test-key\"\"\" test-value\r\n" + r write "ping\r\n" + r flush + assert_error "*unbalanced*" {r read} + } + + set c 0 + foreach seq [list "\x00" "*\x00" "$\x00"] { + incr c + test "Protocol desync regression test #$c" { + set s [socket [srv 0 host] [srv 0 port]] + puts -nonewline $s $seq + set payload [string repeat A 1024]"\n" + set test_start [clock seconds] + set test_time_limit 30 + while 1 { + if {[catch { + puts -nonewline $s payload + flush $s + incr payload_size [string length $payload] + }]} { + set retval [gets $s] + close $s + break + } else { + set elapsed [expr {[clock seconds]-$test_start}] + if {$elapsed > $test_time_limit} { + close $s + error "assertion:Redis did not closed connection after protocol desync" + } + } + } + set retval + } {*Protocol error*} + } + unset c +} + +start_server {tags {"regression"}} { + test "Regression for a crash with blocking ops and pipelining" { + set rd [redis_deferring_client] + set fd [r channel] + set proto "*3\r\n\$5\r\nBLPOP\r\n\$6\r\nnolist\r\n\$1\r\n0\r\n" + puts -nonewline $fd $proto$proto + flush $fd + set res {} + + $rd rpush nolist a + $rd read + $rd rpush nolist a + $rd read + } +} diff --git a/tools/pika_migrate/tests/unit/pubsub.tcl b/tools/pika_migrate/tests/unit/pubsub.tcl new file mode 100644 index 0000000000..16c8c6a5f7 --- /dev/null +++ b/tools/pika_migrate/tests/unit/pubsub.tcl @@ -0,0 +1,399 @@ +start_server {tags {"pubsub"}} { + proc __consume_subscribe_messages {client type channels} { + set numsub -1 + set counts {} + + for {set i [llength $channels]} {$i > 0} {incr i -1} { + set msg [$client read] + assert_equal $type [lindex $msg 0] + + # when receiving subscribe messages the channels names + # are ordered. when receiving unsubscribe messages + # they are unordered + set idx [lsearch -exact $channels [lindex $msg 1]] + if {[string match "*unsubscribe" $type]} { + assert {$idx >= 0} + } else { + assert {$idx == 0} + } + set channels [lreplace $channels $idx $idx] + + # aggregate the subscription count to return to the caller + lappend counts [lindex $msg 2] + } + + # we should have received messages for channels + assert {[llength $channels] == 0} + return $counts + } + + proc subscribe {client channels} { + $client subscribe {*}$channels + __consume_subscribe_messages $client subscribe $channels + } + + proc unsubscribe {client {channels {}}} { + $client unsubscribe {*}$channels + __consume_subscribe_messages $client unsubscribe $channels + } + + proc psubscribe {client channels} { + $client psubscribe {*}$channels + __consume_subscribe_messages $client psubscribe $channels + } + + proc punsubscribe {client {channels {}}} { + $client punsubscribe {*}$channels + __consume_subscribe_messages $client punsubscribe $channels + } + + test "Pub/Sub PING" { + set rd1 [redis_deferring_client] + subscribe $rd1 somechannel + # While subscribed to non-zero channels PING works in Pub/Sub mode. + $rd1 ping + set reply1 [$rd1 read] + unsubscribe $rd1 somechannel + # Now we are unsubscribed, PING should just return PONG. + $rd1 ping + set reply2 [$rd1 read] + $rd1 close + list $reply1 $reply2 + } {PONG PONG} + + test "PUBLISH/SUBSCRIBE basics" { + set rd1 [redis_deferring_client] + + # subscribe to two channels + assert_equal {1 2} [subscribe $rd1 {chan1 chan2}] + assert_equal 1 [r publish chan1 hello] + assert_equal 1 [r publish chan2 world] + assert_equal {message chan1 hello} [$rd1 read] + assert_equal {message chan2 world} [$rd1 read] + + # unsubscribe from one of the channels + unsubscribe $rd1 {chan1} + assert_equal 0 [r publish chan1 hello] + assert_equal 1 [r publish chan2 world] + assert_equal {message chan2 world} [$rd1 read] + + # unsubscribe from the remaining channel + unsubscribe $rd1 {chan2} + assert_equal 0 [r publish chan1 hello] + assert_equal 0 [r publish chan2 world] + + # clean up clients + $rd1 close + } + + test "PUBLISH/SUBSCRIBE with two clients" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + assert_equal {1} [subscribe $rd1 {chan1}] + assert_equal {1} [subscribe $rd2 {chan1}] + assert_equal 2 [r publish chan1 hello] + assert_equal {message chan1 hello} [$rd1 read] + assert_equal {message chan1 hello} [$rd2 read] + + # clean up clients + $rd1 close + $rd2 close + } + + test "PUBLISH/SUBSCRIBE after UNSUBSCRIBE without arguments" { + set rd1 [redis_deferring_client] + assert_equal {1 2 3} [subscribe $rd1 {chan1 chan2 chan3}] + unsubscribe $rd1 + assert_equal 0 [r publish chan1 hello] + assert_equal 0 [r publish chan2 hello] + assert_equal 0 [r publish chan3 hello] + + # clean up clients + $rd1 close + } + + test "SUBSCRIBE to one channel more than once" { + set rd1 [redis_deferring_client] + assert_equal {1 1 1} [subscribe $rd1 {chan1 chan1 chan1}] + assert_equal 1 [r publish chan1 hello] + assert_equal {message chan1 hello} [$rd1 read] + + # clean up clients + $rd1 close + } + + test "UNSUBSCRIBE from non-subscribed channels" { + set rd1 [redis_deferring_client] + assert_equal {0 0 0} [unsubscribe $rd1 {foo bar quux}] + + # clean up clients + $rd1 close + } + + test "PUBLISH/PSUBSCRIBE basics" { + set rd1 [redis_deferring_client] + + # subscribe to two patterns + assert_equal {1 2} [psubscribe $rd1 {foo.* bar.*}] + assert_equal 1 [r publish foo.1 hello] + assert_equal 1 [r publish bar.1 hello] + assert_equal 0 [r publish foo1 hello] + assert_equal 0 [r publish barfoo.1 hello] + assert_equal 0 [r publish qux.1 hello] + assert_equal {pmessage foo.* foo.1 hello} [$rd1 read] + assert_equal {pmessage bar.* bar.1 hello} [$rd1 read] + + # unsubscribe from one of the patterns + assert_equal {1} [punsubscribe $rd1 {foo.*}] + assert_equal 0 [r publish foo.1 hello] + assert_equal 1 [r publish bar.1 hello] + assert_equal {pmessage bar.* bar.1 hello} [$rd1 read] + + # unsubscribe from the remaining pattern + assert_equal {0} [punsubscribe $rd1 {bar.*}] + assert_equal 0 [r publish foo.1 hello] + assert_equal 0 [r publish bar.1 hello] + + # clean up clients + $rd1 close + } + + test "PUBLISH/PSUBSCRIBE with two clients" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + assert_equal {1} [psubscribe $rd1 {chan.*}] + assert_equal {1} [psubscribe $rd2 {chan.*}] + assert_equal 2 [r publish chan.foo hello] + assert_equal {pmessage chan.* chan.foo hello} [$rd1 read] + assert_equal {pmessage chan.* chan.foo hello} [$rd2 read] + + # clean up clients + $rd1 close + $rd2 close + } + + test "PUBLISH/PSUBSCRIBE after PUNSUBSCRIBE without arguments" { + set rd1 [redis_deferring_client] + assert_equal {1 2 3} [psubscribe $rd1 {chan1.* chan2.* chan3.*}] + punsubscribe $rd1 + assert_equal 0 [r publish chan1.hi hello] + assert_equal 0 [r publish chan2.hi hello] + assert_equal 0 [r publish chan3.hi hello] + + # clean up clients + $rd1 close + } + + test "PUNSUBSCRIBE from non-subscribed channels" { + set rd1 [redis_deferring_client] + assert_equal {0 0 0} [punsubscribe $rd1 {foo.* bar.* quux.*}] + + # clean up clients + $rd1 close + } + + test "NUMSUB returns numbers, not strings (#1561)" { + r pubsub numsub abc def + } {abc 0 def 0} + + test "PubSub return value" { + set rd1 [redis_deferring_client] + assert_equal {1} [subscribe $rd1 {foo.bar}] + assert_equal {2} [psubscribe $rd1 {foo.*}] + assert_equal {foo.bar} [r pubsub channels] + assert_equal {1} [r pubsub numpat] + assert_equal {foo.bar 1} [r pubsub numsub foo.bar] + + $rd1 close + } + + test "Mix SUBSCRIBE and PSUBSCRIBE" { + set rd1 [redis_deferring_client] + assert_equal {1} [subscribe $rd1 {foo.bar}] + assert_equal {2} [psubscribe $rd1 {foo.*}] + + assert_equal 2 [r publish foo.bar hello] + assert_equal {message foo.bar hello} [$rd1 read] + assert_equal {pmessage foo.* foo.bar hello} [$rd1 read] + + # clean up clients + $rd1 close + } + + test "PUNSUBSCRIBE and UNSUBSCRIBE should always reply" { + # Make sure we are not subscribed to any channel at all. + r punsubscribe + r unsubscribe + # Now check if the commands still reply correctly. + set reply1 [r punsubscribe] + set reply2 [r unsubscribe] + concat $reply1 $reply2 + } {punsubscribe {} 0 unsubscribe {} 0} + + ### Keyspace events notification tests + +# test "Keyspace notifications: we receive keyspace notifications" { +# r config set notify-keyspace-events KA +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# assert_equal {pmessage * __keyspace@9__:foo set} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: we receive keyevent notifications" { +# r config set notify-keyspace-events EA +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# assert_equal {pmessage * __keyevent@9__:set foo} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: we can receive both kind of events" { +# r config set notify-keyspace-events KEA +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# assert_equal {pmessage * __keyspace@9__:foo set} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:set foo} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: we are able to mask events" { +# r config set notify-keyspace-events KEl +# r del mylist +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# r lpush mylist a +# # No notification for set, because only list commands are enabled. +# assert_equal {pmessage * __keyspace@9__:mylist lpush} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:lpush mylist} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: general events test" { +# r config set notify-keyspace-events KEg +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# r expire foo 1 +# r del foo +# assert_equal {pmessage * __keyspace@9__:foo expire} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:expire foo} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:foo del} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:del foo} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: list events test" { +# r config set notify-keyspace-events KEl +# r del mylist +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r lpush mylist a +# r rpush mylist a +# r rpop mylist +# assert_equal {pmessage * __keyspace@9__:mylist lpush} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:lpush mylist} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:mylist rpush} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:rpush mylist} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:mylist rpop} [$rd1 read] +# assert_equal {pmessage * __keyevent@9__:rpop mylist} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: set events test" { +# r config set notify-keyspace-events Ks +# r del myset +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r sadd myset a b c d +# r srem myset x +# r sadd myset x y z +# r srem myset x +# assert_equal {pmessage * __keyspace@9__:myset sadd} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:myset sadd} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:myset srem} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: zset events test" { +# r config set notify-keyspace-events Kz +# r del myzset +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r zadd myzset 1 a 2 b +# r zrem myzset x +# r zadd myzset 3 x 4 y 5 z +# r zrem myzset x +# assert_equal {pmessage * __keyspace@9__:myzset zadd} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:myzset zadd} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:myzset zrem} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: hash events test" { +# r config set notify-keyspace-events Kh +# r del myhash +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r hmset myhash yes 1 no 0 +# r hincrby myhash yes 10 +# assert_equal {pmessage * __keyspace@9__:myhash hset} [$rd1 read] +# assert_equal {pmessage * __keyspace@9__:myhash hincrby} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: expired events (triggered expire)" { +# r config set notify-keyspace-events Ex +# r del foo +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r psetex foo 100 1 +# wait_for_condition 50 100 { +# [r exists foo] == 0 +# } else { +# fail "Key does not expire?!" +# } +# assert_equal {pmessage * __keyevent@9__:expired foo} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: expired events (background expire)" { +# r config set notify-keyspace-events Ex +# r del foo +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r psetex foo 100 1 +# assert_equal {pmessage * __keyevent@9__:expired foo} [$rd1 read] +# $rd1 close +# } +# +# test "Keyspace notifications: evicted events" { +# r config set notify-keyspace-events Ee +# r config set maxmemory-policy allkeys-lru +# r flushdb +# set rd1 [redis_deferring_client] +# assert_equal {1} [psubscribe $rd1 *] +# r set foo bar +# r config set maxmemory 1 +# assert_equal {pmessage * __keyevent@9__:evicted foo} [$rd1 read] +# r config set maxmemory 0 +# $rd1 close +# } +# +# test "Keyspace notifications: test CONFIG GET/SET of event flags" { +# r config set notify-keyspace-events gKE +# assert_equal {gKE} [lindex [r config get notify-keyspace-events] 1] +# r config set notify-keyspace-events {$lshzxeKE} +# assert_equal {$lshzxeKE} [lindex [r config get notify-keyspace-events] 1] +# r config set notify-keyspace-events KA +# assert_equal {AK} [lindex [r config get notify-keyspace-events] 1] +# r config set notify-keyspace-events EA +# assert_equal {AE} [lindex [r config get notify-keyspace-events] 1] +# } +#} diff --git a/tools/pika_migrate/tests/unit/quit.tcl b/tools/pika_migrate/tests/unit/quit.tcl new file mode 100644 index 0000000000..4cf440abf1 --- /dev/null +++ b/tools/pika_migrate/tests/unit/quit.tcl @@ -0,0 +1,40 @@ +start_server {tags {"quit"}} { + proc format_command {args} { + set cmd "*[llength $args]\r\n" + foreach a $args { + append cmd "$[string length $a]\r\n$a\r\n" + } + set _ $cmd + } + + test "QUIT returns OK" { + reconnect + assert_equal OK [r quit] + assert_error * {r ping} + } + + test "Pipelined commands after QUIT must not be executed" { + reconnect + r write [format_command quit] + r write [format_command set foo bar] + r flush + assert_equal OK [r read] + assert_error * {r read} + + reconnect + assert_equal {} [r get foo] + } + + test "Pipelined commands after QUIT that exceed read buffer size" { + reconnect + r write [format_command quit] + r write [format_command set foo [string repeat "x" 1024]] + r flush + assert_equal OK [r read] + assert_error * {r read} + + reconnect + assert_equal {} [r get foo] + + } +} diff --git a/tools/pika_migrate/tests/unit/scan.tcl b/tools/pika_migrate/tests/unit/scan.tcl new file mode 100644 index 0000000000..1d84f128da --- /dev/null +++ b/tools/pika_migrate/tests/unit/scan.tcl @@ -0,0 +1,239 @@ +start_server {tags {"scan"}} { + test "SCAN basic" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys [lsort -unique $keys] + assert_equal 1000 [llength $keys] + } + + test "SCAN COUNT" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur count 5] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys [lsort -unique $keys] + assert_equal 1000 [llength $keys] + } + + test "SCAN MATCH" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur match "key:1??"] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys [lsort -unique $keys] + assert_equal 100 [llength $keys] + } + + foreach enc {intset hashtable} { + test "SSCAN with encoding $enc" { + # Create the Set + r del set + if {$enc eq {intset}} { + set prefix "" + } else { + set prefix "ele:" + } + set elements {} + for {set j 0} {$j < 100} {incr j} { + lappend elements ${prefix}${j} + } + r sadd set {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding set] eq $enc} + + # Test SSCAN + set cur 0 + set keys {} + while 1 { + set res [r sscan set $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys [lsort -unique $keys] + assert_equal 100 [llength $keys] + } + } + + foreach enc {ziplist hashtable} { + test "HSCAN with encoding $enc" { + # Create the Hash + r del hash + if {$enc eq {ziplist}} { + set count 30 + } else { + set count 1000 + } + set elements {} + for {set j 0} {$j < $count} {incr j} { + lappend elements key:$j $j + } + r hmset hash {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding hash] eq $enc} + + # Test HSCAN + set cur 0 + set keys {} + while 1 { + set res [r hscan hash $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys2 {} + foreach {k v} $keys { + assert {$k eq "key:$v"} + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal $count [llength $keys2] + } + } + + foreach enc {ziplist skiplist} { + test "ZSCAN with encoding $enc" { + # Create the Sorted Set + r del zset + if {$enc eq {ziplist}} { + set count 30 + } else { + set count 1000 + } + set elements {} + for {set j 0} {$j < $count} {incr j} { + lappend elements $j key:$j + } + r zadd zset {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding zset] eq $enc} + + # Test ZSCAN + set cur 0 + set keys {} + while 1 { + set res [r zscan zset $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys2 {} + foreach {k v} $keys { + assert {$k eq "key:$v"} + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal $count [llength $keys2] + } + } + + test "SCAN guarantees check under write load" { + r flushdb + r debug populate 100 + + # We start scanning here, so keys from 0 to 99 should all be + # reported at the end of the iteration. + set keys {} + while 1 { + set res [r scan $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + # Write 10 random keys at every SCAN iteration. + for {set j 0} {$j < 10} {incr j} { + r set addedkey:[randomInt 1000] foo + } + } + + set keys2 {} + foreach k $keys { + if {[string length $k] > 6} continue + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal 100 [llength $keys2] + } + + test "SSCAN with integer encoded object (issue #1345)" { + set objects {1 a} + r del set + r sadd set {*}$objects + set res [r sscan set 0 MATCH *a* COUNT 100] + assert_equal [lsort -unique [lindex $res 1]] {a} + set res [r sscan set 0 MATCH *1* COUNT 100] + assert_equal [lsort -unique [lindex $res 1]] {1} + } + + test "SSCAN with PATTERN" { + r del mykey + r sadd mykey foo fab fiz foobar 1 2 3 4 + set res [r sscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } {foo foobar} + + test "HSCAN with PATTERN" { + r del mykey + r hmset mykey foo 1 fab 2 fiz 3 foobar 10 1 a 2 b 3 c 4 d + set res [r hscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } {1 10 foo foobar} + + test "ZSCAN with PATTERN" { + r del mykey + r zadd mykey 1 foo 2 fab 3 fiz 10 foobar + set res [r zscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } + + test "ZSCAN scores: regression test for issue #2175" { + r del mykey + for {set j 0} {$j < 500} {incr j} { + r zadd mykey 9.8813129168249309e-323 $j + } + set res [lindex [r zscan mykey 0] 1] + set first_score [lindex $res 1] + assert {$first_score != 0} + } +} diff --git a/tools/pika_migrate/tests/unit/scripting.tcl b/tools/pika_migrate/tests/unit/scripting.tcl new file mode 100644 index 0000000000..e1cd2174ba --- /dev/null +++ b/tools/pika_migrate/tests/unit/scripting.tcl @@ -0,0 +1,606 @@ +start_server {tags {"scripting"}} { + test {EVAL - Does Lua interpreter replies to our requests?} { + r eval {return 'hello'} 0 + } {hello} + + test {EVAL - Lua integer -> Redis protocol type conversion} { + r eval {return 100.5} 0 + } {100} + + test {EVAL - Lua string -> Redis protocol type conversion} { + r eval {return 'hello world'} 0 + } {hello world} + + test {EVAL - Lua true boolean -> Redis protocol type conversion} { + r eval {return true} 0 + } {1} + + test {EVAL - Lua false boolean -> Redis protocol type conversion} { + r eval {return false} 0 + } {} + + test {EVAL - Lua status code reply -> Redis protocol type conversion} { + r eval {return {ok='fine'}} 0 + } {fine} + + test {EVAL - Lua error reply -> Redis protocol type conversion} { + catch { + r eval {return {err='this is an error'}} 0 + } e + set _ $e + } {this is an error} + + test {EVAL - Lua table -> Redis protocol type conversion} { + r eval {return {1,2,3,'ciao',{1,2}}} 0 + } {1 2 3 ciao {1 2}} + + test {EVAL - Are the KEYS and ARGV arrays populated correctly?} { + r eval {return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}} 2 a b c d + } {a b c d} + + test {EVAL - is Lua able to call Redis API?} { + r set mykey myval + r eval {return redis.call('get',KEYS[1])} 1 mykey + } {myval} + + test {EVALSHA - Can we call a SHA1 if already defined?} { + r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey + } {myval} + + test {EVALSHA - Can we call a SHA1 in uppercase?} { + r evalsha FD758D1589D044DD850A6F05D52F2EEFD27F033F 1 mykey + } {myval} + + test {EVALSHA - Do we get an error on invalid SHA1?} { + catch {r evalsha NotValidShaSUM 0} e + set _ $e + } {NOSCRIPT*} + + test {EVALSHA - Do we get an error on non defined SHA1?} { + catch {r evalsha ffd632c7d33e571e9f24556ebed26c3479a87130 0} e + set _ $e + } {NOSCRIPT*} + + test {EVAL - Redis integer -> Lua type conversion} { + r eval { + local foo = redis.pcall('incr','x') + return {type(foo),foo} + } 0 + } {number 1} + + test {EVAL - Redis bulk -> Lua type conversion} { + r set mykey myval + r eval { + local foo = redis.pcall('get','mykey') + return {type(foo),foo} + } 0 + } {string myval} + + test {EVAL - Redis multi bulk -> Lua type conversion} { + r del mylist + r rpush mylist a + r rpush mylist b + r rpush mylist c + r eval { + local foo = redis.pcall('lrange','mylist',0,-1) + return {type(foo),foo[1],foo[2],foo[3],# foo} + } 0 + } {table a b c 3} + + test {EVAL - Redis status reply -> Lua type conversion} { + r eval { + local foo = redis.pcall('set','mykey','myval') + return {type(foo),foo['ok']} + } 0 + } {table OK} + + test {EVAL - Redis error reply -> Lua type conversion} { + r set mykey myval + r eval { + local foo = redis.pcall('incr','mykey') + return {type(foo),foo['err']} + } 0 + } {table {ERR value is not an integer or out of range}} + + test {EVAL - Redis nil bulk reply -> Lua type conversion} { + r del mykey + r eval { + local foo = redis.pcall('get','mykey') + return {type(foo),foo == false} + } 0 + } {boolean 1} + + test {EVAL - Is the Lua client using the currently selected DB?} { + r set mykey "this is DB 9" + r select 10 + r set mykey "this is DB 10" + r eval {return redis.pcall('get','mykey')} 0 + } {this is DB 10} + + test {EVAL - SELECT inside Lua should not affect the caller} { + # here we DB 10 is selected + r set mykey "original value" + r eval {return redis.pcall('select','9')} 0 + set res [r get mykey] + r select 9 + set res + } {original value} + + if 0 { + test {EVAL - Script can't run more than configured time limit} { + r config set lua-time-limit 1 + catch { + r eval { + local i = 0 + while true do i=i+1 end + } 0 + } e + set _ $e + } {*execution time*} + } + + test {EVAL - Scripts can't run certain commands} { + set e {} + catch {r eval {return redis.pcall('spop','x')} 0} e + set e + } {*not allowed*} + + test {EVAL - Scripts can't run certain commands} { + set e {} + catch { + r eval "redis.pcall('randomkey'); return redis.pcall('set','x','ciao')" 0 + } e + set e + } {*not allowed after*} + + test {EVAL - No arguments to redis.call/pcall is considered an error} { + set e {} + catch {r eval {return redis.call()} 0} e + set e + } {*one argument*} + + test {EVAL - redis.call variant raises a Lua error on Redis cmd error (1)} { + set e {} + catch { + r eval "redis.call('nosuchcommand')" 0 + } e + set e + } {*Unknown Redis*} + + test {EVAL - redis.call variant raises a Lua error on Redis cmd error (1)} { + set e {} + catch { + r eval "redis.call('get','a','b','c')" 0 + } e + set e + } {*number of args*} + + test {EVAL - redis.call variant raises a Lua error on Redis cmd error (1)} { + set e {} + r set foo bar + catch { + r eval {redis.call('lpush',KEYS[1],'val')} 1 foo + } e + set e + } {*against a key*} + + test {EVAL - JSON numeric decoding} { + # We must return the table as a string because otherwise + # Redis converts floats to ints and we get 0 and 1023 instead + # of 0.0003 and 1023.2 as the parsed output. + r eval {return + table.concat( + cjson.decode( + "[0.0, -5e3, -1, 0.3e-3, 1023.2, 0e10]"), " ") + } 0 + } {0 -5000 -1 0.0003 1023.2 0} + + test {EVAL - JSON string decoding} { + r eval {local decoded = cjson.decode('{"keya": "a", "keyb": "b"}') + return {decoded.keya, decoded.keyb} + } 0 + } {a b} + + test {EVAL - cmsgpack can pack double?} { + r eval {local encoded = cmsgpack.pack(0.1) + local h = "" + for i = 1, #encoded do + h = h .. string.format("%02x",string.byte(encoded,i)) + end + return h + } 0 + } {cb3fb999999999999a} + + test {EVAL - cmsgpack can pack negative int64?} { + r eval {local encoded = cmsgpack.pack(-1099511627776) + local h = "" + for i = 1, #encoded do + h = h .. string.format("%02x",string.byte(encoded,i)) + end + return h + } 0 + } {d3ffffff0000000000} + + test {EVAL - cmsgpack can pack and unpack circular references?} { + r eval {local a = {x=nil,y=5} + local b = {x=a} + a['x'] = b + local encoded = cmsgpack.pack(a) + local h = "" + -- cmsgpack encodes to a depth of 16, but can't encode + -- references, so the encoded object has a deep copy recusive + -- depth of 16. + for i = 1, #encoded do + h = h .. string.format("%02x",string.byte(encoded,i)) + end + -- when unpacked, re.x.x != re because the unpack creates + -- individual tables down to a depth of 16. + -- (that's why the encoded output is so large) + local re = cmsgpack.unpack(encoded) + assert(re) + assert(re.x) + assert(re.x.x.y == re.y) + assert(re.x.x.x.x.y == re.y) + assert(re.x.x.x.x.x.x.y == re.y) + assert(re.x.x.x.x.x.x.x.x.x.x.y == re.y) + -- maximum working depth: + assert(re.x.x.x.x.x.x.x.x.x.x.x.x.x.x.y == re.y) + -- now the last x would be b above and has no y + assert(re.x.x.x.x.x.x.x.x.x.x.x.x.x.x.x) + -- so, the final x.x is at the depth limit and was assigned nil + assert(re.x.x.x.x.x.x.x.x.x.x.x.x.x.x.x.x == nil) + return {h, re.x.x.x.x.x.x.x.x.y == re.y, re.y == 5} + } 0 + } {82a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a17882a17905a17881a178c0 1 1} + + test {EVAL - Numerical sanity check from bitop} { + r eval {assert(0x7fffffff == 2147483647, "broken hex literals"); + assert(0xffffffff == -1 or 0xffffffff == 2^32-1, + "broken hex literals"); + assert(tostring(-1) == "-1", "broken tostring()"); + assert(tostring(0xffffffff) == "-1" or + tostring(0xffffffff) == "4294967295", + "broken tostring()") + } 0 + } {} + + test {EVAL - Verify minimal bitop functionality} { + r eval {assert(bit.tobit(1) == 1); + assert(bit.band(1) == 1); + assert(bit.bxor(1,2) == 3); + assert(bit.bor(1,2,4,8,16,32,64,128) == 255) + } 0 + } {} + + test {SCRIPTING FLUSH - is able to clear the scripts cache?} { + r set mykey myval + set v [r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey] + assert_equal $v myval + set e "" + r script flush + catch {r evalsha fd758d1589d044dd850a6f05d52f2eefd27f033f 1 mykey} e + set e + } {NOSCRIPT*} + + test {SCRIPT EXISTS - can detect already defined scripts?} { + r eval "return 1+1" 0 + r script exists a27e7e8a43702b7046d4f6a7ccf5b60cef6b9bd9 a27e7e8a43702b7046d4f6a7ccf5b60cef6b9bda + } {1 0} + + test {SCRIPT LOAD - is able to register scripts in the scripting cache} { + list \ + [r script load "return 'loaded'"] \ + [r evalsha b534286061d4b9e4026607613b95c06c06015ae8 0] + } {b534286061d4b9e4026607613b95c06c06015ae8 loaded} + + test "In the context of Lua the output of random commands gets ordered" { + r del myset + r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz + r eval {return redis.call('smembers',KEYS[1])} 1 myset + } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} + + test "SORT is normally not alpha re-ordered for the scripting engine" { + r del myset + r sadd myset 1 2 3 4 10 + r eval {return redis.call('sort',KEYS[1],'desc')} 1 myset + } {10 4 3 2 1} + + test "SORT BY output gets ordered for scripting" { + r del myset + r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz + r eval {return redis.call('sort',KEYS[1],'by','_')} 1 myset + } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} + + test "SORT BY with GET gets ordered for scripting" { + r del myset + r sadd myset a b c + r eval {return redis.call('sort',KEYS[1],'by','_','get','#','get','_:*')} 1 myset + } {a {} b {} c {}} + + test "redis.sha1hex() implementation" { + list [r eval {return redis.sha1hex('')} 0] \ + [r eval {return redis.sha1hex('Pizza & Mandolino')} 0] + } {da39a3ee5e6b4b0d3255bfef95601890afd80709 74822d82031af7493c20eefa13bd07ec4fada82f} + + test {Globals protection reading an undeclared global variable} { + catch {r eval {return a} 0} e + set e + } {*ERR*attempted to access unexisting global*} + + test {Globals protection setting an undeclared global*} { + catch {r eval {a=10} 0} e + set e + } {*ERR*attempted to create global*} + + test {Test an example script DECR_IF_GT} { + set decr_if_gt { + local current + + current = redis.call('get',KEYS[1]) + if not current then return nil end + if current > ARGV[1] then + return redis.call('decr',KEYS[1]) + else + return redis.call('get',KEYS[1]) + end + } + r set foo 5 + set res {} + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + set res + } {4 3 2 2 2} + + test {Scripting engine resets PRNG at every script execution} { + set rand1 [r eval {return tostring(math.random())} 0] + set rand2 [r eval {return tostring(math.random())} 0] + assert_equal $rand1 $rand2 + } + + test {Scripting engine PRNG can be seeded correctly} { + set rand1 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 10] + set rand2 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 10] + set rand3 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 20] + assert_equal $rand1 $rand2 + assert {$rand2 ne $rand3} + } + + test {EVAL does not leak in the Lua stack} { + r set x 0 + # Use a non blocking client to speedup the loop. + set rd [redis_deferring_client] + for {set j 0} {$j < 10000} {incr j} { + $rd eval {return redis.call("incr",KEYS[1])} 1 x + } + for {set j 0} {$j < 10000} {incr j} { + $rd read + } + assert {[s used_memory_lua] < 1024*100} + $rd close + r get x + } {10000} + + test {EVAL processes writes from AOF in read-only slaves} { + r flushall + r config set appendonly yes + r eval {redis.call("set",KEYS[1],"100")} 1 foo + r eval {redis.call("incr",KEYS[1])} 1 foo + r eval {redis.call("incr",KEYS[1])} 1 foo + wait_for_condition 50 100 { + [s aof_rewrite_in_progress] == 0 + } else { + fail "AOF rewrite can't complete after CONFIG SET appendonly yes." + } + r config set slave-read-only yes + r slaveof 127.0.0.1 0 + r debug loadaof + set res [r get foo] + r slaveof no one + set res + } {102} + + test {We can call scripts rewriting client->argv from Lua} { + r del myset + r sadd myset a b c + r mset a 1 b 2 c 3 d 4 + assert {[r spop myset] ne {}} + assert {[r spop myset] ne {}} + assert {[r spop myset] ne {}} + assert {[r mget a b c d] eq {1 2 3 4}} + assert {[r spop myset] eq {}} + } + + test {Call Redis command with many args from Lua (issue #1764)} { + r eval { + local i + local x={} + redis.call('del','mylist') + for i=1,100 do + table.insert(x,i) + end + redis.call('rpush','mylist',unpack(x)) + return redis.call('lrange','mylist',0,-1) + } 0 + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100} + + test {Number conversion precision test (issue #1118)} { + r eval { + local value = 9007199254740991 + redis.call("set","foo",value) + return redis.call("get","foo") + } 0 + } {9007199254740991} + + test {String containing number precision test (regression of issue #1118)} { + r eval { + redis.call("set", "key", "12039611435714932082") + return redis.call("get", "key") + } 0 + } {12039611435714932082} + + test {Verify negative arg count is error instead of crash (issue #1842)} { + catch { r eval { return "hello" } -12 } e + set e + } {ERR Number of keys can't be negative} + + test {Correct handling of reused argv (issue #1939)} { + r eval { + for i = 0, 10 do + redis.call('SET', 'a', '1') + redis.call('MGET', 'a', 'b', 'c') + redis.call('EXPIRE', 'a', 0) + redis.call('GET', 'a') + redis.call('MGET', 'a', 'b', 'c') + end + } 0 + } +} + +# Start a new server since the last test in this stanza will kill the +# instance at all. +start_server {tags {"scripting"}} { + test {Timedout read-only scripts can be killed by SCRIPT KILL} { + set rd [redis_deferring_client] + r config set lua-time-limit 10 + $rd eval {while true do end} 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + r script kill + after 200 ; # Give some time to Lua to call the hook again... + assert_equal [r ping] "PONG" + } + + test {Timedout script link is still usable after Lua returns} { + r config set lua-time-limit 10 + r eval {for i=1,100000 do redis.call('ping') end return 'ok'} 0 + r ping + } {PONG} + + test {Timedout scripts that modified data can't be killed by SCRIPT KILL} { + set rd [redis_deferring_client] + r config set lua-time-limit 10 + $rd eval {redis.call('set',KEYS[1],'y'); while true do end} 1 x + after 200 + catch {r ping} e + assert_match {BUSY*} $e + catch {r script kill} e + assert_match {UNKILLABLE*} $e + catch {r ping} e + assert_match {BUSY*} $e + } + + # Note: keep this test at the end of this server stanza because it + # kills the server. + test {SHUTDOWN NOSAVE can kill a timedout script anyway} { + # The server sould be still unresponding to normal commands. + catch {r ping} e + assert_match {BUSY*} $e + catch {r shutdown nosave} + # Make sure the server was killed + catch {set rd [redis_deferring_client]} e + assert_match {*connection refused*} $e + } +} + +start_server {tags {"scripting repl"}} { + start_server {} { + test {Before the slave connects we issue two EVAL commands} { + # One with an error, but still executing a command. + # SHA is: 67164fc43fa971f76fd1aaeeaf60c1c178d25876 + catch { + r eval {redis.call('incr',KEYS[1]); redis.call('nonexisting')} 1 x + } + # One command is correct: + # SHA is: 6f5ade10a69975e903c6d07b10ea44c6382381a5 + r eval {return redis.call('incr',KEYS[1])} 1 x + } {2} + + test {Connect a slave to the main instance} { + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 role] eq {slave} && + [string match {*master_link_status:up*} [r -1 info replication]] + } else { + fail "Can't turn the instance into a slave" + } + } + + test {Now use EVALSHA against the master, with both SHAs} { + # The server should replicate successful and unsuccessful + # commands as EVAL instead of EVALSHA. + catch { + r evalsha 67164fc43fa971f76fd1aaeeaf60c1c178d25876 1 x + } + r evalsha 6f5ade10a69975e903c6d07b10ea44c6382381a5 1 x + } {4} + + test {If EVALSHA was replicated as EVAL, 'x' should be '4'} { + wait_for_condition 50 100 { + [r -1 get x] eq {4} + } else { + fail "Expected 4 in x, but value is '[r -1 get x]'" + } + } + + test {Replication of script multiple pushes to list with BLPOP} { + set rd [redis_deferring_client] + $rd brpop a 0 + r eval { + redis.call("lpush",KEYS[1],"1"); + redis.call("lpush",KEYS[1],"2"); + } 1 a + set res [$rd read] + $rd close + wait_for_condition 50 100 { + [r -1 lrange a 0 -1] eq [r lrange a 0 -1] + } else { + fail "Expected list 'a' in slave and master to be the same, but they are respectively '[r -1 lrange a 0 -1]' and '[r lrange a 0 -1]'" + } + set res + } {a 1} + + test {EVALSHA replication when first call is readonly} { + r del x + r eval {if tonumber(ARGV[1]) > 0 then redis.call('incr', KEYS[1]) end} 1 x 0 + r evalsha 6e0e2745aa546d0b50b801a20983b70710aef3ce 1 x 0 + r evalsha 6e0e2745aa546d0b50b801a20983b70710aef3ce 1 x 1 + wait_for_condition 50 100 { + [r -1 get x] eq {1} + } else { + fail "Expected 1 in x, but value is '[r -1 get x]'" + } + } + + test {Lua scripts using SELECT are replicated correctly} { + r eval { + redis.call("set","foo1","bar1") + redis.call("select","10") + redis.call("incr","x") + redis.call("select","11") + redis.call("incr","z") + } 0 + r eval { + redis.call("set","foo1","bar1") + redis.call("select","10") + redis.call("incr","x") + redis.call("select","11") + redis.call("incr","z") + } 0 + wait_for_condition 50 100 { + [r -1 debug digest] eq [r debug digest] + } else { + fail "Master-Slave desync after Lua script using SELECT." + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/slowlog.tcl b/tools/pika_migrate/tests/unit/slowlog.tcl new file mode 100644 index 0000000000..b25b91e2ce --- /dev/null +++ b/tools/pika_migrate/tests/unit/slowlog.tcl @@ -0,0 +1,70 @@ +start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { + test {SLOWLOG - check that it starts with an empty log} { + r slowlog len + } {0} + + test {SLOWLOG - only logs commands taking more time than specified} { + r config set slowlog-log-slower-than 100000 + r ping + assert_equal [r slowlog len] 0 + r debug sleep 0.2 + assert_equal [r slowlog len] 1 + } + + test {SLOWLOG - max entries is correctly handled} { + r config set slowlog-log-slower-than 0 + r config set slowlog-max-len 10 + for {set i 0} {$i < 100} {incr i} { + r ping + } + r slowlog len + } {10} + + test {SLOWLOG - GET optional argument to limit output len works} { + llength [r slowlog get 5] + } {5} + + test {SLOWLOG - RESET subcommand works} { + r config set slowlog-log-slower-than 100000 + r slowlog reset + r slowlog len + } {0} + + test {SLOWLOG - logged entry sanity check} { + r debug sleep 0.2 + set e [lindex [r slowlog get] 0] + assert_equal [llength $e] 4 + assert_equal [lindex $e 0] 105 + assert_equal [expr {[lindex $e 2] > 100000}] 1 + assert_equal [lindex $e 3] {debug sleep 0.2} + } + + test {SLOWLOG - commands with too many arguments are trimmed} { + r config set slowlog-log-slower-than 0 + r slowlog reset + r sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 + set e [lindex [r slowlog get] 0] + lindex $e 3 + } {sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 {... (2 more arguments)}} + + test {SLOWLOG - too long arguments are trimmed} { + r config set slowlog-log-slower-than 0 + r slowlog reset + set arg [string repeat A 129] + r sadd set foo $arg + set e [lindex [r slowlog get] 0] + lindex $e 3 + } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} + + test {SLOWLOG - EXEC is not logged, just executed commands} { + r config set slowlog-log-slower-than 100000 + r slowlog reset + assert_equal [r slowlog len] 0 + r multi + r debug sleep 0.2 + r exec + assert_equal [r slowlog len] 1 + set e [lindex [r slowlog get] 0] + assert_equal [lindex $e 3] {debug sleep 0.2} + } +} diff --git a/tools/pika_migrate/tests/unit/sort.tcl b/tools/pika_migrate/tests/unit/sort.tcl new file mode 100644 index 0000000000..a25ffeb5ce --- /dev/null +++ b/tools/pika_migrate/tests/unit/sort.tcl @@ -0,0 +1,311 @@ +start_server { + tags {"sort"} + overrides { + "list-max-ziplist-value" 16 + "list-max-ziplist-entries" 32 + "set-max-intset-entries" 32 + } +} { + proc create_random_dataset {num cmd} { + set tosort {} + set result {} + array set seenrand {} + r del tosort + for {set i 0} {$i < $num} {incr i} { + # Make sure all the weights are different because + # Redis does not use a stable sort but Tcl does. + while 1 { + randpath { + set rint [expr int(rand()*1000000)] + } { + set rint [expr rand()] + } + if {![info exists seenrand($rint)]} break + } + set seenrand($rint) x + r $cmd tosort $i + r set weight_$i $rint + r hset wobj_$i weight $rint + lappend tosort [list $i $rint] + } + set sorted [lsort -index 1 -real $tosort] + for {set i 0} {$i < $num} {incr i} { + lappend result [lindex $sorted $i 0] + } + set _ $result + } + + foreach {num cmd enc title} { + 16 lpush ziplist "Ziplist" + 1000 lpush linkedlist "Linked list" + 10000 lpush linkedlist "Big Linked list" + 16 sadd intset "Intset" + 1000 sadd hashtable "Hash table" + 10000 sadd hashtable "Big Hash table" + } { + set result [create_random_dataset $num $cmd] + assert_encoding $enc tosort + + test "$title: SORT BY key" { + assert_equal $result [r sort tosort BY weight_*] + } + + test "$title: SORT BY key with limit" { + assert_equal [lrange $result 5 9] [r sort tosort BY weight_* LIMIT 5 5] + } + + test "$title: SORT BY hash field" { + assert_equal $result [r sort tosort BY wobj_*->weight] + } + } + + set result [create_random_dataset 16 lpush] + test "SORT GET #" { + assert_equal [lsort -integer $result] [r sort tosort GET #] + } + + test "SORT GET " { + r del foo + set res [r sort tosort GET foo] + assert_equal 16 [llength $res] + foreach item $res { assert_equal {} $item } + } + + test "SORT GET (key and hash) with sanity check" { + set l1 [r sort tosort GET # GET weight_*] + set l2 [r sort tosort GET # GET wobj_*->weight] + foreach {id1 w1} $l1 {id2 w2} $l2 { + assert_equal $id1 $id2 + assert_equal $w1 [r get weight_$id1] + assert_equal $w2 [r get weight_$id1] + } + } + + test "SORT BY key STORE" { + r sort tosort BY weight_* store sort-res + assert_equal $result [r lrange sort-res 0 -1] + assert_equal 16 [r llen sort-res] + assert_encoding ziplist sort-res + } + + test "SORT BY hash field STORE" { + r sort tosort BY wobj_*->weight store sort-res + assert_equal $result [r lrange sort-res 0 -1] + assert_equal 16 [r llen sort-res] + assert_encoding ziplist sort-res + } + + test "SORT DESC" { + assert_equal [lsort -decreasing -integer $result] [r sort tosort DESC] + } + + test "SORT ALPHA against integer encoded strings" { + r del mylist + r lpush mylist 2 + r lpush mylist 1 + r lpush mylist 3 + r lpush mylist 10 + r sort mylist alpha + } {1 10 2 3} + + test "SORT sorted set" { + r del zset + r zadd zset 1 a + r zadd zset 5 b + r zadd zset 2 c + r zadd zset 10 d + r zadd zset 3 e + r sort zset alpha desc + } {e d c b a} + + test "SORT sorted set BY nosort should retain ordering" { + r del zset + r zadd zset 1 a + r zadd zset 5 b + r zadd zset 2 c + r zadd zset 10 d + r zadd zset 3 e + r multi + r sort zset by nosort asc + r sort zset by nosort desc + r exec + } {{a c e b d} {d b e c a}} + + test "SORT sorted set BY nosort + LIMIT" { + r del zset + r zadd zset 1 a + r zadd zset 5 b + r zadd zset 2 c + r zadd zset 10 d + r zadd zset 3 e + assert_equal [r sort zset by nosort asc limit 0 1] {a} + assert_equal [r sort zset by nosort desc limit 0 1] {d} + assert_equal [r sort zset by nosort asc limit 0 2] {a c} + assert_equal [r sort zset by nosort desc limit 0 2] {d b} + assert_equal [r sort zset by nosort limit 5 10] {} + assert_equal [r sort zset by nosort limit -10 100] {a c e b d} + } + + test "SORT sorted set BY nosort works as expected from scripts" { + r del zset + r zadd zset 1 a + r zadd zset 5 b + r zadd zset 2 c + r zadd zset 10 d + r zadd zset 3 e + r eval { + return {redis.call('sort',KEYS[1],'by','nosort','asc'), + redis.call('sort',KEYS[1],'by','nosort','desc')} + } 1 zset + } {{a c e b d} {d b e c a}} + + test "SORT sorted set: +inf and -inf handling" { + r del zset + r zadd zset -100 a + r zadd zset 200 b + r zadd zset -300 c + r zadd zset 1000000 d + r zadd zset +inf max + r zadd zset -inf min + r zrange zset 0 -1 + } {min c a b d max} + + test "SORT regression for issue #19, sorting floats" { + r flushdb + set floats {1.1 5.10 3.10 7.44 2.1 5.75 6.12 0.25 1.15} + foreach x $floats { + r lpush mylist $x + } + assert_equal [lsort -real $floats] [r sort mylist] + } + + test "SORT with STORE returns zero if result is empty (github issue 224)" { + r flushdb + r sort foo store bar + } {0} + + test "SORT with STORE does not create empty lists (github issue 224)" { + r flushdb + r lpush foo bar + r sort foo alpha limit 10 10 store zap + r exists zap + } {0} + + test "SORT with STORE removes key if result is empty (github issue 227)" { + r flushdb + r lpush foo bar + r sort emptylist store foo + r exists foo + } {0} + + test "SORT with BY and STORE should still order output" { + r del myset mylist + r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz + r sort myset alpha by _ store mylist + r lrange mylist 0 -1 + } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} + + test "SORT will complain with numerical sorting and bad doubles (1)" { + r del myset + r sadd myset 1 2 3 4 not-a-double + set e {} + catch {r sort myset} e + set e + } {*ERR*double*} + + test "SORT will complain with numerical sorting and bad doubles (2)" { + r del myset + r sadd myset 1 2 3 4 + r mset score:1 10 score:2 20 score:3 30 score:4 not-a-double + set e {} + catch {r sort myset by score:*} e + set e + } {*ERR*double*} + + test "SORT BY sub-sorts lexicographically if score is the same" { + r del myset + r sadd myset a b c d e f g h i l m n o p q r s t u v z aa aaa azz + foreach ele {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} { + set score:$ele 100 + } + r sort myset by score:* + } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} + + test "SORT GET with pattern ending with just -> does not get hash field" { + r del mylist + r lpush mylist a + r set x:a-> 100 + r sort mylist by num get x:*-> + } {100} + + test "SORT by nosort retains native order for lists" { + r del testa + r lpush testa 2 1 4 3 5 + r sort testa by nosort + } {5 3 4 1 2} + + test "SORT by nosort plus store retains native order for lists" { + r del testa + r lpush testa 2 1 4 3 5 + r sort testa by nosort store testb + r lrange testb 0 -1 + } {5 3 4 1 2} + + test "SORT by nosort with limit returns based on original list order" { + r sort testa by nosort limit 0 3 store testb + r lrange testb 0 -1 + } {5 3 4} + + tags {"slow"} { + set num 100 + set res [create_random_dataset $num lpush] + + test "SORT speed, $num element list BY key, 100 times" { + set start [clock clicks -milliseconds] + for {set i 0} {$i < 100} {incr i} { + set sorted [r sort tosort BY weight_* LIMIT 0 10] + } + set elapsed [expr [clock clicks -milliseconds]-$start] + if {$::verbose} { + puts -nonewline "\n Average time to sort: [expr double($elapsed)/100] milliseconds " + flush stdout + } + } + + test "SORT speed, $num element list BY hash field, 100 times" { + set start [clock clicks -milliseconds] + for {set i 0} {$i < 100} {incr i} { + set sorted [r sort tosort BY wobj_*->weight LIMIT 0 10] + } + set elapsed [expr [clock clicks -milliseconds]-$start] + if {$::verbose} { + puts -nonewline "\n Average time to sort: [expr double($elapsed)/100] milliseconds " + flush stdout + } + } + + test "SORT speed, $num element list directly, 100 times" { + set start [clock clicks -milliseconds] + for {set i 0} {$i < 100} {incr i} { + set sorted [r sort tosort LIMIT 0 10] + } + set elapsed [expr [clock clicks -milliseconds]-$start] + if {$::verbose} { + puts -nonewline "\n Average time to sort: [expr double($elapsed)/100] milliseconds " + flush stdout + } + } + + test "SORT speed, $num element list BY , 100 times" { + set start [clock clicks -milliseconds] + for {set i 0} {$i < 100} {incr i} { + set sorted [r sort tosort BY nokey LIMIT 0 10] + } + set elapsed [expr [clock clicks -milliseconds]-$start] + if {$::verbose} { + puts -nonewline "\n Average time to sort: [expr double($elapsed)/100] milliseconds " + flush stdout + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/type/hash.tcl b/tools/pika_migrate/tests/unit/type/hash.tcl new file mode 100644 index 0000000000..55441bd33a --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/hash.tcl @@ -0,0 +1,470 @@ +start_server {tags {"hash"}} { + test {HSET/HLEN - Small hash creation} { + array set smallhash {} + for {set i 0} {$i < 8} {incr i} { + set key [randstring 0 8 alpha] + set val [randstring 0 8 alpha] + if {[info exists smallhash($key)]} { + incr i -1 + continue + } + r hset smallhash $key $val + set smallhash($key) $val + } + list [r hlen smallhash] + } {8} + +# test {Is the small hash encoded with a ziplist?} { +# assert_encoding ziplist smallhash +# } + + test {HSET/HLEN - Big hash creation} { + array set bighash {} + for {set i 0} {$i < 1024} {incr i} { + set key [randstring 0 8 alpha] + set val [randstring 0 8 alpha] + if {[info exists bighash($key)]} { + incr i -1 + continue + } + r hset bighash $key $val + set bighash($key) $val + } + list [r hlen bighash] + } {1024} + +# test {Is the big hash encoded with a ziplist?} { +# assert_encoding hashtable bighash +# } + + test {HGET against the small hash} { + set err {} + foreach k [array names smallhash *] { + if {$smallhash($k) ne [r hget smallhash $k]} { + set err "$smallhash($k) != [r hget smallhash $k]" + break + } + } + set _ $err + } {} + + test {HGET against the big hash} { + set err {} + foreach k [array names bighash *] { + if {$bighash($k) ne [r hget bighash $k]} { + set err "$bighash($k) != [r hget bighash $k]" + break + } + } + set _ $err + } {} + + test {HGET against non existing key} { + set rv {} + lappend rv [r hget smallhash __123123123__] + lappend rv [r hget bighash __123123123__] + set _ $rv + } {{} {}} + + test {HSET in update and insert mode} { + set rv {} + set k [lindex [array names smallhash *] 0] + lappend rv [r hset smallhash $k newval1] + set smallhash($k) newval1 + lappend rv [r hget smallhash $k] + lappend rv [r hset smallhash __foobar123__ newval] + set k [lindex [array names bighash *] 0] + lappend rv [r hset bighash $k newval2] + set bighash($k) newval2 + lappend rv [r hget bighash $k] + lappend rv [r hset bighash __foobar123__ newval] + lappend rv [r hdel smallhash __foobar123__] + lappend rv [r hdel bighash __foobar123__] + set _ $rv + } {0 newval1 1 0 newval2 1 1 1} + + test {HSETNX target key missing - small hash} { + r hsetnx smallhash __123123123__ foo + r hget smallhash __123123123__ + } {foo} + + test {HSETNX target key exists - small hash} { + r hsetnx smallhash __123123123__ bar + set result [r hget smallhash __123123123__] + r hdel smallhash __123123123__ + set _ $result + } {foo} + + test {HSETNX target key missing - big hash} { + r hsetnx bighash __123123123__ foo + r hget bighash __123123123__ + } {foo} + + test {HSETNX target key exists - big hash} { + r hsetnx bighash __123123123__ bar + set result [r hget bighash __123123123__] + r hdel bighash __123123123__ + set _ $result + } {foo} + + test {HMSET wrong number of args} { + catch {r hmset smallhash key1 val1 key2} err + format $err + } {*wrong number*} + + test {HMSET - small hash} { + set args {} + foreach {k v} [array get smallhash] { + set newval [randstring 0 8 alpha] + set smallhash($k) $newval + lappend args $k $newval + } + r hmset smallhash {*}$args + } {OK} + + test {HMSET - big hash} { + set args {} + foreach {k v} [array get bighash] { + set newval [randstring 0 8 alpha] + set bighash($k) $newval + lappend args $k $newval + } + r hmset bighash {*}$args + } {OK} + + test {HMGET against non existing key and fields} { + set rv {} + lappend rv [r hmget doesntexist __123123123__ __456456456__] + lappend rv [r hmget smallhash __123123123__ __456456456__] + lappend rv [r hmget bighash __123123123__ __456456456__] + set _ $rv + } {{{} {}} {{} {}} {{} {}}} + +# test {HMGET against wrong type} { +# r set wrongtype somevalue +# assert_error "*wrong*" {r hmget wrongtype field1 field2} +# } + + test {HMGET - small hash} { + set keys {} + set vals {} + foreach {k v} [array get smallhash] { + lappend keys $k + lappend vals $v + } + set err {} + set result [r hmget smallhash {*}$keys] + if {$vals ne $result} { + set err "$vals != $result" + break + } + set _ $err + } {} + + test {HMGET - big hash} { + set keys {} + set vals {} + foreach {k v} [array get bighash] { + lappend keys $k + lappend vals $v + } + set err {} + set result [r hmget bighash {*}$keys] + if {$vals ne $result} { + set err "$vals != $result" + break + } + set _ $err + } {} + + test {HKEYS - small hash} { + lsort [r hkeys smallhash] + } [lsort [array names smallhash *]] + + test {HKEYS - big hash} { + lsort [r hkeys bighash] + } [lsort [array names bighash *]] + + test {HVALS - small hash} { + set vals {} + foreach {k v} [array get smallhash] { + lappend vals $v + } + set _ [lsort $vals] + } [lsort [r hvals smallhash]] + + test {HVALS - big hash} { + set vals {} + foreach {k v} [array get bighash] { + lappend vals $v + } + set _ [lsort $vals] + } [lsort [r hvals bighash]] + + test {HGETALL - small hash} { + lsort [r hgetall smallhash] + } [lsort [array get smallhash]] + + test {HGETALL - big hash} { + lsort [r hgetall bighash] + } [lsort [array get bighash]] + + test {HDEL and return value} { + set rv {} + lappend rv [r hdel smallhash nokey] + lappend rv [r hdel bighash nokey] + set k [lindex [array names smallhash *] 0] + lappend rv [r hdel smallhash $k] + lappend rv [r hdel smallhash $k] + lappend rv [r hget smallhash $k] + unset smallhash($k) + set k [lindex [array names bighash *] 0] + lappend rv [r hdel bighash $k] + lappend rv [r hdel bighash $k] + lappend rv [r hget bighash $k] + unset bighash($k) + set _ $rv + } {0 0 1 0 {} 1 0 {}} + + test {HDEL - more than a single value} { + set rv {} + r del myhash + r hmset myhash a 1 b 2 c 3 + assert_equal 0 [r hdel myhash x y] + assert_equal 2 [r hdel myhash a c f] + r hgetall myhash + } {b 2} + + test {HDEL - hash becomes empty before deleting all specified fields} { + r del myhash + r hmset myhash a 1 b 2 c 3 + assert_equal 3 [r hdel myhash a b c d e] + assert_equal 0 [r exists myhash] + } + + test {HEXISTS} { + set rv {} + set k [lindex [array names smallhash *] 0] + lappend rv [r hexists smallhash $k] + lappend rv [r hexists smallhash nokey] + set k [lindex [array names bighash *] 0] + lappend rv [r hexists bighash $k] + lappend rv [r hexists bighash nokey] + } {1 0 1 0} + +# test {Is a ziplist encoded Hash promoted on big payload?} { +# r hset smallhash foo [string repeat a 1024] +# r debug object smallhash +# } {*hashtable*} + + test {HINCRBY against non existing database key} { + r del htest + list [r hincrby htest foo 2] + } {2} + + test {HINCRBY against non existing hash key} { + set rv {} + r hdel smallhash tmp + r hdel bighash tmp + lappend rv [r hincrby smallhash tmp 2] + lappend rv [r hget smallhash tmp] + lappend rv [r hincrby bighash tmp 2] + lappend rv [r hget bighash tmp] + } {2 2 2 2} + + test {HINCRBY against hash key created by hincrby itself} { + set rv {} + lappend rv [r hincrby smallhash tmp 3] + lappend rv [r hget smallhash tmp] + lappend rv [r hincrby bighash tmp 3] + lappend rv [r hget bighash tmp] + } {5 5 5 5} + + test {HINCRBY against hash key originally set with HSET} { + r hset smallhash tmp 100 + r hset bighash tmp 100 + list [r hincrby smallhash tmp 2] [r hincrby bighash tmp 2] + } {102 102} + + test {HINCRBY over 32bit value} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrby smallhash tmp 1] [r hincrby bighash tmp 1] + } {17179869185 17179869185} + + test {HINCRBY over 32bit value with over 32bit increment} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrby smallhash tmp 17179869184] [r hincrby bighash tmp 17179869184] + } {34359738368 34359738368} + + test {HINCRBY fails against hash value with spaces (left)} { + r hset smallhash str " 11" + r hset bighash str " 11" + catch {r hincrby smallhash str 1} smallerr + catch {r hincrby smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not an integer*" $smallerr] + lappend rv [string match "ERR*not an integer*" $bigerr] + } {1 1} + + test {HINCRBY fails against hash value with spaces (right)} { + r hset smallhash str "11 " + r hset bighash str "11 " + catch {r hincrby smallhash str 1} smallerr + catch {r hincrby smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not an integer*" $smallerr] + lappend rv [string match "ERR*not an integer*" $bigerr] + } {1 1} + + test {HINCRBY can detect overflows} { + set e {} + r hset hash n -9223372036854775484 + assert {[r hincrby hash n -1] == -9223372036854775485} + catch {r hincrby hash n -10000} e + set e + } {*overflow*} + + test {HINCRBYFLOAT against non existing database key} { + r del htest + list [r hincrbyfloat htest foo 2.5] + } {2.5} + + test {HINCRBYFLOAT against non existing hash key} { + set rv {} + r hdel smallhash tmp + r hdel bighash tmp + lappend rv [roundFloat [r hincrbyfloat smallhash tmp 2.5]] + lappend rv [roundFloat [r hget smallhash tmp]] + lappend rv [roundFloat [r hincrbyfloat bighash tmp 2.5]] + lappend rv [roundFloat [r hget bighash tmp]] + } {2.5 2.5 2.5 2.5} + + test {HINCRBYFLOAT against hash key created by hincrby itself} { + set rv {} + lappend rv [roundFloat [r hincrbyfloat smallhash tmp 3.5]] + lappend rv [roundFloat [r hget smallhash tmp]] + lappend rv [roundFloat [r hincrbyfloat bighash tmp 3.5]] + lappend rv [roundFloat [r hget bighash tmp]] + } {6 6 6 6} + + test {HINCRBYFLOAT against hash key originally set with HSET} { + r hset smallhash tmp 100 + r hset bighash tmp 100 + list [roundFloat [r hincrbyfloat smallhash tmp 2.5]] \ + [roundFloat [r hincrbyfloat bighash tmp 2.5]] + } {102.5 102.5} + + test {HINCRBYFLOAT over 32bit value} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrbyfloat smallhash tmp 1] \ + [r hincrbyfloat bighash tmp 1] + } {17179869185 17179869185} + + test {HINCRBYFLOAT over 32bit value with over 32bit increment} { + r hset smallhash tmp 17179869184 + r hset bighash tmp 17179869184 + list [r hincrbyfloat smallhash tmp 17179869184] \ + [r hincrbyfloat bighash tmp 17179869184] + } {34359738368 34359738368} + + test {HINCRBYFLOAT fails against hash value with spaces (left)} { + r hset smallhash str " 11" + r hset bighash str " 11" + catch {r hincrbyfloat smallhash str 1} smallerr + catch {r hincrbyfloat smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not*float*" $smallerr] + lappend rv [string match "ERR*not*float*" $bigerr] + } {1 1} + + test {HINCRBYFLOAT fails against hash value with spaces (right)} { + r hset smallhash str "11 " + r hset bighash str "11 " + catch {r hincrbyfloat smallhash str 1} smallerr + catch {r hincrbyfloat smallhash str 1} bigerr + set rv {} + lappend rv [string match "ERR*not*float*" $smallerr] + lappend rv [string match "ERR*not*float*" $bigerr] + } {1 1} + + test {Hash ziplist regression test for large keys} { + r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk a + r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk b + r hget hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk + } {b} + + foreach size {10 512} { + test "Hash fuzzing #1 - $size fields" { + for {set times 0} {$times < 10} {incr times} { + catch {unset hash} + array set hash {} + r del hash + + # Create + for {set j 0} {$j < $size} {incr j} { + set field [randomValue] + set value [randomValue] + r hset hash $field $value + set hash($field) $value + } + + # Verify + foreach {k v} [array get hash] { + assert_equal $v [r hget hash $k] + } + assert_equal [array size hash] [r hlen hash] + } + } + + test "Hash fuzzing #2 - $size fields" { + for {set times 0} {$times < 10} {incr times} { + catch {unset hash} + array set hash {} + r del hash + + # Create + for {set j 0} {$j < $size} {incr j} { + randpath { + set field [randomValue] + set value [randomValue] + r hset hash $field $value + set hash($field) $value + } { + set field [randomSignedInt 512] + set value [randomSignedInt 512] + r hset hash $field $value + set hash($field) $value + } { + randpath { + set field [randomValue] + } { + set field [randomSignedInt 512] + } + r hdel hash $field + unset -nocomplain hash($field) + } + } + + # Verify + foreach {k v} [array get hash] { + assert_equal $v [r hget hash $k] + } + assert_equal [array size hash] [r hlen hash] + } + } + } + +# test {Stress test the hash ziplist -> hashtable encoding conversion} { +# r config set hash-max-ziplist-entries 32 +# for {set j 0} {$j < 100} {incr j} { +# r del myhash +# for {set i 0} {$i < 64} {incr i} { +# r hset myhash [randomValue] [randomValue] +# } +# assert {[r object encoding myhash] eq {hashtable}} +# } +# } +} diff --git a/tools/pika_migrate/tests/unit/type/list-2.tcl b/tools/pika_migrate/tests/unit/type/list-2.tcl new file mode 100644 index 0000000000..bf6a055eba --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/list-2.tcl @@ -0,0 +1,44 @@ +start_server { + tags {"list"} + overrides { + "list-max-ziplist-value" 16 + "list-max-ziplist-entries" 256 + } +} { + source "tests/unit/type/list-common.tcl" + + foreach {type large} [array get largevalue] { + tags {"slow"} { + test "LTRIM stress testing - $type" { + set mylist {} + set startlen 32 + r del mylist + + # Start with the large value to ensure the + # right encoding is used. + r rpush mylist $large + lappend mylist $large + + for {set i 0} {$i < $startlen} {incr i} { + set str [randomInt 9223372036854775807] + r rpush mylist $str + lappend mylist $str + } + + for {set i 0} {$i < 1000} {incr i} { + set min [expr {int(rand()*$startlen)}] + set max [expr {$min+int(rand()*$startlen)}] + set mylist [lrange $mylist $min $max] + r ltrim mylist $min $max + assert_equal $mylist [r lrange mylist 0 -1] + + for {set j [r llen mylist]} {$j < $startlen} {incr j} { + set str [randomInt 9223372036854775807] + r rpush mylist $str + lappend mylist $str + } + } + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/type/list-3.tcl b/tools/pika_migrate/tests/unit/type/list-3.tcl new file mode 100644 index 0000000000..94f9a0b797 --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/list-3.tcl @@ -0,0 +1,79 @@ +start_server { + tags {list ziplist} + overrides { + "list-max-ziplist-value" 200000 + "list-max-ziplist-entries" 256 + } +} { + test {Explicit regression for a list bug} { + set mylist {49376042582 {BkG2o\pIC]4YYJa9cJ4GWZalG[4tin;1D2whSkCOW`mX;SFXGyS8sedcff3fQI^tgPCC@^Nu1J6o]meM@Lko]t_jRyotK?tH[\EvWqS]b`o2OCtjg:?nUTwdjpcUm]y:pg5q24q7LlCOwQE^}} + r del l + r rpush l [lindex $mylist 0] + r rpush l [lindex $mylist 1] + assert_equal [r lindex l 0] [lindex $mylist 0] + assert_equal [r lindex l 1] [lindex $mylist 1] + } + + tags {slow} { + test {ziplist implementation: value encoding and backlink} { + if {$::accurate} {set iterations 100} else {set iterations 10} + for {set j 0} {$j < $iterations} {incr j} { + r del l + set l {} + for {set i 0} {$i < 200} {incr i} { + randpath { + set data [string repeat x [randomInt 100000]] + } { + set data [randomInt 65536] + } { + set data [randomInt 4294967296] + } { + set data [randomInt 18446744073709551616] + } { + set data -[randomInt 65536] + if {$data eq {-0}} {set data 0} + } { + set data -[randomInt 4294967296] + if {$data eq {-0}} {set data 0} + } { + set data -[randomInt 18446744073709551616] + if {$data eq {-0}} {set data 0} + } + lappend l $data + r rpush l $data + } + assert_equal [llength $l] [r llen l] + # Traverse backward + for {set i 199} {$i >= 0} {incr i -1} { + if {[lindex $l $i] ne [r lindex l $i]} { + assert_equal [lindex $l $i] [r lindex l $i] + } + } + } + } + + test {ziplist implementation: encoding stress testing} { + for {set j 0} {$j < 200} {incr j} { + r del l + set l {} + set len [randomInt 400] + for {set i 0} {$i < $len} {incr i} { + set rv [randomValue] + randpath { + lappend l $rv + r rpush l $rv + } { + set l [concat [list $rv] $l] + r lpush l $rv + } + } + assert_equal [llength $l] [r llen l] + for {set i 0} {$i < $len} {incr i} { + if {[lindex $l $i] ne [r lindex l $i]} { + assert_equal [lindex $l $i] [r lindex l $i] + } + } + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/type/list-common.tcl b/tools/pika_migrate/tests/unit/type/list-common.tcl new file mode 100644 index 0000000000..ab45f0b31b --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/list-common.tcl @@ -0,0 +1,5 @@ +# We need a value larger than list-max-ziplist-value to make sure +# the list has the right encoding when it is swapped in again. +array set largevalue {} +set largevalue(ziplist) "hello" +set largevalue(linkedlist) [string repeat "hello" 4] diff --git a/tools/pika_migrate/tests/unit/type/list.tcl b/tools/pika_migrate/tests/unit/type/list.tcl new file mode 100644 index 0000000000..17358ae378 --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/list.tcl @@ -0,0 +1,896 @@ +start_server { + tags {"list"} + overrides { + "list-max-ziplist-value" 16 + "list-max-ziplist-entries" 256 + } +} { + source "tests/unit/type/list-common.tcl" + + test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} { + # first lpush then rpush + assert_equal 1 [r lpush myziplist1 a] + assert_equal 2 [r rpush myziplist1 b] + assert_equal 3 [r rpush myziplist1 c] + assert_equal 3 [r llen myziplist1] + assert_equal a [r lindex myziplist1 0] + assert_equal b [r lindex myziplist1 1] + assert_equal c [r lindex myziplist1 2] + assert_equal {} [r lindex myziplist2 3] + assert_equal c [r rpop myziplist1] + assert_equal a [r lpop myziplist1] +# assert_encoding ziplist myziplist1 + + # first rpush then lpush + assert_equal 1 [r rpush myziplist2 a] + assert_equal 2 [r lpush myziplist2 b] + assert_equal 3 [r lpush myziplist2 c] + assert_equal 3 [r llen myziplist2] + assert_equal c [r lindex myziplist2 0] + assert_equal b [r lindex myziplist2 1] + assert_equal a [r lindex myziplist2 2] + assert_equal {} [r lindex myziplist2 3] + assert_equal a [r rpop myziplist2] + assert_equal c [r lpop myziplist2] +# assert_encoding ziplist myziplist2 + } + + test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - regular list} { + # first lpush then rpush + assert_equal 1 [r lpush mylist1 $largevalue(linkedlist)] +# assert_encoding linkedlist mylist1 + assert_equal 2 [r rpush mylist1 b] + assert_equal 3 [r rpush mylist1 c] + assert_equal 3 [r llen mylist1] + assert_equal $largevalue(linkedlist) [r lindex mylist1 0] + assert_equal b [r lindex mylist1 1] + assert_equal c [r lindex mylist1 2] + assert_equal {} [r lindex mylist1 3] + assert_equal c [r rpop mylist1] + assert_equal $largevalue(linkedlist) [r lpop mylist1] + + # first rpush then lpush + assert_equal 1 [r rpush mylist2 $largevalue(linkedlist)] +# assert_encoding linkedlist mylist2 + assert_equal 2 [r lpush mylist2 b] + assert_equal 3 [r lpush mylist2 c] + assert_equal 3 [r llen mylist2] + assert_equal c [r lindex mylist2 0] + assert_equal b [r lindex mylist2 1] + assert_equal $largevalue(linkedlist) [r lindex mylist2 2] + assert_equal {} [r lindex mylist2 3] + assert_equal $largevalue(linkedlist) [r rpop mylist2] + assert_equal c [r lpop mylist2] + } + + test {R/LPOP against empty list} { + r lpop non-existing-list + } {} + + test {Variadic RPUSH/LPUSH} { + r del mylist + assert_equal 4 [r lpush mylist a b c d] + assert_equal 8 [r rpush mylist 0 1 2 3] + assert_equal {d c b a 0 1 2 3} [r lrange mylist 0 -1] + } + + test {DEL a list - ziplist} { + assert_equal 1 [r del myziplist2] + assert_equal 0 [r exists myziplist2] + assert_equal 0 [r llen myziplist2] + } + + test {DEL a list - regular list} { + assert_equal 1 [r del mylist2] + assert_equal 0 [r exists mylist2] + assert_equal 0 [r llen mylist2] + } + + proc create_ziplist {key entries} { + r del $key + foreach entry $entries { r rpush $key $entry } +# assert_encoding ziplist $key + } + + proc create_linkedlist {key entries} { + r del $key + foreach entry $entries { r rpush $key $entry } +# assert_encoding linkedlist $key + } + +# foreach {type large} [array get largevalue] { +# test "BLPOP, BRPOP: single existing list - $type" { +# set rd [redis_deferring_client] +# create_$type blist "a b $large c d" +# +# $rd blpop blist 1 +# assert_equal {blist a} [$rd read] +# $rd brpop blist 1 +# assert_equal {blist d} [$rd read] +# +# $rd blpop blist 1 +# assert_equal {blist b} [$rd read] +# $rd brpop blist 1 +# assert_equal {blist c} [$rd read] +# } +# +# test "BLPOP, BRPOP: multiple existing lists - $type" { +# set rd [redis_deferring_client] +# create_$type blist1 "a $large c" +# create_$type blist2 "d $large f" +# +# $rd blpop blist1 blist2 1 +# assert_equal {blist1 a} [$rd read] +# $rd brpop blist1 blist2 1 +# assert_equal {blist1 c} [$rd read] +# assert_equal 1 [r llen blist1] +# assert_equal 3 [r llen blist2] +# +# $rd blpop blist2 blist1 1 +# assert_equal {blist2 d} [$rd read] +# $rd brpop blist2 blist1 1 +# assert_equal {blist2 f} [$rd read] +# assert_equal 1 [r llen blist1] +# assert_equal 1 [r llen blist2] +# } +# +# test "BLPOP, BRPOP: second list has an entry - $type" { +# set rd [redis_deferring_client] +# r del blist1 +# create_$type blist2 "d $large f" +# +# $rd blpop blist1 blist2 1 +# assert_equal {blist2 d} [$rd read] +# $rd brpop blist1 blist2 1 +# assert_equal {blist2 f} [$rd read] +# assert_equal 0 [r llen blist1] +# assert_equal 1 [r llen blist2] +# } +# +# test "BRPOPLPUSH - $type" { +# r del target +# +# set rd [redis_deferring_client] +# create_$type blist "a b $large c d" +# +# $rd brpoplpush blist target 1 +# assert_equal d [$rd read] +# +# assert_equal d [r rpop target] +# assert_equal "a b $large c" [r lrange blist 0 -1] +# } +# } +# +# test "BLPOP, LPUSH + DEL should not awake blocked client" { +# set rd [redis_deferring_client] +# r del list +# +# $rd blpop list 0 +# r multi +# r lpush list a +# r del list +# r exec +# r del list +# r lpush list b +# $rd read +# } {list b} +# +# test "BLPOP, LPUSH + DEL + SET should not awake blocked client" { +# set rd [redis_deferring_client] +# r del list +# +# $rd blpop list 0 +# r multi +# r lpush list a +# r del list +# r set list foo +# r exec +# r del list +# r lpush list b +# $rd read +# } {list b} +# +# test "BLPOP with same key multiple times should work (issue #801)" { +# set rd [redis_deferring_client] +# r del list1 list2 +# +# # Data arriving after the BLPOP. +# $rd blpop list1 list2 list2 list1 0 +# r lpush list1 a +# assert_equal [$rd read] {list1 a} +# $rd blpop list1 list2 list2 list1 0 +# r lpush list2 b +# assert_equal [$rd read] {list2 b} +# +# # Data already there. +# r lpush list1 a +# r lpush list2 b +# $rd blpop list1 list2 list2 list1 0 +# assert_equal [$rd read] {list1 a} +# $rd blpop list1 list2 list2 list1 0 +# assert_equal [$rd read] {list2 b} +# } +# +# test "MULTI/EXEC is isolated from the point of view of BLPOP" { +# set rd [redis_deferring_client] +# r del list +# $rd blpop list 0 +# r multi +# r lpush list a +# r lpush list b +# r lpush list c +# r exec +# $rd read +# } {list c} +# +# test "BLPOP with variadic LPUSH" { +# set rd [redis_deferring_client] +# r del blist target +# if {$::valgrind} {after 100} +# $rd blpop blist 0 +# if {$::valgrind} {after 100} +# assert_equal 2 [r lpush blist foo bar] +# if {$::valgrind} {after 100} +# assert_equal {blist bar} [$rd read] +# assert_equal foo [lindex [r lrange blist 0 -1] 0] +# } +# +# test "BRPOPLPUSH with zero timeout should block indefinitely" { +# set rd [redis_deferring_client] +# r del blist target +# $rd brpoplpush blist target 0 +# after 1000 +# r rpush blist foo +# assert_equal foo [$rd read] +# assert_equal {foo} [r lrange target 0 -1] +# } +# +# test "BRPOPLPUSH with a client BLPOPing the target list" { +# set rd [redis_deferring_client] +# set rd2 [redis_deferring_client] +# r del blist target +# $rd2 blpop target 0 +# $rd brpoplpush blist target 0 +# after 1000 +# r rpush blist foo +# assert_equal foo [$rd read] +# assert_equal {target foo} [$rd2 read] +# assert_equal 0 [r exists target] +# } +# +# test "BRPOPLPUSH with wrong source type" { +# set rd [redis_deferring_client] +# r del blist target +# r set blist nolist +# $rd brpoplpush blist target 1 +# assert_error "WRONGTYPE*" {$rd read} +# } +# +# test "BRPOPLPUSH with wrong destination type" { +# set rd [redis_deferring_client] +# r del blist target +# r set target nolist +# r lpush blist foo +# $rd brpoplpush blist target 1 +# assert_error "WRONGTYPE*" {$rd read} +# +# set rd [redis_deferring_client] +# r del blist target +# r set target nolist +# $rd brpoplpush blist target 0 +# after 1000 +# r rpush blist foo +# assert_error "WRONGTYPE*" {$rd read} +# assert_equal {foo} [r lrange blist 0 -1] +# } +# +# test "BRPOPLPUSH maintains order of elements after failure" { +# set rd [redis_deferring_client] +# r del blist target +# r set target nolist +# $rd brpoplpush blist target 0 +# r rpush blist a b c +# assert_error "WRONGTYPE*" {$rd read} +# r lrange blist 0 -1 +# } {a b c} +# +# test "BRPOPLPUSH with multiple blocked clients" { +# set rd1 [redis_deferring_client] +# set rd2 [redis_deferring_client] +# r del blist target1 target2 +# r set target1 nolist +# $rd1 brpoplpush blist target1 0 +# $rd2 brpoplpush blist target2 0 +# r lpush blist foo +# +# assert_error "WRONGTYPE*" {$rd1 read} +# assert_equal {foo} [$rd2 read] +# assert_equal {foo} [r lrange target2 0 -1] +# } +# +# test "Linked BRPOPLPUSH" { +# set rd1 [redis_deferring_client] +# set rd2 [redis_deferring_client] +# +# r del list1 list2 list3 +# +# $rd1 brpoplpush list1 list2 0 +# $rd2 brpoplpush list2 list3 0 +# +# r rpush list1 foo +# +# assert_equal {} [r lrange list1 0 -1] +# assert_equal {} [r lrange list2 0 -1] +# assert_equal {foo} [r lrange list3 0 -1] +# } +# +# test "Circular BRPOPLPUSH" { +# set rd1 [redis_deferring_client] +# set rd2 [redis_deferring_client] +# +# r del list1 list2 +# +# $rd1 brpoplpush list1 list2 0 +# $rd2 brpoplpush list2 list1 0 +# +# r rpush list1 foo +# +# assert_equal {foo} [r lrange list1 0 -1] +# assert_equal {} [r lrange list2 0 -1] +# } +# +# test "Self-referential BRPOPLPUSH" { +# set rd [redis_deferring_client] +# +# r del blist +# +# $rd brpoplpush blist blist 0 +# +# r rpush blist foo +# +# assert_equal {foo} [r lrange blist 0 -1] +# } +# +# test "BRPOPLPUSH inside a transaction" { +# r del xlist target +# r lpush xlist foo +# r lpush xlist bar +# +# r multi +# r brpoplpush xlist target 0 +# r brpoplpush xlist target 0 +# r brpoplpush xlist target 0 +# r lrange xlist 0 -1 +# r lrange target 0 -1 +# r exec +# } {foo bar {} {} {bar foo}} +# +# test "PUSH resulting from BRPOPLPUSH affect WATCH" { +# set blocked_client [redis_deferring_client] +# set watching_client [redis_deferring_client] +# r del srclist dstlist somekey +# r set somekey somevalue +# $blocked_client brpoplpush srclist dstlist 0 +# $watching_client watch dstlist +# $watching_client read +# $watching_client multi +# $watching_client read +# $watching_client get somekey +# $watching_client read +# r lpush srclist element +# $watching_client exec +# $watching_client read +# } {} +# +# test "BRPOPLPUSH does not affect WATCH while still blocked" { +# set blocked_client [redis_deferring_client] +# set watching_client [redis_deferring_client] +# r del srclist dstlist somekey +# r set somekey somevalue +# $blocked_client brpoplpush srclist dstlist 0 +# $watching_client watch dstlist +# $watching_client read +# $watching_client multi +# $watching_client read +# $watching_client get somekey +# $watching_client read +# $watching_client exec +# # Blocked BLPOPLPUSH may create problems, unblock it. +# r lpush srclist element +# $watching_client read +# } {somevalue} +# +# test {BRPOPLPUSH timeout} { +# set rd [redis_deferring_client] +# +# $rd brpoplpush foo_list bar_list 1 +# after 2000 +# $rd read +# } {} +# +# test "BLPOP when new key is moved into place" { +# set rd [redis_deferring_client] +# +# $rd blpop foo 5 +# r lpush bob abc def hij +# r rename bob foo +# $rd read +# } {foo hij} +# +# test "BLPOP when result key is created by SORT..STORE" { +# set rd [redis_deferring_client] +# +# # zero out list from previous test without explicit delete +# r lpop foo +# r lpop foo +# r lpop foo +# +# $rd blpop foo 5 +# r lpush notfoo hello hola aguacate konichiwa zanzibar +# r sort notfoo ALPHA store foo +# $rd read +# } {foo aguacate} +# +# foreach {pop} {BLPOP BRPOP} { +# test "$pop: with single empty list argument" { +# set rd [redis_deferring_client] +# r del blist1 +# $rd $pop blist1 1 +# r rpush blist1 foo +# assert_equal {blist1 foo} [$rd read] +# assert_equal 0 [r exists blist1] +# } +# +# test "$pop: with negative timeout" { +# set rd [redis_deferring_client] +# $rd $pop blist1 -1 +# assert_error "ERR*is negative*" {$rd read} +# } +# +# test "$pop: with non-integer timeout" { +# set rd [redis_deferring_client] +# $rd $pop blist1 1.1 +# assert_error "ERR*not an integer*" {$rd read} +# } +# +# test "$pop: with zero timeout should block indefinitely" { +# # To test this, use a timeout of 0 and wait a second. +# # The blocking pop should still be waiting for a push. +# set rd [redis_deferring_client] +# $rd $pop blist1 0 +# after 1000 +# r rpush blist1 foo +# assert_equal {blist1 foo} [$rd read] +# } +# +# test "$pop: second argument is not a list" { +# set rd [redis_deferring_client] +# r del blist1 blist2 +# r set blist2 nolist +# $rd $pop blist1 blist2 1 +# assert_error "WRONGTYPE*" {$rd read} +# } +# +# test "$pop: timeout" { +# set rd [redis_deferring_client] +# r del blist1 blist2 +# $rd $pop blist1 blist2 1 +# assert_equal {} [$rd read] +# } +# +# test "$pop: arguments are empty" { +# set rd [redis_deferring_client] +# r del blist1 blist2 +# +# $rd $pop blist1 blist2 1 +# r rpush blist1 foo +# assert_equal {blist1 foo} [$rd read] +# assert_equal 0 [r exists blist1] +# assert_equal 0 [r exists blist2] +# +# $rd $pop blist1 blist2 1 +# r rpush blist2 foo +# assert_equal {blist2 foo} [$rd read] +# assert_equal 0 [r exists blist1] +# assert_equal 0 [r exists blist2] +# } +# } +# +# test {BLPOP inside a transaction} { +# r del xlist +# r lpush xlist foo +# r lpush xlist bar +# r multi +# r blpop xlist 0 +# r blpop xlist 0 +# r blpop xlist 0 +# r exec +# } {{xlist bar} {xlist foo} {}} + + test {LPUSHX, RPUSHX - generic} { + r del xlist + assert_equal 0 [r lpushx xlist a] + assert_equal 0 [r llen xlist] + assert_equal 0 [r rpushx xlist a] + assert_equal 0 [r llen xlist] + } + + foreach {type large} [array get largevalue] { + test "LPUSHX, RPUSHX - $type" { + create_$type xlist "$large c" + assert_equal 3 [r rpushx xlist d] + assert_equal 4 [r lpushx xlist a] + assert_equal "a $large c d" [r lrange xlist 0 -1] + } + + test "LINSERT - $type" { + create_$type xlist "a $large c d" + assert_equal 5 [r linsert xlist before c zz] + assert_equal "a $large zz c d" [r lrange xlist 0 10] + assert_equal 6 [r linsert xlist after c yy] + assert_equal "a $large zz c yy d" [r lrange xlist 0 10] + assert_equal 7 [r linsert xlist after d dd] + assert_equal -1 [r linsert xlist after bad ddd] + assert_equal "a $large zz c yy d dd" [r lrange xlist 0 10] + assert_equal 8 [r linsert xlist before a aa] + assert_equal -1 [r linsert xlist before bad aaa] + assert_equal "aa a $large zz c yy d dd" [r lrange xlist 0 10] + + # check inserting integer encoded value + assert_equal 9 [r linsert xlist before aa 42] + assert_equal 42 [r lrange xlist 0 0] + } + } + + test {LINSERT raise error on bad syntax} { + catch {[r linsert xlist aft3r aa 42]} e + set e + } {*ERR*syntax*error*} + +# test {LPUSHX, RPUSHX convert from ziplist to list} { +# set large $largevalue(linkedlist) +# +# # convert when a large value is pushed +# create_ziplist xlist a +# assert_equal 2 [r rpushx xlist $large] +# assert_encoding linkedlist xlist +# create_ziplist xlist a +# assert_equal 2 [r lpushx xlist $large] +# assert_encoding linkedlist xlist +# +# # convert when the length threshold is exceeded +# create_ziplist xlist [lrepeat 256 a] +# assert_equal 257 [r rpushx xlist b] +# assert_encoding linkedlist xlist +# create_ziplist xlist [lrepeat 256 a] +# assert_equal 257 [r lpushx xlist b] +# assert_encoding linkedlist xlist +# } + +# test {LINSERT convert from ziplist to list} { +# set large $largevalue(linkedlist) +# +# # convert when a large value is inserted +# create_ziplist xlist a +# assert_equal 2 [r linsert xlist before a $large] +# assert_encoding linkedlist xlist +# create_ziplist xlist a +# assert_equal 2 [r linsert xlist after a $large] +# assert_encoding linkedlist xlist +# +# # convert when the length threshold is exceeded +# create_ziplist xlist [lrepeat 256 a] +# assert_equal 257 [r linsert xlist before a a] +# assert_encoding linkedlist xlist +# create_ziplist xlist [lrepeat 256 a] +# assert_equal 257 [r linsert xlist after a a] +# assert_encoding linkedlist xlist +# +# # don't convert when the value could not be inserted +# create_ziplist xlist [lrepeat 256 a] +# assert_equal -1 [r linsert xlist before foo a] +# assert_encoding ziplist xlist +# create_ziplist xlist [lrepeat 256 a] +# assert_equal -1 [r linsert xlist after foo a] +# assert_encoding ziplist xlist +# } + + foreach {type num} {ziplist 250 linkedlist 500} { + proc check_numbered_list_consistency {key} { + set len [r llen $key] + for {set i 0} {$i < $len} {incr i} { + assert_equal $i [r lindex $key $i] + assert_equal [expr $len-1-$i] [r lindex $key [expr (-$i)-1]] + } + } + + proc check_random_access_consistency {key} { + set len [r llen $key] + for {set i 0} {$i < $len} {incr i} { + set rint [expr int(rand()*$len)] + assert_equal $rint [r lindex $key $rint] + assert_equal [expr $len-1-$rint] [r lindex $key [expr (-$rint)-1]] + } + } + + test "LINDEX consistency test - $type" { + r del mylist + for {set i 0} {$i < $num} {incr i} { + r rpush mylist $i + } +# assert_encoding $type mylist + check_numbered_list_consistency mylist + } + + test "LINDEX random access - $type" { +# assert_encoding $type mylist + check_random_access_consistency mylist + } + +# test "Check if list is still ok after a DEBUG RELOAD - $type" { +# r debug reload +# assert_encoding $type mylist +# check_numbered_list_consistency mylist +# check_random_access_consistency mylist +# } + } + +# test {LLEN against non-list value error} { +# r del mylist +# r set mylist foobar +# assert_error WRONGTYPE* {r llen mylist} +# } + + test {LLEN against non existing key} { + assert_equal 0 [r llen not-a-key] + } + +# test {LINDEX against non-list value error} { +# assert_error WRONGTYPE* {r lindex mylist 0} +# } + + test {LINDEX against non existing key} { + assert_equal "" [r lindex not-a-key 10] + } + +# test {LPUSH against non-list value error} { +# assert_error WRONGTYPE* {r lpush mylist 0} +# } + +# test {RPUSH against non-list value error} { +# assert_error WRONGTYPE* {r rpush mylist 0} +# } + + foreach {type large} [array get largevalue] { + test "RPOPLPUSH base case - $type" { + r del mylist1 mylist2 + create_$type mylist1 "a $large c d" + assert_equal d [r rpoplpush mylist1 mylist2] + assert_equal c [r rpoplpush mylist1 mylist2] + assert_equal "a $large" [r lrange mylist1 0 -1] + assert_equal "c d" [r lrange mylist2 0 -1] +# assert_encoding ziplist mylist2 + } + + test "RPOPLPUSH with the same list as src and dst - $type" { + create_$type mylist "a $large c" + assert_equal "a $large c" [r lrange mylist 0 -1] + assert_equal c [r rpoplpush mylist mylist] + assert_equal "c a $large" [r lrange mylist 0 -1] + } + + foreach {othertype otherlarge} [array get largevalue] { + test "RPOPLPUSH with $type source and existing target $othertype" { + create_$type srclist "a b c $large" + create_$othertype dstlist "$otherlarge" + assert_equal $large [r rpoplpush srclist dstlist] + assert_equal c [r rpoplpush srclist dstlist] + assert_equal "a b" [r lrange srclist 0 -1] + assert_equal "c $large $otherlarge" [r lrange dstlist 0 -1] + + # When we rpoplpush'ed a large value, dstlist should be + # converted to the same encoding as srclist. +# if {$type eq "linkedlist"} { +# assert_encoding linkedlist dstlist +# } + } + } + } + + test {RPOPLPUSH against non existing key} { + r del srclist dstlist + assert_equal {} [r rpoplpush srclist dstlist] + assert_equal 0 [r exists srclist] + assert_equal 0 [r exists dstlist] + } + + test {RPOPLPUSH against non list src key} { + r del srclist dstlist + r set srclist x +# assert_error WRONGTYPE* {r rpoplpush srclist dstlist} +# assert_type string srclist + assert_equal 0 [r exists newlist] + } + + test {RPOPLPUSH against non list dst key} { + create_ziplist srclist {a b c d} + r set dstlist x +# assert_error WRONGTYPE* {r rpoplpush srclist dstlist} +# assert_type string dstlist + assert_equal {a b c d} [r lrange srclist 0 -1] + } + + test {RPOPLPUSH against non existing src key} { + r del srclist dstlist + assert_equal {} [r rpoplpush srclist dstlist] + } {} + + foreach {type large} [array get largevalue] { + test "Basic LPOP/RPOP - $type" { + create_$type mylist "$large 1 2" + assert_equal $large [r lpop mylist] + assert_equal 2 [r rpop mylist] + assert_equal 1 [r lpop mylist] + assert_equal 0 [r llen mylist] + + # pop on empty list + assert_equal {} [r lpop mylist] + assert_equal {} [r rpop mylist] + } + } + +# test {LPOP/RPOP against non list value} { +# r set notalist foo +# assert_error WRONGTYPE* {r lpop notalist} +# assert_error WRONGTYPE* {r rpop notalist} +# } + + foreach {type num} {ziplist 250 linkedlist 500} { + test "Mass RPOP/LPOP - $type" { + r del mylist + set sum1 0 + for {set i 0} {$i < $num} {incr i} { + r lpush mylist $i + incr sum1 $i + } +# assert_encoding $type mylist + set sum2 0 + for {set i 0} {$i < [expr $num/2]} {incr i} { + incr sum2 [r lpop mylist] + incr sum2 [r rpop mylist] + } + assert_equal $sum1 $sum2 + } + } + + foreach {type large} [array get largevalue] { + test "LRANGE basics - $type" { + create_$type mylist "$large 1 2 3 4 5 6 7 8 9" + assert_equal {1 2 3 4 5 6 7 8} [r lrange mylist 1 -2] + assert_equal {7 8 9} [r lrange mylist -3 -1] + assert_equal {4} [r lrange mylist 4 4] + } + + test "LRANGE inverted indexes - $type" { + create_$type mylist "$large 1 2 3 4 5 6 7 8 9" + assert_equal {} [r lrange mylist 6 2] + } + + test "LRANGE out of range indexes including the full list - $type" { + create_$type mylist "$large 1 2 3" + assert_equal "$large 1 2 3" [r lrange mylist -1000 1000] + } + + test "LRANGE out of range negative end index - $type" { + create_$type mylist "$large 1 2 3" + assert_equal $large [r lrange mylist 0 -4] + assert_equal {} [r lrange mylist 0 -5] + } + } + + test {LRANGE against non existing key} { + assert_equal {} [r lrange nosuchkey 0 1] + } + + foreach {type large} [array get largevalue] { + proc trim_list {type min max} { + upvar 1 large large + r del mylist + create_$type mylist "1 2 3 4 $large" + r ltrim mylist $min $max + r lrange mylist 0 -1 + } + + test "LTRIM basics - $type" { + assert_equal "1" [trim_list $type 0 0] + assert_equal "1 2" [trim_list $type 0 1] + assert_equal "1 2 3" [trim_list $type 0 2] + assert_equal "2 3" [trim_list $type 1 2] + assert_equal "2 3 4 $large" [trim_list $type 1 -1] + assert_equal "2 3 4" [trim_list $type 1 -2] + assert_equal "4 $large" [trim_list $type -2 -1] + assert_equal "$large" [trim_list $type -1 -1] + assert_equal "1 2 3 4 $large" [trim_list $type -5 -1] + assert_equal "1 2 3 4 $large" [trim_list $type -10 10] + assert_equal "1 2 3 4 $large" [trim_list $type 0 5] + assert_equal "1 2 3 4 $large" [trim_list $type 0 10] + } + + test "LTRIM out of range negative end index - $type" { + assert_equal {1} [trim_list $type 0 -5] + assert_equal {} [trim_list $type 0 -6] + } + + } + + foreach {type large} [array get largevalue] { + test "LSET - $type" { + create_$type mylist "99 98 $large 96 95" + r lset mylist 1 foo + r lset mylist -1 bar + assert_equal "99 foo $large 96 bar" [r lrange mylist 0 -1] + } + + test "LSET out of range index - $type" { + assert_error ERR*range* {r lset mylist 10 foo} + } + } + + test {LSET against non existing key} { + assert_error ERR*key* {r lset nosuchkey 10 foo} + } + +# test {LSET against non list value} { +# r set nolist foobar +# assert_error WRONGTYPE* {r lset nolist 0 foo} +# } + + foreach {type e} [array get largevalue] { + test "LREM remove all the occurrences - $type" { + create_$type mylist "$e foo bar foobar foobared zap bar test foo" + assert_equal 2 [r lrem mylist 0 bar] + assert_equal "$e foo foobar foobared zap test foo" [r lrange mylist 0 -1] + } + + test "LREM remove the first occurrence - $type" { + assert_equal 1 [r lrem mylist 1 foo] + assert_equal "$e foobar foobared zap test foo" [r lrange mylist 0 -1] + } + + test "LREM remove non existing element - $type" { + assert_equal 0 [r lrem mylist 1 nosuchelement] + assert_equal "$e foobar foobared zap test foo" [r lrange mylist 0 -1] + } + + test "LREM starting from tail with negative count - $type" { + create_$type mylist "$e foo bar foobar foobared zap bar test foo foo" + assert_equal 1 [r lrem mylist -1 bar] + assert_equal "$e foo bar foobar foobared zap test foo foo" [r lrange mylist 0 -1] + } + + test "LREM starting from tail with negative count (2) - $type" { + assert_equal 2 [r lrem mylist -2 foo] + assert_equal "$e foo bar foobar foobared zap test" [r lrange mylist 0 -1] + } + + test "LREM deleting objects that may be int encoded - $type" { + create_$type myotherlist "$e 1 2 3" + assert_equal 1 [r lrem myotherlist 1 2] + assert_equal 3 [r llen myotherlist] + } + } + + test "Regression for bug 593 - chaining BRPOPLPUSH with other blocking cmds" { + set rd1 [redis_deferring_client] + set rd2 [redis_deferring_client] + + $rd1 brpoplpush a b 0 + $rd1 brpoplpush a b 0 + $rd2 brpoplpush b c 0 + after 1000 + r lpush a data + $rd1 close + $rd2 close + r ping + } {PONG} +} diff --git a/tools/pika_migrate/tests/unit/type/set.tcl b/tools/pika_migrate/tests/unit/type/set.tcl new file mode 100644 index 0000000000..de3c493a9c --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/set.tcl @@ -0,0 +1,531 @@ +start_server { + tags {"set"} + overrides { + "set-max-intset-entries" 512 + } +} { + proc create_set {key entries} { + r del $key + foreach entry $entries { r sadd $key $entry } + } + + test {SADD, SCARD, SISMEMBER, SMEMBERS basics - regular set} { + create_set myset {foo} +# assert_encoding hashtable myset + assert_equal 1 [r sadd myset bar] + assert_equal 0 [r sadd myset bar] + assert_equal 2 [r scard myset] + assert_equal 1 [r sismember myset foo] + assert_equal 1 [r sismember myset bar] + assert_equal 0 [r sismember myset bla] + assert_equal {bar foo} [lsort [r smembers myset]] + } + + test {SADD, SCARD, SISMEMBER, SMEMBERS basics - intset} { + create_set myset {17} +# assert_encoding intset myset + assert_equal 1 [r sadd myset 16] + assert_equal 0 [r sadd myset 16] + assert_equal 2 [r scard myset] + assert_equal 1 [r sismember myset 16] + assert_equal 1 [r sismember myset 17] + assert_equal 0 [r sismember myset 18] + assert_equal {16 17} [lsort [r smembers myset]] + } + +# test {SADD against non set} { +# r lpush mylist foo +# assert_error WRONGTYPE* {r sadd mylist bar} +# } + + test "SADD a non-integer against an intset" { + create_set myset {1 2 3} +# assert_encoding intset myset + assert_equal 1 [r sadd myset a] +# assert_encoding hashtable myset + } + + test "SADD an integer larger than 64 bits" { + create_set myset {213244124402402314402033402} +# assert_encoding hashtable myset + assert_equal 1 [r sismember myset 213244124402402314402033402] + } + + test "SADD overflows the maximum allowed integers in an intset" { + r del myset + for {set i 0} {$i < 512} {incr i} { r sadd myset $i } +# assert_encoding intset myset + assert_equal 1 [r sadd myset 512] +# assert_encoding hashtable myset + } + + test {Variadic SADD} { + r del myset + assert_equal 3 [r sadd myset a b c] + assert_equal 2 [r sadd myset A a b c B] + assert_equal [lsort {A a b c B}] [lsort [r smembers myset]] + } + +# test "Set encoding after DEBUG RELOAD" { +# r del myintset myhashset mylargeintset +# for {set i 0} {$i < 100} {incr i} { r sadd myintset $i } +# for {set i 0} {$i < 1280} {incr i} { r sadd mylargeintset $i } +# for {set i 0} {$i < 256} {incr i} { r sadd myhashset [format "i%03d" $i] } +# assert_encoding intset myintset +# assert_encoding hashtable mylargeintset +# assert_encoding hashtable myhashset +# +# r debug reload +# assert_encoding intset myintset +# assert_encoding hashtable mylargeintset +# assert_encoding hashtable myhashset +# } + + test {SREM basics - regular set} { + create_set myset {foo bar ciao} +# assert_encoding hashtable myset + assert_equal 0 [r srem myset qux] + assert_equal 1 [r srem myset foo] + assert_equal {bar ciao} [lsort [r smembers myset]] + } + + test {SREM basics - intset} { + create_set myset {3 4 5} +# assert_encoding intset myset + assert_equal 0 [r srem myset 6] + assert_equal 1 [r srem myset 4] + assert_equal {3 5} [lsort [r smembers myset]] + } + + test {SREM with multiple arguments} { + r del myset + r sadd myset a b c d + assert_equal 0 [r srem myset k k k] + assert_equal 2 [r srem myset b d x y] + lsort [r smembers myset] + } {a c} + + test {SREM variadic version with more args needed to destroy the key} { + r del myset + r sadd myset 1 2 3 + r srem myset 1 2 3 4 5 6 7 8 + } {3} + + foreach {type} {hashtable intset} { + for {set i 1} {$i <= 5} {incr i} { + r del [format "set%d" $i] + } + for {set i 0} {$i < 200} {incr i} { + r sadd set1 $i + r sadd set2 [expr $i+195] + } + foreach i {199 195 1000 2000} { + r sadd set3 $i + } + for {set i 5} {$i < 200} {incr i} { + r sadd set4 $i + } + r sadd set5 0 + + # To make sure the sets are encoded as the type we are testing -- also + # when the VM is enabled and the values may be swapped in and out + # while the tests are running -- an extra element is added to every + # set that determines its encoding. + set large 200 + if {$type eq "hashtable"} { + set large foo + } + + for {set i 1} {$i <= 5} {incr i} { + r sadd [format "set%d" $i] $large + } + +# test "Generated sets must be encoded as $type" { +# for {set i 1} {$i <= 5} {incr i} { +# assert_encoding $type [format "set%d" $i] +# } +# } + + test "SINTER with two sets - $type" { + assert_equal [list 195 196 197 198 199 $large] [lsort [r sinter set1 set2]] + } + + test "SINTERSTORE with two sets - $type" { + r sinterstore setres set1 set2 +# assert_encoding $type setres + assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres]] + } + +# test "SINTERSTORE with two sets, after a DEBUG RELOAD - $type" { +# r debug reload +# r sinterstore setres set1 set2 +# assert_encoding $type setres +# assert_equal [list 195 196 197 198 199 $large] [lsort [r smembers setres]] +# } + + test "SUNION with two sets - $type" { + set expected [lsort -uniq "[r smembers set1] [r smembers set2]"] + assert_equal $expected [lsort [r sunion set1 set2]] + } + + test "SUNIONSTORE with two sets - $type" { + r sunionstore setres set1 set2 +# assert_encoding $type setres + set expected [lsort -uniq "[r smembers set1] [r smembers set2]"] + assert_equal $expected [lsort [r smembers setres]] + } + + test "SINTER against three sets - $type" { + assert_equal [list 195 199 $large] [lsort [r sinter set1 set2 set3]] + } + + test "SINTERSTORE with three sets - $type" { + r sinterstore setres set1 set2 set3 + assert_equal [list 195 199 $large] [lsort [r smembers setres]] + } + + test "SUNION with non existing keys - $type" { + set expected [lsort -uniq "[r smembers set1] [r smembers set2]"] + assert_equal $expected [lsort [r sunion nokey1 set1 set2 nokey2]] + } + + test "SDIFF with two sets - $type" { + assert_equal {0 1 2 3 4} [lsort [r sdiff set1 set4]] + } + + test "SDIFF with three sets - $type" { + assert_equal {1 2 3 4} [lsort [r sdiff set1 set4 set5]] + } + + test "SDIFFSTORE with three sets - $type" { + r sdiffstore setres set1 set4 set5 + # When we start with intsets, we should always end with intsets. +# if {$type eq {intset}} { +# assert_encoding intset setres +# } + assert_equal {1 2 3 4} [lsort [r smembers setres]] + } + } + + test "SDIFF with first set empty" { + r del set1 set2 set3 + r sadd set2 1 2 3 4 + r sadd set3 a b c d + r sdiff set1 set2 set3 + } {} + + test "SDIFF with same set two times" { + r del set1 + r sadd set1 a b c 1 2 3 4 5 6 + r sdiff set1 set1 + } {} + + test "SDIFF fuzzing" { + for {set j 0} {$j < 100} {incr j} { + unset -nocomplain s + array set s {} + set args {} + set num_sets [expr {[randomInt 10]+1}] + for {set i 0} {$i < $num_sets} {incr i} { + set num_elements [randomInt 100] + r del set_$i + lappend args set_$i + while {$num_elements} { + set ele [randomValue] + r sadd set_$i $ele + if {$i == 0} { + set s($ele) x + } else { + unset -nocomplain s($ele) + } + incr num_elements -1 + } + } + set result [lsort [r sdiff {*}$args]] + assert_equal $result [lsort [array names s]] + } + } + +# test "SINTER against non-set should throw error" { +# r set key1 x +# assert_error "WRONGTYPE*" {r sinter key1 noset} +# } + +# test "SUNION against non-set should throw error" { +# r set key1 x +# assert_error "WRONGTYPE*" {r sunion key1 noset} +# } + + test "SINTER should handle non existing key as empty" { + r del set1 set2 set3 + r sadd set1 a b c + r sadd set2 b c d + r sinter set1 set2 set3 + } {} + + test "SINTER with same integer elements but different encoding" { + r del set1 set2 + r sadd set1 1 2 3 + r sadd set2 1 2 3 a + r srem set2 a +# assert_encoding intset set1 +# assert_encoding hashtable set2 + lsort [r sinter set1 set2] + } {1 2 3} + + test "SINTERSTORE against non existing keys should delete dstkey" { + r set setres xxx + assert_equal 0 [r sinterstore setres foo111 bar222] +# assert_equal 0 [r exists setres] + } + + test "SUNIONSTORE against non existing keys should delete dstkey" { + r set setres xxx + assert_equal 0 [r sunionstore setres foo111 bar222] +# assert_equal 0 [r exists setres] + } + + foreach {type contents} {hashtable {a b c} intset {1 2 3}} { + test "SPOP basics - $type" { + create_set myset $contents +# assert_encoding $type myset + assert_equal $contents [lsort [list [r spop myset] [r spop myset] [r spop myset]]] + assert_equal 0 [r scard myset] + } + + test "SRANDMEMBER - $type" { + create_set myset $contents + unset -nocomplain myset + array set myset {} + for {set i 0} {$i < 100} {incr i} { + set myset([r srandmember myset]) 1 + } + assert_equal $contents [lsort [array names myset]] + } + } + + test "SRANDMEMBER with against non existing key" { + r srandmember nonexisting_key 100 + } {} + + foreach {type contents} { + hashtable { + 1 5 10 50 125 50000 33959417 4775547 65434162 + 12098459 427716 483706 2726473884 72615637475 + MARY PATRICIA LINDA BARBARA ELIZABETH JENNIFER MARIA + SUSAN MARGARET DOROTHY LISA NANCY KAREN BETTY HELEN + SANDRA DONNA CAROL RUTH SHARON MICHELLE LAURA SARAH + KIMBERLY DEBORAH JESSICA SHIRLEY CYNTHIA ANGELA MELISSA + BRENDA AMY ANNA REBECCA VIRGINIA KATHLEEN + } + intset { + 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 + 20 21 22 23 24 25 26 27 28 29 + 30 31 32 33 34 35 36 37 38 39 + 40 41 42 43 44 45 46 47 48 49 + } + } { + test "SRANDMEMBER with - $type" { + create_set myset $contents + unset -nocomplain myset + array set myset {} + foreach ele [r smembers myset] { + set myset($ele) 1 + } + assert_equal [lsort $contents] [lsort [array names myset]] + + # Make sure that a count of 0 is handled correctly. + assert_equal [r srandmember myset 0] {} + + # We'll stress different parts of the code, see the implementation + # of SRANDMEMBER for more information, but basically there are + # four different code paths. + # + # PATH 1: Use negative count. + # + # 1) Check that it returns repeated elements. + set res [r srandmember myset -100] + assert_equal [llength $res] 100 + + # 2) Check that all the elements actually belong to the + # original set. + foreach ele $res { + assert {[info exists myset($ele)]} + } + + # 3) Check that eventually all the elements are returned. + unset -nocomplain auxset + set iterations 1000 + while {$iterations != 0} { + incr iterations -1 + set res [r srandmember myset -10] + foreach ele $res { + set auxset($ele) 1 + } + if {[lsort [array names myset]] eq + [lsort [array names auxset]]} { + break; + } + } + assert {$iterations != 0} + + # PATH 2: positive count (unique behavior) with requested size + # equal or greater than set size. + foreach size {50 100} { + set res [r srandmember myset $size] + assert_equal [llength $res] 50 + assert_equal [lsort $res] [lsort [array names myset]] + } + + # PATH 3: Ask almost as elements as there are in the set. + # In this case the implementation will duplicate the original + # set and will remove random elements up to the requested size. + # + # PATH 4: Ask a number of elements definitely smaller than + # the set size. + # + # We can test both the code paths just changing the size but + # using the same code. + + foreach size {45 5} { + set res [r srandmember myset $size] + assert_equal [llength $res] $size + + # 1) Check that all the elements actually belong to the + # original set. + foreach ele $res { + assert {[info exists myset($ele)]} + } + + # 2) Check that eventually all the elements are returned. + unset -nocomplain auxset + set iterations 1000 + while {$iterations != 0} { + incr iterations -1 + set res [r srandmember myset -10] + foreach ele $res { + set auxset($ele) 1 + } + if {[lsort [array names myset]] eq + [lsort [array names auxset]]} { + break; + } + } + assert {$iterations != 0} + } + } + } + + proc setup_move {} { + r del myset3 myset4 + create_set myset1 {1 a b} + create_set myset2 {2 3 4} +# assert_encoding hashtable myset1 +# assert_encoding intset myset2 + } + + test "SMOVE basics - from regular set to intset" { + # move a non-integer element to an intset should convert encoding + setup_move + assert_equal 1 [r smove myset1 myset2 a] + assert_equal {1 b} [lsort [r smembers myset1]] + assert_equal {2 3 4 a} [lsort [r smembers myset2]] +# assert_encoding hashtable myset2 + + # move an integer element should not convert the encoding + setup_move + assert_equal 1 [r smove myset1 myset2 1] + assert_equal {a b} [lsort [r smembers myset1]] + assert_equal {1 2 3 4} [lsort [r smembers myset2]] +# assert_encoding intset myset2 + } + + test "SMOVE basics - from intset to regular set" { + setup_move + assert_equal 1 [r smove myset2 myset1 2] + assert_equal {1 2 a b} [lsort [r smembers myset1]] + assert_equal {3 4} [lsort [r smembers myset2]] + } + + test "SMOVE non existing key" { + setup_move + assert_equal 0 [r smove myset1 myset2 foo] + assert_equal {1 a b} [lsort [r smembers myset1]] + assert_equal {2 3 4} [lsort [r smembers myset2]] + } + + test "SMOVE non existing src set" { + setup_move + assert_equal 0 [r smove noset myset2 foo] + assert_equal {2 3 4} [lsort [r smembers myset2]] + } + + test "SMOVE from regular set to non existing destination set" { + setup_move + assert_equal 1 [r smove myset1 myset3 a] + assert_equal {1 b} [lsort [r smembers myset1]] + assert_equal {a} [lsort [r smembers myset3]] +# assert_encoding hashtable myset3 + } + + test "SMOVE from intset to non existing destination set" { + setup_move + assert_equal 1 [r smove myset2 myset3 2] + assert_equal {3 4} [lsort [r smembers myset2]] + assert_equal {2} [lsort [r smembers myset3]] +# assert_encoding intset myset3 + } + +# test "SMOVE wrong src key type" { +# r set x 10 +# assert_error "WRONGTYPE*" {r smove x myset2 foo} +# } + +# test "SMOVE wrong dst key type" { +# r set x 10 +# assert_error "WRONGTYPE*" {r smove myset2 x foo} +# } + + test "SMOVE with identical source and destination" { + r del set + r sadd set a b c + r smove set set b + lsort [r smembers set] + } {a b c} + + tags {slow} { + test {intsets implementation stress testing} { + for {set j 0} {$j < 20} {incr j} { + unset -nocomplain s + array set s {} + r del s + set len [randomInt 1024] + for {set i 0} {$i < $len} {incr i} { + randpath { + set data [randomInt 65536] + } { + set data [randomInt 4294967296] + } { + set data [randomInt 18446744073709551616] + } + set s($data) {} + r sadd s $data + } + assert_equal [lsort [r smembers s]] [lsort [array names s]] + set len [array size s] + for {set i 0} {$i < $len} {incr i} { + set e [r spop s] + if {![info exists s($e)]} { + puts "Can't find '$e' on local array" + puts "Local array: [lsort [r smembers s]]" + puts "Remote array: [lsort [array names s]]" + error "exception" + } + array unset s $e + } + assert_equal [r scard s] 0 + assert_equal [array size s] 0 + } + } + } +} diff --git a/tools/pika_migrate/tests/unit/type/zset.tcl b/tools/pika_migrate/tests/unit/type/zset.tcl new file mode 100644 index 0000000000..626156c572 --- /dev/null +++ b/tools/pika_migrate/tests/unit/type/zset.tcl @@ -0,0 +1,944 @@ +start_server {tags {"zset"}} { + proc create_zset {key items} { + r del $key + foreach {score entry} $items { + r zadd $key $score $entry + } + } + + proc basics {encoding} { + #if {$encoding == "ziplist"} { + # r config set zset-max-ziplist-entries 128 + # r config set zset-max-ziplist-value 64 + #} elseif {$encoding == "skiplist"} { + # r config set zset-max-ziplist-entries 0 + # r config set zset-max-ziplist-value 0 + #} else { + # puts "Unknown sorted set encoding" + # exit + #} + + test "Check encoding - $encoding" { + r del ztmp + r zadd ztmp 10 x + #assert_encoding $encoding ztmp + } + + test "ZSET basic ZADD and score update - $encoding" { + r del ztmp + r zadd ztmp 10 x + r zadd ztmp 20 y + r zadd ztmp 30 z + assert_equal {x y z} [r zrange ztmp 0 -1] + + r zadd ztmp 1 y + assert_equal {y x z} [r zrange ztmp 0 -1] + } + + test "ZSET element can't be set to NaN with ZADD - $encoding" { + assert_error "*not*float*" {r zadd myzset abcde abc} + } + + test "ZSET element can't be set to NaN with ZINCRBY" { + assert_error "*not*float*" {r zadd myzset abcde abc} + } + + test "ZINCRBY calls leading to NaN result in error" { + r zincrby myzset 999999999 abc + assert_error "*not*float*" {r zincrby myzset abcde abc} + } + + test {ZADD - Variadic version base case} { + r del myzset + list [r zadd myzset 10 a 20 b 30 c] [r zrange myzset 0 -1 withscores] + } {3 {a 10 b 20 c 30}} + + test {ZADD - Return value is the number of actually added items} { + list [r zadd myzset 5 x 20 b 30 c] [r zrange myzset 0 -1 withscores] + } {1 {x 5 a 10 b 20 c 30}} + + test {ZADD - Variadic version does not add nothing on single parsing err} { + r del myzset + catch {r zadd myzset 10 a 20 b 30.badscore c} e + assert_match {*ERR*not*float*} $e + #r exists myzset + } + + test {ZADD - Variadic version will raise error on missing arg} { + r del myzset + catch {r zadd myzset 10 a 20 b 30 c 40} e + assert_match {*ERR*syntax*} $e + } + + test {ZINCRBY does not work variadic even if shares ZADD implementation} { + r del myzset + catch {r zincrby myzset 10 a 20 b 30 c} e + assert_match {*ERR*wrong*number*arg*} $e + } + + test "ZCARD basics - $encoding" { + assert_equal 3 [r zcard ztmp] + assert_equal 0 [r zcard zdoesntexist] + } + + test "ZREM removes key after last element is removed" { + r del ztmp + r zadd ztmp 10 x + r zadd ztmp 20 y + + #assert_equal 1 [r exists ztmp] + assert_equal 0 [r zrem ztmp z] + assert_equal 1 [r zrem ztmp y] + assert_equal 1 [r zrem ztmp x] + #assert_equal 0 [r exists ztmp] + } + + test "ZREM variadic version" { + r del ztmp + r zadd ztmp 10 a 20 b 30 c + assert_equal 2 [r zrem ztmp x y a b k] + assert_equal 0 [r zrem ztmp foo bar] + assert_equal 1 [r zrem ztmp c] + #assert_equal 0 [r exists ztmp] + } + + test "ZREM variadic version -- remove elements after key deletion" { + r del ztmp + r zadd ztmp 10 a 20 b 30 c + r zrem ztmp a b c d e f g + } {3} + + test "ZRANGE basics - $encoding" { + r del ztmp + r zadd ztmp 1 a + r zadd ztmp 2 b + r zadd ztmp 3 c + r zadd ztmp 4 d + + assert_equal {a b c d} [r zrange ztmp 0 -1] + assert_equal {a b c} [r zrange ztmp 0 -2] + assert_equal {b c d} [r zrange ztmp 1 -1] + assert_equal {b c} [r zrange ztmp 1 -2] + assert_equal {c d} [r zrange ztmp -2 -1] + assert_equal {c} [r zrange ztmp -2 -2] + + # out of range start index + assert_equal {a b c} [r zrange ztmp -5 2] + assert_equal {a b} [r zrange ztmp -5 1] + assert_equal {} [r zrange ztmp 5 -1] + assert_equal {} [r zrange ztmp 5 -2] + + # out of range end index + assert_equal {a b c d} [r zrange ztmp 0 5] + assert_equal {b c d} [r zrange ztmp 1 5] + assert_equal {} [r zrange ztmp 0 -5] + assert_equal {} [r zrange ztmp 1 -5] + + # withscores + assert_equal {a 1 b 2 c 3 d 4} [r zrange ztmp 0 -1 withscores] + } + + test "ZREVRANGE basics - $encoding" { + r del ztmp + r zadd ztmp 1 a + r zadd ztmp 2 b + r zadd ztmp 3 c + r zadd ztmp 4 d + + assert_equal {d c b a} [r zrevrange ztmp 0 -1] + assert_equal {d c b} [r zrevrange ztmp 0 -2] + assert_equal {c b a} [r zrevrange ztmp 1 -1] + assert_equal {c b} [r zrevrange ztmp 1 -2] + assert_equal {b a} [r zrevrange ztmp -2 -1] + assert_equal {b} [r zrevrange ztmp -2 -2] + + # out of range start index + assert_equal {d c b} [r zrevrange ztmp -5 2] + assert_equal {d c} [r zrevrange ztmp -5 1] + assert_equal {} [r zrevrange ztmp 5 -1] + assert_equal {} [r zrevrange ztmp 5 -2] + + # out of range end index + assert_equal {d c b a} [r zrevrange ztmp 0 5] + assert_equal {c b a} [r zrevrange ztmp 1 5] + assert_equal {} [r zrevrange ztmp 0 -5] + assert_equal {} [r zrevrange ztmp 1 -5] + + ## withscores + assert_equal {d 4 c 3 b 2 a 1} [r zrevrange ztmp 0 -1 withscores] + } + + test "ZRANK/ZREVRANK basics - $encoding" { + r del zranktmp + r zadd zranktmp 10 x + r zadd zranktmp 20 y + r zadd zranktmp 30 z + assert_equal 0 [r zrank zranktmp x] + assert_equal 1 [r zrank zranktmp y] + assert_equal 2 [r zrank zranktmp z] + assert_equal "" [r zrank zranktmp foo] + assert_equal 2 [r zrevrank zranktmp x] + assert_equal 1 [r zrevrank zranktmp y] + assert_equal 0 [r zrevrank zranktmp z] + assert_equal "" [r zrevrank zranktmp foo] + } + + test "ZRANK - after deletion - $encoding" { + r zrem zranktmp y + assert_equal 0 [r zrank zranktmp x] + assert_equal 1 [r zrank zranktmp z] + } + + test "ZINCRBY - can create a new sorted set - $encoding" { + r del zset + r zincrby zset 1 foo + assert_equal {foo} [r zrange zset 0 -1] + assert_equal 1 [r zscore zset foo] + } + + test "ZINCRBY - increment and decrement - $encoding" { + r zincrby zset 2 foo + r zincrby zset 1 bar + assert_equal {bar foo} [r zrange zset 0 -1] + + r zincrby zset 10 bar + r zincrby zset -5 foo + r zincrby zset -5 bar + assert_equal {foo bar} [r zrange zset 0 -1] + + assert_equal -2 [r zscore zset foo] + assert_equal 6 [r zscore zset bar] + } + + proc create_default_zset {} { + create_zset zset {-999999999 a 1 b 2 c 3 d 4 e 5 f 999999999 g} + } + + test "ZRANGEBYSCORE/ZREVRANGEBYSCORE/ZCOUNT basics" { + create_default_zset + + # inclusive range + assert_equal {a b c} [r zrangebyscore zset -999999999 2] + assert_equal {b c d} [r zrangebyscore zset 0 3] + assert_equal {d e f} [r zrangebyscore zset 3 6] + assert_equal {e f g} [r zrangebyscore zset 4 999999999] + assert_equal {c b a} [r zrevrangebyscore zset 2 -999999999] + assert_equal {d c b} [r zrevrangebyscore zset 3 0] + assert_equal {f e d} [r zrevrangebyscore zset 6 3] + assert_equal {g f e} [r zrevrangebyscore zset 999999999 4] + assert_equal 3 [r zcount zset 0 3] + + # exclusive range + assert_equal {b} [r zrangebyscore zset (-999999999 (2] + assert_equal {b c} [r zrangebyscore zset (0 (3] + assert_equal {e f} [r zrangebyscore zset (3 (6] + assert_equal {f} [r zrangebyscore zset (4 (999999999] + assert_equal {b} [r zrevrangebyscore zset (2 (-999999999] + assert_equal {c b} [r zrevrangebyscore zset (3 (0] + assert_equal {f e} [r zrevrangebyscore zset (6 (3] + assert_equal {f} [r zrevrangebyscore zset (999999999 (4] + assert_equal 2 [r zcount zset (0 (3] + + # test empty ranges + r zrem zset a + r zrem zset g + + # inclusive + assert_equal {} [r zrangebyscore zset 4 2] + assert_equal {} [r zrangebyscore zset 6 999999999] + assert_equal {} [r zrangebyscore zset -999999999 -6] + assert_equal {} [r zrevrangebyscore zset 999999999 6] + assert_equal {} [r zrevrangebyscore zset -6 -999999999] + + # exclusive + assert_equal {} [r zrangebyscore zset (4 (2] + assert_equal {} [r zrangebyscore zset 2 (2] + assert_equal {} [r zrangebyscore zset (2 2] + assert_equal {} [r zrangebyscore zset (6 (999999999] + assert_equal {} [r zrangebyscore zset (-999999999 (-6] + assert_equal {} [r zrevrangebyscore zset (999999999 (6] + assert_equal {} [r zrevrangebyscore zset (-6 (-999999999] + + # empty inner range + assert_equal {} [r zrangebyscore zset 2.4 2.6] + assert_equal {} [r zrangebyscore zset (2.4 2.6] + assert_equal {} [r zrangebyscore zset 2.4 (2.6] + assert_equal {} [r zrangebyscore zset (2.4 (2.6] + } + + test "ZRANGEBYSCORE with WITHSCORES" { + create_default_zset + assert_equal {b 1 c 2 d 3} [r zrangebyscore zset 0 3 withscores] + assert_equal {d 3 c 2 b 1} [r zrevrangebyscore zset 3 0 withscores] + } + + test "ZRANGEBYSCORE with LIMIT" { + create_default_zset + assert_equal {b c} [r zrangebyscore zset 0 10 LIMIT 0 2] + assert_equal {d e f} [r zrangebyscore zset 0 10 LIMIT 2 3] + assert_equal {d e f} [r zrangebyscore zset 0 10 LIMIT 2 10] + assert_equal {} [r zrangebyscore zset 0 10 LIMIT 20 10] + assert_equal {f e} [r zrevrangebyscore zset 10 0 LIMIT 0 2] + assert_equal {d c b} [r zrevrangebyscore zset 10 0 LIMIT 2 3] + assert_equal {d c b} [r zrevrangebyscore zset 10 0 LIMIT 2 10] + assert_equal {} [r zrevrangebyscore zset 10 0 LIMIT 20 10] + } + + test "ZRANGEBYSCORE with LIMIT and WITHSCORES" { + create_default_zset + assert_equal {e 4 f 5} [r zrangebyscore zset 2 5 LIMIT 2 3 WITHSCORES] + assert_equal {d 3 c 2} [r zrevrangebyscore zset 5 2 LIMIT 2 3 WITHSCORES] + } + + test "ZRANGEBYSCORE with non-value min or max" { + assert_error "*not*float*" {r zrangebyscore fooz str 1} + assert_error "*not*float*" {r zrangebyscore fooz 1 str} + assert_error "*not*float*" {r zrangebyscore fooz 1 abcde} + } + + proc create_default_lex_zset {} { + create_zset zset {0 alpha 0 bar 0 cool 0 down + 0 elephant 0 foo 0 great 0 hill + 0 omega} + } + + test "ZRANGEBYLEX/ZREVRANGEBYLEX/ZCOUNT basics" { + create_default_lex_zset + + # inclusive range + assert_equal {alpha bar cool} [r zrangebylex zset - \[cool] + assert_equal {bar cool down} [r zrangebylex zset \[bar \[down] + assert_equal {great hill omega} [r zrangebylex zset \[g +] + assert_equal {cool bar alpha} [r zrevrangebylex zset \[cool -] + assert_equal {down cool bar} [r zrevrangebylex zset \[down \[bar] + assert_equal {omega hill great foo elephant down} [r zrevrangebylex zset + \[d] + assert_equal 3 [r zlexcount zset \[ele \[h] + + # exclusive range + assert_equal {alpha bar} [r zrangebylex zset - (cool] + assert_equal {cool} [r zrangebylex zset (bar (down] + assert_equal {hill omega} [r zrangebylex zset (great +] + assert_equal {bar alpha} [r zrevrangebylex zset (cool -] + assert_equal {cool} [r zrevrangebylex zset (down (bar] + assert_equal {omega hill} [r zrevrangebylex zset + (great] + assert_equal 2 [r zlexcount zset (ele (great] + + # inclusive and exclusive + assert_equal {} [r zrangebylex zset (az (b] + assert_equal {} [r zrangebylex zset (z +] + assert_equal {} [r zrangebylex zset - \[aaaa] + assert_equal {} [r zrevrangebylex zset \[elez \[elex] + assert_equal {} [r zrevrangebylex zset (hill (omega] + } + + test "ZRANGEBYSLEX with LIMIT" { + create_default_lex_zset + assert_equal {alpha bar} [r zrangebylex zset - \[cool LIMIT 0 2] + assert_equal {bar cool} [r zrangebylex zset - \[cool LIMIT 1 2] + assert_equal {} [r zrangebylex zset \[bar \[down LIMIT 0 0] + assert_equal {} [r zrangebylex zset \[bar \[down LIMIT 2 0] + assert_equal {bar} [r zrangebylex zset \[bar \[down LIMIT 0 1] + assert_equal {cool} [r zrangebylex zset \[bar \[down LIMIT 1 1] + assert_equal {bar cool down} [r zrangebylex zset \[bar \[down LIMIT 0 100] + assert_equal {omega hill great foo elephant} [r zrevrangebylex zset + \[d LIMIT 0 5] + assert_equal {omega hill great foo} [r zrevrangebylex zset + \[d LIMIT 0 4] + } + + test "ZRANGEBYLEX with invalid lex range specifiers" { + assert_error "*not*string*" {r zrangebylex fooz foo bar} + assert_error "*not*string*" {r zrangebylex fooz \[foo bar} + assert_error "*not*string*" {r zrangebylex fooz foo \[bar} + assert_error "*not*string*" {r zrangebylex fooz +x \[bar} + assert_error "*not*string*" {r zrangebylex fooz -x \[bar} + } + + test "ZREMRANGEBYSCORE basics" { + proc remrangebyscore {min max} { + create_zset zset {1 a 2 b 3 c 4 d 5 e} + #assert_equal 1 [r exists zset] + r zremrangebyscore zset $min $max + } + + # inner range + assert_equal 3 [remrangebyscore 2 4] + assert_equal {a e} [r zrange zset 0 -1] + + # start underflow + assert_equal 1 [remrangebyscore -10 1] + assert_equal {b c d e} [r zrange zset 0 -1] + + # end overflow + assert_equal 1 [remrangebyscore 5 10] + assert_equal {a b c d} [r zrange zset 0 -1] + + # switch min and max + assert_equal 0 [remrangebyscore 4 2] + assert_equal {a b c d e} [r zrange zset 0 -1] + + # -999999999 to mid + assert_equal 3 [remrangebyscore -999999999 3] + assert_equal {d e} [r zrange zset 0 -1] + + # mid to 999999999 + assert_equal 3 [remrangebyscore 3 999999999] + assert_equal {a b} [r zrange zset 0 -1] + + # -999999999 to 999999999 + assert_equal 5 [remrangebyscore -999999999 999999999] + assert_equal {} [r zrange zset 0 -1] + + # exclusive min + assert_equal 4 [remrangebyscore (1 5] + assert_equal {a} [r zrange zset 0 -1] + assert_equal 3 [remrangebyscore (2 5] + assert_equal {a b} [r zrange zset 0 -1] + + # exclusive max + assert_equal 4 [remrangebyscore 1 (5] + assert_equal {e} [r zrange zset 0 -1] + assert_equal 3 [remrangebyscore 1 (4] + assert_equal {d e} [r zrange zset 0 -1] + + # exclusive min and max + assert_equal 3 [remrangebyscore (1 (5] + assert_equal {a e} [r zrange zset 0 -1] + + # destroy when empty + assert_equal 5 [remrangebyscore 1 5] + # assert_equal 0 [r exists zset] + } + + test "ZREMRANGEBYSCORE with non-value min or max" { + assert_error "*not*float*" {r zremrangebyscore fooz str 1} + assert_error "*not*float*" {r zremrangebyscore fooz 1 str} + assert_error "*not*float*" {r zremrangebyscore fooz 1 abcde} + } + + test "ZREMRANGEBYRANK basics" { + proc remrangebyrank {min max} { + create_zset zset {1 a 2 b 3 c 4 d 5 e} + #assert_equal 1 [r exists zset] + r zremrangebyrank zset $min $max + } + + # inner range + assert_equal 3 [remrangebyrank 1 3] + assert_equal {a e} [r zrange zset 0 -1] + + # start underflow + assert_equal 1 [remrangebyrank -10 0] + assert_equal {b c d e} [r zrange zset 0 -1] + + # start overflow + assert_equal 0 [remrangebyrank 10 -1] + assert_equal {a b c d e} [r zrange zset 0 -1] + + # end underflow + assert_equal 0 [remrangebyrank 0 -10] + assert_equal {a b c d e} [r zrange zset 0 -1] + + # end overflow + assert_equal 5 [remrangebyrank 0 10] + assert_equal {} [r zrange zset 0 -1] + + # destroy when empty + assert_equal 5 [remrangebyrank 0 4] + #assert_equal 0 [r exists zset] + } + + test "ZUNIONSTORE against non-existing key doesn't set destination - $encoding" { + r del zseta + assert_equal 0 [r zunionstore dst_key 1 zseta] + #assert_equal 0 [r exists dst_key] + } + + test "ZUNIONSTORE with empty set - $encoding" { + r del zseta zsetb + r zadd zseta 1 a + r zadd zseta 2 b + r zunionstore zsetc 2 zseta zsetb + r zrange zsetc 0 -1 withscores + } {a 1 b 2} + + test "ZUNIONSTORE basics - $encoding" { + r del zseta zsetb zsetc + r zadd zseta 1 a + r zadd zseta 2 b + r zadd zseta 3 c + r zadd zsetb 1 b + r zadd zsetb 2 c + r zadd zsetb 3 d + + assert_equal 4 [r zunionstore zsetc 2 zseta zsetb] + assert_equal {a 1 b 3 d 3 c 5} [r zrange zsetc 0 -1 withscores] + } + + test "ZUNIONSTORE with weights - $encoding" { + assert_equal 4 [r zunionstore zsetc 2 zseta zsetb weights 2 3] + assert_equal {a 2 b 7 d 9 c 12} [r zrange zsetc 0 -1 withscores] + } + + test "ZUNIONSTORE with a regular set and weights - $encoding" { + r del seta + r sadd seta a + r sadd seta b + r sadd seta c + + # assert_equal 4 [r zunionstore zsetc 2 seta zsetb weights 2 3] + # assert_equal {a 2 b 5 c 8 d 9} [r zrange zsetc 0 -1 withscores] + } + + test "ZUNIONSTORE with AGGREGATE MIN - $encoding" { + assert_equal 4 [r zunionstore zsetc 2 zseta zsetb aggregate min] + assert_equal {a 1 b 1 c 2 d 3} [r zrange zsetc 0 -1 withscores] + } + + test "ZUNIONSTORE with AGGREGATE MAX - $encoding" { + assert_equal 4 [r zunionstore zsetc 2 zseta zsetb aggregate max] + assert_equal {a 1 b 2 c 3 d 3} [r zrange zsetc 0 -1 withscores] + } + + test "ZINTERSTORE basics - $encoding" { + assert_equal 2 [r zinterstore zsetc 2 zseta zsetb] + assert_equal {b 3 c 5} [r zrange zsetc 0 -1 withscores] + } + + test "ZINTERSTORE with weights - $encoding" { + assert_equal 2 [r zinterstore zsetc 2 zseta zsetb weights 2 3] + assert_equal {b 7 c 12} [r zrange zsetc 0 -1 withscores] + } + + test "ZINTERSTORE with a regular set and weights - $encoding" { + r del seta + r sadd seta a + r sadd seta b + r sadd seta c + # assert_equal 2 [r zinterstore zsetc 2 seta zsetb weights 2 3] + # assert_equal {b 5 c 8} [r zrange zsetc 0 -1 withscores] + } + + test "ZINTERSTORE with AGGREGATE MIN - $encoding" { + assert_equal 2 [r zinterstore zsetc 2 zseta zsetb aggregate min] + assert_equal {b 1 c 2} [r zrange zsetc 0 -1 withscores] + } + + test "ZINTERSTORE with AGGREGATE MAX - $encoding" { + assert_equal 2 [r zinterstore zsetc 2 zseta zsetb aggregate max] + assert_equal {b 2 c 3} [r zrange zsetc 0 -1 withscores] + } + + foreach cmd {ZUNIONSTORE ZINTERSTORE} { + # test "$cmd with 999999999/-999999999 scores - $encoding" { + # r del zsetinf1 zsetinf2 + + # r zadd zsetinf1 999999999 key + # r zadd zsetinf2 999999999 key + # r $cmd zsetinf3 2 zsetinf1 zsetinf2 + # assert_equal 999999999 [r zscore zsetinf3 key] + + # r zadd zsetinf1 -999999999 key + # r zadd zsetinf2 999999999 key + # r $cmd zsetinf3 2 zsetinf1 zsetinf2 + # assert_equal 0 [r zscore zsetinf3 key] + + # r zadd zsetinf1 999999999 key + # r zadd zsetinf2 -999999999 key + # r $cmd zsetinf3 2 zsetinf1 zsetinf2 + # assert_equal 0 [r zscore zsetinf3 key] + + # r zadd zsetinf1 -999999999 key + # r zadd zsetinf2 -999999999 key + # r $cmd zsetinf3 2 zsetinf1 zsetinf2 + # assert_equal -999999999 [r zscore zsetinf3 key] + # } + + test "$cmd with NaN weights $encoding" { + r del zsetinf1 zsetinf2 + + r zadd zsetinf1 1.0 key + r zadd zsetinf2 1.0 key + assert_error "*weight*not*float*" { + r $cmd zsetinf3 2 zsetinf1 zsetinf2 weights abcde abcde + } + } + } + } + + basics ziplist + basics skiplist + + test {ZINTERSTORE regression with two sets, intset+hashtable} { + r del seta setb setc + r sadd set1 a + r sadd set2 10 + r zinterstore set3 2 set1 set2 + } {0} + + test {ZUNIONSTORE regression, should not create NaN in scores} { + r zadd z -999999999 neginf + r zunionstore out 1 z weights 0 + r zrange out 0 -1 withscores + } {neginf 0} + + # test {ZINTERSTORE #516 regression, mixed sets and ziplist zsets} { + # r sadd one 100 101 102 103 + # r sadd two 100 200 201 202 + # r zadd three 1 500 1 501 1 502 1 503 1 100 + # r zinterstore to_here 3 one two three WEIGHTS 0 0 1 + # r zrange to_here 0 -1 + # } {100} + + test {ZUNIONSTORE result is sorted} { + # Create two sets with common and not common elements, perform + # the UNION, check that elements are still sorted. + r del one two dest + set cmd1 [list r zadd one] + set cmd2 [list r zadd two] + for {set j 0} {$j < 1000} {incr j} { + lappend cmd1 [expr rand()] [randomInt 1000] + lappend cmd2 [expr rand()] [randomInt 1000] + } + {*}$cmd1 + {*}$cmd2 + assert {[r zcard one] > 100} + assert {[r zcard two] > 100} + r zunionstore dest 2 one two + set oldscore 0 + foreach {ele score} [r zrange dest 0 -1 withscores] { + assert {$score >= $oldscore} + set oldscore $score + } + } + + proc stressers {encoding} { + if {$encoding == "ziplist"} { + # Little extra to allow proper fuzzing in the sorting stresser + #r config set zset-max-ziplist-entries 256 + #r config set zset-max-ziplist-value 64 + set elements 128 + } elseif {$encoding == "skiplist"} { + #r config set zset-max-ziplist-entries 0 + #r config set zset-max-ziplist-value 0 + if {$::accurate} {set elements 1000} else {set elements 100} + } else { + puts "Unknown sorted set encoding" + exit + } + + test "ZSCORE - $encoding" { + r del zscoretest + set aux {} + for {set i 0} {$i < $elements} {incr i} { + set score [expr rand()] + lappend aux $score + r zadd zscoretest $score $i + } + + #assert_encoding $encoding zscoretest + for {set i 0} {$i < $elements} {incr i} { + assert_equal [lindex $aux $i] [r zscore zscoretest $i] + } + } + + test "ZSCORE after a DEBUG RELOAD - $encoding" { + r del zscoretest + set aux {} + for {set i 0} {$i < $elements} {incr i} { + set score [expr rand()] + lappend aux $score + r zadd zscoretest $score $i + } + + #r debug reload + #assert_encoding $encoding zscoretest + for {set i 0} {$i < $elements} {incr i} { + assert_equal [lindex $aux $i] [r zscore zscoretest $i] + } + } + + test "ZSET sorting stresser - $encoding" { + set delta 0 + for {set test 0} {$test < 2} {incr test} { + unset -nocomplain auxarray + array set auxarray {} + set auxlist {} + r del myzset + for {set i 0} {$i < $elements} {incr i} { + if {$test == 0} { + set score [expr rand()] + } else { + set score [expr int(rand()*10)] + } + set auxarray($i) $score + r zadd myzset $score $i + # Random update + if {[expr rand()] < .2} { + set j [expr int(rand()*1000)] + if {$test == 0} { + set score [expr rand()] + } else { + set score [expr int(rand()*10)] + } + set auxarray($j) $score + r zadd myzset $score $j + } + } + foreach {item score} [array get auxarray] { + lappend auxlist [list $score $item] + } + set sorted [lsort -command zlistAlikeSort $auxlist] + set auxlist {} + foreach x $sorted { + lappend auxlist [lindex $x 1] + } + + #assert_encoding $encoding myzset + set fromredis [r zrange myzset 0 -1] + set delta 0 + for {set i 0} {$i < [llength $fromredis]} {incr i} { + if {[lindex $fromredis $i] != [lindex $auxlist $i]} { + incr delta + } + } + } + assert_equal 0 $delta + } + + test "ZRANGEBYSCORE fuzzy test, 100 ranges in $elements element sorted set - $encoding" { + set err {} + r del zset + for {set i 0} {$i < $elements} {incr i} { + r zadd zset [expr rand()] $i + } + + #assert_encoding $encoding zset + for {set i 0} {$i < 100} {incr i} { + set min [expr rand()] + set max [expr rand()] + if {$min > $max} { + set aux $min + set min $max + set max $aux + } + set low [r zrangebyscore zset -999999999 $min] + set ok [r zrangebyscore zset $min $max] + set high [r zrangebyscore zset $max 999999999] + set lowx [r zrangebyscore zset -999999999 ($min] + set okx [r zrangebyscore zset ($min ($max] + set highx [r zrangebyscore zset ($max 999999999] + + if {[r zcount zset -999999999 $min] != [llength $low]} { + append err "Error, len does not match zcount\n" + } + if {[r zcount zset $min $max] != [llength $ok]} { + append err "Error, len does not match zcount\n" + } + if {[r zcount zset $max 999999999] != [llength $high]} { + append err "Error, len does not match zcount\n" + } + if {[r zcount zset -999999999 ($min] != [llength $lowx]} { + append err "Error, len does not match zcount\n" + } + if {[r zcount zset ($min ($max] != [llength $okx]} { + append err "Error, len does not match zcount\n" + } + if {[r zcount zset ($max 999999999] != [llength $highx]} { + append err "Error, len does not match zcount\n" + } + + foreach x $low { + set score [r zscore zset $x] + if {$score > $min} { + append err "Error, score for $x is $score > $min\n" + } + } + foreach x $lowx { + set score [r zscore zset $x] + if {$score >= $min} { + append err "Error, score for $x is $score >= $min\n" + } + } + foreach x $ok { + set score [r zscore zset $x] + if {$score < $min || $score > $max} { + append err "Error, score for $x is $score outside $min-$max range\n" + } + } + foreach x $okx { + set score [r zscore zset $x] + if {$score <= $min || $score >= $max} { + append err "Error, score for $x is $score outside $min-$max open range\n" + } + } + foreach x $high { + set score [r zscore zset $x] + if {$score < $max} { + append err "Error, score for $x is $score < $max\n" + } + } + foreach x $highx { + set score [r zscore zset $x] + if {$score <= $max} { + append err "Error, score for $x is $score <= $max\n" + } + } + } + assert_equal {} $err + } + + test "ZRANGEBYLEX fuzzy test, 100 ranges in $elements element sorted set - $encoding" { + set lexset {} + r del zset + for {set j 0} {$j < $elements} {incr j} { + set e [randstring 1 30 alpha] + lappend lexset $e + r zadd zset 0 $e + } + set lexset [lsort -unique $lexset] + for {set j 0} {$j < 100} {incr j} { + set min [randstring 1 30 alpha] + set max [randstring 1 30 alpha] + set mininc [randomInt 2] + set maxinc [randomInt 2] + if {$mininc} {set cmin "\[$min"} else {set cmin "($min"} + if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"} + set rev [randomInt 2] + if {$rev} { + break + set cmd zrevrangebylex + } else { + set cmd zrangebylex + } + + # Make sure data is the same in both sides + assert {[r zrange zset 0 -1] eq $lexset} + + # Get the Redis output + set output [r $cmd zset $cmin $cmax] + if {$rev} { + set outlen [r zlexcount zset $cmax $cmin] + } else { + set outlen [r zlexcount zset $cmin $cmax] + } + + # Compute the same output via Tcl + set o {} + set copy $lexset + if {(!$rev && [string compare $min $max] > 0) || + ($rev && [string compare $max $min] > 0)} { + # Empty output when ranges are inverted. + } else { + if {$rev} { + # Invert the Tcl array using Redis itself. + set copy [r zrevrange zset 0 -1] + # Invert min / max as well + lassign [list $min $max $mininc $maxinc] \ + max min maxinc mininc + } + foreach e $copy { + set mincmp [string compare $e $min] + set maxcmp [string compare $e $max] + if { + ($mininc && $mincmp >= 0 || !$mininc && $mincmp > 0) + && + ($maxinc && $maxcmp <= 0 || !$maxinc && $maxcmp < 0) + } { + lappend o $e + } + } + } + assert {$o eq $output} + assert {$outlen eq [llength $output]} + } + } + + test "ZREMRANGEBYLEX fuzzy test, 100 ranges in $elements element sorted set - $encoding" { + set lexset {} + r del zset zsetcopy + for {set j 0} {$j < $elements} {incr j} { + set e [randstring 1 30 alpha] + lappend lexset $e + r zadd zset 0 $e + } + set lexset [lsort -unique $lexset] + for {set j 0} {$j < 100} {incr j} { + # Copy... + r zunionstore zsetcopy 1 zset + set lexsetcopy $lexset + + set min [randstring 1 30 alpha] + set max [randstring 1 30 alpha] + set mininc [randomInt 2] + set maxinc [randomInt 2] + if {$mininc} {set cmin "\[$min"} else {set cmin "($min"} + if {$maxinc} {set cmax "\[$max"} else {set cmax "($max"} + + # Make sure data is the same in both sides + assert {[r zrange zset 0 -1] eq $lexset} + + # Get the range we are going to remove + set torem [r zrangebylex zset $cmin $cmax] + set toremlen [r zlexcount zset $cmin $cmax] + r zremrangebylex zsetcopy $cmin $cmax + set output [r zrange zsetcopy 0 -1] + # Remove the range with Tcl from the original list + if {$toremlen} { + set first [lsearch -exact $lexsetcopy [lindex $torem 0]] + set last [expr {$first+$toremlen-1}] + set lexsetcopy [lreplace $lexsetcopy $first $last] + } + assert {$lexsetcopy eq $output} + } + } + + test "ZSETs skiplist implementation backlink consistency test - $encoding" { + set diff 0 + for {set j 0} {$j < $elements} {incr j} { + r zadd myzset [expr rand()] "Element-$j" + r zrem myzset "Element-[expr int(rand()*$elements)]" + } + + #assert_encoding $encoding myzset + set l1 [r zrange myzset 0 -1] + set l2 [r zrevrange myzset 0 -1] + for {set j 0} {$j < [llength $l1]} {incr j} { + if {[lindex $l1 $j] ne [lindex $l2 end-$j]} { + incr diff + } + } + assert_equal 0 $diff + } + + test "ZSETs ZRANK augmented skip list stress testing - $encoding" { + set err {} + r del myzset + for {set k 0} {$k < 2000} {incr k} { + set i [expr {$k % $elements}] + if {[expr rand()] < .2} { + r zrem myzset $i + } else { + set score [expr rand()] + r zadd myzset $score $i + #assert_encoding $encoding myzset + } + + set card [r zcard myzset] + if {$card > 0} { + set index [randomInt $card] + set ele [lindex [r zrange myzset $index $index] 0] + set rank [r zrank myzset $ele] + if {$rank != $index} { + set err "$ele RANK is wrong! ($rank != $index)" + break + } + } + } + assert_equal {} $err + } + } + + tags {"slow"} { + stressers ziplist + stressers skiplist + } +}