feat: cherry-pick code to pika 3.5 release branch

.dockerignore

@@ -33,7 +33,19 @@
 make_config.mk
 log/
 lib/
-tools/
+tools/aof_to_pika
+tools/benchmark_client
+tools/binlog_sender
+tools/codis2pika
+tools/kubeblocks_helm
+tools/manifest_generator
+tools/pika-port
+tools/pika_keys_analysis
+tools/pika_to_txt
+tools/rdb_to_pika
+tools/redis-copy
+tools/txt_to_pika
+docker/
 output/
 
 # DB

.github/workflows/issue-translator.yml

@@ -0,0 +1,15 @@
+name: Issue Translator
+on:
+  issue_comment:
+    types: [created]
+  issues:
+    types: [opened]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: usthe/[email protected]
+        with:
+          IS_MODIFY_TITLE: false
+          CUSTOM_BOT_NOTE: Bot detected the issue body's language is not English, translate it automatically.

.github/workflows/pika.yml

@@ -26,15 +26,6 @@ jobs:
         with:
           go-version: 1.19
 
-      - name: Cache dependencies
-        uses: actions/cache@v3
-        id: cache
-        with:
-          path: |
-            ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
-            ~/.cache/pip
-          key: ${{ runner.os }}-dependencies
-
       - name: Install Deps
         if: ${{ steps.cache.output.cache-hit != 'true' }}
         run: |
@@ -75,11 +66,13 @@ jobs:
         run: ./pikatests.sh all
 
         # master on port 9221, slave on port 9231, all with 2 db
-      - name: Start pika master and slave
+      - name: Start codis, pika master and pika slave
         working-directory: ${{ github.workspace }}/build
         run: |
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
+          chmod +x ../tests/integration/start_codis.sh
+          ../tests/integration/start_codis.sh
 
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
@@ -145,11 +138,13 @@ jobs:
         working-directory: ${{ github.workspace }}
         run: ./pikatests.sh all
 
-      - name: Start pika master and slave
+      - name: Start codis, pika master and pika slave
         working-directory: ${{ github.workspace }}/build
         run: |
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
+          chmod +x ../tests/integration/start_codis.sh
+          ../tests/integration/start_codis.sh
 
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
@@ -174,7 +169,7 @@ jobs:
       - name: Install Deps
         run: |
           brew update
-          brew install --overwrite autoconf protobuf llvm wget git
+          brew install --overwrite [email protected] autoconf protobuf llvm wget git
           brew install gcc@10 automake cmake make binutils
 
       - name: Configure CMake
@@ -205,11 +200,13 @@ jobs:
         run: |
           ./pikatests.sh all
 
-      - name: Start pika master and slave
+      - name: Start codis, pika master and pika slave
         working-directory: ${{ github.workspace }}/build
         run: |
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
+          chmod +x ../tests/integration/start_codis.sh
+          ../tests/integration/start_codis.sh
 
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build

.github/workflows/publish_nightly_docker_image.yml

@@ -39,7 +39,7 @@ jobs:
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          file: ./Dockerfile
+          file: ./docker/Dockerfile_pika
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -81,3 +81,39 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  push_pika_exporter_to_registry:
+    name: Push Pika Exporter Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: pikadb/pika-exporter-dev-nightly
+
+      - name: Build and push Docker image
+        timeout-minutes: 1440
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/Dockerfile_pika_exporter
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/publish_release_docker_image.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           context: .
           platforms: linux/amd64,linux/arm64
-          file: ./Dockerfile
+          file: ./docker/Dockerfile_pika
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
@@ -77,3 +77,39 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  push_pika_exporter_to_registry:
+    name: Push Pika Exporter Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: pikadb/pika-exporter
+
+      - name: Build and push Docker image
+        timeout-minutes: 1440
+        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          file: ./docker/Dockerfile_pika_exporter
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/release.yml

@@ -47,7 +47,7 @@ jobs:
         if: contains(matrix.os, 'macos')
         run: |
           brew update
-          brew install --overwrite python autoconf protobuf llvm wget git
+          brew install --overwrite python@3.12 autoconf protobuf llvm wget git
           brew install gcc@10 automake cmake make binutils
 
       - name: Configure CMake - linux

codis/go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11
 	github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414
 	github.com/spinlock/jemalloc-go v0.0.0-20201010032256-e81523fb8524
+	github.com/stretchr/testify v1.8.0
 	go.etcd.io/etcd/client/v2 v2.305.7
 	golang.org/x/net v0.17.0
 	gopkg.in/alexcesaro/statsd.v2 v2.0.0
@@ -26,10 +27,13 @@ require (
 require (
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.7 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

codis/go.sum

@@ -41,8 +41,11 @@ github.com/samuel/go-zookeeper v0.0.0-20201211165307-7117e9ea2414/go.mod h1:gi+0
 github.com/spinlock/jemalloc-go v0.0.0-20201010032256-e81523fb8524 h1:U+dpuWn15gFCqZkqhpUd5a85X1Oe1Tb+DeGF3nn6Bvs=
 github.com/spinlock/jemalloc-go v0.0.0-20201010032256-e81523fb8524/go.mod h1:A/ik9Cf2cSgEVcmTWlvTfCxyFgoL1UP/WbevsdDeguc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY=
 go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA=
 go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg=
@@ -53,6 +56,8 @@ golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
 golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc=
 gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

codis/pkg/proxy/mapper.go

@@ -96,8 +96,8 @@ func init() {
 		{"DISCARD", FlagNotAllow},
 		{"DUMP", 0},
 		{"ECHO", 0},
-		{"EVAL", FlagWrite},
-		{"EVALSHA", FlagWrite},
+		{"EVAL", FlagNotAllow},
+		{"EVALSHA", FlagNotAllow},
 		{"EXEC", FlagNotAllow},
 		{"EXISTS", 0},
 		{"EXPIRE", FlagWrite},
@@ -161,7 +161,7 @@ func init() {
 		{"PFADD", FlagWrite},
 		{"PFCOUNT", 0},
 		{"PFDEBUG", FlagWrite},
-		{"PFMERGE", FlagWrite},
+		{"PFMERGE", FlagNotAllow},
 		{"PFSELFTEST", 0},
 		{"PING", 0},
 		{"POST", FlagNotAllow},
@@ -183,15 +183,15 @@ func init() {
 		{"RESTORE-ASKING", FlagWrite | FlagNotAllow},
 		{"ROLE", 0},
 		{"RPOP", FlagWrite},
-		{"RPOPLPUSH", FlagWrite},
+		{"RPOPLPUSH", FlagNotAllow},
 		{"RPUSH", FlagWrite},
 		{"RPUSHX", FlagWrite},
 		{"SADD", FlagWrite},
 		{"SAVE", FlagNotAllow},
 		{"SCAN", FlagMasterOnly | FlagNotAllow},
 		{"SCARD", 0},
 		{"SCRIPT", FlagNotAllow},
-		{"SDIFF", 0},
+		{"SDIFF", FlagNotAllow},
 		{"SDIFFSTORE", FlagWrite},
 		{"SELECT", 0},
 		{"SET", FlagWrite},
@@ -200,8 +200,8 @@ func init() {
 		{"SETNX", FlagWrite},
 		{"SETRANGE", FlagWrite},
 		{"SHUTDOWN", FlagNotAllow},
-		{"SINTER", 0},
-		{"SINTERSTORE", FlagWrite},
+		{"SINTER", FlagNotAllow},
+		{"SINTERSTORE", FlagNotAllow},
 		{"SISMEMBER", 0},
 		{"SLAVEOF", FlagNotAllow},
 		{"SLOTSCHECK", FlagNotAllow},
@@ -228,7 +228,7 @@ func init() {
 		{"SLOTSSCAN", FlagMasterOnly},
 		{"SLOWLOG", FlagNotAllow},
 		{"SMEMBERS", 0},
-		{"SMOVE", FlagWrite},
+		{"SMOVE", FlagNotAllow},
 		{"SORT", FlagWrite},
 		{"SPOP", FlagWrite},
 		{"SRANDMEMBER", 0},
@@ -237,8 +237,8 @@ func init() {
 		{"STRLEN", 0},
 		{"SUBSCRIBE", FlagNotAllow},
 		{"SUBSTR", 0},
-		{"SUNION", 0},
-		{"SUNIONSTORE", FlagWrite},
+		{"SUNION", FlagNotAllow},
+		{"SUNIONSTORE", FlagNotAllow},
 		{"SYNC", FlagNotAllow},
 		{"PCONFIG", 0},
 		{"TIME", FlagNotAllow},
@@ -253,7 +253,7 @@ func init() {
 		{"ZCARD", 0},
 		{"ZCOUNT", 0},
 		{"ZINCRBY", FlagWrite},
-		{"ZINTERSTORE", FlagWrite},
+		{"ZINTERSTORE", FlagNotAllow},
 		{"ZLEXCOUNT", 0},
 		{"ZRANGE", 0},
 		{"ZRANGEBYLEX", 0},
@@ -269,7 +269,7 @@ func init() {
 		{"ZREVRANK", 0},
 		{"ZSCAN", FlagMasterOnly},
 		{"ZSCORE", 0},
-		{"ZUNIONSTORE", FlagWrite},
+		{"ZUNIONSTORE", FlagNotAllow},
 	} {
 		opTable[i.Name] = i
 	}

conf/pika.conf

@@ -75,7 +75,7 @@ requirepass :
 # [NOTICE] The value of this parameter must match the "requirepass" setting on the master.
 masterauth :
 
-# The [password of user], which is empty by default.(Deprecated)
+# The [password of user], which is empty by default.
 # [NOTICE] If this user password is the same as admin password (including both being empty),
 # the value of this parameter will be ignored and all users are considered as administrators,
 # in this scenario, users are not subject to the restrictions imposed by the userblacklist.
@@ -87,7 +87,7 @@ masterauth :
 # [Advice] It's recommended to add high-risk commands to this list.
 # [Format] Commands should be separated by ",". For example: FLUSHALL, SHUTDOWN, KEYS, CONFIG
 # By default, this list is empty.
-userblacklist :
+# userblacklist :
 
 # Running Mode of Pika, The current version only supports running in "classic mode".
 # If set to 'classic', Pika will create multiple DBs whose number is the value of configure item "databases".
@@ -249,9 +249,8 @@ disable_auto_compactions : false
 sync-window-size : 9000
 
 # Maximum buffer size of a client connection.
-# Only three values are valid here: [67108864(64MB) | 268435456(256MB) | 536870912(512MB)].
 # [NOTICE] Master and slaves must have exactly the same value for the max-conn-rbuf-size.
-# Supported Units [K|M|G]. Its default unit is in [bytes] and its default value is 268435456(256MB).
+# Supported Units [K|M|G]. Its default unit is in [bytes] and its default value is 268435456(256MB). The value range is [64MB, 1GB].
 max-conn-rbuf-size : 268435456
 
 
@@ -338,6 +337,12 @@ max-bytes-for-level-multiplier : 10
 # slotmigrate  [yes | no]
 slotmigrate : no
 
+# slotmigrate thread num
+slotmigrate-thread-num : 8
+
+# thread-migrate-keys-num  1/8 of the write_buffer_size_
+thread-migrate-keys-num : 64
+
 # BlockBasedTable block_size, default 4k
 # block-size: 4096