Merge branch 'OpenAtomFoundation:unstable' into stream_base

.github/workflows/codeql.yml

@@ -13,10 +13,10 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
   pull_request:
     # The branches below must be a subset of the branches above
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
   schedule:
     - cron: '25 19 * * 6'
 

.github/workflows/codis.yml

@@ -5,9 +5,9 @@ name: Codis
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
   pull_request:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
 
 jobs:
 

.github/workflows/operator.yml

@@ -5,9 +5,9 @@ name: Operator
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
   pull_request:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
 
 jobs:
 

.github/workflows/pika.yml

@@ -2,13 +2,14 @@ name: Pika
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
   pull_request:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
 
 env:
   # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
   BUILD_TYPE: RelWithDebInfo
+  ARTIFACT_PIKA_NAME: artifact-pika
 
 jobs:
   build_on_ubuntu:
@@ -25,7 +26,7 @@ jobs:
         with:
           go-version: 1.19
 
-      - name: cache dependencies
+      - name: Cache dependencies
         uses: actions/cache@v2
         id: cache
         with:
@@ -34,23 +35,37 @@ jobs:
             ~/.cache/pip
           key: ${{ runner.os }}-dependencies
 
-      - name: install Deps
+      - name: Install Deps
         if: ${{ steps.cache.output.cache-hit != 'true' }}
         run: |
           sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
-          sudo apt-get install -y clang-tidy-12 python3-pip
-          python3 -m pip install --upgrade pip
-          python3 -m pip install redis
+          sudo apt-get install -y clang-tidy-12
 
       - name: Configure CMake
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
         run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
+      - name: Cache Build
+        uses: actions/cache@v3
+        id: cache-ubuntu
+        with:
+          key: ${{ runner.os }}-build-ubuntu-${{ hashFiles('**/CMakeLists.txt') }}
+          path: |
+            ${{ github.workspace }}/build
+            ${{ github.workspace }}/buildtrees
+            ${{ github.workspace }}/deps
+
       - name: Build
+        if: ${{ steps.cache-ubuntu.outputs.cache-hit != 'true' }}
         # Build your program with the given configuration
         run: cmake --build build --config ${{ env.BUILD_TYPE }}
 
+      - uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_PIKA_NAME }}
+          path: ${{ github.workspace }}/build/pika
+
       - name: Test
         working-directory: ${{ github.workspace }}/build
         # Execute tests defined by the CMake configuration.
@@ -68,12 +83,6 @@ jobs:
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
 
-      - name: Run Python E2E Tests
-        working-directory: ${{ github.workspace }}/build
-        run: |
-          python3 ../tests/integration/pika_replication_test.py
-          python3 ../tests/unit/Blpop_Brpop_test.py
-
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
         run: |
@@ -89,11 +98,9 @@ jobs:
     steps:
       - name: Install deps
         run: |
-          yum install -y wget git autoconf centos-release-scl
+          yum install -y wget git autoconf centos-release-scl gcc
           yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-make devtoolset-10-bin-util
-          yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which python3
-          python3 -m pip install --upgrade pip
-          python3 -m pip install redis
+          yum install -y llvm-toolset-7 llvm-toolset-7-clang tcl which 
 
       - name: Set up Go
         uses: actions/setup-go@v3
@@ -115,7 +122,18 @@ jobs:
           source /opt/rh/devtoolset-10/enable
           cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
+      - name: Cache Build
+        uses: actions/cache@v3
+        id: cache-centos
+        with:
+          key: ${{ runner.os }}-build-centos-${{ hashFiles('**/CMakeLists.txt') }}
+          path: |
+            ${{ github.workspace }}/build
+            ${{ github.workspace }}/buildtrees
+            ${{ github.workspace }}/deps
+
       - name: Build
+        if: ${{ steps.cache-centos.outputs.cache-hit != 'true' }}
         run: |
           source /opt/rh/devtoolset-10/enable
           cmake --build build --config ${{ env.BUILD_TYPE }}
@@ -134,11 +152,12 @@ jobs:
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
 
-      - name: Run Python E2E Tests
+      - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
         run: |
-          python3 ../tests/integration/pika_replication_test.py
-          python3 ../tests/unit/Blpop_Brpop_test.py
+          cd ../tests/integration/
+          chmod +x integrate_test.sh 
+          sh integrate_test.sh
 
   build_on_macos:
     runs-on: macos-latest
@@ -151,36 +170,35 @@ jobs:
         with:
           go-version: 1.19
 
-      - name: cache dependencies
-        uses: actions/cache@v2
-        id: cache
-        with:
-          path: |
-            ${{ github.workspace }}/${{ env.INSTALL_LOCATION }}
-            ~/.cache/pip
-          key: ${{ runner.os }}-dependencies
-
-      - name: install Deps
-        if: ${{ steps.cache.output.cache-hit != 'true' }}
+      - name: Install Deps
         run: |
           brew update
           brew install --overwrite python autoconf protobuf llvm wget git
           brew install gcc@10 automake cmake make binutils
-          python3 -m pip install --upgrade pip
-          python3 -m pip install redis
 
       - name: Configure CMake
         run: |
           export CC=/usr/local/opt/gcc@10/bin/gcc-10 
-          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
+
+      - name: Cache Build
+        uses: actions/cache@v3
+        id: cache-macos
+        with:
+          key: ${{ runner.os }}-build-macos-${{ hashFiles('**/CMakeLists.txt') }}
+          path: |
+            ${{ github.workspace }}/build
+            ${{ github.workspace }}/buildtrees
+            ${{ github.workspace }}/deps
 
       - name: Build
+        if: ${{ steps.cache-macos.outputs.cache-hit != 'true' }}
         run: |
           cmake --build build --config ${{ env.BUILD_TYPE }}
 
       - name: Test
         working-directory: ${{ github.workspace }}/build
-        run: ctest -C ${{ env.BUILD_TYPE }}
+        run: ctest --rerun-failed --output-on-failure -C ${{ env.BUILD_TYPE }}
 
       - name: Unit Test
         working-directory: ${{ github.workspace }}
@@ -193,12 +211,6 @@ jobs:
           chmod +x ../tests/integration/start_master_and_slave.sh
           ../tests/integration/start_master_and_slave.sh
 
-      - name: Run Python E2E Tests
-        working-directory: ${{ github.workspace }}/build
-        run: |
-          python3 ../tests/integration/pika_replication_test.py
-          python3 ../tests/unit/Blpop_Brpop_test.py
-
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
         run: |
@@ -209,6 +221,7 @@ jobs:
   build_pika_image:
     name: Build Pika Docker image
     runs-on: ubuntu-latest
+    needs: build_on_ubuntu
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -218,18 +231,23 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: pikadb/pika
-
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.ARTIFACT_PIKA_NAME }}
+          path: artifact/
+
       - name: Build Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@v5
         with:
           context: .
-          file: ./Dockerfile
+          file: ./ci/Dockerfile
           push: false
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/publish_nightly_docker_image.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   push_pika_to_registry:
-    timeout-minutes: 600
+    timeout-minutes: 1440
     name: Push Pika Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
@@ -34,6 +34,7 @@ jobs:
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
       
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
@@ -71,6 +72,7 @@ jobs:
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
       
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: ./codis

.github/workflows/publish_release_docker_image.yml

@@ -8,7 +8,7 @@ jobs:
   push_pika_to_registry:
     name: Push Pika Docker image to Docker Hub
     runs-on: ubuntu-latest
-    timeout-minutes: 600
+    timeout-minutes: 1440
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
@@ -32,6 +32,7 @@ jobs:
           images: pikadb/pika
 
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: .
@@ -67,6 +68,7 @@ jobs:
           images: pikadb/codis
 
       - name: Build and push Docker image
+        timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
         with:
           context: ./codis

.github/workflows/tools_go.yml

@@ -2,11 +2,11 @@ name: Tools_go_build
 
 on:
   push:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
     paths:
       - 'tools/**'
   pull_request:
-    branches: [ "unstable", "3.5.0" ]
+    branches: [ "unstable", "3.5.1" ]
     paths:
       - 'tools/**'
 

CHANGELOG.MD

@@ -1,3 +1,48 @@
+# v3.5.1
+
+
+
+## New features
+
+- slow log Adds statistics on queue waiting time [#1997](https://github.com/OpenAtomFoundation/pika/pull/1997) @[wangshao1](https://github.com/wangshao1)
+
+- ReplicationID is used for primary/secondary replication [#1951](https://github.com/OpenAtomFoundation/pika/pull/1951) @[Mixficsol](https://github.com/Mixficsol)
+
+- WAL uses the disablewal command to support dynamic shutdown [#2015](https://github.com/OpenAtomFoundation/pika/pull/2015) @[Mixficsol](https://github.com/Mixficsol)
+
+- The number of threads flushed and the number of threads compaction dynamically adjust into one [#2014](flush线程数和compaction线程数动态调整，合二为一) @[Tianpingan](https://github.com/Tianpingan)
+
+- The RocksDB version was upgraded to v8.3.3 [#1999](https://github.com/OpenAtomFoundation/pika/issues/1999) @[dingxiaoshuai123](https://github.com/dingxiaoshuai123)
+
+- Added documentation for configuring Pika in Macos environment [#2003](https://github.com/OpenAtomFoundation/pika/pull/2003) @[klboke](https://github.com/klboke)
+
+- Added the ability to periodically print the length of the work queue to quickly locate problems when the queue is blocked [#1978](https://github.com/OpenAtomFoundation/pika/pull/1978) @[Tianpingan](https://github.com/Tianpingan)
+
+- Added an indicator to detect the entire cluster using a `pika_exporter` [#1953](https://github.com/OpenAtomFoundation/pika/pull/1953) @[chenbt-hz](https://github.com/chenbt-hz)
+
+- Realize automatic registration of Pika service on K8s environment, and automatically register at startup, so as to realize self-organization of cluster [#1931](https://github.com/OpenAtomFoundation/pika/pull/1931) @[machinly](https://github.com/machinly)
+
+
+## bugfix
+
+- Reduces unnecessary log printing by the exporter, reducing CPU utilization [#1945](https://github.com/OpenAtomFoundation/pika/pull/1945) @[Mixficsol](https://github.com/Mixficsol)
+
+- The sentinel mechanism has been upgraded to carry out logical deletion of primary nodes that fall offline [#1949](https://github.com/OpenAtomFoundation/pika/pull/1949) @[Mixficsol](https://github.com/Mixficsol)
+
+- Adjust the rate_limit parameter to fix the situation that RPS is 0 during pressure measurement [#2009](https://github.com/OpenAtomFoundation/pika/pull/2009) @[chejinge](https://github.com/chejinge)
+
+- Fixed the logical determination of empty path when traversing data files in INFODATA command [#1996](https://github.com/OpenAtomFoundation/pika/pull/1996) @[Mixficsol](https://github.com/Mixficsol)
+
+- Improved the description of some missing parts of the document [#1962](https://github.com/OpenAtomFoundation/pika/pull/1962) @[baerwang](https://github.com/baerwang)
+
+- Use make-j to improve build speed [#1933](https://github.com/OpenAtomFoundation/pika/pull/1933) @[xiezheng-XD](https://github.com/xiezheng-XD)
+
+- Fixed an issue where large burrs appeared on the Codis line [#2016](https://github.com/OpenAtomFoundation/pika/pull/2016) @[chejinge](https://github.com/chejinge)
+
+- Fixed an issue where tools could not be compiled in Macos [#2011](https://github.com/OpenAtomFoundation/pika/pull/2011) @[A2ureStone](https://github.com/A2ureStone)
+
+
+
 # v3.5.0
 
 ## Major features
@@ -26,7 +71,7 @@
 2. Support the INFO COMMANDSTATS command.
 3. Support the HELLO and SETNAME commands.
 4. Support the BLPOP and BRPOP commands.
-5. Added Pika original command DISKRECOVERY   
+5. Added Pika original command DISKRECOVERY
 
 ### 3.RocksDB Version Upgrade and Tiered Compression