Merge branch 'OpenAtomFoundation:unstable' into unstable

.github/workflows/publish_nightly_docker_image.yml

@@ -18,21 +18,21 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: pikadb/pika-dev-nightly
           tags: |
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
-      
+
       - name: Build and push Docker image
         timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
@@ -56,21 +56,21 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: pikadb/codis-dev-nightly
           tags: |
             type=schedule,prefix={{branch}},pattern={{date 'YYYYMMDD'}}
-      
+
       - name: Build and push Docker image
         timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
@@ -81,4 +81,3 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-

.github/workflows/publish_release_docker_image.yml

@@ -12,25 +12,25 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-      
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: pikadb/pika
-      
+
       - name: Build and push Docker image
         timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
@@ -48,25 +48,25 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-      
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: pikadb/codis
-      
+
       - name: Build and push Docker image
         timeout-minutes: 1440
         uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
@@ -77,4 +77,3 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-

.github/workflows/release.yml

@@ -0,0 +1,110 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+env:
+  BUILD_TYPE: Release
+
+jobs:
+  build:
+    name: Build binary
+    strategy:
+      matrix:
+        include:
+          - arch: x86_64-unknown-linux-gnu
+            os: ubuntu-latest
+            file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-amd64
+            file_ext: .tar.gz
+          - arch: aarch64-unknown-linux-gnu
+            os: ubuntu-latest
+            file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-linux-arm64
+            file_ext: .tar.gz
+          - arch: x86_64-apple-darwin
+            os: macos-latest
+            file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-amd64
+            file_ext: .tar.gz
+          - arch: aarch64-apple-darwin
+            os: macos-latest
+            file_name: ${{ github.event.repository.name }}-${{ github.ref_name }}-darwin-arm64
+            file_ext: .tar.gz
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+
+      - name: Install Deps - linux
+        if: contains(matrix.arch, 'linux')
+        run: |
+          sudo apt-get install -y autoconf libprotobuf-dev protobuf-compiler
+          sudo apt-get install -y clang-tidy-12
+
+      - name: Install Deps - darwin
+        if: contains(matrix.os, 'macos')
+        run: |
+          brew update
+          brew install --overwrite python autoconf protobuf llvm wget git
+          brew install gcc@10 automake cmake make binutils
+
+      - name: Configure CMake - linux
+        if: contains(matrix.arch, 'linux')
+        run: cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS="-s" -DCMAKE_EXE_LINKER_FLAGS="-s"
+
+      - name: Configure CMake - darwin
+        if: contains(matrix.os, 'macos')
+        run: |
+          export CC=/usr/local/opt/gcc@10/bin/gcc-10 
+          cmake -B build -DCMAKE_C_COMPILER=/usr/local/opt/gcc@10/bin/gcc-10 -DUSE_PIKA_TOOLS=ON -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }}
+
+      - name: Cache Build - linux
+        if: contains(matrix.arch, 'linux')
+        uses: actions/cache@v3
+        id: cache-ubuntu
+        with:
+          key: ${{ runner.os }}-build-ubuntu-${{ hashFiles('**/CMakeLists.txt') }}
+          path: |
+            ${{ github.workspace }}/buildtrees
+            ${{ github.workspace }}/deps
+
+      - name: Build
+        run: cmake --build build --config ${{ env.BUILD_TYPE }}
+
+      - name: Calculate checksum and rename binary
+        shell: bash
+        run: |
+          cd build/
+          chmod +x ${{ github.event.repository.name }}
+          tar -zcvf ${{ matrix.file_name }}${{ matrix.file_ext }} ${{ github.event.repository.name }}
+          echo $(shasum -a 256 ${{ matrix.file_name }}${{ matrix.file_ext }} | cut -f1 -d' ') > ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file_name }}${{ matrix.file_ext }}
+          path: build/${{ matrix.file_name }}${{ matrix.file_ext }}
+
+      - name: Upload checksum of artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
+          path: build/${{ matrix.file_name }}${{ matrix.file_ext }}.sha256sum
+
+  release:
+    name: Release artifacts
+    needs: [ build ]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+
+      - name: Publish release
+        uses: softprops/action-gh-release@v1
+        with:
+          name: "Release ${{ github.ref_name }}"
+          generate_release_notes: true
+          files: |
+            **/${{ github.event.repository.name }}-*

codis/pkg/topom/topom_stats.go

@@ -66,7 +66,7 @@ func (s *Topom) RefreshRedisStats(timeout time.Duration) (*sync2.Future, error)
 	for _, g := range ctx.group {
 		for _, x := range g.Servers {
 			goStats(x.Addr, func(addr string) (*RedisStats, error) {
-				m, err := s.stats.redisp.InfoFull(addr)
+				m, err := s.stats.redisp.InfoFullv2(addr)
 				if err != nil {
 					return nil, err
 				}

codis/pkg/utils/redis/client.go

@@ -157,6 +157,24 @@ func (c *Client) Info() (map[string]string, error) {
 	return info, nil
 }
 
+func (c *Client) InfoReplicationIpPort() (map[string]string, error) {
+	text, err := redigo.String(c.Do("INFO", "replication"))
+	if err != nil {
+		return nil, errors.Trace(err)
+	}
+	info := make(map[string]string)
+	for _, line := range strings.Split(text, "\n") {
+		kv := strings.SplitN(line, ":", 2)
+		if len(kv) != 2 {
+			continue
+		}
+		if key := strings.TrimSpace(kv[0]); key != "" {
+			info[key] = strings.TrimSpace(kv[1])
+		}
+	}
+	return info, nil
+}
+
 func (c *Client) InfoKeySpace() (map[int]string, error) {
 	text, err := redigo.String(c.Do("INFO", "keyspace"))
 	if err != nil {
@@ -262,6 +280,32 @@ func (c *Client) InfoFull() (map[string]string, error) {
 	}
 }
 
+func (c *Client) InfoFullv2() (map[string]string, error) {
+	if info, err := c.InfoReplicationIpPort(); err != nil {
+		return nil, errors.Trace(err)
+	} else {
+		host := info["master_host"]
+		port := info["master_port"]
+		if host != "" || port != "" {
+			info["master_addr"] = net.JoinHostPort(host, port)
+		}
+		r, err := c.Do("CONFIG", "GET", "maxmemory")
+		if err != nil {
+			return nil, errors.Trace(err)
+		}
+		p, err := redigo.Values(r, nil)
+		if err != nil || len(p) != 2 {
+			return nil, errors.Errorf("invalid response = %v", r)
+		}
+		v, err := redigo.Int(p[1], nil)
+		if err != nil {
+			return nil, errors.Errorf("invalid response = %v", r)
+		}
+		info["maxmemory"] = strconv.Itoa(v)
+		return info, nil
+	}
+}
+
 func (c *Client) SetMaster(master string) error {
 	if master == "" || strings.ToUpper(master) == "NO:ONE" {
 		if _, err := c.Do("SLAVEOF", "NO", "ONE"); err != nil {
@@ -509,6 +553,15 @@ func (p *Pool) InfoFull(addr string) (_ map[string]string, err error) {
 	return c.InfoFull()
 }
 
+func (p *Pool) InfoFullv2(addr string) (_ map[string]string, err error) {
+	c, err := p.GetClient(addr)
+	if err != nil {
+		return nil, err
+	}
+	defer p.PutClient(c)
+	return c.InfoFullv2()
+}
+
 type InfoCache struct {
 	mu sync.Mutex
 

conf/pika.conf

@@ -268,6 +268,7 @@ max-cache-statistic-keys : 0
 # a small compact is triggered automatically if the small compaction feature is enabled.
 # small-compaction-threshold default value is 5000 and the value range is [1, 100000].
 small-compaction-threshold : 5000
+small-compaction-duration-threshold : 10000
 
 # The maximum total size of all live memtables of the RocksDB instance that owned by Pika.
 # Flushing from memtable to disk will be triggered if the actual memory usage of RocksDB

include/pika_admin.h

@@ -261,7 +261,9 @@ class InfoCmd : public Cmd {
   bool rescan_ = false;  // whether to rescan the keyspace
   bool off_ = false;
   std::set<std::string> keyspace_scan_dbs_;
-
+  time_t db_size_last_time_ = 0;
+  uint64_t db_size_ = 0;
+  uint64_t log_size_ = 0;
   const static std::string kInfoSection;
   const static std::string kAllSection;
   const static std::string kServerSection;

include/pika_conf.h

@@ -220,6 +220,10 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return small_compaction_threshold_;
   }
+  int small_compaction_duration_threshold() {
+    std::shared_lock l(rwlock_);
+    return small_compaction_duration_threshold_;
+  }
   int max_background_flushes() {
     std::shared_lock l(rwlock_);
     return max_background_flushes_;
@@ -425,6 +429,11 @@ class PikaConf : public pstd::BaseConf {
     TryPushDiffCommands("small-compaction-threshold", std::to_string(value));
     small_compaction_threshold_ = value;
   }
+  void SetSmallCompactionDurationThreshold(const int value) {
+    std::lock_guard l(rwlock_);
+    TryPushDiffCommands("small-compaction-duration-threshold", std::to_string(value));
+    small_compaction_duration_threshold_ = value;
+  }
   void SetMaxClientResponseSize(const int value) {
     std::lock_guard l(rwlock_);
     TryPushDiffCommands("max-client-response-size", std::to_string(value));
@@ -684,6 +693,7 @@ class PikaConf : public pstd::BaseConf {
 
   int max_cache_statistic_keys_ = 0;
   int small_compaction_threshold_ = 0;
+  int small_compaction_duration_threshold_ = 0;
   int max_background_flushes_ = 0;
   int max_background_compactions_ = 0;
   int max_background_jobs_ = 0;

include/pika_server.h

@@ -195,6 +195,7 @@ class PikaServer : public pstd::noncopyable {
   bool IsDBExist(const std::string& db_name);
   bool IsDBSlotExist(const std::string& db_name, uint32_t slot_id);
   bool IsDBBinlogIoError(const std::string& db_name);
+  std::set<std::string> GetAllDBName();
   pstd::Status DoSameThingSpecificDB(const std::set<std::string>& dbs, const TaskArg& arg);
   std::shared_mutex& GetDBLock() {
     return dbs_rw_;
@@ -218,6 +219,7 @@ class PikaServer : public pstd::noncopyable {
   void PrepareSlotTrySync();
   void SlotSetMaxCacheStatisticKeys(uint32_t max_cache_statistic_keys);
   void SlotSetSmallCompactionThreshold(uint32_t small_compaction_threshold);
+  void SlotSetSmallCompactionDurationThreshold(uint32_t small_compaction_duration_threshold);
   bool GetDBSlotBinlogOffset(const std::string& db_name, uint32_t slot_id, BinlogOffset* boffset);
   std::shared_ptr<Slot> GetSlotByDBName(const std::string& db_name);
   std::shared_ptr<Slot> GetDBSlotById(const std::string& db_name, uint32_t slot_id);