Merge branch 'OpenAtomFoundation:unstable' into HSET

.github/workflows/pika.yml

@@ -84,51 +84,32 @@ jobs:
           chmod +x integrate_test.sh
           sh integrate_test.sh
 
-  build_on_rocky:
+
+  build_on_centos:
     runs-on: ubuntu-latest
     container:
-      image: rockylinux:9
+      image: cheniujh/pika-centos7-ci:v5
 
     steps:
-      - name: Install deps
-        run: |
-          dnf update -y
-          dnf install -y bash cmake wget git autoconf gcc perl-Digest-SHA tcl which tar g++ tar epel-release gcc-c++ libstdc++-devel gcc-toolset-13
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: 1.19
-
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v1
         with:
           fetch-depth: 0
 
       - name: Configure CMake
         run: |
-          source /opt/rh/gcc-toolset-13/enable
-          cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address .
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/deps
-          key: ${{ runner.os }}-rocky-deps-${{ hashFiles('**/CMakeLists.txt') }}
-
-      - uses: actions/cache@v3
-        with:
-          path: ${{ github.workspace }}/buildtrees
-          key: ${{ runner.os }}-rocky-buildtrees-${{ hashFiles('**/CMakeLists.txt') }}
+          source /opt/rh/devtoolset-10/enable
+          cmake -B build -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} -DUSE_PIKA_TOOLS=ON -DCMAKE_CXX_FLAGS_DEBUG=-fsanitize=address
 
       - name: Build
         run: |
-          source /opt/rh/gcc-toolset-13/enable
+          source /opt/rh/devtoolset-10/enable
           cmake --build build --config ${{ env.BUILD_TYPE }}
 
       - name: Cleanup
         run: |
           rm -rf  ./deps
-          rm -rf ./buildtrees
+          rm -rf ./buildtrees    
 
       - name: Test
         working-directory: ${{ github.workspace }}/build
@@ -145,7 +126,6 @@ jobs:
           ../tests/integration/start_master_and_slave.sh
           chmod +x ../tests/integration/start_codis.sh
           ../tests/integration/start_codis.sh
-
       - name: Run Go E2E Tests
         working-directory: ${{ github.workspace }}/build
         run: |
@@ -155,8 +135,10 @@ jobs:
           chmod +x integrate_test.sh
           sh integrate_test.sh
 
+
   build_on_macos:
-    runs-on: macos-12
+
+    runs-on: macos-13
 
     steps:
       - uses: actions/checkout@v4
@@ -169,7 +151,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/[email protected]
         with:
-          key: macos-12
+          key: macos-13
 
       - name: Install Deps
         run: |
@@ -188,8 +170,10 @@ jobs:
 
       - name: Cleanup
         run: |
-          rm -rf  ./deps
-          rm -rf ./buildtrees
+          cp deps/lib/libz.1.dylib .
+          cp deps/lib/libz.1.dylib tests/integration/
+          rm -rf ./deps 
+          rm -rf ./buildtree
 
       - name: Test
         working-directory: ${{ github.workspace }}/build
@@ -201,21 +185,21 @@ jobs:
           ./pikatests.sh all clean
 
       - name: Start codis, pika master and pika slave
-        working-directory: ${{ github.workspace }}/build
+        working-directory: ${{ github.workspace }}
         run: |
-          chmod +x ../tests/integration/start_master_and_slave.sh
-          ../tests/integration/start_master_and_slave.sh
-          chmod +x ../tests/integration/start_codis.sh
-          ../tests/integration/start_codis.sh
+          cd tests/integration/
+          chmod +x start_master_and_slave.sh
+          ./start_master_and_slave.sh
+          chmod +x start_codis.sh
+          ./start_codis.sh
+          
 
       - name: Run Go E2E Tests
-        working-directory: ${{ github.workspace }}/build
+        working-directory: ${{ github.workspace }}
         run: |
-          cd ../tools/pika_keys_analysis/
-          go test -v ./...
-          cd ../../tests/integration/
+          cd tests/integration/
           chmod +x integrate_test.sh
-          sh integrate_test.sh
+   #       sh integrate_test.sh
 
   build_pika_image:
     name: Build Pika Docker image

.gitmodule

@@ -0,0 +1,11 @@
+[submodule "tools/pika_migrate/third/blackwidow"]
+        url = https://github.com/Qihoo360/blackwidow.git
+[submodule "tools/pika_migrate/third/slash"]
+        url = https://github.com/Qihoo360/slash.git
+[submodule "tools/pika_migrate/third/pink"]
+        url = https://github.com/Qihoo360/pink.git
+[submodule "tools/pika_migrate/third/glog"]
+        url = https://github.com/Qihoo360/glog.git
+[submodule "tools/pika_migrate/third/rocksdb"]
+        url = https://github.com/facebook/rocksdb.git
+

codis/config/proxy.toml

@@ -105,10 +105,10 @@ session_break_on_failure = false
 # Slowlog-log-slower-than(us), from receive command to send response, 0 is allways print slow log
 slowlog_log_slower_than = 100000
 
-# quick command list e.g. get, set
-quick_cmd_list = ""
-# slow command list e.g. hgetall, mset
-slow_cmd_list = ""
+# quick command list
+quick_cmd_list = "get,set"
+# slow command list
+slow_cmd_list = "mget, mset"
 
 # Set metrics server (such as http://localhost:28000), proxy will report json formatted metrics to specified server in a predefined period.
 metrics_report_server = ""

conf/pika.conf

@@ -100,7 +100,6 @@ timeout : 60
 
 # The [password of administrator], which is empty by default.
 # [NOTICE] If this admin password is the same as user password (including both being empty),
-# the value of userpass will be ignored and all users are considered as administrators,
 # in this scenario, users are not subject to the restrictions imposed by the userblacklist.
 # PS: "user password" refers to value of the parameter below: userpass.
 requirepass :
@@ -658,3 +657,37 @@ internal-used-unfinished-full-sync :
 # https://github.com/OpenAtomFoundation/pika/issues/2886
 # default value: true
 wash-data: true
+
+# Pika automatic compact compact strategy, a complement to rocksdb compact.
+# Trigger the compact background task periodically according to `compact-interval`
+# Can choose `full-compact` or `obd-compact`.
+# obd-compact https://github.com/OpenAtomFoundation/pika/issues/2255
+compaction-strategy : obd-compact
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+compact-every-num-of-files : 10
+
+# For OBD_Compact 
+# In another search, if the file creation time is 
+# greater than `force-compact-file-age-seconds`, 
+# a compaction of the upper and lower boundaries 
+# of the file will be performed at the same time 
+# `compact-every-num-of-files` -1
+force-compact-file-age-seconds : 300
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+force-compact-min-delete-ratio : 10
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+dont-compact-sst-created-in-seconds : 20
+
+# For OBD_Compact 
+# According to the number of sst files in rocksdb, 
+# compact every `compact-every-num-of-files` file.
+best-delete-min-ratio : 10

include/acl.h

@@ -129,7 +129,7 @@ class AclSelector {
   friend User;
 
  public:
-  explicit AclSelector() : AclSelector(0){};
+  explicit AclSelector() : AclSelector(0) {};
   explicit AclSelector(uint32_t flag);
   explicit AclSelector(const AclSelector& selector);
   ~AclSelector() = default;
@@ -138,7 +138,7 @@ class AclSelector {
   inline bool HasFlags(uint32_t flag) const { return flags_ & flag; };
   inline void AddFlags(uint32_t flag) { flags_ |= flag; };
   inline void DecFlags(uint32_t flag) { flags_ &= ~flag; };
-  bool EqualChannel(const std::vector<std::string> &allChannel);
+  bool EqualChannel(const std::vector<std::string>& allChannel);
 
  private:
   pstd::Status SetSelector(const std::string& op);
@@ -224,8 +224,7 @@ class User {
   ~User() = default;
 
   std::string Name() const;
-
-  //  inline uint32_t Flags() const { return flags_; };
+  // inline uint32_t Flags() const { return flags_; };
   inline bool HasFlags(uint32_t flag) const { return flags_ & flag; };
   inline void AddFlags(uint32_t flag) { flags_ |= flag; };
   inline void DecFlags(uint32_t flag) { flags_ &= ~flag; };

include/pika_client_conn.h

@@ -22,25 +22,15 @@ struct TimeStat {
     before_queue_ts_ = 0;
   }
 
-  uint64_t start_ts() const {
-    return enqueue_ts_;
-  }
+  uint64_t start_ts() const { return enqueue_ts_; }
 
-  uint64_t total_time() const {
-    return process_done_ts_ > enqueue_ts_ ? process_done_ts_ - enqueue_ts_ : 0;
-  }
+  uint64_t total_time() const { return process_done_ts_ > enqueue_ts_ ? process_done_ts_ - enqueue_ts_ : 0; }
 
-  uint64_t queue_time() const {
-    return dequeue_ts_ > enqueue_ts_ ? dequeue_ts_ - enqueue_ts_ : 0;
-  }
+  uint64_t queue_time() const { return dequeue_ts_ > enqueue_ts_ ? dequeue_ts_ - enqueue_ts_ : 0; }
 
-  uint64_t process_time() const {
-    return process_done_ts_ > dequeue_ts_ ? process_done_ts_ - dequeue_ts_ : 0;
-  }
+  uint64_t process_time() const { return process_done_ts_ > dequeue_ts_ ? process_done_ts_ - dequeue_ts_ : 0; }
 
-  uint64_t before_queue_time() const {
-    return process_done_ts_ > dequeue_ts_ ? before_queue_ts_ - enqueue_ts_ : 0;
-  }
+  uint64_t before_queue_time() const { return process_done_ts_ > dequeue_ts_ ? before_queue_ts_ - enqueue_ts_ : 0; }
 
   uint64_t enqueue_ts_;
   uint64_t dequeue_ts_;
@@ -94,7 +84,7 @@ class PikaClientConn : public net::RedisConn {
   void UnAuth(const std::shared_ptr<User>& user);
 
   bool IsAuthed() const;
-
+  void InitUser();
   bool AuthRequired() const;
 
   std::string UserName() const;
@@ -123,6 +113,7 @@ class PikaClientConn : public net::RedisConn {
   std::vector<std::shared_ptr<std::string>> resp_array;
 
   std::shared_ptr<TimeStat> time_stat_;
+
  private:
   net::ServerThread* const server_thread_;
   std::string current_db_;

include/pika_conf.h

@@ -29,6 +29,11 @@ const uint32_t configReplicationIDSize = 50;
 // global class, class members well initialized
 class PikaConf : public pstd::BaseConf {
  public:
+  enum CompactionStrategy {
+    NONE,
+    FullCompact,
+    OldestOrBestDeleteRatioSstCompact
+  };
   PikaConf(const std::string& path);
   ~PikaConf() override = default;
 
@@ -118,6 +123,30 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return max_subcompactions_;
   }
+  int compact_every_num_of_files() {
+    std::shared_lock l(rwlock_);
+    return compact_every_num_of_files_;
+  }
+  int force_compact_file_age_seconds() {
+    std::shared_lock l(rwlock_);
+    return force_compact_file_age_seconds_;
+  }
+  int force_compact_min_delete_ratio() {
+    std::shared_lock l(rwlock_);
+    return force_compact_min_delete_ratio_;
+  }
+  int dont_compact_sst_created_in_seconds() {
+    std::shared_lock l(rwlock_);
+    return dont_compact_sst_created_in_seconds_;
+  }
+  int best_delete_min_ratio() {
+    std::shared_lock l(rwlock_);
+    return best_delete_min_ratio_;
+  }
+  CompactionStrategy compaction_strategy() {
+    std::shared_lock l(rwlock_);
+    return compaction_strategy_;
+  }
   bool disable_auto_compactions() {
     std::shared_lock l(rwlock_);
     return disable_auto_compactions_;
@@ -933,6 +962,15 @@ class PikaConf : public pstd::BaseConf {
   std::string compact_interval_;
   int max_subcompactions_ = 1;
   bool disable_auto_compactions_ = false;
+
+  // for obd_compact
+  int compact_every_num_of_files_;
+  int force_compact_file_age_seconds_;
+  int force_compact_min_delete_ratio_;
+  int dont_compact_sst_created_in_seconds_;
+  int best_delete_min_ratio_;
+  CompactionStrategy compaction_strategy_;
+
   int64_t resume_check_interval_ = 60; // seconds
   int64_t least_free_disk_to_resume_ = 268435456; // 256 MB
   double min_check_resume_ratio_ = 0.7;

include/pika_db.h

@@ -132,6 +132,7 @@ class DB : public std::enable_shared_from_this<DB>, public pstd::noncopyable {
   // Compact use;
   void Compact(const storage::DataType& type);
   void CompactRange(const storage::DataType& type, const std::string& start, const std::string& end);
+  void LongestNotCompactionSstCompact(const storage::DataType& type);
 
   void SetCompactRangeOptions(const bool is_canceled);
 

include/pika_server.h

@@ -59,6 +59,7 @@ enum TaskType {
   kStopKeyScan,
   kBgSave,
   kCompactRangeAll,
+  kCompactOldestOrBestDeleteRatioSst,
 };
 
 struct TaskArg {

pikatests.sh

@@ -45,6 +45,7 @@ function setup_pika_bin {
         exit 1
     fi
     cp $PIKA_BIN src/redis-server
+    cp $PIKA_BIN tests/integration/pika
     cp tests/conf/pika.conf tests/assets/default.conf
 }
 

src/acl.cc

@@ -489,27 +489,27 @@ void Acl::InitLimitUser(const std::string& bl, bool limit_exist) {
   auto u = GetUser(DefaultLimitUser);
   if (limit_exist) {
     if (!bl.empty()) {
-      for(auto& cmd : blacklist) {
+      for (auto& cmd : blacklist) {
         cmd = pstd::StringTrim(cmd, " ");
         u->SetUser("-" + cmd);
       }
       u->SetUser("on");
     }
     if (!pass.empty()) {
-      u->SetUser(">"+pass);
+      u->SetUser(">" + pass);
     }
   } else {
     if (pass.empty()) {
       u->SetUser("nopass");
     } else {
-      u->SetUser(">"+pass);
+      u->SetUser(">" + pass);
     }
     u->SetUser("on");
     u->SetUser("+@all");
     u->SetUser("~*");
     u->SetUser("&*");
 
-    for(auto& cmd : blacklist) {
+    for (auto& cmd : blacklist) {
       cmd = pstd::StringTrim(cmd, " ");
       u->SetUser("-" + cmd);
     }