feat:Added the periodically clearing serverlog function (#2829)

* Added the serverlog periodic reclamation function


---------

Co-authored-by: cheniujh <[email protected]>

conf/pika.conf

@@ -65,6 +65,11 @@ sync-binlog-thread-num : 1
 # is used for replication.
 log-path : ./log/
 
+# log retention time of serverlogs(pika.{hostname}.{username}.log.{loglevel}.YYYYMMDD-HHMMSS) files that stored within log-path.
+# Any serverlogs files that exceed this time will be cleaned up.
+# The unit of serverlogs is in [days] and the default value is 7(days).
+log-retention-time : 7
+
 # Directory to store the data of Pika.
 db-path : ./db/
 

include/pika_conf.h

@@ -76,6 +76,10 @@ class PikaConf : public pstd::BaseConf {
     std::shared_lock l(rwlock_);
     return log_path_;
   }
+  int log_retention_time() {
+    std::shared_lock l(rwlock_);
+    return log_retention_time_;
+  }
   std::string log_level() {
     std::shared_lock l(rwlock_);
     return log_level_;
@@ -828,6 +832,7 @@ class PikaConf : public pstd::BaseConf {
   int db_sync_speed_ = 0;
   std::string slaveof_;
   std::string log_path_;
+  int log_retention_time_;
   std::string log_level_;
   std::string db_path_;
   std::string db_sync_path_;

include/pika_server.h

@@ -518,7 +518,8 @@ class PikaServer : public pstd::noncopyable {
    */
   void DoTimingTask();
   void AutoCompactRange();
-  void AutoPurge();
+  void AutoBinlogPurge();
+  void AutoServerlogPurge();
   void AutoDeleteExpiredDump();
   void AutoUpdateNetworkMetric();
   void PrintThreadPoolQueueStatus();

src/pika_conf.cc

@@ -133,6 +133,10 @@ int PikaConf::Load() {
   if (log_path_[log_path_.length() - 1] != '/') {
     log_path_ += "/";
   }
+  GetConfInt("log-retention-time",&log_retention_time_);
+  if(log_retention_time_ < 0){
+    LOG(FATAL) << "log-retention-time invalid";
+  }
   GetConfStr("loglevel", &log_level_);
   GetConfStr("db-path", &db_path_);
   db_path_ = db_path_.empty() ? "./db/" : db_path_;

src/pika_server.cc

@@ -1113,8 +1113,10 @@ int PikaServer::ClientPubSubChannelPatternSize(const std::shared_ptr<NetConn>& c
 void PikaServer::DoTimingTask() {
   // Maybe schedule compactrange
   AutoCompactRange();
-  // Purge log
-  AutoPurge();
+  // Purge serverlog
+  AutoServerlogPurge();
+  // Purge binlog
+  AutoBinlogPurge();
   // Delete expired dump
   AutoDeleteExpiredDump();
   // Cheek Rsync Status
@@ -1234,7 +1236,88 @@ void PikaServer::AutoCompactRange() {
   }
 }
 
-void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); }
+void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); }
+
+void PikaServer::AutoServerlogPurge() {
+  std::string log_path = g_pika_conf->log_path();
+  int retention_time = g_pika_conf->log_retention_time();
+  if (retention_time < 0) {
+    return;
+  }
+  std::vector<std::string> log_files;
+
+  if (!pstd::FileExists(log_path)) {
+    return;
+  }
+
+  if (pstd::GetChildren(log_path, log_files) != 0) {
+    return;
+  }
+  //Get the current time of system
+  time_t t = time(nullptr);
+  struct tm* now_time = localtime(&t);
+  now_time->tm_hour = 0;
+  now_time->tm_min = 0;
+  now_time->tm_sec = 0;
+  time_t now_timestamp = mktime(now_time);
+
+  std::map<std::string, std::vector<std::pair<std::string, int64_t>>> log_files_by_level;
+
+  //Serverlogformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid]
+  for (const auto& file : log_files) {
+    std::vector<std::string> file_parts;
+    pstd::StringSplit(file, '.', file_parts);
+    if (file_parts.size() < 7) {
+      continue;
+    }
+
+    std::string severity_level = file_parts[4];
+    if (severity_level != "WARNING" && severity_level != "INFO" && severity_level != "ERROR") {
+      continue;
+    }
+
+    int log_year, log_month, log_day;
+    if (sscanf(file_parts[5].c_str(), "%4d%2d%2d", &log_year, &log_month, &log_day) != 3) {
+      continue;
+    }
+
+    //Get the time when the server log file was originally created
+    struct tm log_time;
+    log_time.tm_year = log_year - 1900;
+    log_time.tm_mon = log_month - 1;
+    log_time.tm_mday = log_day;
+    log_time.tm_hour = 0;
+    log_time.tm_min = 0;
+    log_time.tm_sec = 0;
+    log_time.tm_isdst = -1;
+    time_t log_timestamp = mktime(&log_time);
+    log_files_by_level[severity_level].push_back({file, log_timestamp});
+}
+
+  // Process files for each log level
+  for (auto& [level, files] : log_files_by_level) {
+    // Sort by time in descending order
+    std::sort(files.begin(), files.end(),
+              [](const auto& a, const auto& b) { return a.second > b.second; });
+
+    bool has_recent_file = false;
+    for (const auto& [file, log_timestamp] : files) {
+      double diff_seconds = difftime(now_timestamp, log_timestamp);
+      int64_t interval_days = static_cast<int64_t>(diff_seconds / 86400);
+      if (interval_days <= retention_time) {
+        has_recent_file = true;
+        continue;
+      }
+      if (!has_recent_file) {
+        has_recent_file = true;
+        continue;
+      }
+      std::string log_file = log_path + "/" + file;
+      LOG(INFO) << "Deleting out of date log file: " << log_file;
+      if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file;
+    }
+  }
+}
 
 void PikaServer::AutoDeleteExpiredDump() {
   std::string db_sync_prefix = g_pika_conf->bgsave_prefix();