From d6262d7810ead886449cacba6a0a4f313ee61be6 Mon Sep 17 00:00:00 2001 From: XiaoLiang2333 <1159025475@qq.com> Date: Fri, 26 Jul 2024 20:54:30 +0800 Subject: [PATCH 1/8] Added the serverlog periodic reclamation function --- conf/pika.conf | 5 +++ include/pika_conf.h | 5 +++ include/pika_server.h | 3 +- src/pika_server.cc | 83 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index 496d974174..b6a9121aa2 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -64,6 +64,11 @@ sync-binlog-thread-num : 1 # is used for replication. log-path : ./log/ +# log retention time of serverlogs(pika.{hostname}.{username}.log.{loglevel}.YYYYMMDD-HHMMSS) files that stored within log-path. +# Any serverlogs files that exceed this time will be cleaned up. +# The unit of serverlogs is in [days] and the default value is 7(days). +log-retention-time : 7 + # Directory to store the data of Pika. db-path : ./db/ diff --git a/include/pika_conf.h b/include/pika_conf.h index bec1ae14ca..b50500ffae 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -77,6 +77,10 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return log_path_; } + int log_retention_time() { + std::shared_lock l(rwlock_); + return log_retention_time_; + } std::string log_level() { std::shared_lock l(rwlock_); return log_level_; @@ -904,6 +908,7 @@ class PikaConf : public pstd::BaseConf { int db_sync_speed_ = 0; std::string slaveof_; std::string log_path_; + int log_retention_time_; std::string log_level_; std::string db_path_; int db_instance_num_ = 0; diff --git a/include/pika_server.h b/include/pika_server.h index 1374158f88..aa5f8dbc53 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -513,7 +513,8 @@ class PikaServer : public pstd::noncopyable { */ void DoTimingTask(); void AutoCompactRange(); - void AutoPurge(); + void AutoBinlogPurge(); + void AutoServerlogPurge(); void AutoDeleteExpiredDump(); void AutoUpdateNetworkMetric(); void PrintThreadPoolQueueStatus(); diff --git a/src/pika_server.cc b/src/pika_server.cc index ed32bfb077..8172fc4bb9 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1085,8 +1085,10 @@ int PikaServer::ClientPubSubChannelPatternSize(const std::shared_ptr& c void PikaServer::DoTimingTask() { // Maybe schedule compactrange AutoCompactRange(); - // Purge log - AutoPurge(); + // Purge serverlog + AutoServerlogPurge(); + // Purge binlog + AutoBinlogPurge(); // Delete expired dump AutoDeleteExpiredDump(); // Cheek Rsync Status @@ -1206,7 +1208,82 @@ void PikaServer::AutoCompactRange() { } } -void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } +void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } + +void PikaServer::AutoServerlogPurge(){ +std::string log_path = g_pika_conf->log_path(); + int retention_time = g_pika_conf->log_retention_time(); + if (retention_time <= 0) { + return; + } + std::vector log_files; + + if (!pstd::FileExists(log_path)) { + return; + } + + if (pstd::GetChildren(log_path, log_files) != 0) { + return; + } + + time_t t = time(nullptr); + struct tm* now = localtime(&t); + int now_year = now->tm_year + 1900; + int now_month = now->tm_mon + 1; + int now_day = now->tm_mday; + + //logformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid] + for (const auto& file : log_files) { + if (file.substr(0, 5) != "pika.") { + continue; + } + + size_t log_pos = file.find(".log."); + if (log_pos == std::string::npos) { + continue; + } + + // Start at the end of ".log." to find the date and time + size_t date_pos = file.find('.', log_pos + 5); + if (date_pos == std::string::npos || date_pos + 16 > file.length()) { + continue; + } + + std::string date = file.substr(date_pos + 1, 8); + std::string time = file.substr(date_pos + 10, 6); + + int log_year = std::atoi(date.substr(0, 4).c_str()); + int log_month = std::atoi(date.substr(4, 2).c_str()); + int log_day = std::atoi(date.substr(6, 2).c_str()); + + struct tm log_time; + struct tm now_time; + + log_time.tm_year = log_year - 1900; + log_time.tm_mon = log_month - 1; + log_time.tm_mday = log_day; + log_time.tm_hour = 0; + log_time.tm_min = 0; + log_time.tm_sec = 0; + + now_time.tm_year = now_year - 1900; + now_time.tm_mon = now_month - 1; + now_time.tm_mday = now_day; + now_time.tm_hour = 0; + now_time.tm_min = 0; + now_time.tm_sec = 0; + + int64_t log_timestamp = mktime(&log_time); + int64_t now_timestamp = mktime(&now_time); + int64_t interval_days = (now_timestamp - log_timestamp) / 86400; + + if (interval_days > retention_time) { + std::string log_file = log_path + "/" + file; + LOG(INFO) << "Deleting out of date log file: " << log_file; + pstd::DeleteFile(log_file); + } + } +} void PikaServer::AutoDeleteExpiredDump() { std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); From d84e8b705bbaf966fa2479b9bcbd7d13924bf683 Mon Sep 17 00:00:00 2001 From: XiaoLiang2333 <1159025475@qq.com> Date: Fri, 26 Jul 2024 21:10:18 +0800 Subject: [PATCH 2/8] Revert "Added the serverlog periodic reclamation function" This reverts commit d6262d7810ead886449cacba6a0a4f313ee61be6. --- conf/pika.conf | 5 --- include/pika_conf.h | 5 --- include/pika_server.h | 3 +- src/pika_server.cc | 83 ++----------------------------------------- 4 files changed, 4 insertions(+), 92 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index b6a9121aa2..496d974174 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -64,11 +64,6 @@ sync-binlog-thread-num : 1 # is used for replication. log-path : ./log/ -# log retention time of serverlogs(pika.{hostname}.{username}.log.{loglevel}.YYYYMMDD-HHMMSS) files that stored within log-path. -# Any serverlogs files that exceed this time will be cleaned up. -# The unit of serverlogs is in [days] and the default value is 7(days). -log-retention-time : 7 - # Directory to store the data of Pika. db-path : ./db/ diff --git a/include/pika_conf.h b/include/pika_conf.h index b50500ffae..bec1ae14ca 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -77,10 +77,6 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return log_path_; } - int log_retention_time() { - std::shared_lock l(rwlock_); - return log_retention_time_; - } std::string log_level() { std::shared_lock l(rwlock_); return log_level_; @@ -908,7 +904,6 @@ class PikaConf : public pstd::BaseConf { int db_sync_speed_ = 0; std::string slaveof_; std::string log_path_; - int log_retention_time_; std::string log_level_; std::string db_path_; int db_instance_num_ = 0; diff --git a/include/pika_server.h b/include/pika_server.h index aa5f8dbc53..1374158f88 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -513,8 +513,7 @@ class PikaServer : public pstd::noncopyable { */ void DoTimingTask(); void AutoCompactRange(); - void AutoBinlogPurge(); - void AutoServerlogPurge(); + void AutoPurge(); void AutoDeleteExpiredDump(); void AutoUpdateNetworkMetric(); void PrintThreadPoolQueueStatus(); diff --git a/src/pika_server.cc b/src/pika_server.cc index 8172fc4bb9..ed32bfb077 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1085,10 +1085,8 @@ int PikaServer::ClientPubSubChannelPatternSize(const std::shared_ptr& c void PikaServer::DoTimingTask() { // Maybe schedule compactrange AutoCompactRange(); - // Purge serverlog - AutoServerlogPurge(); - // Purge binlog - AutoBinlogPurge(); + // Purge log + AutoPurge(); // Delete expired dump AutoDeleteExpiredDump(); // Cheek Rsync Status @@ -1208,82 +1206,7 @@ void PikaServer::AutoCompactRange() { } } -void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } - -void PikaServer::AutoServerlogPurge(){ -std::string log_path = g_pika_conf->log_path(); - int retention_time = g_pika_conf->log_retention_time(); - if (retention_time <= 0) { - return; - } - std::vector log_files; - - if (!pstd::FileExists(log_path)) { - return; - } - - if (pstd::GetChildren(log_path, log_files) != 0) { - return; - } - - time_t t = time(nullptr); - struct tm* now = localtime(&t); - int now_year = now->tm_year + 1900; - int now_month = now->tm_mon + 1; - int now_day = now->tm_mday; - - //logformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid] - for (const auto& file : log_files) { - if (file.substr(0, 5) != "pika.") { - continue; - } - - size_t log_pos = file.find(".log."); - if (log_pos == std::string::npos) { - continue; - } - - // Start at the end of ".log." to find the date and time - size_t date_pos = file.find('.', log_pos + 5); - if (date_pos == std::string::npos || date_pos + 16 > file.length()) { - continue; - } - - std::string date = file.substr(date_pos + 1, 8); - std::string time = file.substr(date_pos + 10, 6); - - int log_year = std::atoi(date.substr(0, 4).c_str()); - int log_month = std::atoi(date.substr(4, 2).c_str()); - int log_day = std::atoi(date.substr(6, 2).c_str()); - - struct tm log_time; - struct tm now_time; - - log_time.tm_year = log_year - 1900; - log_time.tm_mon = log_month - 1; - log_time.tm_mday = log_day; - log_time.tm_hour = 0; - log_time.tm_min = 0; - log_time.tm_sec = 0; - - now_time.tm_year = now_year - 1900; - now_time.tm_mon = now_month - 1; - now_time.tm_mday = now_day; - now_time.tm_hour = 0; - now_time.tm_min = 0; - now_time.tm_sec = 0; - - int64_t log_timestamp = mktime(&log_time); - int64_t now_timestamp = mktime(&now_time); - int64_t interval_days = (now_timestamp - log_timestamp) / 86400; - - if (interval_days > retention_time) { - std::string log_file = log_path + "/" + file; - LOG(INFO) << "Deleting out of date log file: " << log_file; - pstd::DeleteFile(log_file); - } - } -} +void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } void PikaServer::AutoDeleteExpiredDump() { std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); From 0ab1a51b38880d8d73dc7a03c0fc2b283bde0c6d Mon Sep 17 00:00:00 2001 From: XiaoLiang2333 <1159025475@qq.com> Date: Fri, 26 Jul 2024 21:15:52 +0800 Subject: [PATCH 3/8] Added the serverlog periodic reclamation function --- conf/pika.conf | 5 +++ include/pika_conf.h | 5 +++ include/pika_server.h | 3 +- src/pika_server.cc | 83 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 92 insertions(+), 4 deletions(-) diff --git a/conf/pika.conf b/conf/pika.conf index 496d974174..b6a9121aa2 100644 --- a/conf/pika.conf +++ b/conf/pika.conf @@ -64,6 +64,11 @@ sync-binlog-thread-num : 1 # is used for replication. log-path : ./log/ +# log retention time of serverlogs(pika.{hostname}.{username}.log.{loglevel}.YYYYMMDD-HHMMSS) files that stored within log-path. +# Any serverlogs files that exceed this time will be cleaned up. +# The unit of serverlogs is in [days] and the default value is 7(days). +log-retention-time : 7 + # Directory to store the data of Pika. db-path : ./db/ diff --git a/include/pika_conf.h b/include/pika_conf.h index bec1ae14ca..b50500ffae 100644 --- a/include/pika_conf.h +++ b/include/pika_conf.h @@ -77,6 +77,10 @@ class PikaConf : public pstd::BaseConf { std::shared_lock l(rwlock_); return log_path_; } + int log_retention_time() { + std::shared_lock l(rwlock_); + return log_retention_time_; + } std::string log_level() { std::shared_lock l(rwlock_); return log_level_; @@ -904,6 +908,7 @@ class PikaConf : public pstd::BaseConf { int db_sync_speed_ = 0; std::string slaveof_; std::string log_path_; + int log_retention_time_; std::string log_level_; std::string db_path_; int db_instance_num_ = 0; diff --git a/include/pika_server.h b/include/pika_server.h index 1374158f88..aa5f8dbc53 100644 --- a/include/pika_server.h +++ b/include/pika_server.h @@ -513,7 +513,8 @@ class PikaServer : public pstd::noncopyable { */ void DoTimingTask(); void AutoCompactRange(); - void AutoPurge(); + void AutoBinlogPurge(); + void AutoServerlogPurge(); void AutoDeleteExpiredDump(); void AutoUpdateNetworkMetric(); void PrintThreadPoolQueueStatus(); diff --git a/src/pika_server.cc b/src/pika_server.cc index ed32bfb077..22d3a01b17 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1085,8 +1085,10 @@ int PikaServer::ClientPubSubChannelPatternSize(const std::shared_ptr& c void PikaServer::DoTimingTask() { // Maybe schedule compactrange AutoCompactRange(); - // Purge log - AutoPurge(); + // Purge serverlog + AutoServerlogPurge(); + // Purge binlog + AutoBinlogPurge(); // Delete expired dump AutoDeleteExpiredDump(); // Cheek Rsync Status @@ -1206,7 +1208,82 @@ void PikaServer::AutoCompactRange() { } } -void PikaServer::AutoPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } +void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } + +void PikaServer::AutoServerlogPurge(){ +std::string log_path = g_pika_conf->log_path(); + int retention_time = g_pika_conf->log_retention_time(); + if (retention_time <= 0) { + return; + } + std::vector log_files; + + if (!pstd::FileExists(log_path)) { + return; + } + + if (pstd::GetChildren(log_path, log_files) != 0) { + return; + } + + time_t t = time(nullptr); + struct tm* now = localtime(&t); + int now_year = now->tm_year + 1900; + int now_month = now->tm_mon + 1; + int now_day = now->tm_mday; + + //logformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid] + for (const auto& file : log_files) { + if (file.substr(0, 5) != "pika.") { + continue; + } + + size_t log_pos = file.find(".log."); + if (log_pos == std::string::npos) { + continue; + } + + // Start at the end of ".log." to find the date and time + size_t date_pos = file.find('.', log_pos + 5); + if (date_pos == std::string::npos || date_pos + 16 > file.length()) { + continue; + } + + std::string date = file.substr(date_pos + 1, 8); + std::string time = file.substr(date_pos + 10, 6); + + int log_year = std::atoi(date.substr(0, 4).c_str()); + int log_month = std::atoi(date.substr(4, 2).c_str()); + int log_day = std::atoi(date.substr(6, 2).c_str()); + + struct tm log_time; + struct tm now_time; + + log_time.tm_year = log_year - 1900; + log_time.tm_mon = log_month - 1; + log_time.tm_mday = log_day; + log_time.tm_hour = 0; + log_time.tm_min = 0; + log_time.tm_sec = 0; + + now_time.tm_year = now_year - 1900; + now_time.tm_mon = now_month - 1; + now_time.tm_mday = now_day; + now_time.tm_hour = 0; + now_time.tm_min = 0; + now_time.tm_sec = 0; + + int64_t log_timestamp = mktime(&log_time); + int64_t now_timestamp = mktime(&now_time); + int64_t interval_days = (now_timestamp - log_timestamp) / 86400; + + if (interval_days > retention_time) { + std::string log_file = log_path + "/" + file; + LOG(INFO) << "Deleting out of date log file: " << log_file; + pstd::DeleteFile(log_file); + } + } +} void PikaServer::AutoDeleteExpiredDump() { std::string db_sync_prefix = g_pika_conf->bgsave_prefix(); From b432ca8d28ba64e487559017309a04fafbcb34ae Mon Sep 17 00:00:00 2001 From: DawnBeams <118454556+XiaoLiang2333@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:46:29 +0800 Subject: [PATCH 4/8] Update AutoServerlogPurge() Check return value and print Error Log if deleteFile failed --- src/pika_server.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pika_server.cc b/src/pika_server.cc index 22d3a01b17..9150cfcc64 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1280,7 +1280,9 @@ std::string log_path = g_pika_conf->log_path(); if (interval_days > retention_time) { std::string log_file = log_path + "/" + file; LOG(INFO) << "Deleting out of date log file: " << log_file; - pstd::DeleteFile(log_file); + if (!pstd::DeleteFile(log_file)) { + LOG(ERROR) << "Failed to delete log file: " << log_file; + } } } } From 973f9fa575f0b9a8337f4f9dc95bec85f3e2e5e7 Mon Sep 17 00:00:00 2001 From: DawnBeams <118454556+XiaoLiang2333@users.noreply.github.com> Date: Fri, 2 Aug 2024 02:52:16 +0800 Subject: [PATCH 5/8] Refactor AutoServerlogPurge function 1. retain the newest serverlog file even if it has been expired 2. Make sure that serverlogs of [INFO],[ERROR], and [WARNING] levels are retained --- src/pika_server.cc | 75 +++++++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/src/pika_server.cc b/src/pika_server.cc index 9150cfcc64..c271ad0c67 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1210,8 +1210,8 @@ void PikaServer::AutoCompactRange() { void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } -void PikaServer::AutoServerlogPurge(){ -std::string log_path = g_pika_conf->log_path(); +void PikaServer::AutoServerlogPurge() { + std::string log_path = g_pika_conf->log_path(); int retention_time = g_pika_conf->log_retention_time(); if (retention_time <= 0) { return; @@ -1225,40 +1225,43 @@ std::string log_path = g_pika_conf->log_path(); if (pstd::GetChildren(log_path, log_files) != 0) { return; } - + //Get the current time of system time_t t = time(nullptr); struct tm* now = localtime(&t); int now_year = now->tm_year + 1900; int now_month = now->tm_mon + 1; int now_day = now->tm_mday; - //logformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid] + struct tm now_time; + now_time.tm_year = now_year - 1900; + now_time.tm_mon = now_month - 1; + now_time.tm_mday = now_day; + now_time.tm_hour = 0; + now_time.tm_min = 0; + now_time.tm_sec = 0; + int64_t now_timestamp = mktime(&now_time); + + std::map>> log_files_by_level; + + //Serverlogformat: pika.[hostname].[user name].log.[severity level].[date].[time].[pid] for (const auto& file : log_files) { - if (file.substr(0, 5) != "pika.") { + std::vector file_parts; + pstd::StringSplit(file, '.', file_parts); + if (file_parts.size() < 7) { continue; } - size_t log_pos = file.find(".log."); - if (log_pos == std::string::npos) { + std::string severity_level = file_parts[4]; + if (severity_level != "WARNING" && severity_level != "INFO" && severity_level != "ERROR") { continue; } - // Start at the end of ".log." to find the date and time - size_t date_pos = file.find('.', log_pos + 5); - if (date_pos == std::string::npos || date_pos + 16 > file.length()) { + int log_year, log_month, log_day; + if (sscanf(file_parts[5].c_str(), "%4d%2d%2d", &log_year, &log_month, &log_day) != 3) { continue; } - - std::string date = file.substr(date_pos + 1, 8); - std::string time = file.substr(date_pos + 10, 6); - - int log_year = std::atoi(date.substr(0, 4).c_str()); - int log_month = std::atoi(date.substr(4, 2).c_str()); - int log_day = std::atoi(date.substr(6, 2).c_str()); - + //Get the time when the server log file was originally created struct tm log_time; - struct tm now_time; - log_time.tm_year = log_year - 1900; log_time.tm_mon = log_month - 1; log_time.tm_mday = log_day; @@ -1266,22 +1269,26 @@ std::string log_path = g_pika_conf->log_path(); log_time.tm_min = 0; log_time.tm_sec = 0; - now_time.tm_year = now_year - 1900; - now_time.tm_mon = now_month - 1; - now_time.tm_mday = now_day; - now_time.tm_hour = 0; - now_time.tm_min = 0; - now_time.tm_sec = 0; - int64_t log_timestamp = mktime(&log_time); - int64_t now_timestamp = mktime(&now_time); - int64_t interval_days = (now_timestamp - log_timestamp) / 86400; + log_files_by_level[severity_level].push_back({file, log_timestamp}); + } + + // Process files for each log level + for (auto& [level, files] : log_files_by_level) { + // Sort by time in descending order + std::sort(files.begin(), files.end(), + [](const auto& a, const auto& b) { return a.second > b.second; }); - if (interval_days > retention_time) { - std::string log_file = log_path + "/" + file; - LOG(INFO) << "Deleting out of date log file: " << log_file; - if (!pstd::DeleteFile(log_file)) { - LOG(ERROR) << "Failed to delete log file: " << log_file; + bool keep_newest = false; + for (const auto& [file, log_timestamp] : files) { + int64_t interval_days = (now_timestamp - log_timestamp) / 86400; + + if (interval_days > retention_time && keep_newest) { + std::string log_file = log_path + "/" + file; + LOG(INFO) << "Deleting out of date log file: " << log_file; + if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file; + } else { + keep_newest = true; } } } From a6934fa63b10df860db92b32672df65b7c1bc01b Mon Sep 17 00:00:00 2001 From: DawnBeams <118454556+XiaoLiang2333@users.noreply.github.com> Date: Wed, 7 Aug 2024 21:50:29 +0800 Subject: [PATCH 6/8] Update pika_server.cc Adjust the time calculation method --- src/pika_server.cc | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/pika_server.cc b/src/pika_server.cc index 565d2ef02c..f4b436989c 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1215,7 +1215,7 @@ void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } void PikaServer::AutoServerlogPurge() { std::string log_path = g_pika_conf->log_path(); int retention_time = g_pika_conf->log_retention_time(); - if (retention_time <= 0) { + if (retention_time < 0) { return; } std::vector log_files; @@ -1229,19 +1229,11 @@ void PikaServer::AutoServerlogPurge() { } //Get the current time of system time_t t = time(nullptr); - struct tm* now = localtime(&t); - int now_year = now->tm_year + 1900; - int now_month = now->tm_mon + 1; - int now_day = now->tm_mday; - - struct tm now_time; - now_time.tm_year = now_year - 1900; - now_time.tm_mon = now_month - 1; - now_time.tm_mday = now_day; - now_time.tm_hour = 0; - now_time.tm_min = 0; - now_time.tm_sec = 0; - int64_t now_timestamp = mktime(&now_time); + struct tm* now_time = localtime(&t); + now_time->tm_hour = 0; + now_time->tm_min = 0; + now_time->tm_sec = 0; + time_t now_timestamp = mktime(now_time); std::map>> log_files_by_level; @@ -1271,7 +1263,7 @@ void PikaServer::AutoServerlogPurge() { log_time.tm_min = 0; log_time.tm_sec = 0; - int64_t log_timestamp = mktime(&log_time); + time_t log_timestamp = mktime(&log_time); log_files_by_level[severity_level].push_back({file, log_timestamp}); } @@ -1283,8 +1275,8 @@ void PikaServer::AutoServerlogPurge() { bool keep_newest = false; for (const auto& [file, log_timestamp] : files) { - int64_t interval_days = (now_timestamp - log_timestamp) / 86400; - + double diff_seconds = difftime(now_timestamp, log_timestamp); + int64_t interval_days = static_cast(diff_seconds / 86400); if (interval_days > retention_time && keep_newest) { std::string log_file = log_path + "/" + file; LOG(INFO) << "Deleting out of date log file: " << log_file; From 9b4d63a28cac9e873a874fa314d41ded7fc88960 Mon Sep 17 00:00:00 2001 From: DawnBeams <118454556+XiaoLiang2333@users.noreply.github.com> Date: Thu, 8 Aug 2024 05:54:57 +0800 Subject: [PATCH 7/8] Optimize delete logic --- src/pika_server.cc | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/pika_server.cc b/src/pika_server.cc index f4b436989c..0d7d94ec17 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1254,6 +1254,7 @@ void PikaServer::AutoServerlogPurge() { if (sscanf(file_parts[5].c_str(), "%4d%2d%2d", &log_year, &log_month, &log_day) != 3) { continue; } + //Get the time when the server log file was originally created struct tm log_time; log_time.tm_year = log_year - 1900; @@ -1262,10 +1263,10 @@ void PikaServer::AutoServerlogPurge() { log_time.tm_hour = 0; log_time.tm_min = 0; log_time.tm_sec = 0; - + log_time.tm_isdst = -1; time_t log_timestamp = mktime(&log_time); log_files_by_level[severity_level].push_back({file, log_timestamp}); - } +} // Process files for each log level for (auto& [level, files] : log_files_by_level) { @@ -1273,17 +1274,21 @@ void PikaServer::AutoServerlogPurge() { std::sort(files.begin(), files.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); - bool keep_newest = false; + bool has_recent_file = false; for (const auto& [file, log_timestamp] : files) { double diff_seconds = difftime(now_timestamp, log_timestamp); int64_t interval_days = static_cast(diff_seconds / 86400); - if (interval_days > retention_time && keep_newest) { - std::string log_file = log_path + "/" + file; - LOG(INFO) << "Deleting out of date log file: " << log_file; - if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file; - } else { - keep_newest = true; + if (interval_days <= retention_time) { + has_recent_file = true; + continue; + } + if (!has_recent_file) { + has_recent_file = true; + continue; } + std::string log_file = log_path + "/" + file; + LOG(INFO) << "Deleting out of date log file: " << log_file; + if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file; } } } From 38c0e0ba778e05d3ab551b3cdfeecdf078cd4da1 Mon Sep 17 00:00:00 2001 From: XiaoLiang2333 <1159025475@qq.com> Date: Thu, 8 Aug 2024 05:59:25 +0800 Subject: [PATCH 8/8] update --- src/pika_conf.cc | 4 ++++ src/pika_server.cc | 49 ++++++++++++++++++++++------------------------ 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/pika_conf.cc b/src/pika_conf.cc index 913f669880..ef7d9ab28c 100644 --- a/src/pika_conf.cc +++ b/src/pika_conf.cc @@ -128,6 +128,10 @@ int PikaConf::Load() { if (log_path_[log_path_.length() - 1] != '/') { log_path_ += "/"; } + GetConfInt("log-retention-time",&log_retention_time_); + if(log_retention_time_ < 0){ + LOG(FATAL) << "log-retention-time invalid"; + } GetConfStr("loglevel", &log_level_); GetConfStr("db-path", &db_path_); GetConfInt("db-instance-num", &db_instance_num_); diff --git a/src/pika_server.cc b/src/pika_server.cc index c271ad0c67..0469fbc43e 100644 --- a/src/pika_server.cc +++ b/src/pika_server.cc @@ -1213,7 +1213,7 @@ void PikaServer::AutoBinlogPurge() { DoSameThingEveryDB(TaskType::kPurgeLog); } void PikaServer::AutoServerlogPurge() { std::string log_path = g_pika_conf->log_path(); int retention_time = g_pika_conf->log_retention_time(); - if (retention_time <= 0) { + if (retention_time < 0) { return; } std::vector log_files; @@ -1227,19 +1227,11 @@ void PikaServer::AutoServerlogPurge() { } //Get the current time of system time_t t = time(nullptr); - struct tm* now = localtime(&t); - int now_year = now->tm_year + 1900; - int now_month = now->tm_mon + 1; - int now_day = now->tm_mday; - - struct tm now_time; - now_time.tm_year = now_year - 1900; - now_time.tm_mon = now_month - 1; - now_time.tm_mday = now_day; - now_time.tm_hour = 0; - now_time.tm_min = 0; - now_time.tm_sec = 0; - int64_t now_timestamp = mktime(&now_time); + struct tm* now_time = localtime(&t); + now_time->tm_hour = 0; + now_time->tm_min = 0; + now_time->tm_sec = 0; + time_t now_timestamp = mktime(now_time); std::map>> log_files_by_level; @@ -1260,6 +1252,7 @@ void PikaServer::AutoServerlogPurge() { if (sscanf(file_parts[5].c_str(), "%4d%2d%2d", &log_year, &log_month, &log_day) != 3) { continue; } + //Get the time when the server log file was originally created struct tm log_time; log_time.tm_year = log_year - 1900; @@ -1268,10 +1261,10 @@ void PikaServer::AutoServerlogPurge() { log_time.tm_hour = 0; log_time.tm_min = 0; log_time.tm_sec = 0; - - int64_t log_timestamp = mktime(&log_time); + log_time.tm_isdst = -1; + time_t log_timestamp = mktime(&log_time); log_files_by_level[severity_level].push_back({file, log_timestamp}); - } +} // Process files for each log level for (auto& [level, files] : log_files_by_level) { @@ -1279,17 +1272,21 @@ void PikaServer::AutoServerlogPurge() { std::sort(files.begin(), files.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); - bool keep_newest = false; + bool has_recent_file = false; for (const auto& [file, log_timestamp] : files) { - int64_t interval_days = (now_timestamp - log_timestamp) / 86400; - - if (interval_days > retention_time && keep_newest) { - std::string log_file = log_path + "/" + file; - LOG(INFO) << "Deleting out of date log file: " << log_file; - if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file; - } else { - keep_newest = true; + double diff_seconds = difftime(now_timestamp, log_timestamp); + int64_t interval_days = static_cast(diff_seconds / 86400); + if (interval_days <= retention_time) { + has_recent_file = true; + continue; + } + if (!has_recent_file) { + has_recent_file = true; + continue; } + std::string log_file = log_path + "/" + file; + LOG(INFO) << "Deleting out of date log file: " << log_file; + if(!pstd::DeleteFile(log_file)) LOG(ERROR) << "Failed to delete log file: " << log_file; } } }