From 3501d1236f2b93c961ff4a1bc7ff9d10137204cf Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Wed, 19 May 2021 16:27:36 -0700 Subject: [PATCH 1/4] apply buckets: move reset code to when we reset --- src/catchup/ApplyBucketsWork.cpp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/catchup/ApplyBucketsWork.cpp b/src/catchup/ApplyBucketsWork.cpp index f2f4e3f0ab..743252e06d 100644 --- a/src/catchup/ApplyBucketsWork.cpp +++ b/src/catchup/ApplyBucketsWork.cpp @@ -77,6 +77,14 @@ ApplyBucketsWork::onReset() if (!isAborting()) { + // clear ledgerTxn state of all ledger entries in preparation of bucket + // apply + if (!mApp.getConfig().MODE_USES_IN_MEMORY_LEDGER) + { + auto& lsRoot = mApp.getLedgerTxnRoot(); + lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); + } + auto addBucket = [this](std::shared_ptr const& bucket) { if (bucket->getSize() > 0) { @@ -114,18 +122,6 @@ ApplyBucketsWork::startLevel() bool applySnap = (i.snap != binToHex(level.getSnap()->getHash())); bool applyCurr = (i.curr != binToHex(level.getCurr()->getHash())); - if (!mApplying && !mApp.getConfig().MODE_USES_IN_MEMORY_LEDGER && - (applySnap || applyCurr)) - { - uint32_t oldestLedger = applySnap - ? BucketList::oldestLedgerInSnap( - mApplyState.currentLedger, mLevel) - : BucketList::oldestLedgerInCurr( - mApplyState.currentLedger, mLevel); - auto& lsRoot = mApp.getLedgerTxnRoot(); - lsRoot.deleteObjectsModifiedOnOrAfterLedger(oldestLedger); - } - if (mApplying || applySnap) { mSnapBucket = getBucket(i.snap); From b51ad628e1fcb8c3e97120d7e424b6b97f0a76a4 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Wed, 19 May 2021 16:41:12 -0700 Subject: [PATCH 2/4] refactor code to use resetLedgerState --- src/catchup/ApplyBucketsWork.cpp | 3 +-- src/main/Application.h | 4 ++++ src/main/ApplicationImpl.cpp | 7 +++++++ src/main/ApplicationImpl.h | 2 ++ src/main/CommandLine.cpp | 3 +-- 5 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/catchup/ApplyBucketsWork.cpp b/src/catchup/ApplyBucketsWork.cpp index 743252e06d..eea4f8d872 100644 --- a/src/catchup/ApplyBucketsWork.cpp +++ b/src/catchup/ApplyBucketsWork.cpp @@ -81,8 +81,7 @@ ApplyBucketsWork::onReset() // apply if (!mApp.getConfig().MODE_USES_IN_MEMORY_LEDGER) { - auto& lsRoot = mApp.getLedgerTxnRoot(); - lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); + mApp.resetLedgerState(); } auto addBucket = [this](std::shared_ptr const& bucket) { diff --git a/src/main/Application.h b/src/main/Application.h index ea4ffafdea..57468f018e 100644 --- a/src/main/Application.h +++ b/src/main/Application.h @@ -164,6 +164,10 @@ class Application virtual void initialize(bool createNewDB) = 0; + // reset the ledger state entirely + // (to be used before applying buckets) + virtual void resetLedgerState() = 0; + // Return the time in seconds since the POSIX epoch, according to the // VirtualClock this Application is bound to. Convenience method. virtual uint64_t timeNow() = 0; diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index 897f83440e..fa8e464cc8 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -220,6 +220,13 @@ ApplicationImpl::initialize(bool createNewDB) LOG_DEBUG(DEFAULT_LOG, "Application constructed"); } +void +ApplicationImpl::resetLedgerState() +{ + auto& lsRoot = getLedgerTxnRoot(); + lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); +} + void ApplicationImpl::newDB() { diff --git a/src/main/ApplicationImpl.h b/src/main/ApplicationImpl.h index eba5f58ffc..be3d5ac941 100644 --- a/src/main/ApplicationImpl.h +++ b/src/main/ApplicationImpl.h @@ -44,6 +44,8 @@ class ApplicationImpl : public Application virtual void initialize(bool newDB) override; + virtual void resetLedgerState() override; + virtual uint64_t timeNow() override; virtual Config const& getConfig() override; diff --git a/src/main/CommandLine.cpp b/src/main/CommandLine.cpp index 49cfbcfffc..383e2a964b 100644 --- a/src/main/CommandLine.cpp +++ b/src/main/CommandLine.cpp @@ -774,8 +774,7 @@ runCatchup(CommandLineArgs const& args) LOG_INFO( DEFAULT_LOG, "Resetting ledger state to genesis before catching up"); - auto& lsRoot = app->getLedgerTxnRoot(); - lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); + app->resetLedgerState(); lm.startNewLedger(); } From 8777f10ea2057dc0e40607d9fd41f576f41a2ff4 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Wed, 19 May 2021 17:18:22 -0700 Subject: [PATCH 3/4] implement and use in memory version of resetLedgerState() --- src/catchup/ApplyBucketsWork.cpp | 5 +---- src/main/ApplicationImpl.cpp | 32 ++++++++++++++++++-------------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/catchup/ApplyBucketsWork.cpp b/src/catchup/ApplyBucketsWork.cpp index eea4f8d872..bf64356726 100644 --- a/src/catchup/ApplyBucketsWork.cpp +++ b/src/catchup/ApplyBucketsWork.cpp @@ -79,10 +79,7 @@ ApplyBucketsWork::onReset() { // clear ledgerTxn state of all ledger entries in preparation of bucket // apply - if (!mApp.getConfig().MODE_USES_IN_MEMORY_LEDGER) - { - mApp.resetLedgerState(); - } + mApp.resetLedgerState(); auto addBucket = [this](std::shared_ptr const& bucket) { if (bucket->getSize() > 0) diff --git a/src/main/ApplicationImpl.cpp b/src/main/ApplicationImpl.cpp index fa8e464cc8..386adde0fe 100644 --- a/src/main/ApplicationImpl.cpp +++ b/src/main/ApplicationImpl.cpp @@ -164,19 +164,9 @@ ApplicationImpl::initialize(bool createNewDB) mBanManager = BanManager::create(*this); mStatusManager = std::make_unique(); -#ifdef BEST_OFFER_DEBUGGING - auto const bestOfferDebuggingEnabled = mConfig.BEST_OFFER_DEBUGGING_ENABLED; -#endif - if (getConfig().MODE_USES_IN_MEMORY_LEDGER) { - mLedgerTxnRoot = std::make_unique( -#ifdef BEST_OFFER_DEBUGGING - bestOfferDebuggingEnabled -#endif - ); - mNeverCommittingLedgerTxn = - std::make_unique(*mLedgerTxnRoot); + resetLedgerState(); } else { @@ -191,7 +181,7 @@ ApplicationImpl::initialize(bool createNewDB) *mDatabase, mConfig.ENTRY_CACHE_SIZE, mConfig.PREFETCH_BATCH_SIZE #ifdef BEST_OFFER_DEBUGGING , - bestOfferDebuggingEnabled + mConfig.BEST_OFFER_DEBUGGING_ENABLED #endif ); } @@ -223,8 +213,22 @@ ApplicationImpl::initialize(bool createNewDB) void ApplicationImpl::resetLedgerState() { - auto& lsRoot = getLedgerTxnRoot(); - lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); + if (getConfig().MODE_USES_IN_MEMORY_LEDGER) + { + mNeverCommittingLedgerTxn.reset(); + mLedgerTxnRoot = std::make_unique( +#ifdef BEST_OFFER_DEBUGGING + mConfig.BEST_OFFER_DEBUGGING_ENABLED +#endif + ); + mNeverCommittingLedgerTxn = + std::make_unique(*mLedgerTxnRoot); + } + else + { + auto& lsRoot = getLedgerTxnRoot(); + lsRoot.deleteObjectsModifiedOnOrAfterLedger(0); + } } void From 1e0f0d1dbfe80bec37fd6074e610c194b77ef6d1 Mon Sep 17 00:00:00 2001 From: MonsieurNicolas Date: Wed, 19 May 2021 17:18:38 -0700 Subject: [PATCH 4/4] Windows: warning police --- src/herder/TransactionQueue.cpp | 3 --- src/main/Config.cpp | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/herder/TransactionQueue.cpp b/src/herder/TransactionQueue.cpp index 47a75973a9..282b24c154 100644 --- a/src/herder/TransactionQueue.cpp +++ b/src/herder/TransactionQueue.cpp @@ -740,9 +740,6 @@ TransactionQueue::getMaxOpsToFloodThisPeriod() const opsToFlood = opsToFloodLedger; } releaseAssertOrThrow(opsToFlood >= 0); - releaseAssertOrThrow( - opsToFlood <= - static_cast(std::numeric_limits::max())); return static_cast(opsToFlood); } diff --git a/src/main/Config.cpp b/src/main/Config.cpp index 19e88dfd9e..9d1aae28fc 100644 --- a/src/main/Config.cpp +++ b/src/main/Config.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -733,7 +734,7 @@ Config::processOpApplySleepTimeForTestingConfigs() ret.reserve(100); for (size_t i = 0; i < OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING.size(); i++) { - LOG_INFO(DEFAULT_LOG, "Sleeps for {} {}\% of the time", + LOG_INFO(DEFAULT_LOG, "Sleeps for {} {}% of the time", OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING[i], OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING[i]); for (size_t j = 0; j < OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING[i]; j++)