Merge branch 'develop' into ximinez/fix/validator-cache

* XRPLF/develop:
  chore: Remove empty Taker.h (6984)
  chore: Enable clang-tidy modernize checks (6975)
  ci: Upload clang-tidy git diff (6983)
  fix: Add rounding to Vault invariants (6217) (6955)

cfg/xrpld-example.cfg

@@ -1033,11 +1033,10 @@
 #                           The online delete process checks periodically
 #                           that xrpld is still in sync with the network,
 #                           and that the validated ledger is less than
-#                           'age_threshold_seconds' old, and that all
-#                           recent ledgers are available. If not, then continue
+#                           'age_threshold_seconds' old. If not, then continue
 #                           sleeping for this number of seconds and
 #                           checking until healthy.
-#                           Default is 1.
+#                           Default is 5.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.

include/xrpl/ledger/helpers/NFTokenHelpers.h

@@ -20,6 +20,10 @@ removeTokenOffersWithLimit(
     Keylet const& directory,
     std::size_t maxDeletableOffers);
 
+/** Returns tesSUCCESS if NFToken has few enough offers that it can be burned */
+TER
+notTooManyOffers(ReadView const& view, uint256 const& nftokenID);
+
 /** Finds the specified token in the owner's token directory. */
 std::optional<STObject>
 findToken(ReadView const& view, AccountID const& owner, uint256 const& nftokenID);

src/libxrpl/ledger/helpers/NFTokenHelpers.cpp

@@ -621,6 +621,33 @@ removeTokenOffersWithLimit(ApplyView& view, Keylet const& directory, std::size_t
     return deletedOffersCount;
 }
 
+TER
+notTooManyOffers(ReadView const& view, uint256 const& nftokenID)
+{
+    std::size_t totalOffers = 0;
+
+    {
+        Dir const buys(view, keylet::nft_buys(nftokenID));
+        for (auto iter = buys.begin(); iter != buys.end(); iter.next_page())
+        {
+            totalOffers += iter.page_size();
+            if (totalOffers > maxDeletableTokenOfferEntries)
+                return tefTOO_BIG;
+        }
+    }
+
+    {
+        Dir const sells(view, keylet::nft_sells(nftokenID));
+        for (auto iter = sells.begin(); iter != sells.end(); iter.next_page())
+        {
+            totalOffers += iter.page_size();
+            if (totalOffers > maxDeletableTokenOfferEntries)
+                return tefTOO_BIG;
+        }
+    }
+    return tesSUCCESS;
+}
+
 bool
 deleteTokenOffer(ApplyView& view, std::shared_ptr<SLE> const& offer)
 {

src/test/app/LedgerMaster_test.cpp

@@ -5,21 +5,17 @@
 #include <test/jtx/noop.h>
 
 #include <xrpld/app/ledger/LedgerMaster.h>
-#include <xrpld/app/misc/SHAMapStore.h>
 #include <xrpld/core/Config.h>
 
-#include <xrpl/basics/ToString.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/beast/unit_test/suite.h>
 #include <xrpl/protocol/Feature.h>
-#include <xrpl/protocol/Protocol.h>
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/STTx.h>
 
 #include <cstdint>
 #include <memory>
-#include <sstream>
 #include <vector>
 
 namespace xrpl::test {
@@ -115,73 +111,6 @@ class LedgerMaster_test : public beast::unit_test::suite
         }
     }
 
-    void
-    testCompleteLedgerRange(FeatureBitset features)
-    {
-        // Note that this test is intentionally very similar to
-        // SHAMapStore_test::testLedgerGaps, but has a different
-        // focus.
-
-        testcase("Complete Ledger operations");
-
-        using namespace test::jtx;
-
-        auto const deleteInterval = 8;
-
-        Env env{*this, envconfig([](auto cfg) {
-                    return online_delete(std::move(cfg), deleteInterval);
-                })};
-
-        auto const alice = Account("alice");
-        env.fund(XRP(1000), alice);
-        env.close();
-
-        auto& lm = env.app().getLedgerMaster();
-        LedgerIndex minSeq = 2;
-        LedgerIndex maxSeq = env.closed()->header().seq;
-        auto& store = env.app().getSHAMapStore();
-        store.rendezvous();
-        LedgerIndex lastRotated = store.getLastRotated();
-        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
-        BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
-        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-
-        // Close enough ledgers to rotate a few times
-        for (int i = 0; i < 24; ++i)
-        {
-            for (int t = 0; t < 3; ++t)
-            {
-                env(noop(alice));
-            }
-            env.close();
-            store.rendezvous();
-
-            ++maxSeq;
-
-            if (maxSeq == lastRotated + deleteInterval)
-            {
-                minSeq = lastRotated;
-                lastRotated = maxSeq;
-            }
-            BEAST_EXPECTS(
-                env.closed()->header().seq == maxSeq, to_string(env.closed()->header().seq));
-            BEAST_EXPECTS(store.getLastRotated() == lastRotated, to_string(store.getLastRotated()));
-            std::stringstream expectedRange;
-            expectedRange << minSeq << "-" << maxSeq;
-            BEAST_EXPECTS(lm.getCompleteLedgers() == expectedRange.str(), lm.getCompleteLedgers());
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-        }
-    }
-
 public:
     void
     run() override
@@ -195,7 +124,6 @@ public:
     testWithFeats(FeatureBitset features)
     {
         testTxnIdFromIndex(features);
-        testCompleteLedgerRange(features);
     }
 };
 

src/test/app/SHAMapStore_test.cpp

@@ -1,9 +1,7 @@
 #include <test/jtx/Env.h>
 #include <test/jtx/amount.h>
 #include <test/jtx/envconfig.h>
-#include <test/jtx/noop.h>
 
-#include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/main/Application.h>
 #include <xrpld/app/main/NodeStoreScheduler.h>
 #include <xrpld/app/misc/SHAMapStore.h>
@@ -12,7 +10,6 @@
 #include <xrpld/core/ConfigSections.h>
 
 #include <xrpl/basics/ByteUtilities.h>
-#include <xrpl/basics/Number.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/beast/unit_test/suite.h>
 #include <xrpl/json/json_value.h>
@@ -32,9 +29,7 @@
 #include <map>
 #include <memory>
 #include <optional>
-#include <sstream>
 #include <string>
-#include <thread>
 #include <utility>
 
 namespace xrpl::test {
@@ -46,8 +41,10 @@ class SHAMapStore_test : public beast::unit_test::suite
     static auto
     onlineDelete(std::unique_ptr<Config> cfg)
     {
-        using namespace jtx;
-        return online_delete(std::move(cfg), deleteInterval);
+        cfg->LEDGER_HISTORY = deleteInterval;
+        auto& section = cfg->section(ConfigSection::nodeDatabase());
+        section.set("online_delete", std::to_string(deleteInterval));
+        return cfg;
     }
 
     static auto
@@ -587,160 +584,6 @@ public:
         BEAST_EXPECT(dbr->getName() == "3");
     }
 
-    void
-    testLedgerGaps()
-    {
-        // Note that this test is intentionally very similar to
-        // LedgerMaster_test::testCompleteLedgerRange, but has a different
-        // focus.
-
-        testcase("Wait for ledger gaps to fill in");
-
-        using namespace test::jtx;
-
-        Env env{*this, envconfig(onlineDelete)};
-
-        auto failureMessage = [&](char const* label, auto expected, auto actual) {
-            std::stringstream ss;
-            ss << label << ": Expected: " << expected << ", Got: " << actual;
-            return ss.str();
-        };
-
-        auto const alice = Account("alice");
-        env.fund(XRP(1000), alice);
-        env.close();
-
-        auto& lm = env.app().getLedgerMaster();
-        LedgerIndex minSeq = 2;
-        LedgerIndex maxSeq = env.closed()->header().seq;
-        auto& store = env.app().getSHAMapStore();
-        store.rendezvous();
-        LedgerIndex lastRotated = store.getLastRotated();
-        BEAST_EXPECTS(maxSeq == 3, std::to_string(maxSeq));
-        BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
-        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-
-        // Close enough ledgers to rotate a few times
-        while (maxSeq < 20)
-        {
-            for (int t = 0; t < 3; ++t)
-            {
-                env(noop(alice));
-            }
-            env.close();
-            store.rendezvous();
-
-            ++maxSeq;
-
-            if (maxSeq + 1 == lastRotated + deleteInterval)
-            {
-                using namespace std::chrono_literals;
-
-                // The next ledger will trigger a rotation. Delete the
-                // current ledger from LedgerMaster.
-                std::this_thread::sleep_for(100ms);
-                LedgerIndex const deleteSeq = maxSeq;
-                while (!lm.haveLedger(deleteSeq))
-                {
-                    std::this_thread::sleep_for(100ms);
-                }
-                lm.clearLedger(deleteSeq);
-
-                auto expectedRange = [](auto minSeq, auto deleteSeq, auto maxSeq) {
-                    std::stringstream expectedRange;
-                    expectedRange << minSeq << "-" << (deleteSeq - 1);
-                    if (deleteSeq + 1 == maxSeq)
-                    {
-                        expectedRange << "," << maxSeq;
-                    }
-                    else if (deleteSeq < maxSeq)
-                    {
-                        expectedRange << "," << (deleteSeq + 1) << "-" << maxSeq;
-                    }
-                    return expectedRange.str();
-                };
-                BEAST_EXPECTS(
-                    lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
-                    failureMessage(
-                        "Complete ledgers",
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                        lm.getCompleteLedgers()));
-                BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
-
-                // Close another ledger, which will trigger a rotation, but the
-                // rotation will be stuck until the missing ledger is filled in.
-                env.close();
-                // DO NOT CALL rendezvous()! You'll end up with a deadlock.
-                ++maxSeq;
-
-                // Nothing has changed
-                BEAST_EXPECTS(
-                    store.getLastRotated() == lastRotated,
-                    failureMessage("lastRotated", lastRotated, store.getLastRotated()));
-                BEAST_EXPECTS(
-                    lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
-                    failureMessage(
-                        "Complete ledgers",
-                        expectedRange(minSeq, deleteSeq, maxSeq),
-                        lm.getCompleteLedgers()));
-
-                // Close 5 more ledgers, waiting one second in between to
-                // simulate the ledger making progress while online delete waits
-                // for the missing ledger to be filled in.
-                // This ensures the healthWait check has time to run and
-                // detect the gap.
-                for (int l = 0; l < 5; ++l)
-                {
-                    env.close();
-                    // DO NOT CALL rendezvous()! You'll end up with a deadlock.
-                    ++maxSeq;
-                    // Nothing has changed
-                    BEAST_EXPECTS(
-                        store.getLastRotated() == lastRotated,
-                        failureMessage("lastRotated", lastRotated, store.getLastRotated()));
-                    BEAST_EXPECTS(
-                        lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
-                        failureMessage(
-                            "Complete Ledgers",
-                            expectedRange(minSeq, deleteSeq, maxSeq),
-                            lm.getCompleteLedgers()));
-                    std::this_thread::sleep_for(1s);
-                }
-
-                // Put the missing ledger back in LedgerMaster
-                lm.setLedgerRangePresent(deleteSeq, deleteSeq);
-
-                // Wait for the rotation to finish
-                store.rendezvous();
-
-                minSeq = lastRotated;
-                lastRotated = deleteSeq + 1;
-            }
-            BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
-            BEAST_EXPECTS(
-                env.closed()->header().seq == maxSeq,
-                failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
-            BEAST_EXPECTS(
-                store.getLastRotated() == lastRotated,
-                failureMessage("lastRotated", lastRotated, store.getLastRotated()));
-            std::stringstream expectedRange;
-            expectedRange << minSeq << "-" << maxSeq;
-            BEAST_EXPECTS(
-                lm.getCompleteLedgers() == expectedRange.str(),
-                failureMessage("CompleteLedgers", expectedRange.str(), lm.getCompleteLedgers()));
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
-            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
-        }
-    }
-
     void
     run() override
     {
@@ -748,7 +591,6 @@ public:
         testAutomatic();
         testCanDelete();
         testRotate();
-        testLedgerGaps();
     }
 };
 

src/test/jtx/envconfig.h

@@ -56,17 +56,6 @@ envconfig(F&& modfunc, Args&&... args)
     return modfunc(envconfig(), std::forward<Args>(args)...);
 }
 
-/// @brief adjust config to enable online_delete
-///
-/// @param cfg config instance to be modified
-///
-/// @param deleteInterval how many new ledgers should be available before
-/// rotating. Defaults to 8, because the standalone minimum is 8.
-///
-/// @return unique_ptr to Config instance
-std::unique_ptr<Config>
-online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
-
 /// @brief adjust config so no admin ports are enabled
 ///
 /// this is intended for use with envconfig, as in

src/test/jtx/impl/envconfig.cpp

@@ -6,10 +6,8 @@
 #include <xrpld/core/ConfigSections.h>
 
 #include <atomic>
-#include <cstdint>
 #include <map>
 #include <memory>
-#include <string>
 #include <vector>
 
 namespace xrpl::test {
@@ -61,15 +59,6 @@ setupConfigForUnitTests(Config& cfg)
 
 namespace jtx {
 
-std::unique_ptr<Config>
-online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
-{
-    cfg->LEDGER_HISTORY = deleteInterval;
-    auto& section = cfg->section(ConfigSection::nodeDatabase());
-    section.set("online_delete", std::to_string(deleteInterval));
-    return cfg;
-}
-
 std::unique_ptr<Config>
 no_admin(std::unique_ptr<Config> cfg)
 {

src/xrpld/app/ledger/LedgerMaster.h

@@ -105,10 +105,7 @@ public:
     failedSave(std::uint32_t seq, uint256 const& hash);
 
     std::string
-    getCompleteLedgers() const;
-
-    std::size_t
-    missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
+    getCompleteLedgers();
 
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
@@ -325,7 +322,7 @@ private:
     // A set of transactions to replay during the next close
     std::unique_ptr<LedgerReplay> replayData;
 
-    std::recursive_mutex mutable mCompleteLock;
+    std::recursive_mutex mCompleteLock;
     RangeSet<std::uint32_t> mCompleteLedgers;
 
     // Publish thread is running.

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -1570,34 +1570,12 @@ LedgerMaster::getPublishedLedger()
 }
 
 std::string
-LedgerMaster::getCompleteLedgers() const
+LedgerMaster::getCompleteLedgers()
 {
     std::lock_guard const sl(mCompleteLock);
     return to_string(mCompleteLedgers);
 }
 
-std::size_t
-LedgerMaster::missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const
-{
-    // Make a copy of the range to avoid holding the lock
-    auto const range = [&] {
-        std::lock_guard const sl(mCompleteLock);
-        return mCompleteLedgers;
-    }();
-
-    std::size_t missing = 0;
-
-    for (LedgerIndex idx = first; idx <= last; ++idx)
-    {
-        if (!boost::icl::contains(range, idx))
-        {
-            ++missing;
-        }
-    }
-
-    return missing;
-}
-
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -292,17 +292,6 @@ SHAMapStoreImp::run()
         bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ &&
             canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
 
-        JLOG(journal_.debug()) << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
-
-        {
-            // Note that this is set after the healthWait() check, so that we
-            // don't start the rotation until the validated ledger is fully
-            // processed. It is not guaranteed to be done at this point. It also
-            // allows the testLedgerGaps unit test to work.
-            std::unique_lock<std::mutex> const lock(mutex_);
-            lastGoodValidatedLedger_ = validatedSeq;
-        }
-
         // will delete up to (not including) lastRotated
         if (readyToRotate)
         {
@@ -310,8 +299,7 @@ SHAMapStoreImp::run()
                                   << lastRotated << " deleteInterval " << deleteInterval_
                                   << " canDelete_ " << canDelete_ << " state "
                                   << app_.getOPs().strOperatingMode(false) << " age "
-                                  << ledgerMaster_->getValidatedLedgerAge().count()
-                                  << "s. Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
+                                  << ledgerMaster_->getValidatedLedgerAge().count() << 's';
 
             clearPrior(lastRotated);
             if (healthWait() == stopping)
@@ -371,9 +359,7 @@ SHAMapStoreImp::run()
                     clearCaches(validatedSeq);
                 });
 
-            JLOG(journal_.warn()) << "finished rotation. validatedSeq: " << validatedSeq
-                                  << ", lastRotated: " << lastRotated
-                                  << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
+            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
         }
     }
 }
@@ -613,41 +599,22 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
 SHAMapStoreImp::HealthResult
 SHAMapStoreImp::healthWait()
 {
-    auto index = ledgerMaster_->getValidLedgerIndex();
     auto age = ledgerMaster_->getValidatedLedgerAge();
     OperatingMode mode = netOPs_->getOperatingMode();
     std::unique_lock lock(mutex_);
-
-    auto numMissing =
-        ledgerMaster_->missingFromCompleteLedgerRange(lastGoodValidatedLedger_, index);
-    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
+    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
     {
-        // this value shouldn't change, so grab it while we have the
-        // lock
-        auto const lowerBound = lastGoodValidatedLedger_;
-
         lock.unlock();
-
-        auto const stream =
-            mode != OperatingMode::FULL || age > ageThreshold_ ? journal_.warn() : journal_.info();
-        JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
-                     << "s for node to stabilize. state: "
-                     << app_.getOPs().strOperatingMode(mode, false) << ". age " << age.count()
-                     << "s. Missing ledgers: " << numMissing << ".  Expect: " << lowerBound << "-"
-                     << index << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
+        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
+                              << "s for node to stabilize. state: "
+                              << app_.getOPs().strOperatingMode(mode, false) << ". age "
+                              << age.count() << 's';
         std::this_thread::sleep_for(recoveryWaitTime_);
-        index = ledgerMaster_->getValidLedgerIndex();
         age = ledgerMaster_->getValidatedLedgerAge();
         mode = netOPs_->getOperatingMode();
-        numMissing =
-            lowerBound == 0 ? 0 : ledgerMaster_->missingFromCompleteLedgerRange(lowerBound, index);
-
         lock.lock();
     }
 
-    JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to " << index;
-    lastGoodValidatedLedger_ = index;
-
     return stop_ ? stopping : keepGoing;
 }
 

src/xrpld/app/misc/SHAMapStoreImp.h

@@ -70,11 +70,6 @@ private:
     std::thread thread_;
     bool stop_ = false;
     bool healthy_ = true;
-    // Used to prevent ledger gaps from forming during online deletion. Keeps
-    // track of the last validated ledger that was processed without gaps. There
-    // are no guarantees about gaps while online delete is not running. For
-    // that, use advisory_delete and check for gaps externally.
-    LedgerIndex lastGoodValidatedLedger_ = 0;
     mutable std::condition_variable cond_;
     mutable std::condition_variable rendezvous_;
     mutable std::mutex mutex_;
@@ -88,11 +83,11 @@ private:
     std::uint32_t deleteBatch_ = 100;
     std::chrono::milliseconds backOff_{100};
     std::chrono::seconds ageThreshold_{60};
-    /// If  the node is out of sync, or any recent ledgers are not
-    /// available during an online_delete healthWait() call, sleep
-    /// the thread for this time, and continue checking until recovery.
+    /// If  the node is out of sync during an online_delete healthWait()
+    /// call, sleep the thread for this time, and continue checking until
+    /// recovery.
     /// See also: "recovery_wait_seconds" in xrpld-example.cfg
-    std::chrono::seconds recoveryWaitTime_{1};
+    std::chrono::seconds recoveryWaitTime_{5};
 
     // these do not exist upon SHAMapStore creation, but do exist
     // as of run() or before
@@ -209,8 +204,6 @@ private:
     enum HealthResult { stopping, keepGoing };
     [[nodiscard]] HealthResult
     healthWait();
-    bool
-    hasCompleteRange(LedgerIndex first, LedgerIndex last);
 
 public:
     void

src/xrpld/app/misc/detail/ValidatorSite.cpp

@@ -161,7 +161,11 @@ ValidatorSite::load(
     {
         try
         {
-            sites_.emplace_back(uri);
+            // This is not super efficient, but it doesn't happen often.
+            bool found = std::ranges::any_of(
+                sites_, [&uri](auto const& site) { return site.loadedResource->uri == uri; });
+            if (!found)
+                sites_.emplace_back(uri);
         }
         catch (std::exception const& e)
         {
@@ -222,7 +226,17 @@ ValidatorSite::setTimer(
     std::lock_guard<std::mutex> const& site_lock,
     std::lock_guard<std::mutex> const& state_lock)
 {
-    auto next = std::ranges::min_element(
+    if (!sites_.empty() &&  //
+        std::ranges::all_of(
+            sites_, [](auto const& site) { return site.lastRefreshStatus.has_value(); }))
+    {
+        // If all of the sites have been handled at least once (including
+        // errors and timeouts), call missingSite, which will load the cache
+        // files for any lists that are still unavailable.
+        missingSite(site_lock);
+    }
+
+    auto const next = std::ranges::min_element(
         sites_, [](Site const& a, Site const& b) { return a.nextRefresh < b.nextRefresh; });
 
     if (next != sites_.end())
@@ -333,7 +347,7 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         // processes a network error. Usually, this function runs first,
         // but on extremely rare occasions, the response handler can run
         // first, which will leave activeResource empty.
-        auto const& site = sites_[siteIdx];
+        auto& site = sites_[siteIdx];
         if (site.activeResource)
         {
             JLOG(j_.warn()) << "Request for " << site.activeResource->uri << " took too long";
@@ -341,6 +355,9 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         else
             JLOG(j_.error()) << "Request took too long, but a response has "
                                 "already been processed";
+        if (!site.lastRefreshStatus)
+            site.lastRefreshStatus.emplace(
+                Site::Status{clock_type::now(), ListDisposition::invalid, "timeout"});
     }
 
     std::lock_guard const lock_state{state_mutex_};