Merge branch 'ximinez/online-delete-gaps' into ximinez/online-delete-lastrotated

- First approximation. Good enough and consistent enough for unit tests.
  Not sure about real world, but it'll be an improvement either way.

.github/scripts/strategy-matrix/generate.py

@@ -130,8 +130,8 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
         if os['distro_name'] == 'rhel' and architecture['platform'] == 'linux/arm64':
             continue
 
-        # We skip all clang 20+ on arm64 due to Boost build error.
-        if f'{os['compiler_name']}-{os['compiler_version']}' in ['clang-20', 'clang-21'] and architecture['platform'] == 'linux/arm64':
+        # We skip all clang-20 on arm64 due to boost 1.86 build error
+        if f'{os['compiler_name']}-{os['compiler_version']}' == 'clang-20' and architecture['platform'] == 'linux/arm64':
             continue
 
         # Enable code coverage for Debian Bookworm using GCC 15 in Debug and no

.github/scripts/strategy-matrix/linux.json

@@ -15,91 +15,63 @@
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "0525eae"
-    },
-    {
-      "distro_name": "debian",
-      "distro_version": "trixie",
-      "compiler_name": "gcc",
-      "compiler_version": "14",
-      "image_sha": "0525eae"
-    },
-    {
-      "distro_name": "debian",
-      "distro_version": "trixie",
-      "compiler_name": "gcc",
-      "compiler_version": "15",
-      "image_sha": "0525eae"
-    },
-    {
-      "distro_name": "debian",
-      "distro_version": "trixie",
-      "compiler_name": "clang",
-      "compiler_version": "20",
-      "image_sha": "0525eae"
-    },
-    {
-      "distro_name": "debian",
-      "distro_version": "trixie",
-      "compiler_name": "clang",
-      "compiler_version": "21",
-      "image_sha": "0525eae"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",

cfg/rippled-example.cfg

@@ -1051,10 +1051,11 @@
 #                           The online delete process checks periodically
 #                           that rippled is still in sync with the network,
 #                           and that the validated ledger is less than
-#                           'age_threshold_seconds' old. If not, then continue
+#                           'age_threshold_seconds' old, and that all
+#                           recent ledgers are available. If not, then continue
 #                           sleeping for this number of seconds and
 #                           checking until healthy.
-#                           Default is 5.
+#                           Default is 1.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.

include/xrpl/basics/SHAMapHash.h

@@ -2,6 +2,7 @@
 #define XRPL_BASICS_SHAMAP_HASH_H_INCLUDED
 
 #include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/partitioned_unordered_map.h>
 
 #include <ostream>
 

include/xrpl/basics/TaggedCache.h

@@ -71,6 +71,9 @@ public:
     int
     getCacheSize() const;
 
+    int
+    getTrackSize() const;
+
     float
     getHitRate();
 
@@ -148,6 +151,9 @@ public:
     bool
     retrieve(key_type const& key, T& data);
 
+    mutex_type&
+    peekMutex();
+
     std::vector<key_type>
     getKeys() const;
 
@@ -168,14 +174,11 @@ public:
 
 private:
     SharedPointerType
-    initialFetch(key_type const& key);
+    initialFetch(key_type const& key, std::lock_guard<mutex_type> const& l);
 
     void
     collect_metrics();
 
-    Mutex&
-    lockPartition(key_type const& key) const;
-
 private:
     struct Stats
     {
@@ -278,8 +281,8 @@ private:
         [[maybe_unused]] clock_type::time_point const& now,
         typename KeyValueCacheType::map_type& partition,
         SweptPointersVector& stuffToSweep,
-        std::atomic<int>& allRemoval,
-        Mutex& partitionLock);
+        std::atomic<int>& allRemovals,
+        std::lock_guard<std::recursive_mutex> const&);
 
     [[nodiscard]] std::thread
     sweepHelper(
@@ -288,12 +291,14 @@ private:
         typename KeyOnlyCacheType::map_type& partition,
         SweptPointersVector&,
         std::atomic<int>& allRemovals,
-        Mutex& partitionLock);
+        std::lock_guard<std::recursive_mutex> const&);
 
     beast::Journal m_journal;
     clock_type& m_clock;
     Stats m_stats;
 
+    mutex_type mutable m_mutex;
+
     // Used for logging
     std::string m_name;
 
@@ -304,11 +309,10 @@ private:
     clock_type::duration const m_target_age;
 
     // Number of items cached
-    std::atomic<int> m_cache_count;
+    int m_cache_count;
     cache_type m_cache;  // Hold strong reference to recent objects
-    std::atomic<std::uint64_t> m_hits;
-    std::atomic<std::uint64_t> m_misses;
-    mutable std::vector<mutex_type> partitionLocks_;
+    std::uint64_t m_hits;
+    std::uint64_t m_misses;
 };
 
 }  // namespace ripple

include/xrpl/basics/TaggedCache.ipp

@@ -3,7 +3,6 @@
 
 #include <xrpl/basics/IntrusivePointer.ipp>
 #include <xrpl/basics/TaggedCache.h>
-#include <xrpl/beast/core/CurrentThreadName.h>
 
 namespace ripple {
 
@@ -42,7 +41,6 @@ inline TaggedCache<
     , m_hits(0)
     , m_misses(0)
 {
-    partitionLocks_ = std::vector<mutex_type>(m_cache.partitions());
 }
 
 template <
@@ -88,13 +86,8 @@ TaggedCache<
     KeyEqual,
     Mutex>::size() const
 {
-    std::size_t totalSize = 0;
-    for (size_t i = 0; i < partitionLocks_.size(); ++i)
-    {
-        std::lock_guard<Mutex> lock(partitionLocks_[i]);
-        totalSize += m_cache.map()[i].size();
-    }
-    return totalSize;
+    std::lock_guard lock(m_mutex);
+    return m_cache.size();
 }
 
 template <
@@ -117,7 +110,32 @@ TaggedCache<
     KeyEqual,
     Mutex>::getCacheSize() const
 {
-    return m_cache_count.load(std::memory_order_relaxed);
+    std::lock_guard lock(m_mutex);
+    return m_cache_count;
+}
+
+template <
+    class Key,
+    class T,
+    bool IsKeyCache,
+    class SharedWeakUnionPointer,
+    class SharedPointerType,
+    class Hash,
+    class KeyEqual,
+    class Mutex>
+inline int
+TaggedCache<
+    Key,
+    T,
+    IsKeyCache,
+    SharedWeakUnionPointer,
+    SharedPointerType,
+    Hash,
+    KeyEqual,
+    Mutex>::getTrackSize() const
+{
+    std::lock_guard lock(m_mutex);
+    return m_cache.size();
 }
 
 template <
@@ -140,10 +158,9 @@ TaggedCache<
     KeyEqual,
     Mutex>::getHitRate()
 {
-    auto const hits = m_hits.load(std::memory_order_relaxed);
-    auto const misses = m_misses.load(std::memory_order_relaxed);
-    float const total = float(hits + misses);
-    return hits * (100.0f / std::max(1.0f, total));
+    std::lock_guard lock(m_mutex);
+    auto const total = static_cast<float>(m_hits + m_misses);
+    return m_hits * (100.0f / std::max(1.0f, total));
 }
 
 template <
@@ -166,12 +183,9 @@ TaggedCache<
     KeyEqual,
     Mutex>::clear()
 {
-    for (auto& mutex : partitionLocks_)
-        mutex.lock();
+    std::lock_guard lock(m_mutex);
     m_cache.clear();
-    for (auto& mutex : partitionLocks_)
-        mutex.unlock();
-    m_cache_count.store(0, std::memory_order_relaxed);
+    m_cache_count = 0;
 }
 
 template <
@@ -194,9 +208,11 @@ TaggedCache<
     KeyEqual,
     Mutex>::reset()
 {
-    clear();
-    m_hits.store(0, std::memory_order_relaxed);
-    m_misses.store(0, std::memory_order_relaxed);
+    std::lock_guard lock(m_mutex);
+    m_cache.clear();
+    m_cache_count = 0;
+    m_hits = 0;
+    m_misses = 0;
 }
 
 template <
@@ -220,7 +236,7 @@ TaggedCache<
     KeyEqual,
     Mutex>::touch_if_exists(KeyComparable const& key)
 {
-    std::lock_guard<Mutex> lock(lockPartition(key));
+    std::lock_guard lock(m_mutex);
     auto const iter(m_cache.find(key));
     if (iter == m_cache.end())
     {
@@ -262,6 +278,8 @@ TaggedCache<
 
     auto const start = std::chrono::steady_clock::now();
     {
+        std::lock_guard lock(m_mutex);
+
         if (m_target_size == 0 ||
             (static_cast<int>(m_cache.size()) <= m_target_size))
         {
@@ -293,13 +311,12 @@ TaggedCache<
                 m_cache.map()[p],
                 allStuffToSweep[p],
                 allRemovals,
-                partitionLocks_[p]));
+                lock));
         }
         for (std::thread& worker : workers)
             worker.join();
 
-        int removals = allRemovals.load(std::memory_order_relaxed);
-        m_cache_count.fetch_sub(removals, std::memory_order_relaxed);
+        m_cache_count -= allRemovals;
     }
     // At this point allStuffToSweep will go out of scope outside the lock
     // and decrement the reference count on each strong pointer.
@@ -333,8 +350,7 @@ TaggedCache<
 {
     // Remove from cache, if !valid, remove from map too. Returns true if
     // removed from cache
-
-    std::lock_guard<Mutex> lock(lockPartition(key));
+    std::lock_guard lock(m_mutex);
 
     auto cit = m_cache.find(key);
 
@@ -347,7 +363,7 @@ TaggedCache<
 
     if (entry.isCached())
     {
-        m_cache_count.fetch_sub(1, std::memory_order_relaxed);
+        --m_cache_count;
         entry.ptr.convertToWeak();
         ret = true;
     }
@@ -385,16 +401,17 @@ TaggedCache<
 {
     // Return canonical value, store if needed, refresh in cache
     // Return values: true=we had the data already
+    std::lock_guard lock(m_mutex);
 
-    std::lock_guard<Mutex> lock(lockPartition(key));
     auto cit = m_cache.find(key);
+
     if (cit == m_cache.end())
     {
         m_cache.emplace(
             std::piecewise_construct,
             std::forward_as_tuple(key),
             std::forward_as_tuple(m_clock.now(), data));
-        m_cache_count.fetch_add(1, std::memory_order_relaxed);
+        ++m_cache_count;
         return false;
     }
 
@@ -443,12 +460,12 @@ TaggedCache<
             data = cachedData;
         }
 
-        m_cache_count.fetch_add(1, std::memory_order_relaxed);
+        ++m_cache_count;
         return true;
     }
 
     entry.ptr = data;
-    m_cache_count.fetch_add(1, std::memory_order_relaxed);
+    ++m_cache_count;
 
     return false;
 }
@@ -524,11 +541,10 @@ TaggedCache<
     KeyEqual,
     Mutex>::fetch(key_type const& key)
 {
-    std::lock_guard<Mutex> lock(lockPartition(key));
-
-    auto ret = initialFetch(key);
+    std::lock_guard<mutex_type> l(m_mutex);
+    auto ret = initialFetch(key, l);
     if (!ret)
-        m_misses.fetch_add(1, std::memory_order_relaxed);
+        ++m_misses;
     return ret;
 }
 
@@ -592,8 +608,8 @@ TaggedCache<
     Mutex>::insert(key_type const& key)
     -> std::enable_if_t<IsKeyCache, ReturnType>
 {
+    std::lock_guard lock(m_mutex);
     clock_type::time_point const now(m_clock.now());
-    std::lock_guard<Mutex> lock(lockPartition(key));
     auto [it, inserted] = m_cache.emplace(
         std::piecewise_construct,
         std::forward_as_tuple(key),
@@ -633,6 +649,29 @@ TaggedCache<
     return true;
 }
 
+template <
+    class Key,
+    class T,
+    bool IsKeyCache,
+    class SharedWeakUnionPointer,
+    class SharedPointerType,
+    class Hash,
+    class KeyEqual,
+    class Mutex>
+inline auto
+TaggedCache<
+    Key,
+    T,
+    IsKeyCache,
+    SharedWeakUnionPointer,
+    SharedPointerType,
+    Hash,
+    KeyEqual,
+    Mutex>::peekMutex() -> mutex_type&
+{
+    return m_mutex;
+}
+
 template <
     class Key,
     class T,
@@ -656,13 +695,10 @@ TaggedCache<
     std::vector<key_type> v;
 
     {
+        std::lock_guard lock(m_mutex);
         v.reserve(m_cache.size());
-        for (std::size_t i = 0; i < partitionLocks_.size(); ++i)
-        {
-            std::lock_guard<Mutex> lock(partitionLocks_[i]);
-            for (auto const& entry : m_cache.map()[i])
-                v.push_back(entry.first);
-        }
+        for (auto const& _ : m_cache)
+            v.push_back(_.first);
     }
 
     return v;
@@ -688,12 +724,11 @@ TaggedCache<
     KeyEqual,
     Mutex>::rate() const
 {
-    auto const hits = m_hits.load(std::memory_order_relaxed);
-    auto const misses = m_misses.load(std::memory_order_relaxed);
-    auto const tot = hits + misses;
+    std::lock_guard lock(m_mutex);
+    auto const tot = m_hits + m_misses;
     if (tot == 0)
-        return 0.0;
-    return double(hits) / tot;
+        return 0;
+    return double(m_hits) / tot;
 }
 
 template <
@@ -717,16 +752,18 @@ TaggedCache<
     KeyEqual,
     Mutex>::fetch(key_type const& digest, Handler const& h)
 {
-    std::lock_guard<Mutex> lock(lockPartition(digest));
-
-    if (auto ret = initialFetch(digest))
-        return ret;
+    {
+        std::lock_guard l(m_mutex);
+        if (auto ret = initialFetch(digest, l))
+            return ret;
+    }
 
     auto sle = h();
     if (!sle)
         return {};
 
-    m_misses.fetch_add(1, std::memory_order_relaxed);
+    std::lock_guard l(m_mutex);
+    ++m_misses;
     auto const [it, inserted] =
         m_cache.emplace(digest, Entry(m_clock.now(), std::move(sle)));
     if (!inserted)
@@ -753,10 +790,9 @@ TaggedCache<
     SharedPointerType,
     Hash,
     KeyEqual,
-    Mutex>::initialFetch(key_type const& key)
+    Mutex>::
+    initialFetch(key_type const& key, std::lock_guard<mutex_type> const& l)
 {
-    std::lock_guard<Mutex> lock(lockPartition(key));
-
     auto cit = m_cache.find(key);
     if (cit == m_cache.end())
         return {};
@@ -764,7 +800,7 @@ TaggedCache<
     Entry& entry = cit->second;
     if (entry.isCached())
     {
-        m_hits.fetch_add(1, std::memory_order_relaxed);
+        ++m_hits;
         entry.touch(m_clock.now());
         return entry.ptr.getStrong();
     }
@@ -772,13 +808,12 @@ TaggedCache<
     if (entry.isCached())
     {
         // independent of cache size, so not counted as a hit
-        m_cache_count.fetch_add(1, std::memory_order_relaxed);
+        ++m_cache_count;
         entry.touch(m_clock.now());
         return entry.ptr.getStrong();
     }
 
     m_cache.erase(cit);
-
     return {};
 }
 
@@ -807,11 +842,10 @@ TaggedCache<
     {
         beast::insight::Gauge::value_type hit_rate(0);
         {
-            auto const hits = m_hits.load(std::memory_order_relaxed);
-            auto const misses = m_misses.load(std::memory_order_relaxed);
-            auto const total = hits + misses;
+            std::lock_guard lock(m_mutex);
+            auto const total(m_hits + m_misses);
             if (total != 0)
-                hit_rate = (hits * 100) / total;
+                hit_rate = (m_hits * 100) / total;
         }
         m_stats.hit_rate.set(hit_rate);
     }
@@ -842,16 +876,12 @@ TaggedCache<
         typename KeyValueCacheType::map_type& partition,
         SweptPointersVector& stuffToSweep,
         std::atomic<int>& allRemovals,
-        Mutex& partitionLock)
+        std::lock_guard<std::recursive_mutex> const&)
 {
     return std::thread([&, this]() {
-        beast::setCurrentThreadName("sweep-KVCache");
-
         int cacheRemovals = 0;
         int mapRemovals = 0;
 
-        std::lock_guard<Mutex> lock(partitionLock);
-
         // Keep references to all the stuff we sweep
         // so that we can destroy them outside the lock.
         stuffToSweep.reserve(partition.size());
@@ -935,16 +965,12 @@ TaggedCache<
         typename KeyOnlyCacheType::map_type& partition,
         SweptPointersVector&,
         std::atomic<int>& allRemovals,
-        Mutex& partitionLock)
+        std::lock_guard<std::recursive_mutex> const&)
 {
     return std::thread([&, this]() {
-        beast::setCurrentThreadName("sweep-KCache");
-
         int cacheRemovals = 0;
         int mapRemovals = 0;
 
-        std::lock_guard<Mutex> lock(partitionLock);
-
         // Keep references to all the stuff we sweep
         // so that we can destroy them outside the lock.
         {
@@ -979,29 +1005,6 @@ TaggedCache<
     });
 }
 
-template <
-    class Key,
-    class T,
-    bool IsKeyCache,
-    class SharedWeakUnionPointer,
-    class SharedPointerType,
-    class Hash,
-    class KeyEqual,
-    class Mutex>
-inline Mutex&
-TaggedCache<
-    Key,
-    T,
-    IsKeyCache,
-    SharedWeakUnionPointer,
-    SharedPointerType,
-    Hash,
-    KeyEqual,
-    Mutex>::lockPartition(key_type const& key) const
-{
-    return partitionLocks_[m_cache.partition_index(key)];
-}
-
 }  // namespace ripple
 
 #endif

include/xrpl/basics/partitioned_unordered_map.h

@@ -258,12 +258,6 @@ public:
         return map_;
     }
 
-    partition_map_type const&
-    map() const
-    {
-        return map_;
-    }
-
     iterator
     begin()
     {
@@ -308,12 +302,6 @@ public:
         return cend();
     }
 
-    std::size_t
-    partition_index(key_type const& key) const
-    {
-        return partitioner(key);
-    }
-
 private:
     template <class T>
     void

include/xrpl/protocol/detail/features.macro

@@ -66,6 +66,7 @@ XRPL_FEATURE(XRPFees,                    Supported::yes, VoteBehavior::DefaultNo
 XRPL_FEATURE(DisallowIncoming,           Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (RemoveNFTokenAutoTrustLine, Supported::yes, VoteBehavior::DefaultYes)
 XRPL_FEATURE(FlowSortStrands,            Supported::yes, VoteBehavior::DefaultYes)
+XRPL_FEATURE(NegativeUNL,                Supported::yes, VoteBehavior::DefaultYes)
 XRPL_FEATURE(RequireFullyCanonicalSig,   Supported::yes, VoteBehavior::DefaultYes)
 XRPL_FEATURE(DeletableAccounts,          Supported::yes, VoteBehavior::DefaultYes)
 XRPL_FEATURE(Checks,                     Supported::yes, VoteBehavior::DefaultYes)
@@ -129,7 +130,6 @@ XRPL_RETIRE_FEATURE(HardenedValidations)
 XRPL_RETIRE_FEATURE(ImmediateOfferKilled)
 XRPL_RETIRE_FEATURE(MultiSign)
 XRPL_RETIRE_FEATURE(MultiSignReserve)
-XRPL_RETIRE_FEATURE(NegativeUNL)
 XRPL_RETIRE_FEATURE(NonFungibleTokensV1_1)
 XRPL_RETIRE_FEATURE(PayChan)
 XRPL_RETIRE_FEATURE(SortedDirectories)

src/test/app/LedgerMaster_test.cpp

@@ -2,6 +2,7 @@
 #include <test/jtx/Env.h>
 
 #include <xrpld/app/ledger/LedgerMaster.h>
+#include <xrpld/app/misc/SHAMapStore.h>
 
 namespace ripple {
 namespace test {
@@ -100,6 +101,89 @@ class LedgerMaster_test : public beast::unit_test::suite
         }
     }
 
+    void
+    testCompleteLedgerRange(FeatureBitset features)
+    {
+        // Note that this test is intentionally very similar to
+        // SHAMapStore_test::testLedgerGaps, but has a different
+        // focus.
+
+        testcase("Complete Ledger operations");
+
+        using namespace test::jtx;
+
+        auto const deleteInterval = 8;
+
+        Env env{*this, envconfig([](auto cfg) {
+                    return online_delete(std::move(cfg), deleteInterval);
+                })};
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->info().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
+        BEAST_EXPECTS(
+            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        for (int i = 0; i < 24; ++i)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq == lastRotated + deleteInterval)
+            {
+                minSeq = lastRotated;
+                lastRotated = store.getLastRotated();
+                BEAST_EXPECT(lastRotated = maxSeq + 2);
+            }
+            BEAST_EXPECTS(
+                env.closed()->info().seq == maxSeq,
+                to_string(env.closed()->info().seq));
+            BEAST_EXPECTS(
+                store.getLastRotated() == lastRotated,
+                to_string(store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(
+                lm.getCompleteLedgers() == expectedRange.str(),
+                lm.getCompleteLedgers());
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
 public:
     void
     run() override
@@ -113,6 +197,7 @@ public:
     testWithFeats(FeatureBitset features)
     {
         testTxnIdFromIndex(features);
+        testCompleteLedgerRange(features);
     }
 };
 

src/test/app/SHAMapStore_test.cpp

@@ -1,6 +1,7 @@
 #include <test/jtx.h>
 #include <test/jtx/envconfig.h>
 
+#include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/main/Application.h>
 #include <xrpld/app/main/NodeStoreScheduler.h>
 #include <xrpld/app/misc/SHAMapStore.h>
@@ -10,6 +11,8 @@
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
 #include <xrpl/protocol/jss.h>
 
+#include <thread>
+
 namespace ripple {
 namespace test {
 
@@ -20,10 +23,8 @@ class SHAMapStore_test : public beast::unit_test::suite
     static auto
     onlineDelete(std::unique_ptr<Config> cfg)
     {
-        cfg->LEDGER_HISTORY = deleteInterval;
-        auto& section = cfg->section(ConfigSection::nodeDatabase());
-        section.set("online_delete", std::to_string(deleteInterval));
-        return cfg;
+        using namespace jtx;
+        return online_delete(std::move(cfg), deleteInterval);
     }
 
     static auto
@@ -226,9 +227,10 @@ public:
 
         store.rendezvous();
 
-        BEAST_EXPECT(store.getLastRotated() == deleteInterval + 3);
+        BEAST_EXPECT(env.closed()->info().seq == deleteInterval + 3);
         lastRotated = store.getLastRotated();
-        BEAST_EXPECT(lastRotated == 11);
+        BEAST_EXPECT(lastRotated == deleteInterval + 5);
+        BEAST_EXPECT(lastRotated == 13);
 
         // That took care of the fake hashes
         ledgerCheck(env, deleteInterval + 1, 3);
@@ -236,9 +238,11 @@ public:
         accountTransactionCheck(env, 2 * deleteInterval);
 
         // The last iteration of this loop should trigger a rotate
-        for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1;
+        for (auto i = lastRotated - 3; i < lastRotated + deleteInterval - 1;
              ++i)
         {
+            BEAST_EXPECT(store.getLastRotated() == lastRotated);
+
             env.close();
 
             ledgerTmp = env.rpc("ledger", "current");
@@ -256,7 +260,8 @@ public:
 
         store.rendezvous();
 
-        BEAST_EXPECT(store.getLastRotated() == deleteInterval + lastRotated);
+        BEAST_EXPECT(
+            store.getLastRotated() == deleteInterval + lastRotated + 2);
 
         ledgerCheck(env, deleteInterval + 1, lastRotated);
         transactionCheck(env, 0);
@@ -397,8 +402,8 @@ public:
 
         ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
 
-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);
 
         for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
         {
@@ -425,10 +430,10 @@ public:
 
         store.rendezvous();
 
-        ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
+        ledgerCheck(env, ledgerSeq - firstBatch - 2, firstBatch + 2);
 
-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);
 
         // This does not kick off a cleanup
         canDelete = env.rpc("can_delete", "always");
@@ -464,13 +469,13 @@ public:
 
         ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
 
-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);
 
         // This does not kick off a cleanup
         canDelete = env.rpc("can_delete", "now");
         BEAST_EXPECT(!RPC::contains_error(canDelete[jss::result]));
-        BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
+        BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == lastRotated);
 
         for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
         {
@@ -499,8 +504,8 @@ public:
 
         ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);
 
-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);
     }
 
     std::unique_ptr<NodeStore::Backend>
@@ -626,6 +631,212 @@ public:
         BEAST_EXPECT(dbr->getName() == "3");
     }
 
+    void
+    testLedgerGaps()
+    {
+        // Note that this test is intentionally very similar to
+        // LedgerMaster_test::testCompleteLedgerRange, but has a different
+        // focus.
+
+        testcase("Wait for ledger gaps to fill in");
+
+        using namespace test::jtx;
+
+        Env env{*this, envconfig(onlineDelete)};
+
+        auto& ns = env.app().getNodeStore();
+        std::map<LedgerIndex, uint256> hashes;
+        auto storeHash = [&]() {
+            hashes.emplace(
+                env.current()->info().seq, env.current()->info().hash);
+
+            auto const& root =
+                ns.fetchNodeObject(hashes[env.current()->info().seq]);
+            BEAST_EXPECT(root);
+        };
+
+        auto failureMessage = [&](char const* label,
+                                  auto expected,
+                                  auto actual) {
+            std::stringstream ss;
+            ss << label << ": Expected: " << expected << ", Got: " << actual;
+            return ss.str();
+        };
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+        storeHash();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->info().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
+        BEAST_EXPECTS(
+            lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(
+            lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        while (maxSeq < 28)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            storeHash();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq + 1 == lastRotated + deleteInterval)
+            {
+                using namespace std::chrono_literals;
+
+                // The next ledger will trigger a rotation. Delete the
+                // current ledger from LedgerMaster.
+                std::this_thread::sleep_for(100ms);
+                LedgerIndex const deleteSeq = maxSeq;
+                while (!lm.haveLedger(deleteSeq))
+                {
+                    std::this_thread::sleep_for(100ms);
+                }
+                lm.clearLedger(deleteSeq);
+
+                auto expectedRange =
+                    [](auto minSeq, auto deleteSeq, auto maxSeq) {
+                        std::stringstream expectedRange;
+                        expectedRange << minSeq << "-" << (deleteSeq - 1);
+                        if (deleteSeq + 1 == maxSeq)
+                            expectedRange << "," << maxSeq;
+                        else if (deleteSeq < maxSeq)
+                            expectedRange << "," << (deleteSeq + 1) << "-"
+                                          << maxSeq;
+                        return expectedRange.str();
+                    };
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() ==
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+                BEAST_EXPECT(
+                    lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
+
+                // Close another ledger, which will trigger a rotation, but the
+                // rotation will be stuck until the missing ledger is filled in.
+                env.close();
+                storeHash();
+                // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                ++maxSeq;
+
+                // Nothing has changed
+                BEAST_EXPECTS(
+                    store.getLastRotated() == lastRotated,
+                    failureMessage(
+                        "lastRotated", lastRotated, store.getLastRotated()));
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() ==
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+
+                // Close 5 more ledgers, waiting one second in between to
+                // simulate the ledger making progress while online delete waits
+                // for the missing ledger to be filled in.
+                // This ensures the healthWait check has time to run and
+                // detect the gap.
+                for (int l = 0; l < 5; ++l)
+                {
+                    env.close();
+                    storeHash();
+                    // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                    ++maxSeq;
+                    // Nothing has changed
+                    BEAST_EXPECTS(
+                        store.getLastRotated() == lastRotated,
+                        failureMessage(
+                            "lastRotated",
+                            lastRotated,
+                            store.getLastRotated()));
+                    BEAST_EXPECTS(
+                        lm.getCompleteLedgers() ==
+                            expectedRange(minSeq, deleteSeq, maxSeq),
+                        failureMessage(
+                            "Complete Ledgers",
+                            expectedRange(minSeq, deleteSeq, maxSeq),
+                            lm.getCompleteLedgers()));
+                    std::this_thread::sleep_for(1s);
+                }
+
+                // Put the missing ledger back in LedgerMaster
+                lm.setLedgerRangePresent(deleteSeq, deleteSeq);
+
+                // Wait for the rotation to finish
+                store.rendezvous();
+
+                minSeq = lastRotated;
+                lastRotated = maxSeq + 2;
+
+                // Bypass caching and try to load the ledger roots from the node
+                // store
+                for (auto const& [seq, hash] : hashes)
+                {
+                    auto const nodeObject = ns.fetchNodeObject(hash);
+                    std::stringstream ss;
+                    ss << "minSeq: " << minSeq << ", maxSeq: " << maxSeq
+                       << ", search: " << seq << ". Should "
+                       << (seq < minSeq ? "NOT " : "") << "be found";
+                    if (seq < minSeq)
+                        BEAST_EXPECTS(!nodeObject, ss.str());
+                    else
+                        BEAST_EXPECTS(nodeObject, ss.str());
+                }
+            }
+            BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
+            BEAST_EXPECTS(
+                env.closed()->info().seq == maxSeq,
+                failureMessage("maxSeq", maxSeq, env.closed()->info().seq));
+            BEAST_EXPECTS(
+                store.getLastRotated() == lastRotated,
+                failureMessage(
+                    "lastRotated", lastRotated, store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(
+                lm.getCompleteLedgers() == expectedRange.str(),
+                failureMessage(
+                    "CompleteLedgers",
+                    expectedRange.str(),
+                    lm.getCompleteLedgers()));
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(
+                lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
     void
     run() override
     {
@@ -633,6 +844,7 @@ public:
         testAutomatic();
         testCanDelete();
         testRotate();
+        testLedgerGaps();
     }
 };
 

src/test/basics/TaggedCache_test.cpp

@@ -39,10 +39,10 @@ public:
         // Insert an item, retrieve it, and age it so it gets purged.
         {
             BEAST_EXPECT(c.getCacheSize() == 0);
-            BEAST_EXPECT(c.size() == 0);
+            BEAST_EXPECT(c.getTrackSize() == 0);
             BEAST_EXPECT(!c.insert(1, "one"));
             BEAST_EXPECT(c.getCacheSize() == 1);
-            BEAST_EXPECT(c.size() == 1);
+            BEAST_EXPECT(c.getTrackSize() == 1);
 
             {
                 std::string s;
@@ -53,7 +53,7 @@ public:
             ++clock;
             c.sweep();
             BEAST_EXPECT(c.getCacheSize() == 0);
-            BEAST_EXPECT(c.size() == 0);
+            BEAST_EXPECT(c.getTrackSize() == 0);
         }
 
         // Insert an item, maintain a strong pointer, age it, and
@@ -61,7 +61,7 @@ public:
         {
             BEAST_EXPECT(!c.insert(2, "two"));
             BEAST_EXPECT(c.getCacheSize() == 1);
-            BEAST_EXPECT(c.size() == 1);
+            BEAST_EXPECT(c.getTrackSize() == 1);
 
             {
                 auto p = c.fetch(2);
@@ -69,14 +69,14 @@ public:
                 ++clock;
                 c.sweep();
                 BEAST_EXPECT(c.getCacheSize() == 0);
-                BEAST_EXPECT(c.size() == 1);
+                BEAST_EXPECT(c.getTrackSize() == 1);
             }
 
             // Make sure its gone now that our reference is gone
             ++clock;
             c.sweep();
             BEAST_EXPECT(c.getCacheSize() == 0);
-            BEAST_EXPECT(c.size() == 0);
+            BEAST_EXPECT(c.getTrackSize() == 0);
         }
 
         // Insert the same key/value pair and make sure we get the same result
@@ -92,7 +92,7 @@ public:
             ++clock;
             c.sweep();
             BEAST_EXPECT(c.getCacheSize() == 0);
-            BEAST_EXPECT(c.size() == 0);
+            BEAST_EXPECT(c.getTrackSize() == 0);
         }
 
         // Put an object in but keep a strong pointer to it, advance the clock a
@@ -102,24 +102,24 @@ public:
             // Put an object in
             BEAST_EXPECT(!c.insert(4, "four"));
             BEAST_EXPECT(c.getCacheSize() == 1);
-            BEAST_EXPECT(c.size() == 1);
+            BEAST_EXPECT(c.getTrackSize() == 1);
 
             {
                 // Keep a strong pointer to it
                 auto const p1 = c.fetch(4);
                 BEAST_EXPECT(p1 != nullptr);
                 BEAST_EXPECT(c.getCacheSize() == 1);
-                BEAST_EXPECT(c.size() == 1);
+                BEAST_EXPECT(c.getTrackSize() == 1);
                 // Advance the clock a lot
                 ++clock;
                 c.sweep();
                 BEAST_EXPECT(c.getCacheSize() == 0);
-                BEAST_EXPECT(c.size() == 1);
+                BEAST_EXPECT(c.getTrackSize() == 1);
                 // Canonicalize a new object with the same key
                 auto p2 = std::make_shared<std::string>("four");
                 BEAST_EXPECT(c.canonicalize_replace_client(4, p2));
                 BEAST_EXPECT(c.getCacheSize() == 1);
-                BEAST_EXPECT(c.size() == 1);
+                BEAST_EXPECT(c.getTrackSize() == 1);
                 // Make sure we get the original object
                 BEAST_EXPECT(p1.get() == p2.get());
             }
@@ -127,7 +127,7 @@ public:
             ++clock;
             c.sweep();
             BEAST_EXPECT(c.getCacheSize() == 0);
-            BEAST_EXPECT(c.size() == 0);
+            BEAST_EXPECT(c.getTrackSize() == 0);
         }
     }
 };

src/test/consensus/NegativeUNL_test.cpp

@@ -15,6 +15,7 @@ namespace test {
 /*
  * This file implements the following negative UNL related tests:
  * -- test filling and applying ttUNL_MODIFY Tx and ledger update
+ * -- test ttUNL_MODIFY Tx failure without featureNegativeUNL amendment
  * -- test the NegativeUNLVote class. The test cases are split to multiple
  *    test classes to allow parallel execution.
  * -- test the negativeUNLFilter function
@@ -207,7 +208,7 @@ class NegativeUNL_test : public beast::unit_test::suite
 
         testcase("Create UNLModify Tx and apply to ledgers");
 
-        jtx::Env env(*this, jtx::testable_amendments());
+        jtx::Env env(*this, jtx::testable_amendments() | featureNegativeUNL);
         std::vector<PublicKey> publicKeys = createPublicKeys(3);
         // genesis ledger
         auto l = std::make_shared<Ledger>(
@@ -215,6 +216,7 @@ class NegativeUNL_test : public beast::unit_test::suite
             env.app().config(),
             std::vector<uint256>{},
             env.app().getNodeFamily());
+        BEAST_EXPECT(l->rules().enabled(featureNegativeUNL));
 
         // Record the public keys and ledger sequences of expected negative UNL
         // validators when we build the ledger history
@@ -498,6 +500,44 @@ class NegativeUNL_test : public beast::unit_test::suite
     }
 };
 
+class NegativeUNLNoAmendment_test : public beast::unit_test::suite
+{
+    void
+    testNegativeUNLNoAmendment()
+    {
+        testcase("No negative UNL amendment");
+
+        jtx::Env env(*this, jtx::testable_amendments() - featureNegativeUNL);
+        std::vector<PublicKey> publicKeys = createPublicKeys(1);
+        // genesis ledger
+        auto l = std::make_shared<Ledger>(
+            create_genesis,
+            env.app().config(),
+            std::vector<uint256>{},
+            env.app().getNodeFamily());
+        BEAST_EXPECT(!l->rules().enabled(featureNegativeUNL));
+
+        // generate more ledgers
+        for (auto i = 0; i < 256 - 1; ++i)
+        {
+            l = std::make_shared<Ledger>(
+                *l, env.app().timeKeeper().closeTime());
+        }
+        BEAST_EXPECT(l->seq() == 256);
+        auto txDisable_0 = createTx(true, l->seq(), publicKeys[0]);
+        OpenView accum(&*l);
+        BEAST_EXPECT(applyAndTestResult(env, accum, txDisable_0, false));
+        accum.apply(*l);
+        BEAST_EXPECT(negUnlSizeTest(l, 0, false, false));
+    }
+
+    void
+    run() override
+    {
+        testNegativeUNLNoAmendment();
+    }
+};
+
 /**
  * Utility class for creating validators and ledger history
  */
@@ -523,7 +563,7 @@ struct NetworkHistory
     };
 
     NetworkHistory(beast::unit_test::suite& suite, Parameter const& p)
-        : env(suite, jtx::testable_amendments())
+        : env(suite, jtx::testable_amendments() | featureNegativeUNL)
         , param(p)
         , validations(env.app().getValidations())
     {
@@ -1827,6 +1867,7 @@ class NegativeUNLVoteFilterValidations_test : public beast::unit_test::suite
 };
 
 BEAST_DEFINE_TESTSUITE(NegativeUNL, consensus, ripple);
+BEAST_DEFINE_TESTSUITE(NegativeUNLNoAmendment, consensus, ripple);
 
 BEAST_DEFINE_TESTSUITE(NegativeUNLVoteInternal, consensus, ripple);
 BEAST_DEFINE_TESTSUITE_MANUAL(NegativeUNLVoteScoreTable, consensus, ripple);

src/test/jtx/envconfig.h

@@ -58,6 +58,17 @@ envconfig(F&& modfunc, Args&&... args)
     return modfunc(envconfig(), std::forward<Args>(args)...);
 }
 
+/// @brief adjust config to enable online_delete
+///
+/// @param cfg config instance to be modified
+///
+/// @param deleteInterval how many new ledgers should be available before
+/// rotating. Defaults to 8, because the standalone minimum is 8.
+///
+/// @return unique_ptr to Config instance
+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
+
 /// @brief adjust config so no admin ports are enabled
 ///
 /// this is intended for use with envconfig, as in

src/test/jtx/impl/envconfig.cpp

@@ -53,6 +53,15 @@ setupConfigForUnitTests(Config& cfg)
 
 namespace jtx {
 
+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
+{
+    cfg->LEDGER_HISTORY = deleteInterval;
+    auto& section = cfg->section(ConfigSection::nodeDatabase());
+    section.set("online_delete", std::to_string(deleteInterval));
+    return cfg;
+}
+
 std::unique_ptr<Config>
 no_admin(std::unique_ptr<Config> cfg)
 {

src/test/rpc/Feature_test.cpp

@@ -124,8 +124,7 @@ class Feature_test : public beast::unit_test::suite
             featureToName(fixRemoveNFTokenAutoTrustLine) ==
             "fixRemoveNFTokenAutoTrustLine");
         BEAST_EXPECT(featureToName(featureFlow) == "Flow");
-        BEAST_EXPECT(
-            featureToName(featureDeletableAccounts) == "DeletableAccounts");
+        BEAST_EXPECT(featureToName(featureNegativeUNL) == "NegativeUNL");
         BEAST_EXPECT(
             featureToName(fixIncludeKeyletFields) == "fixIncludeKeyletFields");
         BEAST_EXPECT(featureToName(featureTokenEscrow) == "TokenEscrow");

src/xrpld/app/consensus/RCLConsensus.cpp

@@ -346,7 +346,9 @@ RCLConsensus::Adaptor::onClose(
                     prevLedger, validations, initialSet, j_);
             }
         }
-        else if (prevLedger->isVotingLedger())
+        else if (
+            prevLedger->isVotingLedger() &&
+            prevLedger->rules().enabled(featureNegativeUNL))
         {
             // previous ledger was a voting ledger,
             // so the current consensus session is for a flag ledger,
@@ -1013,7 +1015,8 @@ RCLConsensus::Adaptor::preStartRound(
     inboundTransactions_.newRound(prevLgr.seq());
 
     // Notify NegativeUNLVote that new validators are added
-    if (!nowTrusted.empty())
+    if (prevLgr.ledger_->rules().enabled(featureNegativeUNL) &&
+        !nowTrusted.empty())
         nUnlVote_.newValidators(prevLgr.seq() + 1, nowTrusted);
 
     // propose only if we're in sync with the network (and validating)

src/xrpld/app/ledger/LedgerHistory.cpp

@@ -44,6 +44,8 @@ LedgerHistory::insert(
         ledger->stateMap().getHash().isNonZero(),
         "ripple::LedgerHistory::insert : nonzero hash");
 
+    std::unique_lock sl(m_ledgers_by_hash.peekMutex());
+
     bool const alreadyHad = m_ledgers_by_hash.canonicalize_replace_cache(
         ledger->info().hash, ledger);
     if (validated)
@@ -55,6 +57,7 @@ LedgerHistory::insert(
 LedgerHash
 LedgerHistory::getLedgerHash(LedgerIndex index)
 {
+    std::unique_lock sl(m_ledgers_by_hash.peekMutex());
     if (auto it = mLedgersByIndex.find(index); it != mLedgersByIndex.end())
         return it->second;
     return {};
@@ -64,11 +67,13 @@ std::shared_ptr<Ledger const>
 LedgerHistory::getLedgerBySeq(LedgerIndex index)
 {
     {
+        std::unique_lock sl(m_ledgers_by_hash.peekMutex());
         auto it = mLedgersByIndex.find(index);
 
         if (it != mLedgersByIndex.end())
         {
             uint256 hash = it->second;
+            sl.unlock();
             return getLedgerByHash(hash);
         }
     }
@@ -84,6 +89,7 @@ LedgerHistory::getLedgerBySeq(LedgerIndex index)
 
     {
         // Add this ledger to the local tracking by index
+        std::unique_lock sl(m_ledgers_by_hash.peekMutex());
 
         XRPL_ASSERT(
             ret->isImmutable(),
@@ -433,6 +439,8 @@ LedgerHistory::builtLedger(
     XRPL_ASSERT(
         !hash.isZero(), "ripple::LedgerHistory::builtLedger : nonzero hash");
 
+    std::unique_lock sl(m_consensus_validated.peekMutex());
+
     auto entry = std::make_shared<cv_entry>();
     m_consensus_validated.canonicalize_replace_client(index, entry);
 
@@ -473,6 +481,8 @@ LedgerHistory::validatedLedger(
         !hash.isZero(),
         "ripple::LedgerHistory::validatedLedger : nonzero hash");
 
+    std::unique_lock sl(m_consensus_validated.peekMutex());
+
     auto entry = std::make_shared<cv_entry>();
     m_consensus_validated.canonicalize_replace_client(index, entry);
 
@@ -506,9 +516,10 @@ LedgerHistory::validatedLedger(
 bool
 LedgerHistory::fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash)
 {
-    auto ledger = m_ledgers_by_hash.fetch(ledgerHash);
+    std::unique_lock sl(m_ledgers_by_hash.peekMutex());
     auto it = mLedgersByIndex.find(ledgerIndex);
-    if (ledger && (it != mLedgersByIndex.end()) && (it->second != ledgerHash))
+
+    if ((it != mLedgersByIndex.end()) && (it->second != ledgerHash))
     {
         it->second = ledgerHash;
         return false;

src/xrpld/app/ledger/LedgerMaster.h

@@ -108,7 +108,10 @@ public:
     failedSave(std::uint32_t seq, uint256 const& hash);
 
     std::string
-    getCompleteLedgers();
+    getCompleteLedgers() const;
+
+    std::size_t
+    missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;
 
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
@@ -325,7 +328,7 @@ private:
     // A set of transactions to replay during the next close
     std::unique_ptr<LedgerReplay> replayData;
 
-    std::recursive_mutex mCompleteLock;
+    std::recursive_mutex mutable mCompleteLock;
     RangeSet<std::uint32_t> mCompleteLedgers;
 
     // Publish thread is running.

src/xrpld/app/ledger/detail/BuildLedger.cpp

@@ -28,7 +28,7 @@ buildLedgerImpl(
 {
     auto built = std::make_shared<Ledger>(*parent, closeTime);
 
-    if (built->isFlagLedger())
+    if (built->isFlagLedger() && built->rules().enabled(featureNegativeUNL))
     {
         built->updateNegativeUNL();
     }

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -1571,12 +1571,36 @@ LedgerMaster::getPublishedLedger()
 }
 
 std::string
-LedgerMaster::getCompleteLedgers()
+LedgerMaster::getCompleteLedgers() const
 {
     std::lock_guard sl(mCompleteLock);
     return to_string(mCompleteLedgers);
 }
 
+std::size_t
+LedgerMaster::missingFromCompleteLedgerRange(
+    LedgerIndex first,
+    LedgerIndex last) const
+{
+    // Make a copy of the range to avoid holding the lock
+    auto const range = [&] {
+        std::lock_guard sl(mCompleteLock);
+        return mCompleteLedgers;
+    }();
+
+    std::size_t missing = 0;
+
+    for (LedgerIndex idx = first; idx <= last; ++idx)
+    {
+        if (!boost::icl::contains(range, idx))
+        {
+            ++missing;
+        }
+    }
+
+    return missing;
+}
+
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {

src/xrpld/app/misc/NetworkOPs.cpp

@@ -2063,7 +2063,8 @@ NetworkOPsImp::beginConsensus(
         "ripple::NetworkOPsImp::beginConsensus : closedLedger parent matches "
         "hash");
 
-    app_.validators().setNegativeUNL(prevLedger->negativeUNL());
+    if (prevLedger->rules().enabled(featureNegativeUNL))
+        app_.validators().setNegativeUNL(prevLedger->negativeUNL());
     TrustChanges const changes = app_.validators().updateTrusted(
         app_.getValidations().getCurrentNodeIDs(),
         closingInfo.parentCloseTime,

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -289,6 +289,18 @@ SHAMapStoreImp::run()
             validatedSeq >= lastRotated + deleteInterval_ &&
             canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;
 
+        JLOG(journal_.debug())
+            << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
+
+        {
+            // Note that this is set after the healthWait() check, so that we
+            // don't start the rotation until the validated ledger is fully
+            // processed. It is not guaranteed to be done at this point. It also
+            // allows the testLedgerGaps unit test to work.
+            std::unique_lock<std::mutex> lock(mutex_);
+            lastGoodValidatedLedger_ = validatedSeq;
+        }
+
         // will delete up to (not including) lastRotated
         if (readyToRotate)
         {
@@ -297,7 +309,9 @@ SHAMapStoreImp::run()
                 << lastRotated << " deleteInterval " << deleteInterval_
                 << " canDelete_ " << canDelete_ << " state "
                 << app_.getOPs().strOperatingMode(false) << " age "
-                << ledgerMaster_->getValidatedLedgerAge().count() << 's';
+                << ledgerMaster_->getValidatedLedgerAge().count()
+                << "s. Complete ledgers: "
+                << ledgerMaster_->getCompleteLedgers();
 
             clearPrior(lastRotated);
             if (healthWait() == stopping)
@@ -345,7 +359,13 @@ SHAMapStoreImp::run()
             if (healthWait() == stopping)
                 return;
 
-            lastRotated = validatedSeq;
+            // The rotation process takes time, and more ledgers - possibly many
+            // more - may have been written to the now-archive database before
+            // the writable database was created. Update the validatedSeq to the
+            // current validated ledger sequence plus a buffer to ensure that
+            // all modified nodes for that ledger forward are all written to the
+            // new writable database.
+            lastRotated = ledgerMaster_->getValidLedgerIndex() + 2;
 
             dbRotating_->rotate(
                 std::move(newBackend),
@@ -360,7 +380,10 @@ SHAMapStoreImp::run()
                     clearCaches(validatedSeq);
                 });
 
-            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
+            JLOG(journal_.warn())
+                << "finished rotation. validatedSeq: " << validatedSeq
+                << ", lastRotated: " << lastRotated << ". Complete ledgers: "
+                << ledgerMaster_->getCompleteLedgers();
         }
     }
 }
@@ -615,22 +638,47 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
 SHAMapStoreImp::HealthResult
 SHAMapStoreImp::healthWait()
 {
+    auto index = ledgerMaster_->getValidLedgerIndex();
     auto age = ledgerMaster_->getValidatedLedgerAge();
     OperatingMode mode = netOPs_->getOperatingMode();
     std::unique_lock lock(mutex_);
-    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
+
+    auto numMissing = ledgerMaster_->missingFromCompleteLedgerRange(
+        lastGoodValidatedLedger_, index);
+    while (
+        !stop_ &&
+        (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
     {
+        // this value shouldn't change, so grab it while we have the
+        // lock
+        auto const lastGood = lastGoodValidatedLedger_;
+
         lock.unlock();
-        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
-                              << "s for node to stabilize. state: "
-                              << app_.getOPs().strOperatingMode(mode, false)
-                              << ". age " << age.count() << 's';
+
+        auto const stream = mode != OperatingMode::FULL || age > ageThreshold_
+            ? journal_.warn()
+            : journal_.info();
+        JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
+                     << "s for node to stabilize. state: "
+                     << app_.getOPs().strOperatingMode(mode, false) << ". age "
+                     << age.count() << "s. Missing ledgers: " << numMissing
+                     << ". Expect: " << lastGood << "-" << index
+                     << ". Complete ledgers: "
+                     << ledgerMaster_->getCompleteLedgers();
         std::this_thread::sleep_for(recoveryWaitTime_);
+        index = ledgerMaster_->getValidLedgerIndex();
         age = ledgerMaster_->getValidatedLedgerAge();
         mode = netOPs_->getOperatingMode();
+        numMissing =
+            ledgerMaster_->missingFromCompleteLedgerRange(lastGood, index);
+
         lock.lock();
     }
 
+    JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to "
+                           << index;
+    lastGoodValidatedLedger_ = index;
+
     return stop_ ? stopping : keepGoing;
 }
 

src/xrpld/app/misc/SHAMapStoreImp.h

@@ -71,6 +71,11 @@ private:
     std::thread thread_;
     bool stop_ = false;
     bool healthy_ = true;
+    // Used to prevent ledger gaps from forming during online deletion. Keeps
+    // track of the last validated ledger that was processed without gaps. There
+    // are no guarantees about gaps while online delete is not running. For
+    // that, use advisory_delete and check for gaps externally.
+    LedgerIndex lastGoodValidatedLedger_ = 0;
     mutable std::condition_variable cond_;
     mutable std::condition_variable rendezvous_;
     mutable std::mutex mutex_;
@@ -84,11 +89,11 @@ private:
     std::uint32_t deleteBatch_ = 100;
     std::chrono::milliseconds backOff_{100};
     std::chrono::seconds ageThreshold_{60};
-    /// If  the node is out of sync during an online_delete healthWait()
-    /// call, sleep the thread for this time, and continue checking until
-    /// recovery.
+    /// If  the node is out of sync, or any recent ledgers are not
+    /// available during an online_delete healthWait() call, sleep
+    /// the thread for this time, and continue checking until recovery.
     /// See also: "recovery_wait_seconds" in rippled-example.cfg
-    std::chrono::seconds recoveryWaitTime_{5};
+    std::chrono::seconds recoveryWaitTime_{1};
 
     // these do not exist upon SHAMapStore creation, but do exist
     // as of run() or before
@@ -212,6 +217,8 @@ private:
     enum HealthResult { stopping, keepGoing };
     [[nodiscard]] HealthResult
     healthWait();
+    bool
+    hasCompleteRange(LedgerIndex first, LedgerIndex last);
 
 public:
     void

src/xrpld/app/tx/detail/Change.cpp

@@ -51,6 +51,13 @@ Transactor::invokePreflight<Change>(PreflightContext const& ctx)
         return temBAD_SEQUENCE;
     }
 
+    if (ctx.tx.getTxnType() == ttUNL_MODIFY &&
+        !ctx.rules.enabled(featureNegativeUNL))
+    {
+        JLOG(ctx.j.warn()) << "Change: NegativeUNL not enabled";
+        return temDISABLED;
+    }
+
     return tesSUCCESS;
 }
 

src/xrpld/rpc/handlers/GetCounts.cpp

@@ -95,7 +95,7 @@ getCountsJson(Application& app, int minObjectCount)
     ret[jss::treenode_cache_size] =
         app.getNodeFamily().getTreeNodeCache()->getCacheSize();
     ret[jss::treenode_track_size] =
-        static_cast<int>(app.getNodeFamily().getTreeNodeCache()->size());
+        app.getNodeFamily().getTreeNodeCache()->getTrackSize();
 
     std::string uptime;
     auto s = UptimeClock::now();