Merge branch 'ximinez/online-delete-gaps' into ximinez/online-delete-lastrotated

Merge branch 'develop' into ximinez/online-delete-gaps
2026-04-29 15:37:57 +00:00 · 2026-04-28 16:31:50 -04:00 · 2026-04-28 16:23:08 -04:00 · 2026-04-25 14:46:13 -04:00 · 2026-04-25 14:41:57 -04:00 · 2026-04-23 15:56:35 -04:00
9 changed files with 389 additions and 35 deletions
--- a/cfg/xrpld-example.cfg
+++ b/cfg/xrpld-example.cfg
@@ -1033,10 +1033,11 @@
 #                           The online delete process checks periodically
 #                           that xrpld is still in sync with the network,
 #                           and that the validated ledger is less than
-#                           'age_threshold_seconds' old. If not, then continue
+#                           'age_threshold_seconds' old, and that all
+#                           recent ledgers are available. If not, then continue
 #                           sleeping for this number of seconds and
 #                           checking until healthy.
-#                           Default is 5.
+#                           Default is 1.
 #
 #   Notes:
 #       The 'node_db' entry configures the primary, persistent storage.
--- a/src/test/app/LedgerMaster_test.cpp
+++ b/src/test/app/LedgerMaster_test.cpp
@@ -5,17 +5,21 @@
 #include <test/jtx/noop.h>

 #include <xrpld/app/ledger/LedgerMaster.h>
+#include <xrpld/app/misc/SHAMapStore.h>
 #include <xrpld/core/Config.h>

+#include <xrpl/basics/ToString.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/beast/unit_test/suite.h>
 #include <xrpl/protocol/Feature.h>
+#include <xrpl/protocol/Protocol.h>
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/STTx.h>

 #include <cstdint>
 #include <memory>
+#include <sstream>
 #include <vector>

 namespace xrpl::test {
@@ -111,6 +115,74 @@ class LedgerMaster_test : public beast::unit_test::suite
        }
    }

+    void
+    testCompleteLedgerRange(FeatureBitset features)
+    {
+        // Note that this test is intentionally very similar to
+        // SHAMapStore_test::testLedgerGaps, but has a different
+        // focus.
+
+        testcase("Complete Ledger operations");
+
+        using namespace test::jtx;
+
+        auto const deleteInterval = 8;
+
+        Env env{*this, envconfig([](auto cfg) {
+                    return online_delete(std::move(cfg), deleteInterval);
+                })};
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->header().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, to_string(maxSeq));
+        BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        for (int i = 0; i < 24; ++i)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq == lastRotated + deleteInterval)
+            {
+                minSeq = lastRotated;
+                lastRotated = store.getLastRotated();
+                BEAST_EXPECT(lastRotated = maxSeq + 2);
+            }
+            BEAST_EXPECTS(
+                env.closed()->header().seq == maxSeq, to_string(env.closed()->header().seq));
+            BEAST_EXPECTS(store.getLastRotated() == lastRotated, to_string(store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(lm.getCompleteLedgers() == expectedRange.str(), lm.getCompleteLedgers());
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
 public:
    void
    run() override
@@ -124,6 +196,7 @@ public:
    testWithFeats(FeatureBitset features)
    {
        testTxnIdFromIndex(features);
+        testCompleteLedgerRange(features);
    }
 };

--- a/src/test/app/SHAMapStore_test.cpp
+++ b/src/test/app/SHAMapStore_test.cpp
@@ -1,7 +1,9 @@
 #include <test/jtx/Env.h>
 #include <test/jtx/amount.h>
 #include <test/jtx/envconfig.h>
+#include <test/jtx/noop.h>

+#include <xrpld/app/ledger/LedgerMaster.h>
 #include <xrpld/app/main/Application.h>
 #include <xrpld/app/main/NodeStoreScheduler.h>
 #include <xrpld/app/misc/SHAMapStore.h>
@@ -29,7 +31,9 @@
 #include <map>
 #include <memory>
 #include <optional>
+#include <sstream>
 #include <string>
+#include <thread>
 #include <utility>

 namespace xrpl::test {
@@ -41,10 +45,8 @@ class SHAMapStore_test : public beast::unit_test::suite
    static auto
    onlineDelete(std::unique_ptr<Config> cfg)
    {
-        cfg->LEDGER_HISTORY = deleteInterval;
-        auto& section = cfg->section(ConfigSection::nodeDatabase());
-        section.set("online_delete", std::to_string(deleteInterval));
-        return cfg;
+        using namespace jtx;
+        return online_delete(std::move(cfg), deleteInterval);
    }

    static auto
@@ -228,9 +230,10 @@ public:

        store.rendezvous();

-        BEAST_EXPECT(store.getLastRotated() == deleteInterval + 3);
+        BEAST_EXPECT(env.closed()->info().seq == deleteInterval + 3);
        lastRotated = store.getLastRotated();
-        BEAST_EXPECT(lastRotated == 11);
+        BEAST_EXPECT(lastRotated == deleteInterval + 5);
+        BEAST_EXPECT(lastRotated == 13);

        // That took care of the fake hashes
        ledgerCheck(env, deleteInterval + 1, 3);
@@ -238,8 +241,10 @@ public:
        accountTransactionCheck(env, 2 * deleteInterval);

        // The last iteration of this loop should trigger a rotate
-        for (auto i = lastRotated - 1; i < lastRotated + deleteInterval - 1; ++i)
+        for (auto i = lastRotated - 3; i < lastRotated + deleteInterval - 1; ++i)
        {
+            BEAST_EXPECT(store.getLastRotated() == lastRotated);
+
            env.close();

            ledgerTmp = env.rpc("ledger", "current");
@@ -255,7 +260,7 @@ public:

        store.rendezvous();

-        BEAST_EXPECT(store.getLastRotated() == deleteInterval + lastRotated);
+        BEAST_EXPECT(store.getLastRotated() == deleteInterval + lastRotated + 2);

        ledgerCheck(env, deleteInterval + 1, lastRotated);
        transactionCheck(env, 0);
@@ -388,8 +393,8 @@ public:

        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);

-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);

        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
        {
@@ -414,10 +419,10 @@ public:

        store.rendezvous();

-        ledgerCheck(env, ledgerSeq - firstBatch, firstBatch);
+        ledgerCheck(env, ledgerSeq - firstBatch - 2, firstBatch + 2);

-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);

        // This does not kick off a cleanup
        canDelete = env.rpc("can_delete", "always");
@@ -450,13 +455,13 @@ public:

        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);

-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);

        // This does not kick off a cleanup
        canDelete = env.rpc("can_delete", "now");
        BEAST_EXPECT(!RPC::contains_error(canDelete[jss::result]));
-        BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == ledgerSeq - 1);
+        BEAST_EXPECT(canDelete[jss::result][jss::can_delete] == lastRotated);

        for (; ledgerSeq < lastRotated + deleteInterval; ++ledgerSeq)
        {
@@ -483,8 +488,8 @@ public:

        ledgerCheck(env, ledgerSeq - lastRotated, lastRotated);

-        BEAST_EXPECT(store.getLastRotated() == ledgerSeq - 1);
-        lastRotated = ledgerSeq - 1;
+        lastRotated = store.getLastRotated();
+        BEAST_EXPECT(lastRotated == ledgerSeq + 1);
    }

    std::unique_ptr<NodeStore::Backend>
@@ -584,6 +589,187 @@ public:
        BEAST_EXPECT(dbr->getName() == "3");
    }

+    void
+    testLedgerGaps()
+    {
+        // Note that this test is intentionally very similar to
+        // LedgerMaster_test::testCompleteLedgerRange, but has a different
+        // focus.
+
+        testcase("Wait for ledger gaps to fill in");
+
+        using namespace test::jtx;
+
+        Env env{*this, envconfig(onlineDelete)};
+
+        auto& ns = env.app().getNodeStore();
+        std::map<LedgerIndex, uint256> hashes;
+        auto storeHash = [&]() {
+            hashes.emplace(env.current()->info().seq, env.current()->info().hash);
+
+            auto const& root = ns.fetchNodeObject(hashes[env.current()->info().seq]);
+            BEAST_EXPECT(root);
+        };
+
+        auto failureMessage = [&](char const* label, auto expected, auto actual) {
+            std::stringstream ss;
+            ss << label << ": Expected: " << expected << ", Got: " << actual;
+            return ss.str();
+        };
+
+        auto const alice = Account("alice");
+        env.fund(XRP(1000), alice);
+        env.close();
+        storeHash();
+
+        auto& lm = env.app().getLedgerMaster();
+        LedgerIndex minSeq = 2;
+        LedgerIndex maxSeq = env.closed()->header().seq;
+        auto& store = env.app().getSHAMapStore();
+        store.rendezvous();
+        LedgerIndex lastRotated = store.getLastRotated();
+        BEAST_EXPECTS(maxSeq == 3, std::to_string(maxSeq));
+        BEAST_EXPECTS(lm.getCompleteLedgers() == "2-3", lm.getCompleteLedgers());
+        BEAST_EXPECTS(lastRotated == 3, to_string(lastRotated));
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+        BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+
+        // Close enough ledgers to rotate a few times
+        while (maxSeq < 28)
+        {
+            for (int t = 0; t < 3; ++t)
+            {
+                env(noop(alice));
+            }
+            env.close();
+            storeHash();
+            store.rendezvous();
+
+            ++maxSeq;
+
+            if (maxSeq + 1 == lastRotated + deleteInterval)
+            {
+                using namespace std::chrono_literals;
+
+                // The next ledger will trigger a rotation. Delete the
+                // current ledger from LedgerMaster.
+                std::this_thread::sleep_for(100ms);
+                LedgerIndex const deleteSeq = maxSeq;
+                while (!lm.haveLedger(deleteSeq))
+                {
+                    std::this_thread::sleep_for(100ms);
+                }
+                lm.clearLedger(deleteSeq);
+
+                auto expectedRange = [](auto minSeq, auto deleteSeq, auto maxSeq) {
+                    std::stringstream expectedRange;
+                    expectedRange << minSeq << "-" << (deleteSeq - 1);
+                    if (deleteSeq + 1 == maxSeq)
+                    {
+                        expectedRange << "," << maxSeq;
+                    }
+                    else if (deleteSeq < maxSeq)
+                    {
+                        expectedRange << "," << (deleteSeq + 1) << "-" << maxSeq;
+                    }
+                    return expectedRange.str();
+                };
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+                BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 1);
+
+                // Close another ledger, which will trigger a rotation, but the
+                // rotation will be stuck until the missing ledger is filled in.
+                env.close();
+                storeHash();
+                // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                ++maxSeq;
+
+                // Nothing has changed
+                BEAST_EXPECTS(
+                    store.getLastRotated() == lastRotated,
+                    failureMessage("lastRotated", lastRotated, store.getLastRotated()));
+                BEAST_EXPECTS(
+                    lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
+                    failureMessage(
+                        "Complete ledgers",
+                        expectedRange(minSeq, deleteSeq, maxSeq),
+                        lm.getCompleteLedgers()));
+
+                // Close 5 more ledgers, waiting one second in between to
+                // simulate the ledger making progress while online delete waits
+                // for the missing ledger to be filled in.
+                // This ensures the healthWait check has time to run and
+                // detect the gap.
+                for (int l = 0; l < 5; ++l)
+                {
+                    env.close();
+                    storeHash();
+                    // DO NOT CALL rendezvous()! You'll end up with a deadlock.
+                    ++maxSeq;
+                    // Nothing has changed
+                    BEAST_EXPECTS(
+                        store.getLastRotated() == lastRotated,
+                        failureMessage("lastRotated", lastRotated, store.getLastRotated()));
+                    BEAST_EXPECTS(
+                        lm.getCompleteLedgers() == expectedRange(minSeq, deleteSeq, maxSeq),
+                        failureMessage(
+                            "Complete Ledgers",
+                            expectedRange(minSeq, deleteSeq, maxSeq),
+                            lm.getCompleteLedgers()));
+                    std::this_thread::sleep_for(1s);
+                }
+
+                // Put the missing ledger back in LedgerMaster
+                lm.setLedgerRangePresent(deleteSeq, deleteSeq);
+
+                // Wait for the rotation to finish
+                store.rendezvous();
+
+                minSeq = lastRotated;
+                lastRotated = maxSeq + 2;
+
+                // Bypass caching and try to load the ledger roots from the node
+                // store
+                for (auto const& [seq, hash] : hashes)
+                {
+                    auto const nodeObject = ns.fetchNodeObject(hash);
+                    std::stringstream ss;
+                    ss << "minSeq: " << minSeq << ", maxSeq: " << maxSeq << ", search: " << seq
+                       << ". Should " << (seq < minSeq ? "NOT " : "") << "be found";
+                    if (seq < minSeq)
+                        BEAST_EXPECTS(!nodeObject, ss.str());
+                    else
+                        BEAST_EXPECTS(nodeObject, ss.str());
+                }
+            }
+            BEAST_EXPECT(maxSeq != lastRotated + deleteInterval);
+            BEAST_EXPECTS(
+                env.closed()->header().seq == maxSeq,
+                failureMessage("maxSeq", maxSeq, env.closed()->header().seq));
+            BEAST_EXPECTS(
+                store.getLastRotated() == lastRotated,
+                failureMessage("lastRotated", lastRotated, store.getLastRotated()));
+            std::stringstream expectedRange;
+            expectedRange << minSeq << "-" << maxSeq;
+            BEAST_EXPECTS(
+                lm.getCompleteLedgers() == expectedRange.str(),
+                failureMessage("CompleteLedgers", expectedRange.str(), lm.getCompleteLedgers()));
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq, maxSeq) == 0);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 1, maxSeq - 1) == 0);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 1, maxSeq + 1) == 2);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq - 2, maxSeq - 2) == 2);
+            BEAST_EXPECT(lm.missingFromCompleteLedgerRange(minSeq + 2, maxSeq + 2) == 2);
+        }
+    }
+
    void
    run() override
    {
@@ -591,6 +777,7 @@ public:
        testAutomatic();
        testCanDelete();
        testRotate();
+        testLedgerGaps();
    }
 };

--- a/src/test/jtx/envconfig.h
+++ b/src/test/jtx/envconfig.h
@@ -56,6 +56,17 @@ envconfig(F&& modfunc, Args&&... args)
    return modfunc(envconfig(), std::forward<Args>(args)...);
 }

+/// @brief adjust config to enable online_delete
+///
+/// @param cfg config instance to be modified
+///
+/// @param deleteInterval how many new ledgers should be available before
+/// rotating. Defaults to 8, because the standalone minimum is 8.
+///
+/// @return unique_ptr to Config instance
+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval = 8);
+
 /// @brief adjust config so no admin ports are enabled
 ///
 /// this is intended for use with envconfig, as in
--- a/src/test/jtx/impl/envconfig.cpp
+++ b/src/test/jtx/impl/envconfig.cpp
@@ -6,8 +6,10 @@
 #include <xrpld/core/ConfigSections.h>

 #include <atomic>
+#include <cstdint>
 #include <map>
 #include <memory>
+#include <string>
 #include <vector>

 namespace xrpl::test {
@@ -59,6 +61,15 @@ setupConfigForUnitTests(Config& cfg)

 namespace jtx {

+std::unique_ptr<Config>
+online_delete(std::unique_ptr<Config> cfg, std::uint32_t deleteInterval)
+{
+    cfg->LEDGER_HISTORY = deleteInterval;
+    auto& section = cfg->section(ConfigSection::nodeDatabase());
+    section.set("online_delete", std::to_string(deleteInterval));
+    return cfg;
+}
+
 std::unique_ptr<Config>
 no_admin(std::unique_ptr<Config> cfg)
 {
--- a/src/xrpld/app/ledger/LedgerMaster.h
+++ b/src/xrpld/app/ledger/LedgerMaster.h
@@ -105,7 +105,10 @@ public:
    failedSave(std::uint32_t seq, uint256 const& hash);

    std::string
-    getCompleteLedgers();
+    getCompleteLedgers() const;
+
+    std::size_t
+    missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const;

    /** Apply held transactions to the open ledger
        This is normally called as we close the ledger.
@@ -322,7 +325,7 @@ private:
    // A set of transactions to replay during the next close
    std::unique_ptr<LedgerReplay> replayData;

-    std::recursive_mutex mCompleteLock;
+    std::recursive_mutex mutable mCompleteLock;
    RangeSet<std::uint32_t> mCompleteLedgers;

    // Publish thread is running.
--- a/src/xrpld/app/ledger/detail/LedgerMaster.cpp
+++ b/src/xrpld/app/ledger/detail/LedgerMaster.cpp
@@ -1570,12 +1570,34 @@ LedgerMaster::getPublishedLedger()
 }

 std::string
-LedgerMaster::getCompleteLedgers()
+LedgerMaster::getCompleteLedgers() const
 {
    std::lock_guard const sl(mCompleteLock);
    return to_string(mCompleteLedgers);
 }

+std::size_t
+LedgerMaster::missingFromCompleteLedgerRange(LedgerIndex first, LedgerIndex last) const
+{
+    // Make a copy of the range to avoid holding the lock
+    auto const range = [&] {
+        std::lock_guard const sl(mCompleteLock);
+        return mCompleteLedgers;
+    }();
+
+    std::size_t missing = 0;
+
+    for (LedgerIndex idx = first; idx <= last; ++idx)
+    {
+        if (!boost::icl::contains(range, idx))
+        {
+            ++missing;
+        }
+    }
+
+    return missing;
+}
+
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {
--- a/src/xrpld/app/misc/SHAMapStoreImp.cpp
+++ b/src/xrpld/app/misc/SHAMapStoreImp.cpp
@@ -292,6 +292,17 @@ SHAMapStoreImp::run()
        bool const readyToRotate = validatedSeq >= lastRotated + deleteInterval_ &&
            canDelete_ >= lastRotated - 1 && healthWait() == keepGoing;

+        JLOG(journal_.debug()) << "run: Setting lastGoodValidatedLedger_ to " << validatedSeq;
+
+        {
+            // Note that this is set after the healthWait() check, so that we
+            // don't start the rotation until the validated ledger is fully
+            // processed. It is not guaranteed to be done at this point. It also
+            // allows the testLedgerGaps unit test to work.
+            std::unique_lock<std::mutex> const lock(mutex_);
+            lastGoodValidatedLedger_ = validatedSeq;
+        }
+
        // will delete up to (not including) lastRotated
        if (readyToRotate)
        {
@@ -299,7 +310,8 @@ SHAMapStoreImp::run()
                                  << lastRotated << " deleteInterval " << deleteInterval_
                                  << " canDelete_ " << canDelete_ << " state "
                                  << app_.getOPs().strOperatingMode(false) << " age "
-                                  << ledgerMaster_->getValidatedLedgerAge().count() << 's';
+                                  << ledgerMaster_->getValidatedLedgerAge().count()
+                                  << "s. Complete ledgers: " << ledgerMaster_->getCompleteLedgers();

            clearPrior(lastRotated);
            if (healthWait() == stopping)
@@ -345,7 +357,13 @@ SHAMapStoreImp::run()
            if (healthWait() == stopping)
                return;

-            lastRotated = validatedSeq;
+            // The rotation process takes time, and more ledgers - possibly many
+            // more - may have been written to the now-archive database before
+            // the writable database was created. Update the validatedSeq to the
+            // current validated ledger sequence plus a buffer to ensure that
+            // all modified nodes for that ledger forward are all written to the
+            // new writable database.
+            lastRotated = ledgerMaster_->getValidLedgerIndex() + 2;

            dbRotating_->rotate(
                std::move(newBackend),
@@ -359,7 +377,9 @@ SHAMapStoreImp::run()
                    clearCaches(validatedSeq);
                });

-            JLOG(journal_.warn()) << "finished rotation " << validatedSeq;
+            JLOG(journal_.warn()) << "finished rotation. validatedSeq: " << validatedSeq
+                                  << ", lastRotated: " << lastRotated
+                                  << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
        }
    }
 }
@@ -599,22 +619,41 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
 SHAMapStoreImp::HealthResult
 SHAMapStoreImp::healthWait()
 {
+    auto index = ledgerMaster_->getValidLedgerIndex();
    auto age = ledgerMaster_->getValidatedLedgerAge();
    OperatingMode mode = netOPs_->getOperatingMode();
    std::unique_lock lock(mutex_);
-    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_))
+
+    auto numMissing =
+        ledgerMaster_->missingFromCompleteLedgerRange(lastGoodValidatedLedger_, index);
+    while (!stop_ && (mode != OperatingMode::FULL || age > ageThreshold_ || numMissing > 0))
    {
+        // this value shouldn't change, so grab it while we have the
+        // lock
+        auto const lowerBound = lastGoodValidatedLedger_;
+
        lock.unlock();
-        JLOG(journal_.warn()) << "Waiting " << recoveryWaitTime_.count()
-                              << "s for node to stabilize. state: "
-                              << app_.getOPs().strOperatingMode(mode, false) << ". age "
-                              << age.count() << 's';
+
+        auto const stream =
+            mode != OperatingMode::FULL || age > ageThreshold_ ? journal_.warn() : journal_.info();
+        JLOG(stream) << "Waiting " << recoveryWaitTime_.count()
+                     << "s for node to stabilize. state: "
+                     << app_.getOPs().strOperatingMode(mode, false) << ". age " << age.count()
+                     << "s. Missing ledgers: " << numMissing << ".  Expect: " << lowerBound << "-"
+                     << index << ". Complete ledgers: " << ledgerMaster_->getCompleteLedgers();
        std::this_thread::sleep_for(recoveryWaitTime_);
+        index = ledgerMaster_->getValidLedgerIndex();
        age = ledgerMaster_->getValidatedLedgerAge();
        mode = netOPs_->getOperatingMode();
+        numMissing =
+            lowerBound == 0 ? 0 : ledgerMaster_->missingFromCompleteLedgerRange(lowerBound, index);
+
        lock.lock();
    }

+    JLOG(journal_.debug()) << "healthWait: Setting lastGoodValidatedLedger_ to " << index;
+    lastGoodValidatedLedger_ = index;
+
    return stop_ ? stopping : keepGoing;
 }

--- a/src/xrpld/app/misc/SHAMapStoreImp.h
+++ b/src/xrpld/app/misc/SHAMapStoreImp.h
@@ -70,6 +70,11 @@ private:
    std::thread thread_;
    bool stop_ = false;
    bool healthy_ = true;
+    // Used to prevent ledger gaps from forming during online deletion. Keeps
+    // track of the last validated ledger that was processed without gaps. There
+    // are no guarantees about gaps while online delete is not running. For
+    // that, use advisory_delete and check for gaps externally.
+    LedgerIndex lastGoodValidatedLedger_ = 0;
    mutable std::condition_variable cond_;
    mutable std::condition_variable rendezvous_;
    mutable std::mutex mutex_;
@@ -83,11 +88,11 @@ private:
    std::uint32_t deleteBatch_ = 100;
    std::chrono::milliseconds backOff_{100};
    std::chrono::seconds ageThreshold_{60};
-    /// If  the node is out of sync during an online_delete healthWait()
-    /// call, sleep the thread for this time, and continue checking until
-    /// recovery.
+    /// If  the node is out of sync, or any recent ledgers are not
+    /// available during an online_delete healthWait() call, sleep
+    /// the thread for this time, and continue checking until recovery.
    /// See also: "recovery_wait_seconds" in xrpld-example.cfg
-    std::chrono::seconds recoveryWaitTime_{5};
+    std::chrono::seconds recoveryWaitTime_{1};

    // these do not exist upon SHAMapStore creation, but do exist
    // as of run() or before
@@ -204,6 +209,8 @@ private:
    enum HealthResult { stopping, keepGoing };
    [[nodiscard]] HealthResult
    healthWait();
+    bool
+    hasCompleteRange(LedgerIndex first, LedgerIndex last);

 public:
    void