Improve node store rotation

- Add a special case to healthWait() to not pause when the server is
  DISCONNECTED.
- Log sequence differences at the start and end of rotation.
- Limit copy-forward to ledgers we're not about to delete, and unknown.
  (Changes rotationInFlight_ from a bool to a LedgerIndex.)
- Set the "rotation in flight" index right at the beginning of the rotation
    - Because the node copy process can take a long time, other ledgers may
      get validated. Any reads for those ledgers have the potential to be
      served by the archive DB and thus lost, too. Why wait?
- Add an assertion suggested on @vlntb in #7763.
- Don't wait as long for ledgers that should be built soon.
- Rescue nodes from the tree node cache, too.

include/xrpl/nodestore/DatabaseRotating.h

@@ -4,6 +4,7 @@
 #include <xrpl/nodestore/Backend.h>
 #include <xrpl/nodestore/Database.h>
 #include <xrpl/nodestore/Scheduler.h>
+#include <xrpl/protocol/Protocol.h>
 
 #include <functional>
 #include <memory>
@@ -45,15 +46,15 @@ public:
     /**
      * Marks an online-delete rotation as in progress (or completed).
      *
-     * While in flight, a read served by the archive backend is copied
-     * forward into the writable backend even for ordinary
-     * (duplicate == false) fetches: the archive is about to be deleted,
-     * and a node body canonicalized into caches during the rotation
-     * window would otherwise survive only in RAM once the archive is
-     * dropped.
+     * While in flight, a read for ledgers after the inFlight value served by the archive backend is
+     * copied forward into the writable backend even for ordinary (duplicate == false) fetches: the
+     * archive is about to be deleted, and a node body canonicalized into caches during the rotation
+     * window would otherwise survive only in RAM once the archive is dropped.
      */
     virtual void
-    setRotationInFlight(bool inFlight) = 0;
+    setRotationInFlight(LedgerIndex inFlight) = 0;
+    virtual LedgerIndex
+    getRotationInFlight() const = 0;
 };
 
 }  // namespace xrpl::NodeStore

include/xrpl/nodestore/detail/DatabaseRotatingImp.h

@@ -8,6 +8,7 @@
 #include <xrpl/nodestore/DatabaseRotating.h>
 #include <xrpl/nodestore/NodeObject.h>
 #include <xrpl/nodestore/Scheduler.h>
+#include <xrpl/protocol/Protocol.h>
 
 #include <atomic>
 #include <cstdint>
@@ -71,20 +72,27 @@ public:
     sweep() override;
 
     void
-    setRotationInFlight(bool inFlight) override;
+    setRotationInFlight(LedgerIndex inFlight) override;
+    LedgerIndex
+    getRotationInFlight() const override;
 
 private:
     std::shared_ptr<Backend> writableBackend_;
     std::shared_ptr<Backend> archiveBackend_;
     mutable std::mutex mutex_;
 
-    // True between SHAMapStore starting the cache-freshen phase and the
-    // completion of rotate(). While true, archive hits on ordinary
-    // (duplicate == false) fetches are copied forward into the writable
-    // backend; copyForwardCount_ tallies them per rotation for the
+    // Set to the index of the last rotated ledger between SHAMapStore
+    // starting the cache-freshen phase and the completion of rotate().
+    // While non-zero, archive hits on ordinary (duplicate == false)
+    // fetches are copied forward into the writable backend if they are
+    // for that ledger or later, since those are the ones we'll keep.
+    // To be safe, copy forward if the provided ledger index is 0.
+    // copyForwardCount_ tallies them per rotation for the
     // summary line logged at swap.
-    std::atomic<bool> rotationInFlight_{false};
+    // copyRejectCount_ tallies the ones that weren't copied.
+    std::atomic<LedgerIndex> rotationInFlight_{0};
     std::atomic<std::uint64_t> copyForwardCount_{0};
+    std::atomic<std::uint64_t> copyRejectCount_{0};
 
     std::shared_ptr<NodeObject>
     fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)