Merge branch 'develop' into ximinez/parse-guard

Co-authored-by: xrplf-ai-reviewer[bot] <266832837+xrplf-ai-reviewer[bot]@users.noreply.github.com>

.github/workflows/on-pr.yml

@@ -25,18 +25,11 @@ on:
       - unlabeled
 
 concurrency:
-  # Use a per-ref group so a newer run (a push, or a change to a label below)
-  # supersedes the in-progress one for that ref. Label events we don't act on get
-  # their own unique group (per run id) instead, keeping them out of the shared
-  # group so real builds keep running. Keep this list in sync with `should-run`.
-  group: >-
-    ${{ github.workflow }}-${{ github.ref }}${{
-      ((github.event.action == 'labeled' || github.event.action == 'unlabeled')
-      && github.event.label.name != 'Ready to merge'
-      && github.event.label.name != 'DraftRunCI'
-      && github.event.label.name != 'Full CI build')
-      && format('-{0}', github.run_id) || ''
-    }}
+  # A single per-ref group with cancel-in-progress means any newer run (a push
+  # or a label change) supersedes the in-progress one for that ref. Keeping
+  # exactly one authoritative run per ref ensures a fast do-nothing run can never
+  # mask a real build's checks.
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 defaults:
@@ -44,21 +37,17 @@ defaults:
     shell: bash
 
 jobs:
-  # This job determines whether the rest of the workflow should run. It runs
-  # when the PR is not a draft (which should also cover merge-group) or has the
-  # 'DraftRunCI' or 'Full CI build' label. For label events it only runs when the
-  # label added or removed is one we act on ('Ready to merge', 'DraftRunCI' or
-  # 'Full CI build'), so unrelated label changes do not trigger a redundant run.
+  # This job determines whether the rest of the workflow should run at all,
+  # based on the current set of labels: it runs when the PR is not a draft
+  # (which should also cover merge-group) or has the 'DraftRunCI' or
+  # 'Full CI build' label. Whether a build then happens, and whether it is the
+  # minimal or full matrix, is decided further below and in the strategy matrix.
   should-run:
     if: >-
       ${{
-        ((github.event.action != 'labeled' && github.event.action != 'unlabeled')
-          || github.event.label.name == 'Ready to merge'
-          || github.event.label.name == 'DraftRunCI'
-          || github.event.label.name == 'Full CI build')
-        && (!github.event.pull_request.draft
-          || contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
-          || contains(github.event.pull_request.labels.*.name, 'Full CI build'))
+        !github.event.pull_request.draft
+        || contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
+        || contains(github.event.pull_request.labels.*.name, 'Full CI build')
       }}
     runs-on: ubuntu-latest
     steps:

include/xrpl/basics/TaggedCache.ipp

@@ -3,6 +3,9 @@
 #include <xrpl/basics/IntrusivePointer.ipp>
 #include <xrpl/basics/Log.h>  // IWYU pragma: keep
 #include <xrpl/basics/TaggedCache.h>
+#include <xrpl/basics/scope.h>
+
+#include <algorithm>
 
 namespace xrpl {
 
@@ -601,8 +604,42 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
     std::vector<key_type> v;
 
     {
-        std::scoped_lock const lock(mutex_);
-        v.reserve(cache_.size());
+        // Keep track of how many iterations are needed. Exit the loop if the number of retries gets
+        // absurd. (Note that if this somehow ever happens, one more allocation will be done under
+        // lock, which is undesirable, but really should be almost impossible.)
+        std::size_t allocationIterations = 0;
+        std::unique_lock lock(mutex_);
+        for (auto size = cache_.size(); v.capacity() < size && allocationIterations < 20;
+             size = cache_.size())
+        {
+            ScopeUnlock const unlock(lock);
+            if (allocationIterations > 0)
+            {
+                JLOG(journal_.info())
+                    << "getKeys(): Cache grew beyond allocated capacity after "
+                    << allocationIterations << " prior attempt(s). Have " << v.capacity()
+                    << ", need " << size << ". Retrying allocation";
+            }
+            // Allocate the current size plus a little extra, in case the cache grows while
+            // allocating. Each time another allocation is needed, the extra also gets bigger until
+            // it ultimately doubles the size + 1.
+            constexpr std::size_t baseShift = 5;
+            auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
+            auto const bufferShift = baseShift - bufferOffset;
+            size += (size >> bufferShift) + 1;
+            v.reserve(size);
+            ++allocationIterations;
+        }
+        if (v.capacity() < cache_.size())
+        {
+            // LCOV_EXCL_START
+            UNREACHABLE("xrpl::TaggedCache::getKeys(): failed to allocate sufficient capacity");
+            v.reserve(cache_.size());
+            // LCOV_EXCL_STOP
+        }
+        XRPL_ASSERT(lock.owns_lock(), "xrpl::TaggedCache::getKeys(): owns lock");
+        XRPL_ASSERT(
+            v.capacity() >= cache_.size(), "xrpl::TaggedCache::getKeys(): sufficient capacity");
         for (auto const& _ : cache_)
             v.push_back(_.first);
     }

include/xrpl/nodestore/DatabaseRotating.h

@@ -41,6 +41,19 @@ public:
         std::unique_ptr<NodeStore::Backend>&& newBackend,
         std::function<void(std::string const& writableName, std::string const& archiveName)> const&
             f) = 0;
+
+    /**
+     * Marks an online-delete rotation as in progress (or completed).
+     *
+     * While in flight, a read served by the archive backend is copied
+     * forward into the writable backend even for ordinary
+     * (duplicate == false) fetches: the archive is about to be deleted,
+     * and a node body canonicalized into caches during the rotation
+     * window would otherwise survive only in RAM once the archive is
+     * dropped.
+     */
+    virtual void
+    setRotationInFlight(bool inFlight) = 0;
 };
 
 }  // namespace xrpl::NodeStore

include/xrpl/nodestore/detail/DatabaseRotatingImp.h

@@ -9,6 +9,7 @@
 #include <xrpl/nodestore/NodeObject.h>
 #include <xrpl/nodestore/Scheduler.h>
 
+#include <atomic>
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -69,11 +70,22 @@ public:
     void
     sweep() override;
 
+    void
+    setRotationInFlight(bool inFlight) override;
+
 private:
     std::shared_ptr<Backend> writableBackend_;
     std::shared_ptr<Backend> archiveBackend_;
     mutable std::mutex mutex_;
 
+    // True between SHAMapStore starting the cache-freshen phase and the
+    // completion of rotate(). While true, archive hits on ordinary
+    // (duplicate == false) fetches are copied forward into the writable
+    // backend; copyForwardCount_ tallies them per rotation for the
+    // summary line logged at swap.
+    std::atomic<bool> rotationInFlight_{false};
+    std::atomic<std::uint64_t> copyForwardCount_{0};
+
     std::shared_ptr<NodeObject>
     fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)
         override;

src/libxrpl/nodestore/DatabaseRotatingImp.cpp

@@ -13,6 +13,7 @@
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/Types.h>
 
+#include <atomic>
 #include <cstdint>
 #include <exception>
 #include <functional>
@@ -52,6 +53,7 @@ DatabaseRotatingImp::rotate(
     // callback finishes. Only then will the archive directory be
     // deleted.
     std::shared_ptr<NodeStore::Backend> oldArchiveBackend;
+    std::uint64_t copyForwards = 0;
     {
         std::scoped_lock const lock(mutex_);
 
@@ -62,11 +64,28 @@ DatabaseRotatingImp::rotate(
         newArchiveBackendName = archiveBackend_->getName();
 
         writableBackend_ = std::move(newBackend);
+
+        copyForwards = copyForwardCount_.exchange(0, std::memory_order_relaxed);
+    }
+
+    if (copyForwards > 0)
+    {
+        JLOG(j_.warn()) << "Rotating: copied forward " << copyForwards
+                        << " archive-served reads into the writable backend "
+                           "during the rotation window";
     }
 
     f(newWritableBackendName, newArchiveBackendName);
 }
 
+void
+DatabaseRotatingImp::setRotationInFlight(bool inFlight)
+{
+    rotationInFlight_.store(inFlight, std::memory_order_release);
+    JLOG(j_.debug()) << "Rotating: copy-forward on archive reads "
+                     << (inFlight ? "enabled" : "disabled");
+}
+
 std::string
 DatabaseRotatingImp::getName() const
 {
@@ -177,9 +196,18 @@ DatabaseRotatingImp::fetchNodeObject(
                 writable = writableBackend_;
             }
 
-            // Update writable backend with data from the archive backend
-            if (duplicate)
+            // Update writable backend with data from the archive backend.
+            // While a rotation is in flight, ordinary (duplicate == false)
+            // reads served by the archive are copied forward too: the
+            // archive is about to be deleted, and a body canonicalized
+            // into the cache after the freshen getKeys() snapshot would
+            // otherwise survive only in RAM once the archive is dropped.
+            if (duplicate || rotationInFlight_.load(std::memory_order_acquire))
+            {
+                if (!duplicate)
+                    copyForwardCount_.fetch_add(1, std::memory_order_relaxed);
                 writable->store(nodeObject);
+            }
         }
     }
 

src/libxrpl/tx/transactors/check/CheckCancel.cpp

@@ -1,10 +1,12 @@
 #include <xrpl/tx/transactors/check/CheckCancel.h>
 
 #include <xrpl/basics/Log.h>
+#include <xrpl/beast/utility/Zero.h>
 #include <xrpl/ledger/ApplyView.h>
 #include <xrpl/ledger/View.h>
 #include <xrpl/ledger/helpers/AccountRootHelpers.h>
 #include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/Feature.h>
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STLedgerEntry.h>
@@ -19,6 +21,9 @@ namespace xrpl {
 NotTEC
 CheckCancel::preflight(PreflightContext const& ctx)
 {
+    if (ctx.rules.enabled(fixCleanup3_3_0) && ctx.tx[sfCheckID] == beast::kZero)
+        return temMALFORMED;
+
     return tesSUCCESS;
 }
 

src/libxrpl/tx/transactors/check/CheckCash.cpp

@@ -2,6 +2,7 @@
 
 #include <xrpl/basics/Log.h>
 #include <xrpl/basics/scope.h>
+#include <xrpl/beast/utility/Zero.h>
 #include <xrpl/core/ServiceRegistry.h>
 #include <xrpl/ledger/ApplyView.h>
 #include <xrpl/ledger/PaymentSandbox.h>
@@ -51,6 +52,9 @@ CheckCash::checkExtraFeatures(xrpl::PreflightContext const& ctx)
 NotTEC
 CheckCash::preflight(PreflightContext const& ctx)
 {
+    if (ctx.rules.enabled(fixCleanup3_3_0) && ctx.tx[sfCheckID] == beast::kZero)
+        return temMALFORMED;
+
     // Exactly one of Amount or DeliverMin must be present.
     auto const optAmount = ctx.tx[~sfAmount];
     auto const optDeliverMin = ctx.tx[~sfDeliverMin];

src/test/app/Batch_test.cpp

@@ -498,7 +498,7 @@ class Batch_test : public beast::unit_test::Suite
             auto const batchFee = batch::calcBatchFee(env, 0, 2);
             auto tx1 = batch::Inner(pay(alice, bob, XRP(1)), seq + 1);
             tx1[jss::Fee] = "1.5";
-            env.setParseFailureExpected(true);
+            auto const g = env.getParseFailureGuard(true);
             try
             {
                 env(batch::outer(alice, seq, batchFee, tfAllOrNothing),
@@ -510,7 +510,6 @@ class Batch_test : public beast::unit_test::Suite
             {
                 BEAST_EXPECT(true);
             }
-            env.setParseFailureExpected(false);
         }
 
         // temSEQ_AND_TICKET: Batch: inner txn cannot have both Sequence

src/test/app/Check_test.cpp

@@ -1257,6 +1257,13 @@ class Check_test : public beast::unit_test::Suite
             env.close();
         }
 
+        // Can't run pre-amendment behavior due to assertion failure.
+        if (features[fixCleanup3_3_0])
+        {
+            env(check::cash(bob, uint256{}, usd(20)), Ter(temMALFORMED));
+            env.close();
+        }
+
         // alice creates her checks ahead of time.
         uint256 const chkIdU{getCheckIndex(alice, env.seq(alice))};
         env(check::create(alice, bob, usd(20)));
@@ -1704,6 +1711,13 @@ class Check_test : public beast::unit_test::Suite
         // Non-existent check.
         env(check::cancel(bob, getCheckIndex(alice, env.seq(alice))), Ter(tecNO_ENTRY));
         env.close();
+
+        // Can't run pre-amendment behavior due to assertion failure.
+        if (features[fixCleanup3_3_0])
+        {
+            env(check::cancel(bob, uint256{}), Ter(temMALFORMED));
+            env.close();
+        }
     }
 
     void

src/test/app/Vault_test.cpp

@@ -5524,9 +5524,9 @@ class Vault_test : public beast::unit_test::Suite
             env.close();
 
             // 2. Mantissa larger than uint64 max
-            env.setParseFailureExpected(true);
             try
             {
+                auto const g = env.getParseFailureGuard(true);
                 tx[sfAssetsMaximum] = "18446744073709551617e5";  // uint64 max + 1
                 env(tx);
                 BEAST_EXPECTS(false, "Expected parse_error for mantissa larger than uint64 max");
@@ -5537,7 +5537,6 @@ class Vault_test : public beast::unit_test::Suite
                 BEAST_EXPECT(
                     e.what() == "invalidParamsField 'tx_json.AssetsMaximum' has invalid data."s);
             }
-            env.setParseFailureExpected(false);
         }
     }
 

src/test/jtx/Env.h

@@ -514,6 +514,48 @@ public:
         parseFailureExpected_ = b;
     }
 
+    /**
+     * RAII class to set and restore the parse failure flag (setParseFailureExpected).
+     *
+     * Can be created directly, or through the `getParseFailureGuard(bool)` function.
+     */
+    class ParseFailureGuard final
+    {
+        Env& self_;
+        bool oldExpected_ = self_.parseFailureExpected_;
+
+    public:
+        ParseFailureGuard(Env& self, bool b) : self_(self)
+        {
+            self_.setParseFailureExpected(b);
+        }
+
+        ~ParseFailureGuard()
+        {
+            self_.setParseFailureExpected(oldExpected_);
+        }
+
+        // No copy, no move
+        ParseFailureGuard(ParseFailureGuard const&) = delete;
+        ParseFailureGuard&
+        operator=(ParseFailureGuard const&) = delete;
+        ParseFailureGuard(ParseFailureGuard&& other) = delete;
+        ParseFailureGuard&
+        operator=(ParseFailureGuard&&) = delete;
+    };
+
+    /**
+     * Gets an RAII guard to set and restore the parse failure flag
+     *
+     * Usage:
+     * auto const guard = env.getParseFailureGuard(true/false);
+     */
+    [[nodiscard]] ParseFailureGuard
+    getParseFailureGuard(bool b)
+    {
+        return ParseFailureGuard{*this, b};
+    }
+
     /**
      * Turn off signature checks.
      */

src/test/jtx/impl/Env.cpp

@@ -629,7 +629,7 @@ Env::autofill(JTx& jt)
     catch (ParseError const&)
     {
         if (!parseFailureExpected_)
-            test.log << "parse failed:\n" << pretty(jv) << std::endl;
+            test.log << "parse failure:\n" << pretty(jv) << std::endl;
         rethrow();
     }
 }

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -16,9 +16,11 @@
 #include <xrpl/ledger/Ledger.h>
 #include <xrpl/nodestore/Database.h>
 #include <xrpl/nodestore/Manager.h>
+#include <xrpl/nodestore/NodeObject.h>
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
 #include <xrpl/protocol/Protocol.h>
+#include <xrpl/protocol/Serializer.h>
 #include <xrpl/server/NetworkOPs.h>
 #include <xrpl/server/State.h>
 #include <xrpl/shamap/SHAMapMissingNode.h>
@@ -254,8 +256,24 @@ bool
 SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
 {
     // Copy a single record from node to dbRotating_
-    dbRotating_->fetchNodeObject(
+    auto obj = dbRotating_->fetchNodeObject(
         node.getHash().asUInt256(), 0, NodeStore::FetchType::Synchronous, true);
+    if (!obj)
+    {
+        XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean");
+        // Reachable from the validated state map in memory, but present in
+        // neither backend: its only on-disk copy lived in a backend removed by
+        // an earlier rotation, and it was never rewritten because it is clean
+        // (cowid == 0, so flushDirty skips it). Persist the in-memory body
+        // directly into the writable backend so it survives this rotation
+        // instead of later surfacing as an unresolvable SHAMapMissingNode.
+        auto const hash = node.getHash().asUInt256();
+        Serializer s;
+        node.serializeWithPrefix(s);
+        dbRotating_->store(NodeObjectType::AccountNode, std::move(s.modData()), hash, 0);
+        JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash="
+                              << hash << " type=" << static_cast<int>(node.getType());
+    }
     if ((++nodeCount % checkHealthInterval_) == 0u)
     {
         if (healthWait() == HealthResult::Stopping)
@@ -348,6 +366,23 @@ SHAMapStoreImp::run()
             JLOG(journal_.debug())
                 << "copied ledger " << validatedSeq << " nodecount " << nodeCount;
 
+            // Close the getKeys()->swap exposure window: from here until
+            // rotate() completes, an ordinary read served by the archive is
+            // copied forward into the writable backend, so a node fetched
+            // from the doomed archive cannot be left RAM-only when the
+            // archive is deleted. RAII so the early returns below (and any
+            // exception) also clear the flag.
+            struct RotationExposureGuard
+            {
+                NodeStore::DatabaseRotating& db;
+                ~RotationExposureGuard()
+                {
+                    db.setRotationInFlight(false);
+                }
+            };
+            RotationExposureGuard const rotationExposureGuard{*dbRotating_};
+            dbRotating_->setRotationInFlight(true);
+
             JLOG(journal_.debug()) << "freshening caches";
             freshenCaches();
             if (healthWait() == HealthResult::Stopping)

src/xrpld/app/misc/detail/ValidatorSite.cpp

@@ -160,11 +160,7 @@ ValidatorSite::load(
     {
         try
         {
-            // This is not super efficient, but it doesn't happen often.
-            bool found = std::ranges::any_of(
-                sites_, [&uri](auto const& site) { return site.loadedResource->uri == uri; });
-            if (!found)
-                sites_.emplace_back(uri);
+            sites_.emplace_back(uri);
         }
         catch (std::exception const& e)
         {
@@ -225,17 +221,7 @@ ValidatorSite::setTimer(
     std::scoped_lock<std::mutex> const& siteLock,
     std::scoped_lock<std::mutex> const& stateLock)
 {
-    if (!sites_.empty() &&  //
-        std::ranges::all_of(
-            sites_, [](auto const& site) { return site.lastRefreshStatus.has_value(); }))
-    {
-        // If all of the sites have been handled at least once (including
-        // errors and timeouts), call missingSite, which will load the cache
-        // files for any lists that are still unavailable.
-        missingSite(siteLock);
-    }
-
-    auto const next = std::ranges::min_element(
+    auto next = std::ranges::min_element(
         sites_, [](Site const& a, Site const& b) { return a.nextRefresh < b.nextRefresh; });
 
     if (next != sites_.end())
@@ -346,7 +332,7 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
         // processes a network error. Usually, this function runs first,
         // but on extremely rare occasions, the response handler can run
         // first, which will leave activeResource empty.
-        auto& site = sites_[siteIdx];
+        auto const& site = sites_[siteIdx];
         if (site.activeResource)
         {
             JLOG(j_.warn()) << "Request for " << site.activeResource->uri << " took too long";
@@ -356,9 +342,6 @@ ValidatorSite::onRequestTimeout(std::size_t siteIdx, error_code const& ec)
             JLOG(j_.error()) << "Request took too long, but a response has "
                                 "already been processed";
         }
-        if (!site.lastRefreshStatus)
-            site.lastRefreshStatus.emplace(
-                Site::Status{clock_type::now(), ListDisposition::Invalid, "timeout"});
     }
 
     std::scoped_lock const lockState{stateMutex_};