Merge branch 'develop' into ximinez/acquireAsyncDispatch

* XRPLF/develop:
  ci: Rewrite clang-tidy workflow(s) in a reusable manner (7062)
  chore: Ignore identifier-naming update in git blame (7066)
  refactor: Enable clang-tidy `readability-identifier-naming` check (6571)

.github/workflows/on-pr.yml

@@ -25,11 +25,18 @@ on:
       - unlabeled
 
 concurrency:
-  # A single per-ref group with cancel-in-progress means any newer run (a push
-  # or a label change) supersedes the in-progress one for that ref. Keeping
-  # exactly one authoritative run per ref ensures a fast do-nothing run can never
-  # mask a real build's checks.
-  group: ${{ github.workflow }}-${{ github.ref }}
+  # Use a per-ref group so a newer run (a push, or a change to a label below)
+  # supersedes the in-progress one for that ref. Label events we don't act on get
+  # their own unique group (per run id) instead, keeping them out of the shared
+  # group so real builds keep running. Keep this list in sync with `should-run`.
+  group: >-
+    ${{ github.workflow }}-${{ github.ref }}${{
+      ((github.event.action == 'labeled' || github.event.action == 'unlabeled')
+      && github.event.label.name != 'Ready to merge'
+      && github.event.label.name != 'DraftRunCI'
+      && github.event.label.name != 'Full CI build')
+      && format('-{0}', github.run_id) || ''
+    }}
   cancel-in-progress: true
 
 defaults:
@@ -37,17 +44,21 @@ defaults:
     shell: bash
 
 jobs:
-  # This job determines whether the rest of the workflow should run at all,
-  # based on the current set of labels: it runs when the PR is not a draft
-  # (which should also cover merge-group) or has the 'DraftRunCI' or
-  # 'Full CI build' label. Whether a build then happens, and whether it is the
-  # minimal or full matrix, is decided further below and in the strategy matrix.
+  # This job determines whether the rest of the workflow should run. It runs
+  # when the PR is not a draft (which should also cover merge-group) or has the
+  # 'DraftRunCI' or 'Full CI build' label. For label events it only runs when the
+  # label added or removed is one we act on ('Ready to merge', 'DraftRunCI' or
+  # 'Full CI build'), so unrelated label changes do not trigger a redundant run.
   should-run:
     if: >-
       ${{
-        !github.event.pull_request.draft
-        || contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
-        || contains(github.event.pull_request.labels.*.name, 'Full CI build')
+        ((github.event.action != 'labeled' && github.event.action != 'unlabeled')
+          || github.event.label.name == 'Ready to merge'
+          || github.event.label.name == 'DraftRunCI'
+          || github.event.label.name == 'Full CI build')
+        && (!github.event.pull_request.draft
+          || contains(github.event.pull_request.labels.*.name, 'DraftRunCI')
+          || contains(github.event.pull_request.labels.*.name, 'Full CI build'))
       }}
     runs-on: ubuntu-latest
     steps:

include/xrpl/basics/CanProcess.h

@@ -0,0 +1,117 @@
+#pragma once
+
+#include <functional>
+#include <mutex>
+#include <set>
+
+/** RAII class to check if an Item is already being processed on another thread,
+ * as indicated by it's presence in a Collection.
+ *
+ * If the Item is not in the Collection, it will be added under lock in the
+ * ctor, and removed under lock in the dtor. The object will be considered
+ * "usable" and evaluate to `true`.
+ *
+ * If the Item is in the Collection, no changes will be made to the collection,
+ * and the CanProcess object will be considered "unusable".
+ *
+ * It's up to the caller to decide what "usable" and "unusable" mean. (e.g.
+ * Process or skip a block of code, or set a flag.)
+ *
+ * The current use is to avoid lock contention that would be involved in
+ * processing something associated with the Item.
+ *
+ * Examples:
+ *
+ * void IncomingLedgers::acquireAsync(LedgerHash const& hash, ...)
+ * {
+ *    if (CanProcess check{acquiresMutex_, pendingAcquires_, hash})
+ *    {
+ *      acquire(hash, ...);
+ *    }
+ * }
+ *
+ * bool
+ * NetworkOPsImp::recvValidation(
+ *   std::shared_ptr<STValidation> const& val,
+ *   std::string const& source)
+ *   {
+ *       CanProcess check(
+ *           validationsMutex_, pendingValidations_, val->getLedgerHash());
+ *       BypassAccept bypassAccept =
+ *           check ? BypassAccept::no : BypassAccept::yes;
+ *       handleNewValidation(app_, val, source, bypassAccept, m_journal);
+ *   }
+ *
+ */
+class CanProcess
+{
+public:
+    template <class Mutex, class Collection, class Item>
+    CanProcess(Mutex& mtx, Collection& collection, Item const& item)
+        : cleanup_(insert(mtx, collection, item))
+    {
+    }
+
+    ~CanProcess()
+    {
+        if (cleanup_)
+            cleanup_();
+    }
+
+    CanProcess(CanProcess const&) = delete;
+
+    CanProcess&
+    operator=(CanProcess const&) = delete;
+
+    explicit
+    operator bool() const
+    {
+        return static_cast<bool>(cleanup_);
+    }
+
+private:
+    template <bool useIterator, class Mutex, class Collection, class Item>
+    std::function<void()>
+    doInsert(Mutex& mtx, Collection& collection, Item const& item)
+    {
+        std::unique_lock<Mutex> lock(mtx);
+        // TODO: Use structured binding once LLVM 16 is the minimum supported
+        // version. See also: https://github.com/llvm/llvm-project/issues/48582
+        // https://github.com/llvm/llvm-project/commit/127bf44385424891eb04cff8e52d3f157fc2cb7c
+        auto const insertResult = collection.insert(item);
+        auto const it = insertResult.first;
+        if (!insertResult.second)
+            return {};
+        if constexpr (useIterator)
+            return [&, it]() {
+                std::unique_lock<Mutex> lock(mtx);
+                collection.erase(it);
+            };
+        else
+            return [&]() {
+                std::unique_lock<Mutex> lock(mtx);
+                collection.erase(item);
+            };
+    }
+
+    // Generic insert() function doesn't use iterators because they may get
+    // invalidated
+    template <class Mutex, class Collection, class Item>
+    std::function<void()>
+    insert(Mutex& mtx, Collection& collection, Item const& item)
+    {
+        return doInsert<false>(mtx, collection, item);
+    }
+
+    // Specialize insert() for std::set, which does not invalidate iterators for
+    // insert and erase
+    template <class Mutex, class Item>
+    std::function<void()>
+    insert(Mutex& mtx, std::set<Item>& collection, Item const& item)
+    {
+        return doInsert<true>(mtx, collection, item);
+    }
+
+    // If set, then the item is "usable"
+    std::function<void()> cleanup_;
+};

include/xrpl/basics/TaggedCache.ipp

@@ -3,9 +3,6 @@
 #include <xrpl/basics/IntrusivePointer.ipp>
 #include <xrpl/basics/Log.h>  // IWYU pragma: keep
 #include <xrpl/basics/TaggedCache.h>
-#include <xrpl/basics/scope.h>
-
-#include <algorithm>
 
 namespace xrpl {
 
@@ -604,42 +601,8 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
     std::vector<key_type> v;
 
     {
-        // Keep track of how many iterations are needed. Exit the loop if the number of retries gets
-        // absurd. (Note that if this somehow ever happens, one more allocation will be done under
-        // lock, which is undesirable, but really should be almost impossible.)
-        std::size_t allocationIterations = 0;
-        std::unique_lock lock(mutex_);
-        for (auto size = cache_.size(); v.capacity() < size && allocationIterations < 20;
-             size = cache_.size())
-        {
-            ScopeUnlock const unlock(lock);
-            if (allocationIterations > 0)
-            {
-                JLOG(journal_.info())
-                    << "getKeys(): Cache grew beyond allocated capacity after "
-                    << allocationIterations << " prior attempt(s). Have " << v.capacity()
-                    << ", need " << size << ". Retrying allocation";
-            }
-            // Allocate the current size plus a little extra, in case the cache grows while
-            // allocating. Each time another allocation is needed, the extra also gets bigger until
-            // it ultimately doubles the size + 1.
-            constexpr std::size_t baseShift = 5;
-            auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
-            auto const bufferShift = baseShift - bufferOffset;
-            size += (size >> bufferShift) + 1;
-            v.reserve(size);
-            ++allocationIterations;
-        }
-        if (v.capacity() < cache_.size())
-        {
-            // LCOV_EXCL_START
-            UNREACHABLE("xrpl::TaggedCache::getKeys(): failed to allocate sufficient capacity");
-            v.reserve(cache_.size());
-            // LCOV_EXCL_STOP
-        }
-        XRPL_ASSERT(lock.owns_lock(), "xrpl::TaggedCache::getKeys(): owns lock");
-        XRPL_ASSERT(
-            v.capacity() >= cache_.size(), "xrpl::TaggedCache::getKeys(): sufficient capacity");
+        std::scoped_lock const lock(mutex_);
+        v.reserve(cache_.size());
         for (auto const& _ : cache_)
             v.push_back(_.first);
     }

include/xrpl/core/HashRouter.h

@@ -218,7 +218,7 @@ public:
     /**
      * Add a suppression peer and get message's relay status.
      * Return pair:
-     * element 1: true if the peer is added.
+     * element 1: true if the key is added.
      * element 2: optional is seated to the relay time point or
      * is unseated if has not relayed yet.
      */

include/xrpl/nodestore/DatabaseRotating.h

@@ -41,19 +41,6 @@ public:
         std::unique_ptr<NodeStore::Backend>&& newBackend,
         std::function<void(std::string const& writableName, std::string const& archiveName)> const&
             f) = 0;
-
-    /**
-     * Marks an online-delete rotation as in progress (or completed).
-     *
-     * While in flight, a read served by the archive backend is copied
-     * forward into the writable backend even for ordinary
-     * (duplicate == false) fetches: the archive is about to be deleted,
-     * and a node body canonicalized into caches during the rotation
-     * window would otherwise survive only in RAM once the archive is
-     * dropped.
-     */
-    virtual void
-    setRotationInFlight(bool inFlight) = 0;
 };
 
 }  // namespace xrpl::NodeStore

include/xrpl/nodestore/detail/DatabaseRotatingImp.h

@@ -9,7 +9,6 @@
 #include <xrpl/nodestore/NodeObject.h>
 #include <xrpl/nodestore/Scheduler.h>
 
-#include <atomic>
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -70,22 +69,11 @@ public:
     void
     sweep() override;
 
-    void
-    setRotationInFlight(bool inFlight) override;
-
 private:
     std::shared_ptr<Backend> writableBackend_;
     std::shared_ptr<Backend> archiveBackend_;
     mutable std::mutex mutex_;
 
-    // True between SHAMapStore starting the cache-freshen phase and the
-    // completion of rotate(). While true, archive hits on ordinary
-    // (duplicate == false) fetches are copied forward into the writable
-    // backend; copyForwardCount_ tallies them per rotation for the
-    // summary line logged at swap.
-    std::atomic<bool> rotationInFlight_{false};
-    std::atomic<std::uint64_t> copyForwardCount_{0};
-
     std::shared_ptr<NodeObject>
     fetchNodeObject(uint256 const& hash, std::uint32_t, FetchReport& fetchReport, bool duplicate)
         override;

include/xrpl/protocol/LedgerHeader.h

@@ -40,6 +40,8 @@ struct LedgerHeader
 
     // If validated is false, it means "not yet validated."
     // Once validated is true, it will never be set false at a later time.
+    // NOTE: If you are accessing this directly, you are probably doing it
+    //   wrong. Use LedgerMaster::isValidated().
     // VFALCO TODO Make this not mutable
     bool mutable validated = false;
     bool accepted = false;

include/xrpl/server/NetworkOPs.h

@@ -196,7 +196,7 @@ public:
     virtual bool
     isFull() = 0;
     virtual void
-    setMode(OperatingMode om) = 0;
+    setMode(OperatingMode om, char const* reason) = 0;
     virtual bool
     isBlocked() = 0;
     virtual bool

src/libxrpl/nodestore/DatabaseRotatingImp.cpp

@@ -13,7 +13,6 @@
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/Types.h>
 
-#include <atomic>
 #include <cstdint>
 #include <exception>
 #include <functional>
@@ -53,7 +52,6 @@ DatabaseRotatingImp::rotate(
     // callback finishes. Only then will the archive directory be
     // deleted.
     std::shared_ptr<NodeStore::Backend> oldArchiveBackend;
-    std::uint64_t copyForwards = 0;
     {
         std::scoped_lock const lock(mutex_);
 
@@ -64,28 +62,11 @@ DatabaseRotatingImp::rotate(
         newArchiveBackendName = archiveBackend_->getName();
 
         writableBackend_ = std::move(newBackend);
-
-        copyForwards = copyForwardCount_.exchange(0, std::memory_order_relaxed);
-    }
-
-    if (copyForwards > 0)
-    {
-        JLOG(j_.warn()) << "Rotating: copied forward " << copyForwards
-                        << " archive-served reads into the writable backend "
-                           "during the rotation window";
     }
 
     f(newWritableBackendName, newArchiveBackendName);
 }
 
-void
-DatabaseRotatingImp::setRotationInFlight(bool inFlight)
-{
-    rotationInFlight_.store(inFlight, std::memory_order_release);
-    JLOG(j_.debug()) << "Rotating: copy-forward on archive reads "
-                     << (inFlight ? "enabled" : "disabled");
-}
-
 std::string
 DatabaseRotatingImp::getName() const
 {
@@ -196,18 +177,9 @@ DatabaseRotatingImp::fetchNodeObject(
                 writable = writableBackend_;
             }
 
-            // Update writable backend with data from the archive backend.
-            // While a rotation is in flight, ordinary (duplicate == false)
-            // reads served by the archive are copied forward too: the
-            // archive is about to be deleted, and a body canonicalized
-            // into the cache after the freshen getKeys() snapshot would
-            // otherwise survive only in RAM once the archive is dropped.
-            if (duplicate || rotationInFlight_.load(std::memory_order_acquire))
-            {
-                if (!duplicate)
-                    copyForwardCount_.fetch_add(1, std::memory_order_relaxed);
+            // Update writable backend with data from the archive backend
+            if (duplicate)
                 writable->store(nodeObject);
-            }
         }
     }
 

src/libxrpl/protocol/BuildInfo.cpp

@@ -23,7 +23,7 @@ namespace {
 //------------------------------------------------------------------------------
 // clang-format off
 // NOLINTNEXTLINE(readability-identifier-naming)
-char const* const versionString = "3.3.0-rc1"
+char const* const versionString = "3.3.0-b1"
     // clang-format on
     ;
 

src/test/app/LedgerReplay_test.cpp

@@ -175,7 +175,12 @@ public:
     }
 
     void
-    acquireAsync(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason reason) override
+    acquireAsync(
+        JobType type,
+        std::string const& name,
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) override
     {
     }
 

src/test/basics/CanProcess_test.cpp

@@ -0,0 +1,165 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012-2016 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <xrpl/basics/CanProcess.h>
+#include <xrpl/beast/unit_test.h>
+
+#include <memory>
+
+namespace ripple {
+namespace test {
+
+struct CanProcess_test : beast::unit_test::Suite
+{
+    template <class Mutex, class Collection, class Item>
+    void
+    test(
+        std::string const& name,
+        Mutex& mtx,
+        Collection& collection,
+        std::vector<Item> const& items)
+    {
+        testcase(name);
+
+        if (!BEAST_EXPECT(!items.empty()))
+            return;
+        if (!BEAST_EXPECT(collection.empty()))
+            return;
+
+        // CanProcess objects can't be copied or moved. To make that easier,
+        // store shared_ptrs
+        std::vector<std::shared_ptr<CanProcess>> trackers;
+        // Fill up the vector with two CanProcess for each Item. The first
+        // inserts the item into the collection and is "good". The second does
+        // not and is "bad".
+        for (int i = 0; i < items.size(); ++i)
+        {
+            {
+                auto const& good =
+                    trackers.emplace_back(std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(*good);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * i) + 1);
+            BEAST_EXPECT(collection.size() == i + 1);
+            {
+                auto const& bad =
+                    trackers.emplace_back(std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(!*bad);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * (i + 1));
+            BEAST_EXPECT(collection.size() == i + 1);
+        }
+        BEAST_EXPECT(collection.size() == items.size());
+        // Now remove the items from the vector<CanProcess> two at a time, and
+        // try to get another CanProcess for that item.
+        for (int i = 0; i < items.size(); ++i)
+        {
+            // Remove the "bad" one in the second position
+            // This will have no effect on the collection
+            {
+                auto const iter = trackers.begin() + 1;
+                BEAST_EXPECT(!**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * items.size()) - 1);
+            BEAST_EXPECT(collection.size() == items.size());
+            {
+                // Append a new "bad" one
+                auto const& bad =
+                    trackers.emplace_back(std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(!*bad);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * items.size());
+            BEAST_EXPECT(collection.size() == items.size());
+
+            // Remove the "good" one from the front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * items.size()) - 1);
+            BEAST_EXPECT(collection.size() == items.size() - 1);
+            {
+                // Append a new "good" one
+                auto const& good =
+                    trackers.emplace_back(std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(*good);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * items.size());
+            BEAST_EXPECT(collection.size() == items.size());
+        }
+        // Now remove them all two at a time
+        for (int i = items.size() - 1; i >= 0; --i)
+        {
+            // Remove the "bad" one from the front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(!**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * i) + 1);
+            BEAST_EXPECT(collection.size() == i + 1);
+            // Remove the "good" one now in front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * i);
+            BEAST_EXPECT(collection.size() == i);
+        }
+        BEAST_EXPECT(trackers.empty());
+        BEAST_EXPECT(collection.empty());
+    }
+
+    void
+    run() override
+    {
+        {
+            std::mutex m;
+            std::set<int> collection;
+            std::vector<int> const items{1, 2, 3, 4, 5};
+            test("set of int", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::set<std::string> collection;
+            std::vector<std::string> const items{"one", "two", "three", "four", "five"};
+            test("set of string", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::unordered_set<char> collection;
+            std::vector<char> const items{'1', '2', '3', '4', '5'};
+            test("unorderd_set of char", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::unordered_set<std::uint64_t> collection;
+            std::vector<std::uint64_t> const items{100u, 1000u, 150u, 4u, 0u};
+            test("unordered_set of uint64_t", m, collection, items);
+        }
+    }
+};
+
+BEAST_DEFINE_TESTSUITE(CanProcess, ripple_basics, ripple);
+
+}  // namespace test
+}  // namespace ripple

src/xrpld/app/consensus/RCLConsensus.cpp

@@ -159,10 +159,8 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& hash)
             // Tell the ledger acquire system that we need the consensus ledger
             acquiringLedger_ = hash;
 
-            app_.getJobQueue().addJob(JtAdvance, "GetConsL1", [id = hash, &app = app_, this]() {
-                JLOG(j_.debug()) << "JOB advanceLedger getConsensusLedger1 started";
-                app.getInboundLedgers().acquireAsync(id, 0, InboundLedger::Reason::CONSENSUS);
-            });
+            app_.getInboundLedgers().acquireAsync(
+                JtAdvance, "GetConsL1", hash, 0, InboundLedger::Reason::CONSENSUS);
         }
         return std::nullopt;
     }
@@ -1058,7 +1056,7 @@ void
 RCLConsensus::Adaptor::updateOperatingMode(std::size_t const positions) const
 {
     if ((positions == 0u) && app_.getOPs().isFull())
-        app_.getOPs().setMode(OperatingMode::CONNECTED);
+        app_.getOPs().setMode(OperatingMode::CONNECTED, "updateOperatingMode: no positions");
 }
 
 void

src/xrpld/app/consensus/RCLValidations.cpp

@@ -130,12 +130,8 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash)
     {
         JLOG(j_.warn()) << "Need validated ledger for preferred ledger analysis " << hash;
 
-        Application* pApp = &app_;
-
-        app_.getJobQueue().addJob(JtAdvance, "GetConsL2", [pApp, hash, this]() {
-            JLOG(j_.debug()) << "JOB advanceLedger getConsensusLedger2 started";
-            pApp->getInboundLedgers().acquireAsync(hash, 0, InboundLedger::Reason::CONSENSUS);
-        });
+        app_.getInboundLedgers().acquireAsync(
+            JtAdvance, "GetConsL2", hash, 0, InboundLedger::Reason::CONSENSUS);
         return std::nullopt;
     }
 

src/xrpld/app/ledger/InboundLedgers.h

@@ -41,7 +41,12 @@ public:
     // Queue. TODO review whether all callers of acquire() can use this
     // instead. Inbound ledger acquisition is asynchronous anyway.
     virtual void
-    acquireAsync(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason reason) = 0;
+    acquireAsync(
+        JobType type,
+        std::string const& name,
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) = 0;
 
     virtual std::shared_ptr<InboundLedger>
     find(LedgerHash const& hash) = 0;

src/xrpld/app/ledger/detail/InboundLedger.cpp

@@ -367,7 +367,14 @@ InboundLedger::onTimer(bool wasProgress, ScopedLockType&)
 
     if (!wasProgress)
     {
-        checkLocal();
+        if (checkLocal())
+        {
+            // Done. Something else (probably consensus) built the ledger
+            // locally while waiting for data (or possibly before requesting)
+            XRPL_ASSERT(isDone(), "ripple::InboundLedger::onTimer : done");
+            JLOG(journal_.info()) << "Finished while waiting " << hash_;
+            return;
+        }
 
         byHash_ = true;
 

src/xrpld/app/ledger/detail/InboundLedgers.cpp

@@ -6,6 +6,7 @@
 #include <xrpld/overlay/PeerSet.h>
 
 #include <xrpl/basics/Blob.h>
+#include <xrpl/basics/CanProcess.h>
 #include <xrpl/basics/DecayingSample.h>
 #include <xrpl/basics/Log.h>
 #include <xrpl/basics/Slice.h>
@@ -85,12 +86,15 @@ public:
                 (reason != InboundLedger::Reason::CONSENSUS))
                 return {};
 
+            std::stringstream ss;
+
             bool isNew = true;
             std::shared_ptr<InboundLedger> inbound;
             {
                 ScopedLockType sl(lock_);
                 if (stopping_)
                 {
+                    JLOG(j_.debug()) << "Abort(stopping): " << ss.str();
                     return {};
                 }
 
@@ -109,47 +113,61 @@ public:
                     ++counter_;
                 }
             }
+            ss << " IsNew: " << (isNew ? "true" : "false");
 
             if (inbound->isFailed())
+            {
+                JLOG(j_.debug()) << "Abort(failed): " << ss.str();
                 return {};
+            }
 
             if (!isNew)
                 inbound->update(seq);
 
             if (!inbound->isComplete())
+            {
+                JLOG(j_.debug()) << "InProgress: " << ss.str();
                 return {};
+            }
 
+            JLOG(j_.debug()) << "Complete: " << ss.str();
             return inbound->getLedger();
         };
         using namespace std::chrono_literals;
-        std::shared_ptr<Ledger const> ledger =
-            perf::measureDurationAndLog(doAcquire, "InboundLedgersImp::acquire", 500ms, j_);
-
-        return ledger;
+        return perf::measureDurationAndLog(doAcquire, "InboundLedgersImp::acquire", 500ms, j_);
     }
 
     void
-    acquireAsync(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason reason) override
+    acquireAsync(
+        JobType type,
+        std::string const& name,
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) override
     {
-        std::unique_lock lock(acquiresMutex_);
-        try
+        if (auto check = std::make_shared<CanProcess const>(acquiresMutex_, pendingAcquires_, hash);
+            *check)
         {
-            if (pendingAcquires_.contains(hash))
-                return;
-            pendingAcquires_.insert(hash);
-            ScopeUnlock const unlock(lock);
-            acquire(hash, seq, reason);
+            app_.getJobQueue().addJob(type, name, [check, name, hash, seq, reason, this]() {
+                JLOG(j_.debug()) << "JOB acquireAsync " << name << " started ";
+                try
+                {
+                    acquire(hash, seq, reason);
+                }
+                catch (std::exception const& e)
+                {
+                    JLOG(j_.warn()) << "Exception thrown for acquiring new "
+                                       "inbound ledger "
+                                    << hash << ": " << e.what();
+                }
+                catch (...)
+                {
+                    JLOG(j_.warn()) << "Unknown exception thrown for acquiring new "
+                                       "inbound ledger "
+                                    << hash;
+                }
+            });
         }
-        catch (std::exception const& e)
-        {
-            JLOG(j_.warn()) << "Exception thrown for acquiring new inbound ledger " << hash << ": "
-                            << e.what();
-        }
-        catch (...)
-        {
-            JLOG(j_.warn()) << "Unknown exception thrown for acquiring new inbound ledger " << hash;
-        }
-        pendingAcquires_.erase(hash);
     }
 
     std::shared_ptr<InboundLedger>

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -970,8 +970,9 @@ LedgerMaster::checkAccept(std::shared_ptr<Ledger const> const& ledger)
         return;
     }
 
-    JLOG(journal_.info()) << "Advancing accepted ledger to " << ledger->header().seq
-                          << " with >= " << minVal << " validations";
+    JLOG(journal_.info()) << "Advancing accepted ledger to " << ledger->header().seq << " ("
+                          << toShortString(ledger->header().hash) << ") with >= " << minVal
+                          << " validations";
 
     ledger->setValidated();
     ledger->setFull();

src/xrpld/app/ledger/detail/TimeoutCounter.cpp

@@ -25,7 +25,8 @@ TimeoutCounter::TimeoutCounter(
     QueueJobParameter&& jobParameter,
     beast::Journal journal)
     : app_(app)
-    , journal_(journal)
+    , sink_(journal, toShortString(hash) + " ")
+    , journal_(sink_)
     , hash_(hash)
     , timerInterval_(interval)
     , queueJobParameter_(std::move(jobParameter))
@@ -41,6 +42,7 @@ TimeoutCounter::setTimer(ScopedLockType& sl)
 {
     if (isDone())
         return;
+    JLOG(journal_.debug()) << "Setting timer for " << timerInterval_.count() << "ms";
     timer_.expires_after(timerInterval_);
     timer_.async_wait([wptr = pmDowncast()](boost::system::error_code const& ec) {
         if (ec == boost::asio::error::operation_aborted)
@@ -48,6 +50,10 @@ TimeoutCounter::setTimer(ScopedLockType& sl)
 
         if (auto ptr = wptr.lock())
         {
+            JLOG(ptr->journal_.debug())
+                << "timer: ec: " << ec
+                << " (operation_aborted: " << boost::asio::error::operation_aborted << " - "
+                << (ec == boost::asio::error::operation_aborted ? "aborted" : "other") << ")";
             ScopedLockType sl(ptr->mtx_);
             ptr->queueJob(sl);
         }

src/xrpld/app/ledger/detail/TimeoutCounter.h

@@ -4,6 +4,7 @@
 
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/beast/utility/Journal.h>
+#include <xrpl/beast/utility/WrappedSink.h>
 #include <xrpl/core/Job.h>
 
 #include <boost/asio/basic_waitable_timer.hpp>
@@ -117,6 +118,7 @@ protected:
     // Used in this class for access to boost::asio::io_context and
     // xrpl::Overlay. Used in subtypes for the kitchen sink.
     Application& app_;
+    beast::WrappedSink sink_;
     beast::Journal journal_;
     mutable std::recursive_mutex mtx_;
 

src/xrpld/app/misc/NetworkOPs.cpp

@@ -34,6 +34,7 @@
 #include <xrpld/rpc/MPTokenIssuanceID.h>
 #include <xrpld/rpc/ServerHandler.h>
 
+#include <xrpl/basics/CanProcess.h>
 #include <xrpl/basics/Log.h>
 #include <xrpl/basics/ToString.h>
 #include <xrpl/basics/UnorderedContainers.h>
@@ -488,7 +489,7 @@ public:
     isFull() override;
 
     void
-    setMode(OperatingMode om) override;
+    setMode(OperatingMode om, char const* reason) override;
 
     bool
     isBlocked() override;
@@ -975,7 +976,7 @@ NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const
 inline void
 NetworkOPsImp::setStandAlone()
 {
-    setMode(OperatingMode::FULL);
+    setMode(OperatingMode::FULL, "setStandAlone");
 }
 
 inline void
@@ -1118,7 +1119,7 @@ NetworkOPsImp::processHeartbeatTimer()
         {
             if (mode_ != OperatingMode::DISCONNECTED)
             {
-                setMode(OperatingMode::DISCONNECTED);
+                setMode(OperatingMode::DISCONNECTED, "Heartbeat: insufficient peers");
                 std::stringstream ss;
                 ss << "Node count (" << numPeers << ") has fallen "
                    << "below required minimum (" << minPeerCount_ << ").";
@@ -1142,7 +1143,7 @@ NetworkOPsImp::processHeartbeatTimer()
 
         if (mode_ == OperatingMode::DISCONNECTED)
         {
-            setMode(OperatingMode::CONNECTED);
+            setMode(OperatingMode::CONNECTED, "Heartbeat: sufficient peers");
             JLOG(journal_.info()) << "Node count (" << numPeers << ") is sufficient.";
             CLOG(clog.ss()) << "setting mode to CONNECTED based on " << numPeers << " peers. ";
         }
@@ -1153,11 +1154,11 @@ NetworkOPsImp::processHeartbeatTimer()
         CLOG(clog.ss()) << "mode: " << strOperatingMode(origMode, true);
         if (mode_ == OperatingMode::SYNCING)
         {
-            setMode(OperatingMode::SYNCING);
+            setMode(OperatingMode::SYNCING, "Heartbeat: check syncing");
         }
         else if (mode_ == OperatingMode::CONNECTED)
         {
-            setMode(OperatingMode::CONNECTED);
+            setMode(OperatingMode::CONNECTED, "Heartbeat: check connected");
         }
         auto newMode = mode_.load();
         if (origMode != newMode)
@@ -1865,7 +1866,7 @@ void
 NetworkOPsImp::setAmendmentBlocked()
 {
     amendmentBlocked_ = true;
-    setMode(OperatingMode::CONNECTED);
+    setMode(OperatingMode::CONNECTED, "setAmendmentBlocked");
 }
 
 inline bool
@@ -1896,7 +1897,7 @@ void
 NetworkOPsImp::setUNLBlocked()
 {
     unlBlocked_ = true;
-    setMode(OperatingMode::CONNECTED);
+    setMode(OperatingMode::CONNECTED, "setUNLBlocked");
 }
 
 inline void
@@ -1996,7 +1997,7 @@ NetworkOPsImp::checkLastClosedLedger(Overlay::PeerSequence const& peerList, uint
 
     if ((mode_ == OperatingMode::TRACKING) || (mode_ == OperatingMode::FULL))
     {
-        setMode(OperatingMode::CONNECTED);
+        setMode(OperatingMode::CONNECTED, "check LCL: not on consensus ledger");
     }
 
     if (consensus)
@@ -2084,8 +2085,8 @@ NetworkOPsImp::beginConsensus(
         // this shouldn't happen unless we jump ledgers
         if (mode_ == OperatingMode::FULL)
         {
-            JLOG(journal_.warn()) << "Don't have LCL, going to tracking";
-            setMode(OperatingMode::TRACKING);
+            JLOG(journal_.warn()) << "beginConsensus Don't have LCL, going to tracking";
+            setMode(OperatingMode::TRACKING, "beginConsensus: No LCL");
             CLOG(clog) << "beginConsensus Don't have LCL, going to tracking. ";
         }
 
@@ -2213,7 +2214,7 @@ NetworkOPsImp::endConsensus(std::unique_ptr<std::stringstream> const& clog)
         // validations we have for LCL.  If the ledger is good enough, go to
         // TRACKING - TODO
         if (!needNetworkLedger_)
-            setMode(OperatingMode::TRACKING);
+            setMode(OperatingMode::TRACKING, "endConsensus: check tracking");
     }
 
     if (((mode_ == OperatingMode::CONNECTED) || (mode_ == OperatingMode::TRACKING)) &&
@@ -2226,7 +2227,7 @@ NetworkOPsImp::endConsensus(std::unique_ptr<std::stringstream> const& clog)
         if (registry_.get().getTimeKeeper().now() <
             (current->header().parentCloseTime + 2 * current->header().closeTimeResolution))
         {
-            setMode(OperatingMode::FULL);
+            setMode(OperatingMode::FULL, "endConsensus: check full");
         }
     }
 
@@ -2238,7 +2239,7 @@ NetworkOPsImp::consensusViewChange()
 {
     if ((mode_ == OperatingMode::FULL) || (mode_ == OperatingMode::TRACKING))
     {
-        setMode(OperatingMode::CONNECTED);
+        setMode(OperatingMode::CONNECTED, "consensusViewChange");
     }
 }
 
@@ -2542,7 +2543,7 @@ NetworkOPsImp::pubPeerStatus(std::function<json::Value(void)> const& func)
 }
 
 void
-NetworkOPsImp::setMode(OperatingMode om)
+NetworkOPsImp::setMode(OperatingMode om, char const* reason)
 {
     using namespace std::chrono_literals;
     if (om == OperatingMode::CONNECTED)
@@ -2562,11 +2563,12 @@ NetworkOPsImp::setMode(OperatingMode om)
     if (mode_ == om)
         return;
 
+    auto const sink = om < mode_ ? journal_.warn() : journal_.info();
     mode_ = om;
 
     accounting_.mode(om);
 
-    JLOG(journal_.info()) << "STATE->" << strOperatingMode();
+    JLOG(sink) << "STATE->" << strOperatingMode() << " - " << reason;
     pubServer();
 }
 
@@ -2575,36 +2577,24 @@ NetworkOPsImp::recvValidation(std::shared_ptr<STValidation> const& val, std::str
 {
     JLOG(journal_.trace()) << "recvValidation " << val->getLedgerHash() << " from " << source;
 
-    std::unique_lock lock(validationsMutex_);
-    BypassAccept bypassAccept = BypassAccept::No;
-    try
     {
-        if (pendingValidations_.contains(val->getLedgerHash()))
+        CanProcess const check(validationsMutex_, pendingValidations_, val->getLedgerHash());
+        try
         {
-            bypassAccept = BypassAccept::Yes;
+            BypassAccept bypassAccept = check ? BypassAccept::No : BypassAccept::Yes;
+            handleNewValidation(registry_.get().getApp(), val, source, bypassAccept, journal_);
         }
-        else
+        catch (std::exception const& e)
         {
-            pendingValidations_.insert(val->getLedgerHash());
+            JLOG(journal_.warn()) << "Exception thrown for handling new validation "
+                                  << val->getLedgerHash() << ": " << e.what();
+        }
+        catch (...)
+        {
+            JLOG(journal_.warn()) << "Unknown exception thrown for handling new validation "
+                                  << val->getLedgerHash();
         }
-        ScopeUnlock const unlock(lock);
-        handleNewValidation(registry_.get().getApp(), val, source, bypassAccept, journal_);
     }
-    catch (std::exception const& e)
-    {
-        JLOG(journal_.warn()) << "Exception thrown for handling new validation "
-                              << val->getLedgerHash() << ": " << e.what();
-    }
-    catch (...)
-    {
-        JLOG(journal_.warn()) << "Unknown exception thrown for handling new validation "
-                              << val->getLedgerHash();
-    }
-    if (bypassAccept == BypassAccept::No)
-    {
-        pendingValidations_.erase(val->getLedgerHash());
-    }
-    lock.unlock();
 
     pubValidation(val);
 

src/xrpld/app/misc/SHAMapStoreImp.cpp

@@ -16,11 +16,9 @@
 #include <xrpl/ledger/Ledger.h>
 #include <xrpl/nodestore/Database.h>
 #include <xrpl/nodestore/Manager.h>
-#include <xrpl/nodestore/NodeObject.h>
 #include <xrpl/nodestore/Scheduler.h>
 #include <xrpl/nodestore/detail/DatabaseRotatingImp.h>
 #include <xrpl/protocol/Protocol.h>
-#include <xrpl/protocol/Serializer.h>
 #include <xrpl/server/NetworkOPs.h>
 #include <xrpl/server/State.h>
 #include <xrpl/shamap/SHAMapMissingNode.h>
@@ -256,24 +254,8 @@ bool
 SHAMapStoreImp::copyNode(std::uint64_t& nodeCount, SHAMapTreeNode const& node)
 {
     // Copy a single record from node to dbRotating_
-    auto obj = dbRotating_->fetchNodeObject(
+    dbRotating_->fetchNodeObject(
         node.getHash().asUInt256(), 0, NodeStore::FetchType::Synchronous, true);
-    if (!obj)
-    {
-        XRPL_ASSERT(node.cowid() == 0, "SHAMapStoreImp::copyNode : rescued node must be clean");
-        // Reachable from the validated state map in memory, but present in
-        // neither backend: its only on-disk copy lived in a backend removed by
-        // an earlier rotation, and it was never rewritten because it is clean
-        // (cowid == 0, so flushDirty skips it). Persist the in-memory body
-        // directly into the writable backend so it survives this rotation
-        // instead of later surfacing as an unresolvable SHAMapMissingNode.
-        auto const hash = node.getHash().asUInt256();
-        Serializer s;
-        node.serializeWithPrefix(s);
-        dbRotating_->store(NodeObjectType::AccountNode, std::move(s.modData()), hash, 0);
-        JLOG(journal_.warn()) << "copyNode: re-stored node missing from both backends, hash="
-                              << hash << " type=" << static_cast<int>(node.getType());
-    }
     if ((++nodeCount % checkHealthInterval_) == 0u)
     {
         if (healthWait() == HealthResult::Stopping)
@@ -366,23 +348,6 @@ SHAMapStoreImp::run()
             JLOG(journal_.debug())
                 << "copied ledger " << validatedSeq << " nodecount " << nodeCount;
 
-            // Close the getKeys()->swap exposure window: from here until
-            // rotate() completes, an ordinary read served by the archive is
-            // copied forward into the writable backend, so a node fetched
-            // from the doomed archive cannot be left RAM-only when the
-            // archive is deleted. RAII so the early returns below (and any
-            // exception) also clear the flag.
-            struct RotationExposureGuard
-            {
-                NodeStore::DatabaseRotating& db;
-                ~RotationExposureGuard()
-                {
-                    db.setRotationInFlight(false);
-                }
-            };
-            RotationExposureGuard const rotationExposureGuard{*dbRotating_};
-            dbRotating_->setRotationInFlight(true);
-
             JLOG(journal_.debug()) << "freshening caches";
             freshenCaches();
             if (healthWait() == HealthResult::Stopping)

src/xrpld/app/misc/detail/AmendmentTable.cpp

@@ -202,11 +202,11 @@ public:
                                            iter->second.upVotes |
                                                boost::adaptors::transformed(to_string<256, void>),
                                            ", ");
-                    // TODO: Maybe transform using to_short_string once #5126 is
+                    // TODO: Maybe transform using toShortString once #5126 is
                     // merged
                     //
                     // iter->second.upVotes |
-                    // boost::adaptors::transformed(to_short_string<256, void>)
+                    // boost::adaptors::transformed(toShortString<256, void>)
                 }
                 else
                 {