Merge branch 'develop' into ximinez/acquireAsyncDispatch

This change updates a script in the documentation to automatically infer the version of a patched Conan dependency from the conan.lock file.

.github/scripts/strategy-matrix/linux.json

@@ -15,196 +15,196 @@
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "21",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "jammy",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     }
   ],
   "build_type": ["Debug", "Release"],

.github/workflows/reusable-build-test-config.yml

@@ -116,16 +116,6 @@ jobs:
             ${CMAKE_ARGS} \
             ..
 
-      - name: Restore build objects (Linux)
-        if: ${{ runner.os == 'Linux' }}
-        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: ${{ inputs.config_name }}
-
-      - name: Print out ccache statistics before building
-        run: ccache -s
-
       - name: Build the binary
         working-directory: ${{ inputs.build_dir }}
         env:
@@ -139,22 +129,14 @@ jobs:
             --parallel "${BUILD_NPROC}" \
             --target "${CMAKE_TARGET}"
 
-      - name: Print out ccache statistics after building
-        run: ccache -s
-
-      - name: Save build objects (Linux)
-        if: ${{ runner.os == 'Linux' }}
-        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: ${{ inputs.config_name }}
-
       - name: Upload rippled artifact (Linux)
         if: ${{ github.repository_owner == 'XRPLF' && runner.os == 'Linux' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        env:
+          BUILD_DIR: ${{ inputs.build_dir }}
         with:
           name: rippled-${{ inputs.config_name }}
-          path: ${{ inputs.build_dir }}/rippled
+          path: ${{ env.BUILD_DIR }}/rippled
           retention-days: 3
           if-no-files-found: error
 

.gitignore

@@ -111,3 +111,6 @@ bld.rippled/
 
 # Suggested in-tree build directory
 /.build*/
+
+# Locally patched Conan recipes
+external/conan-center-index/

BUILD.md

@@ -141,21 +141,37 @@ Alternatively, you can pull the patched recipes into the repository and use them
 locally:
 
 ```bash
+# Extract the version number from the lockfile.
+function extract_version {
+  version=$(cat conan.lock | sed -nE "s@.+${1}/(.+)#.+@\1@p" | head -n1)
+  echo ${version}
+}
+
+# Define which recipes to export.
+recipes=(ed25519 grpc secp256k1 snappy soci)
+
+# Selectively check out the recipes from our CCI fork.
 cd external
+mkdir -p conan-center-index
+cd conan-center-index
 git init
 git remote add origin git@github.com:XRPLF/conan-center-index.git
 git sparse-checkout init
-git sparse-checkout set recipes/ed25519
+for recipe in ${recipes[@]}; do
-git sparse-checkout add recipes/secp256k1
+  echo "Checking out ${recipe}..."
-git sparse-checkout add recipes/snappy
+  git sparse-checkout add recipes/${recipe}/all
-git sparse-checkout add recipes/soci
+done
 git fetch origin master
 git checkout master
-conan export --version 2015.03 recipes/ed25519/all
+cd ../..
-conan export --version 0.7.0 recipes/secp256k1/all
+
-conan export --version 1.1.10 recipes/snappy/all
+# Export the recipes into the local cache.
-conan export --version 4.0.3 recipes/soci/all
+for recipe in ${recipes[@]}; do
-rm -rf .git
+  version=$(extract_version ${recipe})
+  echo "Exporting ${recipe}/${version}..."
+  conan export --version $(extract_version ${recipe}) \
+    external/conan-center-index/recipes/${recipe}/all
+done
 ```
 
 In the case we switch to a newer version of a dependency that still requires a
@@ -385,19 +401,6 @@ tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
    `--settings build_type=$BUILD_TYPE` or in the profile itself,
    under the section `[settings]` with the key `build_type`.
 
-   If you are using a Microsoft Visual C++ compiler,
-   then you will need to ensure consistency between the `build_type` setting
-   and the `compiler.runtime` setting.
-
-   When `build_type` is `Release`, `compiler.runtime` should be `MT`.
-
-   When `build_type` is `Debug`, `compiler.runtime` should be `MTd`.
-
-   ```
-   conan install .. --output-folder . --build missing --settings build_type=Release --settings compiler.runtime=MT
-   conan install .. --output-folder . --build missing --settings build_type=Debug --settings compiler.runtime=MTd
-   ```
-
 3. Configure CMake and pass the toolchain file generated by Conan, located at
    `$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
 

cmake/XrplCompiler.cmake

@@ -2,12 +2,6 @@
    setup project-wide compiler settings
 #]===================================================================]
 
-find_program(CCACHE_PATH "ccache")
-if (CCACHE_PATH)
-  set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PATH}")
-  message(STATUS "Using ccache: ${CCACHE_PATH}")
-endif ()
-
 #[=========================================================[
    TODO some/most of these common settings belong in a
    toolchain file, especially the ABI-impacting ones

conan.lock

@@ -9,7 +9,7 @@
         "secp256k1/0.7.0#9210e4c655d0a665400378a663960d17%1764261276.764",
         "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535",
         "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594",
         "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909",
         "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999",
@@ -17,19 +17,19 @@
         "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03",
         "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142",
         "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244",
-        "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1756234248.958",
+        "grpc/1.72.0#f244a57bff01e708c55a1100b12e1589%1763158050.628",
-        "ed25519/2015.03#17c1f1910e769f368025267a53c23c13%1764259445.491",
+        "ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1764270189.893",
         "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757",
         "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32",
         "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429",
         "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429",
         "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326",
-        "abseil/20230802.1#90ba607d4ee8fb5fb157c3db540671fc%1764175359.429"
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "build_requires": [
         "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
         "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901",
         "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857",
         "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846",
@@ -37,13 +37,13 @@
         "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429",
         "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28",
         "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
-        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86"
+        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86",
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "python_requires": [],
     "overrides": {
-        "protobuf/3.21.12": [
+        "protobuf/5.27.0": [
-            null,
+            "protobuf/6.32.1"
-            "protobuf/3.21.12"
         ],
         "lz4/1.9.4": [
             "lz4/1.10.0"

conan/profiles/default

@@ -20,12 +20,6 @@ compiler.libcxx={{detect_api.detect_libcxx(compiler, version, compiler_exe)}}
 {% endif %}
 
 [conf]
-{% if compiler == "clang" and compiler_version >= 19 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
-{% if compiler == "apple-clang" and compiler_version >= 17 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
 {% if compiler == "gcc" and compiler_version < 13 %}
 tools.build:cxxflags+=['-Wno-restrict']
 {% endif %}

conanfile.py

@@ -29,7 +29,7 @@ class Xrpl(ConanFile):
 
     requires = [
         "ed25519/2015.03",
-        "grpc/1.50.1",
+        "grpc/1.72.0",
         "libarchive/3.8.1",
         "nudb/2.0.9",
         "openssl/3.5.4",
@@ -43,7 +43,7 @@ class Xrpl(ConanFile):
     ]
 
     tool_requires = [
-        "protobuf/3.21.12",
+        "protobuf/6.32.1",
     ]
 
     default_options = {
@@ -61,6 +61,16 @@ class Xrpl(ConanFile):
         "ed25519/*:shared": False,
         "grpc/*:shared": False,
         "grpc/*:secure": True,
+        "grpc/*:codegen": True,
+        "grpc/*:cpp_plugin": True,
+        "grpc/*:csharp_ext": False,
+        "grpc/*:csharp_plugin": False,
+        "grpc/*:node_plugin": False,
+        "grpc/*:objective_c_plugin": False,
+        "grpc/*:php_plugin": False,
+        "grpc/*:python_plugin": False,
+        "grpc/*:ruby_plugin": False,
+        "grpc/*:otel_plugin": False,
         "libarchive/*:shared": False,
         "libarchive/*:with_acl": False,
         "libarchive/*:with_bzip2": False,
@@ -118,7 +128,7 @@ class Xrpl(ConanFile):
         self.requires("boost/1.88.0", force=True, **transitive_headers_opt)
         self.requires("date/3.0.4", **transitive_headers_opt)
         self.requires("lz4/1.10.0", force=True)
-        self.requires("protobuf/3.21.12", force=True)
+        self.requires("protobuf/6.32.1", force=True)
         self.requires("sqlite3/3.49.1", force=True)
         if self.options.jemalloc:
             self.requires("jemalloc/5.3.0")

include/xrpl/basics/CanProcess.h

@@ -0,0 +1,139 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2024 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_BASICS_CANPROCESS_H_INCLUDED
+#define RIPPLE_BASICS_CANPROCESS_H_INCLUDED
+
+#include <functional>
+#include <mutex>
+#include <set>
+
+/** RAII class to check if an Item is already being processed on another thread,
+ * as indicated by it's presence in a Collection.
+ *
+ * If the Item is not in the Collection, it will be added under lock in the
+ * ctor, and removed under lock in the dtor. The object will be considered
+ * "usable" and evaluate to `true`.
+ *
+ * If the Item is in the Collection, no changes will be made to the collection,
+ * and the CanProcess object will be considered "unusable".
+ *
+ * It's up to the caller to decide what "usable" and "unusable" mean. (e.g.
+ * Process or skip a block of code, or set a flag.)
+ *
+ * The current use is to avoid lock contention that would be involved in
+ * processing something associated with the Item.
+ *
+ * Examples:
+ *
+ * void IncomingLedgers::acquireAsync(LedgerHash const& hash, ...)
+ * {
+ *    if (CanProcess check{acquiresMutex_, pendingAcquires_, hash})
+ *    {
+ *      acquire(hash, ...);
+ *    }
+ * }
+ *
+ * bool
+ * NetworkOPsImp::recvValidation(
+ *   std::shared_ptr<STValidation> const& val,
+ *   std::string const& source)
+ *   {
+ *       CanProcess check(
+ *           validationsMutex_, pendingValidations_, val->getLedgerHash());
+ *       BypassAccept bypassAccept =
+ *           check ? BypassAccept::no : BypassAccept::yes;
+ *       handleNewValidation(app_, val, source, bypassAccept, m_journal);
+ *   }
+ *
+ */
+class CanProcess
+{
+public:
+    template <class Mutex, class Collection, class Item>
+    CanProcess(Mutex& mtx, Collection& collection, Item const& item)
+        : cleanup_(insert(mtx, collection, item))
+    {
+    }
+
+    ~CanProcess()
+    {
+        if (cleanup_)
+            cleanup_();
+    }
+
+    CanProcess(CanProcess const&) = delete;
+
+    CanProcess&
+    operator=(CanProcess const&) = delete;
+
+    explicit
+    operator bool() const
+    {
+        return static_cast<bool>(cleanup_);
+    }
+
+private:
+    template <bool useIterator, class Mutex, class Collection, class Item>
+    std::function<void()>
+    doInsert(Mutex& mtx, Collection& collection, Item const& item)
+    {
+        std::unique_lock<Mutex> lock(mtx);
+        // TODO: Use structured binding once LLVM 16 is the minimum supported
+        // version. See also: https://github.com/llvm/llvm-project/issues/48582
+        // https://github.com/llvm/llvm-project/commit/127bf44385424891eb04cff8e52d3f157fc2cb7c
+        auto const insertResult = collection.insert(item);
+        auto const it = insertResult.first;
+        if (!insertResult.second)
+            return {};
+        if constexpr (useIterator)
+            return [&, it]() {
+                std::unique_lock<Mutex> lock(mtx);
+                collection.erase(it);
+            };
+        else
+            return [&]() {
+                std::unique_lock<Mutex> lock(mtx);
+                collection.erase(item);
+            };
+    }
+
+    // Generic insert() function doesn't use iterators because they may get
+    // invalidated
+    template <class Mutex, class Collection, class Item>
+    std::function<void()>
+    insert(Mutex& mtx, Collection& collection, Item const& item)
+    {
+        return doInsert<false>(mtx, collection, item);
+    }
+
+    // Specialize insert() for std::set, which does not invalidate iterators for
+    // insert and erase
+    template <class Mutex, class Item>
+    std::function<void()>
+    insert(Mutex& mtx, std::set<Item>& collection, Item const& item)
+    {
+        return doInsert<true>(mtx, collection, item);
+    }
+
+    // If set, then the item is "usable"
+    std::function<void()> cleanup_;
+};
+
+#endif

include/xrpl/protocol/LedgerHeader.h

@@ -36,6 +36,8 @@ struct LedgerHeader
 
     // If validated is false, it means "not yet validated."
     // Once validated is true, it will never be set false at a later time.
+    // NOTE: If you are accessing this directly, you are probably doing it
+    //   wrong. Use LedgerMaster::isValidated().
     // VFALCO TODO Make this not mutable
     bool mutable validated = false;
     bool accepted = false;

src/test/app/LedgerReplay_test.cpp

@@ -91,6 +91,8 @@ public:
 
     virtual void
     acquireAsync(
+        JobType type,
+        std::string const& name,
         uint256 const& hash,
         std::uint32_t seq,
         InboundLedger::Reason reason) override

src/test/basics/CanProcess_test.cpp

@@ -0,0 +1,166 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright (c) 2012-2016 Ripple Labs Inc.
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include <xrpl/basics/CanProcess.h>
+#include <xrpl/beast/unit_test.h>
+
+#include <memory>
+
+namespace ripple {
+namespace test {
+
+struct CanProcess_test : beast::unit_test::suite
+{
+    template <class Mutex, class Collection, class Item>
+    void
+    test(
+        std::string const& name,
+        Mutex& mtx,
+        Collection& collection,
+        std::vector<Item> const& items)
+    {
+        testcase(name);
+
+        if (!BEAST_EXPECT(!items.empty()))
+            return;
+        if (!BEAST_EXPECT(collection.empty()))
+            return;
+
+        // CanProcess objects can't be copied or moved. To make that easier,
+        // store shared_ptrs
+        std::vector<std::shared_ptr<CanProcess>> trackers;
+        // Fill up the vector with two CanProcess for each Item. The first
+        // inserts the item into the collection and is "good". The second does
+        // not and is "bad".
+        for (int i = 0; i < items.size(); ++i)
+        {
+            {
+                auto const& good = trackers.emplace_back(
+                    std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(*good);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * i) + 1);
+            BEAST_EXPECT(collection.size() == i + 1);
+            {
+                auto const& bad = trackers.emplace_back(
+                    std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(!*bad);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * (i + 1));
+            BEAST_EXPECT(collection.size() == i + 1);
+        }
+        BEAST_EXPECT(collection.size() == items.size());
+        // Now remove the items from the vector<CanProcess> two at a time, and
+        // try to get another CanProcess for that item.
+        for (int i = 0; i < items.size(); ++i)
+        {
+            // Remove the "bad" one in the second position
+            // This will have no effect on the collection
+            {
+                auto const iter = trackers.begin() + 1;
+                BEAST_EXPECT(!**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * items.size()) - 1);
+            BEAST_EXPECT(collection.size() == items.size());
+            {
+                // Append a new "bad" one
+                auto const& bad = trackers.emplace_back(
+                    std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(!*bad);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * items.size());
+            BEAST_EXPECT(collection.size() == items.size());
+
+            // Remove the "good" one from the front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * items.size()) - 1);
+            BEAST_EXPECT(collection.size() == items.size() - 1);
+            {
+                // Append a new "good" one
+                auto const& good = trackers.emplace_back(
+                    std::make_shared<CanProcess>(mtx, collection, items[i]));
+                BEAST_EXPECT(*good);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * items.size());
+            BEAST_EXPECT(collection.size() == items.size());
+        }
+        // Now remove them all two at a time
+        for (int i = items.size() - 1; i >= 0; --i)
+        {
+            // Remove the "bad" one from the front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(!**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == (2 * i) + 1);
+            BEAST_EXPECT(collection.size() == i + 1);
+            // Remove the "good" one now in front
+            {
+                auto const iter = trackers.begin();
+                BEAST_EXPECT(**iter);
+                trackers.erase(iter);
+            }
+            BEAST_EXPECT(trackers.size() == 2 * i);
+            BEAST_EXPECT(collection.size() == i);
+        }
+        BEAST_EXPECT(trackers.empty());
+        BEAST_EXPECT(collection.empty());
+    }
+
+    void
+    run() override
+    {
+        {
+            std::mutex m;
+            std::set<int> collection;
+            std::vector<int> const items{1, 2, 3, 4, 5};
+            test("set of int", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::set<std::string> collection;
+            std::vector<std::string> const items{
+                "one", "two", "three", "four", "five"};
+            test("set of string", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::unordered_set<char> collection;
+            std::vector<char> const items{'1', '2', '3', '4', '5'};
+            test("unorderd_set of char", m, collection, items);
+        }
+        {
+            std::mutex m;
+            std::unordered_set<std::uint64_t> collection;
+            std::vector<std::uint64_t> const items{100u, 1000u, 150u, 4u, 0u};
+            test("unordered_set of uint64_t", m, collection, items);
+        }
+    }
+};
+
+BEAST_DEFINE_TESTSUITE(CanProcess, ripple_basics, ripple);
+
+}  // namespace test
+}  // namespace ripple

src/xrpld/app/consensus/RCLConsensus.cpp

@@ -118,15 +118,12 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& hash)
             // Tell the ledger acquire system that we need the consensus ledger
             acquiringLedger_ = hash;
 
-            app_.getJobQueue().addJob(
+            app_.getInboundLedgers().acquireAsync(
                 jtADVANCE,
                 "getConsensusLedger1",
-                [id = hash, &app = app_, this]() {
+                hash,
-                    JLOG(j_.debug())
+                0,
-                        << "JOB advanceLedger getConsensusLedger1 started";
+                InboundLedger::Reason::CONSENSUS);
-                    app.getInboundLedgers().acquireAsync(
-                        id, 0, InboundLedger::Reason::CONSENSUS);
-                });
         }
         return std::nullopt;
     }
@@ -1056,7 +1053,8 @@ void
 RCLConsensus::Adaptor::updateOperatingMode(std::size_t const positions) const
 {
     if (!positions && app_.getOPs().isFull())
-        app_.getOPs().setMode(OperatingMode::CONNECTED);
+        app_.getOPs().setMode(
+            OperatingMode::CONNECTED, "updateOperatingMode: no positions");
 }
 
 void

src/xrpld/app/consensus/RCLValidations.cpp

@@ -120,15 +120,12 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash)
         JLOG(j_.warn())
             << "Need validated ledger for preferred ledger analysis " << hash;
 
-        Application* pApp = &app_;
+        app_.getInboundLedgers().acquireAsync(
-
+            jtADVANCE,
-        app_.getJobQueue().addJob(
+            "getConsensusLedger2",
-            jtADVANCE, "getConsensusLedger2", [pApp, hash, this]() {
+            hash,
-                JLOG(j_.debug())
+            0,
-                    << "JOB advanceLedger getConsensusLedger2 started";
+            InboundLedger::Reason::CONSENSUS);
-                pApp->getInboundLedgers().acquireAsync(
-                    hash, 0, InboundLedger::Reason::CONSENSUS);
-            });
         return std::nullopt;
     }
 

src/xrpld/app/ledger/InboundLedgers.h

@@ -28,6 +28,8 @@ public:
     // instead. Inbound ledger acquisition is asynchronous anyway.
     virtual void
     acquireAsync(
+        JobType type,
+        std::string const& name,
         uint256 const& hash,
         std::uint32_t seq,
         InboundLedger::Reason reason) = 0;

src/xrpld/app/ledger/detail/InboundLedger.cpp

@@ -373,7 +373,14 @@ InboundLedger::onTimer(bool wasProgress, ScopedLockType&)
 
     if (!wasProgress)
     {
-        checkLocal();
+        if (checkLocal())
+        {
+            // Done. Something else (probably consensus) built the ledger
+            // locally while waiting for data (or possibly before requesting)
+            XRPL_ASSERT(isDone(), "ripple::InboundLedger::onTimer : done");
+            JLOG(journal_.info()) << "Finished while waiting " << hash_;
+            return;
+        }
 
         mByHash = true;
 

src/xrpld/app/ledger/detail/InboundLedgers.cpp

@@ -5,9 +5,9 @@
 #include <xrpld/core/JobQueue.h>
 #include <xrpld/perflog/PerfLog.h>
 
+#include <xrpl/basics/CanProcess.h>
 #include <xrpl/basics/DecayingSample.h>
 #include <xrpl/basics/Log.h>
-#include <xrpl/basics/scope.h>
 #include <xrpl/beast/container/aged_map.h>
 #include <xrpl/protocol/jss.h>
 
@@ -64,12 +64,15 @@ public:
                 (reason != InboundLedger::Reason::CONSENSUS))
                 return {};
 
+            std::stringstream ss;
+
             bool isNew = true;
             std::shared_ptr<InboundLedger> inbound;
             {
                 ScopedLockType sl(mLock);
                 if (stopping_)
                 {
+                    JLOG(j_.debug()) << "Abort(stopping): " << ss.str();
                     return {};
                 }
 
@@ -93,53 +96,66 @@ public:
                     ++mCounter;
                 }
             }
+            ss << " IsNew: " << (isNew ? "true" : "false");
 
             if (inbound->isFailed())
+            {
+                JLOG(j_.debug()) << "Abort(failed): " << ss.str();
                 return {};
+            }
 
             if (!isNew)
                 inbound->update(seq);
 
             if (!inbound->isComplete())
+            {
+                JLOG(j_.debug()) << "InProgress: " << ss.str();
                 return {};
+            }
 
+            JLOG(j_.debug()) << "Complete: " << ss.str();
             return inbound->getLedger();
         };
         using namespace std::chrono_literals;
-        std::shared_ptr<Ledger const> ledger = perf::measureDurationAndLog(
+        return perf::measureDurationAndLog(
             doAcquire, "InboundLedgersImp::acquire", 500ms, j_);
-
-        return ledger;
     }
 
     void
     acquireAsync(
+        JobType type,
+        std::string const& name,
         uint256 const& hash,
         std::uint32_t seq,
         InboundLedger::Reason reason) override
     {
-        std::unique_lock lock(acquiresMutex_);
+        if (auto check = std::make_shared<CanProcess const>(
-        try
+                acquiresMutex_, pendingAcquires_, hash);
+            *check)
         {
-            if (pendingAcquires_.contains(hash))
+            app_.getJobQueue().addJob(
-                return;
+                type, name, [check, name, hash, seq, reason, this]() {
-            pendingAcquires_.insert(hash);
+                    JLOG(j_.debug())
-            scope_unlock unlock(lock);
+                        << "JOB acquireAsync " << name << " started ";
-            acquire(hash, seq, reason);
+                    try
+                    {
+                        acquire(hash, seq, reason);
+                    }
+                    catch (std::exception const& e)
+                    {
+                        JLOG(j_.warn()) << "Exception thrown for acquiring new "
+                                           "inbound ledger "
+                                        << hash << ": " << e.what();
+                    }
+                    catch (...)
+                    {
+                        JLOG(j_.warn())
+                            << "Unknown exception thrown for acquiring new "
+                               "inbound ledger "
+                            << hash;
+                    }
+                });
         }
-        catch (std::exception const& e)
-        {
-            JLOG(j_.warn())
-                << "Exception thrown for acquiring new inbound ledger " << hash
-                << ": " << e.what();
-        }
-        catch (...)
-        {
-            JLOG(j_.warn())
-                << "Unknown exception thrown for acquiring new inbound ledger "
-                << hash;
-        }
-        pendingAcquires_.erase(hash);
     }
 
     std::shared_ptr<InboundLedger>

src/xrpld/app/ledger/detail/LedgerMaster.cpp

@@ -942,8 +942,9 @@ LedgerMaster::checkAccept(std::shared_ptr<Ledger const> const& ledger)
     }
 
     JLOG(m_journal.info()) << "Advancing accepted ledger to "
-                           << ledger->info().seq << " with >= " << minVal
+                           << ledger->info().seq << " ("
-                           << " validations";
+                           << to_short_string(ledger->info().hash)
+                           << ") with >= " << minVal << " validations";
 
     ledger->setValidated();
     ledger->setFull();

src/xrpld/app/ledger/detail/TimeoutCounter.cpp

@@ -12,7 +12,8 @@ TimeoutCounter::TimeoutCounter(
     QueueJobParameter&& jobParameter,
     beast::Journal journal)
     : app_(app)
-    , journal_(journal)
+    , sink_(journal, to_short_string(hash) + " ")
+    , journal_(sink_)
     , hash_(hash)
     , timeouts_(0)
     , complete_(false)
@@ -32,6 +33,8 @@ TimeoutCounter::setTimer(ScopedLockType& sl)
 {
     if (isDone())
         return;
+    JLOG(journal_.debug()) << "Setting timer for " << timerInterval_.count()
+                           << "ms";
     timer_.expires_after(timerInterval_);
     timer_.async_wait(
         [wptr = pmDowncast()](boost::system::error_code const& ec) {
@@ -40,6 +43,12 @@ TimeoutCounter::setTimer(ScopedLockType& sl)
 
             if (auto ptr = wptr.lock())
             {
+                JLOG(ptr->journal_.debug())
+                    << "timer: ec: " << ec << " (operation_aborted: "
+                    << boost::asio::error::operation_aborted << " - "
+                    << (ec == boost::asio::error::operation_aborted ? "aborted"
+                                                                    : "other")
+                    << ")";
                 ScopedLockType sl(ptr->mtx_);
                 ptr->queueJob(sl);
             }

src/xrpld/app/ledger/detail/TimeoutCounter.h

@@ -5,6 +5,7 @@
 #include <xrpld/core/Job.h>
 
 #include <xrpl/beast/utility/Journal.h>
+#include <xrpl/beast/utility/WrappedSink.h>
 
 #include <boost/asio/basic_waitable_timer.hpp>
 
@@ -104,6 +105,7 @@ protected:
     // Used in this class for access to boost::asio::io_context and
     // ripple::Overlay. Used in subtypes for the kitchen sink.
     Application& app_;
+    beast::WrappedSink sink_;
     beast::Journal journal_;
     mutable std::recursive_mutex mtx_;
 

src/xrpld/app/misc/HashRouter.h

@@ -203,7 +203,7 @@ public:
 
     /** Add a suppression peer and get message's relay status.
      * Return pair:
-     * element 1: true if the peer is added.
+     * element 1: true if the key is added.
      * element 2: optional is seated to the relay time point or
      * is unseated if has not relayed yet. */
     std::pair<bool, std::optional<Stopwatch::time_point>>

src/xrpld/app/misc/NetworkOPs.cpp

@@ -34,10 +34,10 @@
 #include <xrpld/rpc/MPTokenIssuanceID.h>
 #include <xrpld/rpc/ServerHandler.h>
 
+#include <xrpl/basics/CanProcess.h>
 #include <xrpl/basics/UptimeClock.h>
 #include <xrpl/basics/mulDiv.h>
 #include <xrpl/basics/safe_cast.h>
-#include <xrpl/basics/scope.h>
 #include <xrpl/beast/utility/rngfill.h>
 #include <xrpl/crypto/RFC1751.h>
 #include <xrpl/crypto/csprng.h>
@@ -408,7 +408,7 @@ public:
     isFull() override;
 
     void
-    setMode(OperatingMode om) override;
+    setMode(OperatingMode om, char const* reason) override;
 
     bool
     isBlocked() override;
@@ -886,7 +886,7 @@ NetworkOPsImp::strOperatingMode(bool const admin /* = false */) const
 inline void
 NetworkOPsImp::setStandAlone()
 {
-    setMode(OperatingMode::FULL);
+    setMode(OperatingMode::FULL, "setStandAlone");
 }
 
 inline void
@@ -1036,7 +1036,9 @@ NetworkOPsImp::processHeartbeatTimer()
         {
             if (mMode != OperatingMode::DISCONNECTED)
             {
-                setMode(OperatingMode::DISCONNECTED);
+                setMode(
+                    OperatingMode::DISCONNECTED,
+                    "Heartbeat: insufficient peers");
                 std::stringstream ss;
                 ss << "Node count (" << numPeers << ") has fallen "
                    << "below required minimum (" << minPeerCount_ << ").";
@@ -1061,7 +1063,7 @@ NetworkOPsImp::processHeartbeatTimer()
 
         if (mMode == OperatingMode::DISCONNECTED)
         {
-            setMode(OperatingMode::CONNECTED);
+            setMode(OperatingMode::CONNECTED, "Heartbeat: sufficient peers");
             JLOG(m_journal.info())
                 << "Node count (" << numPeers << ") is sufficient.";
             CLOG(clog.ss()) << "setting mode to CONNECTED based on " << numPeers
@@ -1073,9 +1075,9 @@ NetworkOPsImp::processHeartbeatTimer()
         auto origMode = mMode.load();
         CLOG(clog.ss()) << "mode: " << strOperatingMode(origMode, true);
         if (mMode == OperatingMode::SYNCING)
-            setMode(OperatingMode::SYNCING);
+            setMode(OperatingMode::SYNCING, "Heartbeat: check syncing");
         else if (mMode == OperatingMode::CONNECTED)
-            setMode(OperatingMode::CONNECTED);
+            setMode(OperatingMode::CONNECTED, "Heartbeat: check connected");
         auto newMode = mMode.load();
         if (origMode != newMode)
         {
@@ -1825,7 +1827,7 @@ void
 NetworkOPsImp::setAmendmentBlocked()
 {
     amendmentBlocked_ = true;
-    setMode(OperatingMode::CONNECTED);
+    setMode(OperatingMode::CONNECTED, "setAmendmentBlocked");
 }
 
 inline bool
@@ -1856,7 +1858,7 @@ void
 NetworkOPsImp::setUNLBlocked()
 {
     unlBlocked_ = true;
-    setMode(OperatingMode::CONNECTED);
+    setMode(OperatingMode::CONNECTED, "setUNLBlocked");
 }
 
 inline void
@@ -1957,7 +1959,7 @@ NetworkOPsImp::checkLastClosedLedger(
 
     if ((mMode == OperatingMode::TRACKING) || (mMode == OperatingMode::FULL))
     {
-        setMode(OperatingMode::CONNECTED);
+        setMode(OperatingMode::CONNECTED, "check LCL: not on consensus ledger");
     }
 
     if (consensus)
@@ -2046,8 +2048,9 @@ NetworkOPsImp::beginConsensus(
         // this shouldn't happen unless we jump ledgers
         if (mMode == OperatingMode::FULL)
         {
-            JLOG(m_journal.warn()) << "Don't have LCL, going to tracking";
+            JLOG(m_journal.warn())
-            setMode(OperatingMode::TRACKING);
+                << "beginConsensus Don't have LCL, going to tracking";
+            setMode(OperatingMode::TRACKING, "beginConsensus: No LCL");
             CLOG(clog) << "beginConsensus Don't have LCL, going to tracking. ";
         }
 
@@ -2182,7 +2185,7 @@ NetworkOPsImp::endConsensus(std::unique_ptr<std::stringstream> const& clog)
         // validations we have for LCL.  If the ledger is good enough, go to
         // TRACKING - TODO
         if (!needNetworkLedger_)
-            setMode(OperatingMode::TRACKING);
+            setMode(OperatingMode::TRACKING, "endConsensus: check tracking");
     }
 
     if (((mMode == OperatingMode::CONNECTED) ||
@@ -2196,7 +2199,7 @@ NetworkOPsImp::endConsensus(std::unique_ptr<std::stringstream> const& clog)
         if (app_.timeKeeper().now() < (current->info().parentCloseTime +
                                        2 * current->info().closeTimeResolution))
         {
-            setMode(OperatingMode::FULL);
+            setMode(OperatingMode::FULL, "endConsensus: check full");
         }
     }
 
@@ -2208,7 +2211,7 @@ NetworkOPsImp::consensusViewChange()
 {
     if ((mMode == OperatingMode::FULL) || (mMode == OperatingMode::TRACKING))
     {
-        setMode(OperatingMode::CONNECTED);
+        setMode(OperatingMode::CONNECTED, "consensusViewChange");
     }
 }
 
@@ -2527,7 +2530,7 @@ NetworkOPsImp::pubPeerStatus(std::function<Json::Value(void)> const& func)
 }
 
 void
-NetworkOPsImp::setMode(OperatingMode om)
+NetworkOPsImp::setMode(OperatingMode om, char const* reason)
 {
     using namespace std::chrono_literals;
     if (om == OperatingMode::CONNECTED)
@@ -2547,11 +2550,12 @@ NetworkOPsImp::setMode(OperatingMode om)
     if (mMode == om)
         return;
 
+    auto const sink = om < mMode ? m_journal.warn() : m_journal.info();
     mMode = om;
 
     accounting_.mode(om);
 
-    JLOG(m_journal.info()) << "STATE->" << strOperatingMode();
+    JLOG(sink) << "STATE->" << strOperatingMode() << " - " << reason;
     pubServer();
 }
 
@@ -2563,34 +2567,28 @@ NetworkOPsImp::recvValidation(
     JLOG(m_journal.trace())
         << "recvValidation " << val->getLedgerHash() << " from " << source;
 
-    std::unique_lock lock(validationsMutex_);
-    BypassAccept bypassAccept = BypassAccept::no;
-    try
     {
-        if (pendingValidations_.contains(val->getLedgerHash()))
+        CanProcess const check(
-            bypassAccept = BypassAccept::yes;
+            validationsMutex_, pendingValidations_, val->getLedgerHash());
-        else
+        try
-            pendingValidations_.insert(val->getLedgerHash());
+        {
-        scope_unlock unlock(lock);
+            BypassAccept bypassAccept =
-        handleNewValidation(app_, val, source, bypassAccept, m_journal);
+                check ? BypassAccept::no : BypassAccept::yes;
+            handleNewValidation(app_, val, source, bypassAccept, m_journal);
+        }
+        catch (std::exception const& e)
+        {
+            JLOG(m_journal.warn())
+                << "Exception thrown for handling new validation "
+                << val->getLedgerHash() << ": " << e.what();
+        }
+        catch (...)
+        {
+            JLOG(m_journal.warn())
+                << "Unknown exception thrown for handling new validation "
+                << val->getLedgerHash();
+        }
     }
-    catch (std::exception const& e)
-    {
-        JLOG(m_journal.warn())
-            << "Exception thrown for handling new validation "
-            << val->getLedgerHash() << ": " << e.what();
-    }
-    catch (...)
-    {
-        JLOG(m_journal.warn())
-            << "Unknown exception thrown for handling new validation "
-            << val->getLedgerHash();
-    }
-    if (bypassAccept == BypassAccept::no)
-    {
-        pendingValidations_.erase(val->getLedgerHash());
-    }
-    lock.unlock();
 
     pubValidation(val);
 

src/xrpld/app/misc/NetworkOPs.h

@@ -191,7 +191,7 @@ public:
     virtual bool
     isFull() = 0;
     virtual void
-    setMode(OperatingMode om) = 0;
+    setMode(OperatingMode om, char const* reason) = 0;
     virtual bool
     isBlocked() = 0;
     virtual bool