Merge branch 'develop' into pratik/Charge-higher-fees-for-and-reject-heavy-TMGetObjectByHash-queries

This PR updates protobuf and grpc to their latest versions. The latest protobuf version no longer requires patches, so we can use it directly from the official Conan Center Index, while the latest grpc still needed a patch, which was added to our own Conan Center Index fork in XRPLF/conan-center-index#8.

.github/scripts/strategy-matrix/linux.json

@@ -15,196 +15,196 @@
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "bookworm",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "gcc",
       "compiler_version": "15",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "20",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "debian",
       "distro_version": "trixie",
       "compiler_name": "clang",
       "compiler_version": "21",
-      "image_sha": "ca4517d"
+      "image_sha": "0525eae"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "8",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "9",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "rhel",
       "distro_version": "10",
       "compiler_name": "clang",
       "compiler_version": "any",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "jammy",
       "compiler_name": "gcc",
       "compiler_version": "12",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "13",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "gcc",
       "compiler_version": "14",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "16",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "17",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "18",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     },
     {
       "distro_name": "ubuntu",
       "distro_version": "noble",
       "compiler_name": "clang",
       "compiler_version": "19",
-      "image_sha": "ca4517d"
+      "image_sha": "e1782cd"
     }
   ],
   "build_type": ["Debug", "Release"],

.github/workflows/reusable-build-test-config.yml

@@ -116,16 +116,6 @@ jobs:
             ${CMAKE_ARGS} \
             ..
 
-      - name: Restore build objects (Linux)
-        if: ${{ runner.os == 'Linux' }}
-        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: ${{ inputs.config_name }}
-
-      - name: Print out ccache statistics before building
-        run: ccache -s
-
       - name: Build the binary
         working-directory: ${{ inputs.build_dir }}
         env:
@@ -139,22 +129,14 @@ jobs:
             --parallel "${BUILD_NPROC}" \
             --target "${CMAKE_TARGET}"
 
-      - name: Print out ccache statistics after building
-        run: ccache -s
-
-      - name: Save build objects (Linux)
-        if: ${{ runner.os == 'Linux' }}
-        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: ${{ env.CCACHE_DIR }}
-          key: ${{ inputs.config_name }}
-
       - name: Upload rippled artifact (Linux)
         if: ${{ github.repository_owner == 'XRPLF' && runner.os == 'Linux' }}
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        env:
+          BUILD_DIR: ${{ inputs.build_dir }}
         with:
           name: rippled-${{ inputs.config_name }}
-          path: ${{ inputs.build_dir }}/rippled
+          path: ${{ env.BUILD_DIR }}/rippled
           retention-days: 3
           if-no-files-found: error
 

.gitignore

@@ -111,3 +111,6 @@ bld.rippled/
 
 # Suggested in-tree build directory
 /.build*/
+
+# Locally patched Conan recipes
+external/conan-center-index/

BUILD.md

@@ -142,20 +142,25 @@ locally:
 
 ```bash
 cd external
+mkdir -p conan-center-index
+cd conan-center-index
 git init
 git remote add origin git@github.com:XRPLF/conan-center-index.git
 git sparse-checkout init
 git sparse-checkout set recipes/ed25519
+git sparse-checkout add recipes/grpc
 git sparse-checkout add recipes/secp256k1
 git sparse-checkout add recipes/snappy
 git sparse-checkout add recipes/soci
 git fetch origin master
 git checkout master
-conan export --version 2015.03 recipes/ed25519/all
-conan export --version 0.7.0 recipes/secp256k1/all
-conan export --version 1.1.10 recipes/snappy/all
-conan export --version 4.0.3 recipes/soci/all
 rm -rf .git
+cd ../..
+conan export --version 2015.03 external/conan-center-index/recipes/ed25519/all
+conan export --version 1.72.0 external/conan-center-index/recipes/grpc/all
+conan export --version 0.7.0 external/conan-center-index/recipes/secp256k1/all
+conan export --version 1.1.10 external/conan-center-index/recipes/snappy/all
+conan export --version 4.0.3 external/conan-center-index/recipes/soci/all
 ```
 
 In the case we switch to a newer version of a dependency that still requires a
@@ -385,19 +390,6 @@ tools.build:cxxflags=['-DBOOST_ASIO_DISABLE_CONCEPTS']
    `--settings build_type=$BUILD_TYPE` or in the profile itself,
    under the section `[settings]` with the key `build_type`.
 
-   If you are using a Microsoft Visual C++ compiler,
-   then you will need to ensure consistency between the `build_type` setting
-   and the `compiler.runtime` setting.
-
-   When `build_type` is `Release`, `compiler.runtime` should be `MT`.
-
-   When `build_type` is `Debug`, `compiler.runtime` should be `MTd`.
-
-   ```
-   conan install .. --output-folder . --build missing --settings build_type=Release --settings compiler.runtime=MT
-   conan install .. --output-folder . --build missing --settings build_type=Debug --settings compiler.runtime=MTd
-   ```
-
 3. Configure CMake and pass the toolchain file generated by Conan, located at
    `$OUTPUT_FOLDER/build/generators/conan_toolchain.cmake`.
 

cmake/XrplCompiler.cmake

@@ -2,12 +2,6 @@
    setup project-wide compiler settings
 #]===================================================================]
 
-find_program(CCACHE_PATH "ccache")
-if (CCACHE_PATH)
-  set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PATH}")
-  message(STATUS "Using ccache: ${CCACHE_PATH}")
-endif ()
-
 #[=========================================================[
    TODO some/most of these common settings belong in a
    toolchain file, especially the ABI-impacting ones

conan.lock

@@ -9,7 +9,7 @@
         "secp256k1/0.7.0#9210e4c655d0a665400378a663960d17%1764261276.764",
         "rocksdb/10.5.1#4a197eca381a3e5ae8adf8cffa5aacd0%1762797952.535",
         "re2/20230301#ca3b241baec15bd31ea9187150e0b333%1764175362.029",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "openssl/3.5.4#a1d5835cc6ed5c5b8f3cd5b9b5d24205%1760106486.594",
         "nudb/2.0.9#fb8dfd1a5557f5e0528114c2da17721e%1763150366.909",
         "lz4/1.10.0#59fc63cac7f10fbe8e05c7e62c2f3504%1756234228.999",
@@ -17,19 +17,19 @@
         "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1756230911.03",
         "libarchive/3.8.1#ffee18995c706e02bf96e7a2f7042e0d%1764175360.142",
         "jemalloc/5.3.0#e951da9cf599e956cebc117880d2d9f8%1729241615.244",
-        "grpc/1.50.1#02291451d1e17200293a409410d1c4e1%1756234248.958",
-        "ed25519/2015.03#17c1f1910e769f368025267a53c23c13%1764259445.491",
+        "grpc/1.72.0#f244a57bff01e708c55a1100b12e1589%1763158050.628",
+        "ed25519/2015.03#ae761bdc52730a843f0809bdf6c1b1f6%1764270189.893",
         "doctest/2.4.12#eb9fb352fb2fdfc8abb17ec270945165%1762797941.757",
         "date/3.0.4#862e11e80030356b53c2c38599ceb32b%1763584497.32",
         "c-ares/1.34.5#5581c2b62a608b40bb85d965ab3ec7c8%1764175359.429",
         "bzip2/1.0.8#c470882369c2d95c5c77e970c0c7e321%1764175359.429",
         "boost/1.88.0#8852c0b72ce8271fb8ff7c53456d4983%1756223752.326",
-        "abseil/20230802.1#90ba607d4ee8fb5fb157c3db540671fc%1764175359.429"
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "build_requires": [
         "zlib/1.3.1#b8bc2603263cf7eccbd6e17e66b0ed76%1756234269.497",
         "strawberryperl/5.32.1.1#707032463aa0620fa17ec0d887f5fe41%1756234281.733",
-        "protobuf/3.21.12#44ee56c0a6eea0c19aeeaca680370b88%1764175361.456",
+        "protobuf/6.32.1#f481fd276fc23a33b85a3ed1e898b693%1759134873.45",
         "nasm/2.16.01#31e26f2ee3c4346ecd347911bd126904%1756234232.901",
         "msys2/cci.latest#5b73b10144f73cc5bfe0572ed9be39e1%1751977009.857",
         "m4/1.4.19#70dc8bbb33e981d119d2acc0175cf381%1763158052.846",
@@ -37,13 +37,13 @@
         "cmake/3.31.10#313d16a1aa16bbdb2ca0792467214b76%1764175359.429",
         "b2/5.3.3#107c15377719889654eb9a162a673975%1756234226.28",
         "automake/1.16.5#b91b7c384c3deaa9d535be02da14d04f%1755524470.56",
-        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86"
+        "autoconf/2.71#51077f068e61700d65bb05541ea1e4b0%1731054366.86",
+        "abseil/20250127.0#9e8e8cfc89a1324139fc0ee3bd4d8c8c%1753819045.301"
     ],
     "python_requires": [],
     "overrides": {
-        "protobuf/3.21.12": [
-            null,
-            "protobuf/3.21.12"
+        "protobuf/5.27.0": [
+            "protobuf/6.32.1"
         ],
         "lz4/1.9.4": [
             "lz4/1.10.0"

conan/profiles/default

@@ -20,12 +20,6 @@ compiler.libcxx={{detect_api.detect_libcxx(compiler, version, compiler_exe)}}
 {% endif %}
 
 [conf]
-{% if compiler == "clang" and compiler_version >= 19 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
-{% if compiler == "apple-clang" and compiler_version >= 17 %}
-grpc/1.50.1:tools.build:cxxflags+=['-Wno-missing-template-arg-list-after-template-kw']
-{% endif %}
 {% if compiler == "gcc" and compiler_version < 13 %}
 tools.build:cxxflags+=['-Wno-restrict']
 {% endif %}

conanfile.py

@@ -29,7 +29,7 @@ class Xrpl(ConanFile):
 
     requires = [
         "ed25519/2015.03",
-        "grpc/1.50.1",
+        "grpc/1.72.0",
         "libarchive/3.8.1",
         "nudb/2.0.9",
         "openssl/3.5.4",
@@ -43,7 +43,7 @@ class Xrpl(ConanFile):
     ]
 
     tool_requires = [
-        "protobuf/3.21.12",
+        "protobuf/6.32.1",
     ]
 
     default_options = {
@@ -61,6 +61,16 @@ class Xrpl(ConanFile):
         "ed25519/*:shared": False,
         "grpc/*:shared": False,
         "grpc/*:secure": True,
+        "grpc/*:codegen": True,
+        "grpc/*:cpp_plugin": True,
+        "grpc/*:csharp_ext": False,
+        "grpc/*:csharp_plugin": False,
+        "grpc/*:node_plugin": False,
+        "grpc/*:objective_c_plugin": False,
+        "grpc/*:php_plugin": False,
+        "grpc/*:python_plugin": False,
+        "grpc/*:ruby_plugin": False,
+        "grpc/*:otel_plugin": False,
         "libarchive/*:shared": False,
         "libarchive/*:with_acl": False,
         "libarchive/*:with_bzip2": False,
@@ -118,7 +128,7 @@ class Xrpl(ConanFile):
         self.requires("boost/1.88.0", force=True, **transitive_headers_opt)
         self.requires("date/3.0.4", **transitive_headers_opt)
         self.requires("lz4/1.10.0", force=True)
-        self.requires("protobuf/3.21.12", force=True)
+        self.requires("protobuf/6.32.1", force=True)
         self.requires("sqlite3/3.49.1", force=True)
         if self.options.jemalloc:
             self.requires("jemalloc/5.3.0")

include/xrpl/protocol/detail/features.macro

@@ -67,6 +67,7 @@ XRPL_FEATURE(Clawback,                   Supported::yes, VoteBehavior::DefaultNo
 XRPL_FIX    (UniversalNumber,            Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FEATURE(XRPFees,                    Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (RemoveNFTokenAutoTrustLine, Supported::yes, VoteBehavior::DefaultYes)
+XRPL_FIX    (TMGetObjectByHashLimit,     Supported::yes, VoteBehavior::DefaultYes)
 
 // The following amendments are obsolete, but must remain supported
 // because they could potentially get enabled.

src/xrpld/overlay/detail/PeerImp.cpp

@@ -18,6 +18,7 @@
 #include <xrpl/basics/base64.h>
 #include <xrpl/basics/random.h>
 #include <xrpl/basics/safe_cast.h>
+#include <xrpl/protocol/Feature.h>
 #include <xrpl/protocol/TxFlags.h>
 #include <xrpl/protocol/digest.h>
 
@@ -2589,9 +2590,51 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMGetObjectByHash> const& m)
             reply.set_ledgerhash(packet.ledgerhash());
         }
 
-        fee_.update(
-            Resource::feeModerateBurdenPeer,
-            " received a get object by hash request");
+        // Get validated rules to check if the fix is enabled
+        auto const rules = app_.getLedgerMaster().getValidatedRules();
+
+        // Charge resource fee based on request size when fix is enabled
+        if (rules.enabled(fixTMGetObjectByHashLimit))
+        {
+            // Enforce per-request object cap
+            if (packet.objects_size() > Tuning::maxGetObjectByHash)
+            {
+                fee_.update(Resource::feeMalformedRequest, "too many objects");
+                return;
+            }
+
+            // Charge heavier fee for large requests (>256 objects)
+            if (packet.objects_size() > 256)
+            {
+                fee_.update(
+                    Resource::feeHeavyBurdenPeer,
+                    "large get object by hash request");
+            }
+            else if (packet.objects_size() > 64)
+            {
+                fee_.update(
+                    Resource::feeModerateBurdenPeer,
+                    "moderate get object by hash request");
+            }
+            else
+            {
+                fee_.update(
+                    Resource::feeTrivialPeer,
+                    "small get object by hash request");
+            }
+        }
+        else
+        {
+            // Legacy behavior: charge moderate fee for all requests
+            fee_.update(
+                Resource::feeModerateBurdenPeer,
+                "received a get object by hash request");
+        }
+
+        // Track reply bytes and stop when over budget (16 MiB) when fix is enabled
+        std::size_t replyBudgetBytes =
+            rules.enabled(fixTMGetObjectByHashLimit) ? megabytes(16) : 0;
+        std::size_t replyBytes = 0;
 
         // This is a very minimal implementation
         for (int i = 0; i < packet.objects_size(); ++i)
@@ -2606,17 +2649,28 @@ PeerImp::onMessage(std::shared_ptr<protocol::TMGetObjectByHash> const& m)
                 auto nodeObject{app_.getNodeStore().fetchNodeObject(hash, seq)};
                 if (nodeObject)
                 {
+                    auto const dataSz = nodeObject->getData().size();
+                    // Check if adding this object would exceed the reply budget
+                    // (only when fix is enabled)
+                    if (replyBudgetBytes > 0 &&
+                        replyBytes + dataSz + 64 > replyBudgetBytes)
+                        break;
+
                     protocol::TMIndexedObject& newObj = *reply.add_objects();
                     newObj.set_hash(hash.begin(), hash.size());
                     newObj.set_data(
                         &nodeObject->getData().front(),
-                        nodeObject->getData().size());
+                        dataSz);
 
                     if (obj.has_nodeid())
                         newObj.set_index(obj.nodeid());
                     if (obj.has_ledgerseq())
                         newObj.set_ledgerseq(obj.ledgerseq());
 
+                    // Track reply bytes when fix is enabled
+                    if (replyBudgetBytes > 0)
+                        replyBytes += dataSz + 64;  // include modest overhead estimate
+
                     // VFALCO NOTE "seq" in the message is obsolete
                 }
             }

src/xrpld/overlay/detail/Tuning.h

@@ -22,6 +22,9 @@ enum {
     /** The hard cap on the number of ledger entries in a single reply. */
     hardMaxReplyNodes = 12288,
 
+    /** Hard cap on TMGetObjectByHash objects per request (non-TRANSACTIONS). */
+    maxGetObjectByHash = 1024,
+
     /** How many timer intervals a sendq has to stay large before we disconnect
      */
     sendqIntervals = 4,