fix: Add upper bound to limit

fixes: #1743

.githooks/check-format

@@ -26,12 +26,12 @@ sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "18.0.0" > "$version" ]]; then
+if [[ "19.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 18 of `which clang-format` is required.
+            A minimum of version 19 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------

.githooks/pre-push

@@ -42,7 +42,7 @@ verify_tag_signed() {
 while read local_ref local_oid remote_ref remote_oid; do
     # Check some things if we're pushing a branch called "release/"
     if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
-        version=$(echo $remote_ref | awk -F/ '{print $NF}')
+        version=$(git tag --points-at HEAD)
         echo "Looks like you're trying to push a $version release..."
         echo "Making sure you've signed and tagged it."
         if verify_commit_signed && verify_tag && verify_tag_signed ; then

.github/workflows/build.yml

@@ -149,13 +149,6 @@ jobs:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
           path: build/clio_*tests
 
-      - name: Upload test data
-        if: ${{ !matrix.code_coverage }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
-          path: build/tests/unit/test_data
-
       - name: Save cache
         uses: ./.github/actions/save_cache
         with:
@@ -219,11 +212,6 @@ jobs:
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
-          path: tests/unit/test_data
-
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/check_pr_title.yml

@@ -10,7 +10,7 @@ jobs:
     # permissions:
     #   pull-requests: write
     steps:
-      - uses: ytanikin/PRConventionalCommits@1.2.0
+      - uses: ytanikin/PRConventionalCommits@1.3.0
         with:
           task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
           add_label: false

.github/workflows/clang-tidy.yml

@@ -60,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-18 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -99,7 +99,7 @@ jobs:
 
       - name: Create PR with fixes
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}

.github/workflows/nightly.yml

@@ -71,12 +71,6 @@ jobs:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
           path: build/clio_*tests
 
-      - name: Upload test data
-        uses: actions/upload-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/tests/unit/test_data
-
       - name: Compress clio_server
         shell: bash
         run: |
@@ -130,11 +124,6 @@ jobs:
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}
-          path: tests/unit/test_data
-
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/upload_coverage_report.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v3.5.0
+        uses: wandalen/wretry.action@v3.7.2
         with:
           action: codecov/codecov-action@v4
           with: |

CONTRIBUTING.md

@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 18.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -105,7 +105,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 18, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-18" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

conanfile.py

@@ -28,7 +28,8 @@ class Clio(ConanFile):
         'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.3.0-b1',
+        'xrpl/2.3.0',
+        'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]
 

docker/ci/dockerfile

@@ -7,7 +7,7 @@ USER root
 WORKDIR /root
 
 ENV CCACHE_VERSION=4.10.2 \
-    LLVM_TOOLS_VERSION=18 \
+    LLVM_TOOLS_VERSION=19 \
     GH_VERSION=2.40.0 \
     DOXYGEN_VERSION=1.12.0
  

docs/examples/config/example-config.json

@@ -39,6 +39,9 @@
         "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
         "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
     },
+    "rpc": {
+        "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -67,7 +70,14 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_admin": false
+        "local_admin": false,
+        "processing_policy": "parallel", // Could be "sequent" or "parallel".
+        // For sequent policy request from one client connection will be processed one by one and the next one will not be read before
+        // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
+        // send a reply for each request whenever it is ready.
+        "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
+        // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
+        "ws_max_sending_queue_size": 1500
     },
     // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
     "graceful_period": 10.0,

docs/metrics-and-static-analysis.md

@@ -14,7 +14,7 @@ You can find an example docker-compose file, with Prometheus and Grafana configs
 
 ## Using `clang-tidy` for static analysis
 
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 17.0.
+The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
 
 Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
 
@@ -26,5 +26,5 @@ By default Cmake will try to find `clang-tidy` automatically in your system.
 To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
 ```

src/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(etl)
+add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
 target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)

src/app/ClioApplication.cpp

@@ -101,9 +101,7 @@ ClioApplication::run()
     auto backend = data::make_Backend(config_);
 
     // Manages clients subscribed to streams
-    auto subscriptionsRunner = feed::SubscriptionManagerRunner(config_, backend);
-
-    auto const subscriptions = subscriptionsRunner.getManager();
+    auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config_, backend);
 
     // Tracks which ledgers have been validated by the network
     auto ledgers = etl::NetworkValidatedLedgers::make_ValidatedLedgers();
@@ -123,12 +121,14 @@ ClioApplication::run()
     auto const handlerProvider = std::make_shared<rpc::impl::ProductionHandlerProvider const>(
         config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
     );
+
+    using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
     auto const rpcEngine =
-        rpc::RPCEngine::make_RPCEngine(backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+        RPCEngineType::make_RPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
 
     // Init the web server
     auto handler =
-        std::make_shared<web::RPCServerHandler<rpc::RPCEngine, etl::ETLService>>(config_, backend, rpcEngine, etl);
+        std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
     auto const httpServer = web::make_HttpServer(config_, ioc, dosGuard, handler);
 
     // Blocks until stopped.

src/data/AmendmentCenter.hpp

@@ -124,6 +124,13 @@ struct Amendments {
     REGISTER(NFTokenMintOffer);
     REGISTER(fixReducedOffersV2);
     REGISTER(fixEnforceNFTokenTrustline);
+    REGISTER(fixInnerObjTemplate2);
+    REGISTER(fixNFTokenPageLinks);
+    REGISTER(InvariantsV1_1);
+    REGISTER(MPTokensV1);
+    REGISTER(fixAMMv1_2);
+    REGISTER(AMMClawback);
+    REGISTER(Credentials);
 
     // Obsolete but supported by libxrpl
     REGISTER(CryptoConditionsSuite);

src/data/BackendInterface.hpp

@@ -364,6 +364,25 @@ public:
         boost::asio::yield_context yield
     ) const = 0;
 
+    /**
+     * @brief Fetches all holders' balances for a MPTIssuanceID
+     *
+     * @param mptID MPTIssuanceID you wish you query.
+     * @param limit Paging limit.
+     * @param cursorIn Optional cursor to allow us to pick up from where we last left off.
+     * @param ledgerSequence The ledger sequence to fetch for
+     * @param yield Currently executing coroutine.
+     * @return std::vector<Blob> of MPToken balances and an optional marker
+     */
+    virtual MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const = 0;
+
     /**
      * @brief Fetches a specific ledger object.
      *
@@ -617,6 +636,14 @@ public:
     virtual void
     writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;
 
+    /**
+     * @brief Write accounts that started holding onto a MPT.
+     *
+     * @param data A vector of MPT ID and account pairs
+     */
+    virtual void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) = 0;
+
     /**
      * @brief Write a new successor.
      *

src/data/CassandraBackend.hpp

@@ -547,6 +547,45 @@ public:
         return ret;
     }
 
+    MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
+    {
+        auto const holderEntries = executor_.read(
+            yield, schema_->selectMPTHolders, mptID, cursorIn.value_or(ripple::AccountID(0)), Limit{limit}
+        );
+
+        auto const& holderResults = holderEntries.value();
+        if (not holderResults.hasRows()) {
+            LOG(log_.debug()) << "No rows returned";
+            return {};
+        }
+
+        std::vector<ripple::uint256> mptKeys;
+        std::optional<ripple::AccountID> cursor;
+        for (auto const [holder] : extract<ripple::AccountID>(holderResults)) {
+            mptKeys.push_back(ripple::keylet::mptoken(mptID, holder).key);
+            cursor = holder;
+        }
+
+        auto mptObjects = doFetchLedgerObjects(mptKeys, ledgerSequence, yield);
+
+        auto it = std::remove_if(mptObjects.begin(), mptObjects.end(), [](Blob const& mpt) { return mpt.empty(); });
+
+        mptObjects.erase(it, mptObjects.end());
+
+        ASSERT(mptKeys.size() <= limit, "Number of keys can't exceed the limit");
+        if (mptKeys.size() == limit)
+            return {mptObjects, cursor};
+
+        return {mptObjects, {}};
+    }
+
     std::optional<Blob>
     doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
         const override
@@ -905,6 +944,17 @@ public:
         executor_.write(std::move(statements));
     }
 
+    void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) override
+    {
+        std::vector<Statement> statements;
+        statements.reserve(data.size());
+        for (auto [mptId, holder] : data)
+            statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
+
+        executor_.write(std::move(statements));
+    }
+
     void
     startWrites() const override
     {

src/data/DBHelpers.hpp

@@ -172,6 +172,14 @@ struct NFTsData {
     }
 };
 
+/**
+ * @brief Represents an MPT and holder pair
+ */
+struct MPTHolderData {
+    ripple::uint192 mptID;
+    ripple::AccountID holder;
+};
+
 /**
  * @brief Check whether the supplied object is an offer.
  *

src/data/Types.hpp

@@ -233,6 +233,14 @@ struct NFTsAndCursor {
     std::optional<ripple::uint256> cursor;
 };
 
+/**
+ * @brief Represents an array of MPTokens
+ */
+struct MPTHoldersAndCursor {
+    std::vector<Blob> mptokens;
+    std::optional<ripple::AccountID> cursor;
+};
+
 /**
  * @brief Stores a range of sequences as a min and max pair.
  */

src/data/cassandra/Schema.hpp

@@ -257,6 +257,19 @@ public:
             qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                    mpt_id blob,
+                    holder blob,
+                   PRIMARY KEY (mpt_id, holder)
+                  ) 
+             WITH CLUSTERING ORDER BY (holder ASC)
+            )",
+            qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+        ));
+
         return statements;
     }();
 
@@ -393,6 +406,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMPTHolder = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {} 
+                       (mpt_id, holder)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement insertLedgerHeader = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(
@@ -687,6 +711,20 @@ public:
             ));
         }();
 
+        PreparedStatement selectMPTHolders = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT holder
+                  FROM {}    
+                 WHERE mpt_id = ?
+                   AND holder > ?
+              ORDER BY holder ASC
+                 LIMIT ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement selectLedgerByHash = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(

src/data/cassandra/impl/Statement.hpp

@@ -106,9 +106,9 @@ public:
         using UintByteTupleType = std::tuple<uint32_t, ripple::uint256>;
         using ByteVectorType = std::vector<ripple::uint256>;
 
-        if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
+        if constexpr (std::is_same_v<DecayedType, ripple::uint256> || std::is_same_v<DecayedType, ripple::uint192>) {
             auto const rc = bindBytes(value.data(), value.size());
-            throwErrorIfNeeded(rc, "Bind ripple::uint256");
+            throwErrorIfNeeded(rc, "Bind ripple::base_uint");
         } else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
             auto const rc = bindBytes(value.data(), value.size());
             throwErrorIfNeeded(rc, "Bind ripple::AccountID");

src/etl/CMakeLists.txt

@@ -10,8 +10,8 @@ target_sources(
           NetworkValidatedLedgers.cpp
           NFTHelpers.cpp
           Source.cpp
+          MPTHelpers.cpp
           impl/AmendmentBlockHandler.cpp
-          impl/ForwardingCache.cpp
           impl/ForwardingSource.cpp
           impl/GrpcSource.cpp
           impl/SubscriptionSource.cpp

src/etl/LoadBalancer.cpp

@@ -27,6 +27,7 @@
 #include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
 #include "util/Random.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
@@ -34,6 +35,7 @@
 #include <boost/json/array.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
 #include <fmt/core.h>
 
 #include <algorithm>
@@ -79,7 +81,10 @@ LoadBalancer::LoadBalancer(
 {
     auto const forwardingCacheTimeout = config.valueOr<float>("forwarding.cache_timeout", 0.f);
     if (forwardingCacheTimeout > 0.f) {
-        forwardingCache_ = impl::ForwardingCache{Config::toMilliseconds(forwardingCacheTimeout)};
+        forwardingCache_ = util::ResponseExpirationCache{
+            Config::toMilliseconds(forwardingCacheTimeout),
+            {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
+        };
     }
 
     static constexpr std::uint32_t MAX_DOWNLOAD = 256;
@@ -111,10 +116,13 @@ LoadBalancer::LoadBalancer(
             validatedLedgers,
             forwardingTimeout,
             [this]() {
-                if (not hasForwardingSource_)
+                if (not hasForwardingSource_.lock().get())
+                    chooseForwardingSource();
+            },
+            [this](bool wasForwarding) {
+                if (wasForwarding)
                     chooseForwardingSource();
             },
-            [this]() { chooseForwardingSource(); },
             [this]() {
                 if (forwardingCache_.has_value())
                     forwardingCache_->invalidate();
@@ -221,8 +229,12 @@ LoadBalancer::forwardToRippled(
     boost::asio::yield_context yield
 )
 {
+    if (not request.contains("command"))
+        return std::unexpected{rpc::ClioError::rpcCOMMAND_IS_MISSING};
+
+    auto const cmd = boost::json::value_to<std::string>(request.at("command"));
     if (forwardingCache_) {
-        if (auto cachedResponse = forwardingCache_->get(request); cachedResponse) {
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
             return std::move(cachedResponse).value();
         }
     }
@@ -250,7 +262,7 @@ LoadBalancer::forwardToRippled(
 
     if (response) {
         if (forwardingCache_ and not response->contains("error"))
-            forwardingCache_->put(request, *response);
+            forwardingCache_->put(cmd, *response);
         return std::move(response).value();
     }
 
@@ -322,11 +334,13 @@ LoadBalancer::getETLState() noexcept
 void
 LoadBalancer::chooseForwardingSource()
 {
-    hasForwardingSource_ = false;
+    LOG(log_.info()) << "Choosing a new source to forward subscriptions";
+    auto hasForwardingSourceLock = hasForwardingSource_.lock();
+    hasForwardingSourceLock.get() = false;
     for (auto& source : sources_) {
-        if (not hasForwardingSource_ and source->isConnected()) {
+        if (not hasForwardingSourceLock.get() and source->isConnected()) {
             source->setForwarding(true);
-            hasForwardingSource_ = true;
+            hasForwardingSourceLock.get() = true;
         } else {
             source->setForwarding(false);
         }

src/etl/LoadBalancer.hpp

@@ -23,9 +23,10 @@
 #include "etl/ETLState.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
-#include "etl/impl/ForwardingCache.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
+#include "util/Mutex.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 
@@ -39,7 +40,6 @@
 #include <org/xrpl/rpc/v1/ledger.pb.h>
 #include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <atomic>
 #include <chrono>
 #include <cstdint>
 #include <expected>
@@ -69,14 +69,17 @@ private:
 
     util::Logger log_{"ETL"};
     // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
-    std::optional<impl::ForwardingCache> forwardingCache_;
+    std::optional<util::ResponseExpirationCache> forwardingCache_;
     std::optional<std::string> forwardingXUserValue_;
 
     std::vector<SourcePtr> sources_;
     std::optional<ETLState> etlState_;
     std::uint32_t downloadRanges_ =
         DEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
-    std::atomic_bool hasForwardingSource_{false};
+
+    // Using mutext instead of atomic_bool because choosing a new source to
+    // forward messages should be done with a mutual exclusion otherwise there will be a race condition
+    util::Mutex<bool> hasForwardingSource_{false};
 
 public:
     /**

src/etl/MPTHelpers.cpp

@@ -0,0 +1,83 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/DBHelpers.hpp"
+
+#include <ripple/protocol/STBase.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/LedgerFormats.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/TER.h>
+#include <xrpl/protocol/TxFormats.h>
+
+#include <optional>
+#include <string>
+
+namespace etl {
+
+/**
+ * @brief Get the MPToken created from a transaction
+ *
+ * @param txMeta Transaction metadata
+ * @return MPT and holder account pair
+ */
+static std::optional<MPTHolderData>
+getMPTokenAuthorize(ripple::TxMeta const& txMeta)
+{
+    for (ripple::STObject const& node : txMeta.getNodes()) {
+        if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+            continue;
+
+        if (node.getFName() == ripple::sfCreatedNode) {
+            auto const& newMPT = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
+            return MPTHolderData{newMPT[ripple::sfMPTokenIssuanceID], newMPT.getAccountID(ripple::sfAccount)};
+        }
+    }
+    return {};
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
+{
+    if (txMeta.getResultTER() != ripple::tesSUCCESS || sttx.getTxnType() != ripple::TxType::ttMPTOKEN_AUTHORIZE)
+        return {};
+
+    return getMPTokenAuthorize(txMeta);
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob)
+{
+    ripple::STLedgerEntry const sle =
+        ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
+
+    if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+        return {};
+
+    auto const mptIssuanceID = sle[ripple::sfMPTokenIssuanceID];
+    auto const holder = sle.getAccountID(ripple::sfAccount);
+
+    return MPTHolderData{mptIssuanceID, holder};
+}
+
+}  // namespace etl

src/etl/MPTHelpers.hpp

@@ -0,0 +1,50 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include "data/DBHelpers.hpp"
+
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+
+namespace etl {
+
+/**
+ * @brief Pull MPT data from TX via ETLService.
+ *
+ * @param txMeta Transaction metadata
+ * @param sttx The transaction
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
+
+/**
+ * @brief Pull MPT data from ledger object via loadInitialLedger.
+ *
+ * @param key The owner key
+ * @param blob Object data as blob
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob);
+
+}  // namespace etl

src/etl/Source.hpp

@@ -53,7 +53,7 @@ namespace etl {
 class SourceBase {
 public:
     using OnConnectHook = std::function<void()>;
-    using OnDisconnectHook = std::function<void()>;
+    using OnDisconnectHook = std::function<void(bool)>;
     using OnLedgerClosedHook = std::function<void()>;
 
     virtual ~SourceBase() = default;

src/etl/impl/AsyncData.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/ETLHelpers.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
@@ -154,6 +155,11 @@ public:
                     backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
                 lastKey_ = obj.key();
                 backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
+
+                auto const maybeMPTHolder = getMPTHolderFromObj(obj.key(), obj.data());
+                if (maybeMPTHolder)
+                    backend.writeMPTHolders({*maybeMPTHolder});
+
                 backend.writeLedgerObject(
                     std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data())
                 );

src/etl/impl/GrpcSource.cpp

@@ -47,7 +47,7 @@
 namespace etl::impl {
 
 GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::shared_ptr<BackendInterface> backend)
-    : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
+    : log_(fmt::format("GrpcSource[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
 {
     try {
         boost::asio::io_context ctx;

src/etl/impl/LedgerLoader.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
@@ -55,6 +56,7 @@ struct FormattedTransactionsData {
     std::vector<AccountTransactionsData> accountTxData;
     std::vector<NFTTransactionsData> nfTokenTxData;
     std::vector<NFTsData> nfTokensData;
+    std::vector<MPTHolderData> mptHoldersData;
 };
 
 namespace etl::impl {
@@ -124,6 +126,10 @@ public:
             if (maybeNFT)
                 result.nfTokensData.push_back(*maybeNFT);
 
+            auto const maybeMPTHolder = getMPTHolderFromTx(txMeta, sttx);
+            if (maybeMPTHolder)
+                result.mptHoldersData.push_back(*maybeMPTHolder);
+
             result.accountTxData.emplace_back(txMeta, sttx.getTransactionID());
             static constexpr std::size_t KEY_SIZE = 32;
             std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), KEY_SIZE};
@@ -240,6 +246,7 @@ public:
                 backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
                 backend_->writeNFTs(insertTxResult.nfTokensData);
                 backend_->writeNFTTransactions(insertTxResult.nfTokenTxData);
+                backend_->writeMPTHolders(insertTxResult.mptHoldersData);
             }
 
             backend_->finishWrites(sequence);

src/etl/impl/SubscriptionSource.cpp

@@ -24,6 +24,8 @@
 #include "rpc/JS.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 #include "util/requests/Types.hpp"
 
 #include <boost/algorithm/string/classification.hpp>
@@ -66,22 +68,28 @@ SubscriptionSource::SubscriptionSource(
     OnConnectHook onConnect,
     OnDisconnectHook onDisconnect,
     OnLedgerClosedHook onLedgerClosed,
-    std::chrono::steady_clock::duration const connectionTimeout,
+    std::chrono::steady_clock::duration const wsTimeout,
     std::chrono::steady_clock::duration const retryDelay
 )
-    : log_(fmt::format("GrpcSource[{}:{}]", ip, wsPort))
+    : log_(fmt::format("SubscriptionSource[{}:{}]", ip, wsPort))
     , wsConnectionBuilder_(ip, wsPort)
     , validatedLedgers_(std::move(validatedLedgers))
     , subscriptions_(std::move(subscriptions))
     , strand_(boost::asio::make_strand(ioContext))
+    , wsTimeout_(wsTimeout)
     , retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))
+    , lastMessageTimeSecondsSinceEpoch_(PrometheusService::gaugeInt(
+          "subscription_source_last_message_time",
+          util::prometheus::Labels({{"source", fmt::format("{}:{}", ip, wsPort)}}),
+          "Seconds since epoch of the last message received from rippled subscription streams"
+      ))
 {
     wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
         .addHeader({"X-User", "clio-client"})
-        .setConnectionTimeout(connectionTimeout);
+        .setConnectionTimeout(wsTimeout_);
 }
 
 SubscriptionSource::~SubscriptionSource()
@@ -133,6 +141,7 @@ void
 SubscriptionSource::setForwarding(bool isForwarding)
 {
     isForwarding_ = isForwarding;
+    LOG(log_.info()) << "Forwarding set to " << isForwarding_;
 }
 
 std::chrono::steady_clock::time_point
@@ -166,20 +175,22 @@ SubscriptionSource::subscribe()
             }
 
             wsConnection_ = std::move(connection).value();
-            isConnected_ = true;
-            onConnect_();
 
             auto const& subscribeCommand = getSubscribeCommandJson();
-            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield);
+            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield, wsTimeout_);
             if (writeErrorOpt) {
                 handleError(writeErrorOpt.value(), yield);
                 return;
             }
 
+            isConnected_ = true;
+            LOG(log_.info()) << "Connected";
+            onConnect_();
+
             retry_.reset();
 
             while (!stop_) {
-                auto const message = wsConnection_->read(yield);
+                auto const message = wsConnection_->read(yield, wsTimeout_);
                 if (not message) {
                     handleError(message.error(), yield);
                     return;
@@ -224,10 +235,11 @@ SubscriptionSource::handleMessage(std::string const& message)
                 auto validatedLedgers = boost::json::value_to<std::string>(result.at(JS(validated_ledgers)));
                 setValidatedRange(std::move(validatedLedgers));
             }
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message on ledger subscription stream. Message: " << object;
 
         } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_LedgerClosed) {
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message of type 'ledgerClosed' on ledger subscription stream. Message: "
+                              << object;
             if (object.contains(JS(ledger_index))) {
                 ledgerIndex = object.at(JS(ledger_index)).as_int64();
             }
@@ -245,10 +257,13 @@ SubscriptionSource::handleMessage(std::string const& message)
                 // 2 - Validated transaction
                 // Only forward proposed transaction, validated transactions are sent by Clio itself
                 if (object.contains(JS(transaction)) and !object.contains(JS(meta))) {
+                    LOG(log_.debug()) << "Forwarding proposed transaction: " << object;
                     subscriptions_->forwardProposedTransaction(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ValidationReceived) {
+                    LOG(log_.debug()) << "Forwarding validation: " << object;
                     subscriptions_->forwardValidation(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ManifestReceived) {
+                    LOG(log_.debug()) << "Forwarding manifest: " << object;
                     subscriptions_->forwardManifest(object);
                 }
             }
@@ -261,7 +276,7 @@ SubscriptionSource::handleMessage(std::string const& message)
 
         return std::nullopt;
     } catch (std::exception const& e) {
-        LOG(log_.error()) << "Exception in handleMessage : " << e.what();
+        LOG(log_.error()) << "Exception in handleMessage: " << e.what();
         return util::requests::RequestError{fmt::format("Error handling message: {}", e.what())};
     }
 }
@@ -270,16 +285,14 @@ void
 SubscriptionSource::handleError(util::requests::RequestError const& error, boost::asio::yield_context yield)
 {
     isConnected_ = false;
-    isForwarding_ = false;
+    bool const wasForwarding = isForwarding_.exchange(false);
     if (not stop_) {
-        onDisconnect_();
+        LOG(log_.info()) << "Disconnected";
+        onDisconnect_(wasForwarding);
     }
 
     if (wsConnection_ != nullptr) {
-        auto const err = wsConnection_->close(yield);
-        if (err) {
-            LOG(log_.error()) << "Error closing websocket connection: " << err->message();
-        }
+        wsConnection_->close(yield);
         wsConnection_.reset();
     }
 
@@ -306,7 +319,11 @@ SubscriptionSource::logError(util::requests::RequestError const& error) const
 void
 SubscriptionSource::setLastMessageTime()
 {
-    lastMessageTime_.lock().get() = std::chrono::steady_clock::now();
+    lastMessageTimeSecondsSinceEpoch_.get().set(
+        std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()
+    );
+    auto lock = lastMessageTime_.lock();
+    lock.get() = std::chrono::steady_clock::now();
 }
 
 void

src/etl/impl/SubscriptionSource.hpp

@@ -25,6 +25,7 @@
 #include "util/Mutex.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 #include "util/requests/Types.hpp"
 #include "util/requests/WsConnection.hpp"
 
@@ -37,6 +38,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <functional>
 #include <future>
 #include <memory>
 #include <optional>
@@ -71,6 +73,8 @@ private:
 
     boost::asio::strand<boost::asio::io_context::executor_type> strand_;
 
+    std::chrono::steady_clock::duration wsTimeout_;
+
     util::Retry retry_;
 
     OnConnectHook onConnect_;
@@ -83,9 +87,11 @@ private:
 
     util::Mutex<std::chrono::steady_clock::time_point> lastMessageTime_;
 
+    std::reference_wrapper<util::prometheus::GaugeInt> lastMessageTimeSecondsSinceEpoch_;
+
     std::future<void> runFuture_;
 
-    static constexpr std::chrono::seconds CONNECTION_TIMEOUT{30};
+    static constexpr std::chrono::seconds WS_TIMEOUT{30};
     static constexpr std::chrono::seconds RETRY_MAX_DELAY{30};
     static constexpr std::chrono::seconds RETRY_DELAY{1};
 
@@ -103,7 +109,7 @@ public:
      * @param onDisconnect The onDisconnect hook. Called when the connection is lost
      * @param onLedgerClosed The onLedgerClosed hook. Called when the ledger is closed but only if the source is
      * forwarding
-     * @param connectionTimeout The connection timeout. Defaults to 30 seconds
+     * @param wsTimeout A timeout for websocket operations. Defaults to 30 seconds
      * @param retryDelay The retry delay. Defaults to 1 second
      */
     SubscriptionSource(
@@ -115,7 +121,7 @@ public:
         OnConnectHook onConnect,
         OnDisconnectHook onDisconnect,
         OnLedgerClosedHook onLedgerClosed,
-        std::chrono::steady_clock::duration const connectionTimeout = CONNECTION_TIMEOUT,
+        std::chrono::steady_clock::duration const wsTimeout = WS_TIMEOUT,
         std::chrono::steady_clock::duration const retryDelay = RETRY_DELAY
     );
 

src/etl/impl/Transformer.hpp

@@ -213,6 +213,7 @@ private:
         backend_->writeAccountTransactions(std::move(insertTxResultOp->accountTxData));
         backend_->writeNFTs(insertTxResultOp->nfTokensData);
         backend_->writeNFTTransactions(insertTxResultOp->nfTokenTxData);
+        backend_->writeMPTHolders(insertTxResultOp->mptHoldersData);
 
         auto [success, duration] =
             ::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });

src/etlng/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(clio_etlng INTERFACE)
+
+# target_sources(clio_etlng PRIVATE )
+
+target_link_libraries(clio_etlng INTERFACE clio_data)

src/etlng/Models.hpp

@@ -0,0 +1,129 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/Concepts.hpp"
+
+#include <boost/json/object.hpp>
+#include <fmt/core.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxFormats.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::model {
+
+/**
+ * @brief A specification for the Registry.
+ *
+ * This specification simply defines the transaction types that are to be filtered out from the incoming transactions by
+ * the Registry for its `onTransaction` and `onInitialTransaction` hooks.
+ * It's a compilation error to list the same transaction type more than once.
+ */
+template <ripple::TxType... Types>
+    requires(util::hasNoDuplicates(Types...))
+struct Spec {
+    static constexpr bool SpecTag = true;
+
+    /**
+     * @brief Checks if the transaction type was requested.
+     *
+     * @param type The transaction type
+     * @return true if the transaction was requested; false otherwise
+     */
+    [[nodiscard]] constexpr static bool
+    wants(ripple::TxType type) noexcept
+    {
+        return ((Types == type) || ...);
+    }
+};
+
+/**
+ * @brief Represents a single transaction on the ledger.
+ */
+struct Transaction {
+    std::string raw;  // raw binary blob
+    std::string metaRaw;
+
+    // unpacked blob and meta
+    ripple::STTx sttx;
+    ripple::TxMeta meta;
+
+    // commonly used stuff
+    ripple::uint256 id;
+    std::string key;  // key is the above id as a string of 32 characters
+    ripple::TxType type;
+};
+
+/**
+ * @brief Represents a single object on the ledger.
+ */
+struct Object {
+    /**
+     * @brief Modification type for the object.
+     */
+    enum class ModType : int {
+        Unspecified = 0,
+        Created = 1,
+        Modified = 2,
+        Deleted = 3,
+    };
+
+    ripple::uint256 key;
+    std::string keyRaw;
+    ripple::Blob data;
+    std::string dataRaw;
+    std::string successor;
+    std::string predecessor;
+
+    ModType type;
+};
+
+/**
+ * @brief Represents a book successor.
+ */
+struct BookSuccessor {
+    std::string firstBook;
+    std::string bookBase;
+};
+
+/**
+ * @brief Represents an entire ledger diff worth of transactions and objects.
+ */
+struct LedgerData {
+    std::vector<Transaction> transactions;
+    std::vector<Object> objects;
+    std::optional<std::vector<BookSuccessor>> successors;
+
+    ripple::LedgerHeader header;
+    std::string rawHeader;
+    uint32_t seq;
+};
+
+}  // namespace etlng::model

src/etlng/RegistryInterface.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief The interface for a registry that can dispatch transactions and objects to extensions.
+ *
+ * This class defines the interface for dispatching data through to extensions.
+ *
+ * @note
+ * The registry itself consists of Extensions.
+ * Each extension must define at least one valid hook:
+ * - for ongoing ETL dispatch:
+ *   - void onLedgerData(etlng::model::LedgerData const&)
+ *   - void onTransaction(uint32_t, etlng::model::Transaction const&)
+ *   - void onObject(uint32_t, etlng::model::Object const&)
+ * - for initial ledger load
+ *   - void onInitialData(etlng::model::LedgerData const&)
+ *   - void onInitialTransaction(uint32_t, etlng::model::Transaction const&)
+ * - for initial objects (called for each downloaded batch)
+ *   - void onInitialObjects(uint32_t, std::vector<etlng::model::Object> const&, std::string)
+ *   - void onInitialObject(uint32_t, etlng::model::Object const&)
+ *
+ * When the registry dispatches (initial)data or objects, each of the above hooks will be called in order on each
+ * registered extension.
+ * This means that the order of execution is from left to right (hooks) and top to bottom (registered extensions).
+ *
+ * If either `onTransaction` or `onInitialTransaction` are defined, the extension will have to additionally define a
+ * Specification. The specification lists transaction types to filter from the incoming data such that `onTransaction`
+ * and `onInitialTransaction` are only called for the transactions that are of interest for the given extension.
+ *
+ * The specification is setup like so:
+ * @code{.cpp}
+ * struct Ext {
+ *   using spec = etlng::model::Spec<
+ *     ripple::TxType::ttNFTOKEN_BURN,
+ *     ripple::TxType::ttNFTOKEN_ACCEPT_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CREATE_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CANCEL_OFFER,
+ *     ripple::TxType::ttNFTOKEN_MINT>;
+ *
+ *   static void
+ *   onInitialTransaction(uint32_t, etlng::model::Transaction const&);
+ * };
+ * @endcode
+ */
+struct RegistryInterface {
+    virtual ~RegistryInterface() = default;
+
+    /**
+     * @brief Dispatch initial objects.
+     *
+     * These objects are received during initial ledger load.
+     *
+     * @param seq The sequence
+     * @param data The objects to dispatch
+     * @param lastKey The predcessor of the first object in data if known; an empty string otherwise
+     */
+    virtual void
+    dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) = 0;
+
+    /**
+     * @brief Dispatch initial ledger data.
+     *
+     * The transactions, header and edge keys are received during initial ledger load.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatchInitialData(model::LedgerData const& data) = 0;
+
+    /**
+     * @brief Dispatch an entire ledger diff.
+     *
+     * This is used to dispatch incoming diffs through the extensions.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatch(model::LedgerData const& data) = 0;
+};
+
+}  // namespace etlng

src/etlng/impl/Registry.hpp

@@ -0,0 +1,219 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+#include "etlng/RegistryInterface.hpp"
+
+#include <xrpl/protocol/TxFormats.h>
+
+#include <concepts>
+#include <cstdint>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+template <typename T>
+concept HasLedgerDataHook = requires(T p) {
+    { p.onLedgerData(std::declval<etlng::model::LedgerData>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialDataHook = requires(T p) {
+    { p.onInitialData(std::declval<etlng::model::LedgerData>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasTransactionHook = requires(T p) {
+    { p.onTransaction(uint32_t{}, std::declval<etlng::model::Transaction>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasObjectHook = requires(T p) {
+    { p.onObject(uint32_t{}, std::declval<etlng::model::Object>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialTransactionHook = requires(T p) {
+    { p.onInitialTransaction(uint32_t{}, std::declval<etlng::model::Transaction>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialObjectsHook = requires(T p) {
+    {
+        p.onInitialObjects(uint32_t{}, std::declval<std::vector<etlng::model::Object>>(), std::string{})
+    } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialObjectHook = requires(T p) {
+    { p.onInitialObject(uint32_t{}, std::declval<etlng::model::Object>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept ContainsSpec = std::decay_t<T>::spec::SpecTag;
+
+template <typename T>
+concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or
+    (HasTransactionHook<T> and ContainsSpec<T>) or (HasInitialTransactionHook<T> and ContainsSpec<T>) or
+    HasObjectHook<T> or HasInitialObjectsHook<T> or HasInitialObjectHook<T>;
+
+template <typename T>
+concept NoTwoOfKind = not(HasLedgerDataHook<T> and HasTransactionHook<T>) and
+    not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and not(HasInitialDataHook<T> and HasObjectHook<T>) and
+    not(HasInitialObjectsHook<T> and HasInitialObjectHook<T>);
+
+template <typename T>
+concept SomeExtension = NoTwoOfKind<T> and ContainsValidHook<T>;
+
+template <SomeExtension... Ps>
+class Registry : public RegistryInterface {
+    std::tuple<Ps...> store_;
+
+    static_assert(
+        (((not HasTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and ...),
+        "Spec must be specified when 'onTransaction' function exists."
+    );
+
+    static_assert(
+        (((not HasInitialTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and ...),
+        "Spec must be specified when 'onInitialTransaction' function exists."
+    );
+
+public:
+    explicit constexpr Registry(SomeExtension auto&&... exts)
+        requires(std::is_same_v<std::decay_t<decltype(exts)>, std::decay_t<Ps>> and ...)
+        : store_(std::forward<Ps>(exts)...)
+    {
+    }
+
+    ~Registry() override = default;
+    Registry(Registry const&) = delete;
+    Registry(Registry&&) = default;
+    Registry&
+    operator=(Registry const&) = delete;
+    Registry&
+    operator=(Registry&&) = default;
+
+    constexpr void
+    dispatch(model::LedgerData const& data) override
+    {
+        // send entire batch of data at once
+        {
+            auto const expand = [&](auto& p) {
+                if constexpr (requires { p.onLedgerData(data); }) {
+                    p.onLedgerData(data);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send filtered transactions
+        {
+            auto const expand = [&]<typename P>(P& p, model::Transaction const& t) {
+                if constexpr (requires { p.onTransaction(data.seq, t); }) {
+                    if (std::decay_t<P>::spec::wants(t.type))
+                        p.onTransaction(data.seq, t);
+                }
+            };
+
+            for (auto const& t : data.transactions) {
+                std::apply([&expand, &t](auto&&... xs) { (expand(xs, t), ...); }, store_);
+            }
+        }
+
+        // send per object path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
+                if constexpr (requires { p.onObject(data.seq, o); }) {
+                    p.onObject(data.seq, o);
+                }
+            };
+
+            for (auto const& obj : data.objects) {
+                std::apply([&expand, &obj](auto&&... xs) { (expand(xs, obj), ...); }, store_);
+            }
+        }
+    }
+
+    constexpr void
+    dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) override
+    {
+        // send entire vector path
+        {
+            auto const expand = [&](auto&& p) {
+                if constexpr (requires { p.onInitialObjects(seq, data, lastKey); }) {
+                    p.onInitialObjects(seq, data, lastKey);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send per object path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
+                if constexpr (requires { p.onInitialObject(seq, o); }) {
+                    p.onInitialObject(seq, o);
+                }
+            };
+
+            for (auto const& obj : data) {
+                std::apply([&expand, &obj](auto&&... xs) { (expand(xs, obj), ...); }, store_);
+            }
+        }
+    }
+
+    constexpr void
+    dispatchInitialData(model::LedgerData const& data) override
+    {
+        // send entire batch path
+        {
+            auto const expand = [&](auto&& p) {
+                if constexpr (requires { p.onInitialData(data); }) {
+                    p.onInitialData(data);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send per tx path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Transaction const& tx) {
+                if constexpr (requires { p.onInitialTransaction(data.seq, tx); }) {
+                    if (std::decay_t<P>::spec::wants(tx.type))
+                        p.onInitialTransaction(data.seq, tx);
+                }
+            };
+
+            for (auto const& tx : data.transactions) {
+                std::apply([&expand, &tx](auto&&... xs) { (expand(xs, tx), ...); }, store_);
+            }
+        }
+    }
+};
+
+}  // namespace etlng::impl

src/feed/SubscriptionManager.hpp

@@ -30,6 +30,7 @@
 #include "feed/impl/TransactionFeed.hpp"
 #include "util/async/AnyExecutionContext.hpp"
 #include "util/async/context/BasicExecutionContext.hpp"
+#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
@@ -44,6 +45,7 @@
 #include <cstdint>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 /**
@@ -67,16 +69,36 @@ class SubscriptionManager : public SubscriptionManagerInterface {
     impl::ProposedTransactionFeed proposedTransactionFeed_;
 
 public:
+    /**
+     * @brief Factory function to create a new SubscriptionManager with a PoolExecutionContext.
+     *
+     * @param config The configuration to use
+     * @param backend The backend to use
+     * @return A shared pointer to a new instance of SubscriptionManager
+     */
+    static std::shared_ptr<SubscriptionManager>
+    make_SubscriptionManager(util::Config const& config, std::shared_ptr<data::BackendInterface const> const& backend)
+    {
+        auto const workersNum = config.valueOr<std::uint64_t>("subscription_workers", 1);
+
+        util::Logger const logger{"Subscriptions"};
+        LOG(logger.info()) << "Starting subscription manager with " << workersNum << " workers";
+
+        return std::make_shared<feed::SubscriptionManager>(util::async::PoolExecutionContext(workersNum), backend);
+    }
+
     /**
      * @brief Construct a new Subscription Manager object
      *
      * @param executor The executor to use to publish the feeds
      * @param backend The backend to use
      */
-    template <class ExecutorCtx>
-    SubscriptionManager(ExecutorCtx& executor, std::shared_ptr<data::BackendInterface const> const& backend)
+    SubscriptionManager(
+        util::async::AnyExecutionContext&& executor,
+        std::shared_ptr<data::BackendInterface const> const& backend
+    )
         : backend_(backend)
-        , ctx_(executor)
+        , ctx_(std::move(executor))
         , manifestFeed_(ctx_, "manifest")
         , validationsFeed_(ctx_, "validations")
         , ledgerFeed_(ctx_)
@@ -291,41 +313,4 @@ public:
     report() const final;
 };
 
-/**
- * @brief The help class to run the subscription manager. The container of PoolExecutionContext which is used to publish
- * the feeds.
- */
-class SubscriptionManagerRunner {
-    std::uint64_t workersNum_;
-    using ActualExecutionCtx = util::async::PoolExecutionContext;
-    ActualExecutionCtx ctx_;
-    std::shared_ptr<SubscriptionManager> subscriptionManager_;
-    util::Logger logger_{"Subscriptions"};
-
-public:
-    /**
-     * @brief Construct a new Subscription Manager Runner object
-     *
-     * @param config The configuration
-     * @param backend The backend to use
-     */
-    SubscriptionManagerRunner(util::Config const& config, std::shared_ptr<data::BackendInterface> const& backend)
-        : workersNum_(config.valueOr<std::uint64_t>("subscription_workers", 1))
-        , ctx_(workersNum_)
-        , subscriptionManager_(std::make_shared<SubscriptionManager>(ctx_, backend))
-    {
-        LOG(logger_.info()) << "Starting subscription manager with " << workersNum_ << " workers";
-    }
-
-    /**
-     * @brief Get the subscription manager
-     *
-     * @return The subscription manager
-     */
-    std::shared_ptr<SubscriptionManager>
-    getManager()
-    {
-        return subscriptionManager_;
-    }
-};
 }  // namespace feed

src/feed/impl/TransactionFeed.cpp

@@ -203,6 +203,7 @@ TransactionFeed::pub(
         pubObj[JS(meta)] = rpc::toJson(*meta);
         rpc::insertDeliveredAmount(pubObj[JS(meta)].as_object(), tx, meta, txMeta.date);
         rpc::insertDeliverMaxAlias(pubObj[txKey].as_object(), version);
+        rpc::insertMPTIssuanceID(pubObj[JS(meta)].as_object(), tx, meta);
 
         pubObj[JS(type)] = "transaction";
         pubObj[JS(validated)] = true;

src/rpc/CMakeLists.txt

@@ -6,6 +6,7 @@ target_sources(
           Factories.cpp
           AMMHelpers.cpp
           RPCHelpers.cpp
+          CredentialHelpers.cpp
           Counters.cpp
           WorkQueue.cpp
           common/Specs.cpp
@@ -33,6 +34,7 @@ target_sources(
           handlers/LedgerEntry.cpp
           handlers/LedgerIndex.cpp
           handlers/LedgerRange.cpp
+          handlers/MPTHolders.cpp
           handlers/NFTsByIssuer.cpp
           handlers/NFTBuyOffers.cpp
           handlers/NFTHistory.cpp

src/rpc/CredentialHelpers.cpp

@@ -0,0 +1,161 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/BackendInterface.hpp"
+#include "rpc/Errors.hpp"
+#include "rpc/JS.hpp"
+#include "rpc/common/Types.hpp"
+#include "util/Assert.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/value_to.hpp>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/StringUtilities.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/chrono.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/Indexes.h>
+#include <xrpl/protocol/LedgerFormats.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STArray.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/jss.h>
+
+#include <cstdint>
+#include <expected>
+#include <optional>
+#include <set>
+#include <string>
+#include <string_view>
+#include <unordered_set>
+#include <utility>
+
+namespace rpc::credentials {
+
+bool
+checkExpired(ripple::SLE const& sleCred, ripple::LedgerHeader const& ledger)
+{
+    if (sleCred.isFieldPresent(ripple::sfExpiration)) {
+        std::uint32_t const exp = sleCred.getFieldU32(ripple::sfExpiration);
+        std::uint32_t const now = ledger.parentCloseTime.time_since_epoch().count();
+        return now > exp;
+    }
+    return false;
+}
+
+std::set<std::pair<ripple::AccountID, ripple::Slice>>
+createAuthCredentials(ripple::STArray const& in)
+{
+    std::set<std::pair<ripple::AccountID, ripple::Slice>> out;
+    for (auto const& cred : in)
+        out.insert({cred[ripple::sfIssuer], cred[ripple::sfCredentialType]});
+
+    return out;
+}
+
+ripple::STArray
+parseAuthorizeCredentials(boost::json::array const& jv)
+{
+    ripple::STArray arr;
+    for (auto const& jo : jv) {
+        ASSERT(
+            jo.at(JS(issuer)).is_string(),
+            "issuer must be string, should already be checked in AuthorizeCredentialValidator"
+        );
+        auto const issuer =
+            ripple::parseBase58<ripple::AccountID>(static_cast<std::string>(jo.at(JS(issuer)).as_string()));
+        ASSERT(
+            issuer.has_value(), "issuer must be present, should already be checked in AuthorizeCredentialValidator."
+        );
+
+        ASSERT(
+            jo.at(JS(credential_type)).is_string(),
+            "credential_type must be string, should already be checked in AuthorizeCredentialValidator"
+        );
+        auto const credentialType = ripple::strUnHex(static_cast<std::string>(jo.at(JS(credential_type)).as_string()));
+        ASSERT(
+            credentialType.has_value(),
+            "credential_type must be present, should already be checked in AuthorizeCredentialValidator."
+        );
+
+        auto credential = ripple::STObject::makeInnerObject(ripple::sfCredential);
+        credential.setAccountID(ripple::sfIssuer, *issuer);
+        credential.setFieldVL(ripple::sfCredentialType, *credentialType);
+        arr.push_back(std::move(credential));
+    }
+
+    return arr;
+}
+
+std::expected<ripple::STArray, Status>
+fetchCredentialArray(
+    std::optional<boost::json::array> const& credID,
+    ripple::AccountID const& srcAcc,
+    BackendInterface const& backend,
+    ripple::LedgerHeader const& info,
+    boost::asio::yield_context const& yield
+)
+{
+    ripple::STArray authCreds;
+    std::unordered_set<std::string_view> elems;
+    for (auto const& elem : credID.value()) {
+        ASSERT(elem.is_string(), "should already be checked in validators.hpp that elem is a string.");
+
+        if (elems.contains(elem.as_string()))
+            return Error{Status{RippledError::rpcBAD_CREDENTIALS, "duplicates in credentials."}};
+        elems.insert(elem.as_string());
+
+        ripple::uint256 credHash;
+        ASSERT(
+            credHash.parseHex(boost::json::value_to<std::string>(elem)),
+            "should already be checked in validators.hpp that elem is a uint256 hex"
+        );
+
+        auto const credKeylet = ripple::keylet::credential(credHash).key;
+        auto const credLedgerObject = backend.fetchLedgerObject(credKeylet, info.seq, yield);
+        if (!credLedgerObject)
+            return Error{Status{RippledError::rpcBAD_CREDENTIALS, "credentials don't exist."}};
+
+        auto credIt = ripple::SerialIter{credLedgerObject->data(), credLedgerObject->size()};
+        auto const sleCred = ripple::SLE{credIt, credKeylet};
+
+        if ((sleCred.getType() != ripple::ltCREDENTIAL) ||
+            ((sleCred.getFieldU32(ripple::sfFlags) & ripple::lsfAccepted) == 0u))
+            return Error{Status{RippledError::rpcBAD_CREDENTIALS, "credentials aren't accepted"}};
+
+        if (credentials::checkExpired(sleCred, info))
+            return Error{Status{RippledError::rpcBAD_CREDENTIALS, "credentials are expired"}};
+
+        if (sleCred.getAccountID(ripple::sfSubject) != srcAcc)
+            return Error{Status{RippledError::rpcBAD_CREDENTIALS, "credentials don't belong to the root account"}};
+
+        auto credential = ripple::STObject::makeInnerObject(ripple::sfCredential);
+        credential.setAccountID(ripple::sfIssuer, sleCred.getAccountID(ripple::sfIssuer));
+        credential.setFieldVL(ripple::sfCredentialType, sleCred.getFieldVL(ripple::sfCredentialType));
+        authCreds.push_back(std::move(credential));
+    }
+
+    return authCreds;
+}
+
+}  // namespace rpc::credentials

src/rpc/CredentialHelpers.hpp

@@ -0,0 +1,89 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "rpc/Errors.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json/array.hpp>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/chrono.h>
+#include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/Protocol.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+
+#include <expected>
+#include <optional>
+#include <set>
+#include <utility>
+
+namespace rpc::credentials {
+
+/**
+ * @brief Check if credential is expired
+ *
+ * @param sleCred The credential to check
+ * @param ledger The ledger to check the closed time of
+ * @return true if credential not expired, false otherwise
+ */
+bool
+checkExpired(ripple::SLE const& sleCred, ripple::LedgerHeader const& ledger);
+
+/**
+ * @brief Creates authentication credential field (which is a set of pairs of AccountID and Credential ID)
+ *
+ * @param in The array of Credential objects to check
+ * @return Auth Credential array
+ */
+std::set<std::pair<ripple::AccountID, ripple::Slice>>
+createAuthCredentials(ripple::STArray const& in);
+
+/**
+ * @brief Parses each credential object and makes sure the credential type and values are correct
+ *
+ * @param jv The boost json array of credentials to parse
+ * @return Array of credentials after parsing
+ */
+ripple::STArray
+parseAuthorizeCredentials(boost::json::array const& jv);
+
+/**
+ * @brief Get Array of Credential objects
+ *
+ * @param credID Array of CredentialID's to parse
+ * @param srcAcc The Source Account
+ * @param backend backend interface
+ * @param info The ledger header
+ * @param yield The coroutine context
+ * @return Array of credential objects, error if failed otherwise
+ */
+std::expected<ripple::STArray, Status>
+fetchCredentialArray(
+    std::optional<boost::json::array> const& credID,
+    ripple::AccountID const& srcAcc,
+    BackendInterface const& backend,
+    ripple::LedgerHeader const& info,
+    boost::asio::yield_context const& yield
+);
+
+}  // namespace rpc::credentials

src/rpc/Errors.cpp

@@ -83,6 +83,9 @@ getErrorInfo(ClioError code)
         {ClioError::rpcUNKNOWN_OPTION, "unknownOption", "Unknown option."},
         {ClioError::rpcFIELD_NOT_FOUND_TRANSACTION, "fieldNotFoundTransaction", "Missing field."},
         {ClioError::rpcMALFORMED_ORACLE_DOCUMENT_ID, "malformedDocumentID", "Malformed oracle_document_id."},
+        {ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS,
+         "malformedAuthorizedCredentials",
+         "Malformed authorized credentials."},
         // special system errors
         {ClioError::rpcINVALID_API_VERSION, JS(invalid_API_version), "Invalid API version."},
         {ClioError::rpcCOMMAND_IS_MISSING, JS(missingCommand), "Method is not specified or is not a string."},

src/rpc/Errors.hpp

@@ -43,6 +43,7 @@ enum class ClioError {
     rpcUNKNOWN_OPTION = 5005,
     rpcFIELD_NOT_FOUND_TRANSACTION = 5006,
     rpcMALFORMED_ORACLE_DOCUMENT_ID = 5007,
+    rpcMALFORMED_AUTHORIZED_CREDENTIALS = 5008,
 
     // special system errors start with 6000
     rpcINVALID_API_VERSION = 6000,

src/rpc/RPCEngine.hpp

@@ -20,19 +20,22 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
-#include "rpc/Counters.hpp"
 #include "rpc/Errors.hpp"
+#include "rpc/RPCHelpers.hpp"
 #include "rpc/WorkQueue.hpp"
 #include "rpc/common/HandlerProvider.hpp"
 #include "rpc/common/Types.hpp"
 #include "rpc/common/impl/ForwardingProxy.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
 #include "web/Context.hpp"
 #include "web/dosguard/DOSGuardInterface.hpp"
 
 #include <boost/asio/spawn.hpp>
+#include <boost/iterator/transform_iterator.hpp>
 #include <boost/json.hpp>
 #include <fmt/core.h>
+#include <fmt/format.h>
 #include <xrpl/protocol/ErrorCodes.h>
 
 #include <chrono>
@@ -41,14 +44,9 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <unordered_set>
 #include <utility>
 
-// forward declarations
-namespace etl {
-class LoadBalancer;
-class ETLService;
-}  // namespace etl
-
 /**
  * @brief This namespace contains all the RPC logic and handlers.
  */
@@ -57,6 +55,7 @@ namespace rpc {
 /**
  * @brief The RPC engine that ties all RPC-related functionality together.
  */
+template <typename LoadBalancerType, typename CountersType>
 class RPCEngine {
     util::Logger perfLog_{"Performance"};
     util::Logger log_{"RPC"};
@@ -64,16 +63,19 @@ class RPCEngine {
     std::shared_ptr<BackendInterface> backend_;
     std::reference_wrapper<web::dosguard::DOSGuardInterface const> dosGuard_;
     std::reference_wrapper<WorkQueue> workQueue_;
-    std::reference_wrapper<Counters> counters_;
+    std::reference_wrapper<CountersType> counters_;
 
     std::shared_ptr<HandlerProvider const> handlerProvider_;
 
-    impl::ForwardingProxy<etl::LoadBalancer, Counters, HandlerProvider> forwardingProxy_;
+    impl::ForwardingProxy<LoadBalancerType, CountersType, HandlerProvider> forwardingProxy_;
+
+    std::optional<util::ResponseExpirationCache> responseCache_;
 
 public:
     /**
      * @brief Construct a new RPCEngine object
      *
+     * @param config The config to use
      * @param backend The backend to use
      * @param balancer The load balancer to use
      * @param dosGuard The DOS guard to use
@@ -82,11 +84,12 @@ public:
      * @param handlerProvider The handler provider to use
      */
     RPCEngine(
+        util::Config const& config,
         std::shared_ptr<BackendInterface> const& backend,
-        std::shared_ptr<etl::LoadBalancer> const& balancer,
+        std::shared_ptr<LoadBalancerType> const& balancer,
         web::dosguard::DOSGuardInterface const& dosGuard,
         WorkQueue& workQueue,
-        Counters& counters,
+        CountersType& counters,
         std::shared_ptr<HandlerProvider const> const& handlerProvider
     )
         : backend_{backend}
@@ -96,11 +99,22 @@ public:
         , handlerProvider_{handlerProvider}
         , forwardingProxy_{balancer, counters, handlerProvider}
     {
+        // Let main thread catch the exception if config type is wrong
+        auto const cacheTimeout = config.valueOr<float>("rpc.cache_timeout", 0.f);
+
+        if (cacheTimeout > 0.f) {
+            LOG(log_.info()) << fmt::format("Init RPC Cache, timeout: {} seconds", cacheTimeout);
+
+            responseCache_.emplace(
+                util::Config::toMilliseconds(cacheTimeout), std::unordered_set<std::string>{"server_info"}
+            );
+        }
     }
 
     /**
      * @brief Factory function to create a new instance of the RPC engine.
      *
+     * @param config The config to use
      * @param backend The backend to use
      * @param balancer The load balancer to use
      * @param dosGuard The DOS guard to use
@@ -111,15 +125,16 @@ public:
      */
     static std::shared_ptr<RPCEngine>
     make_RPCEngine(
+        util::Config const& config,
         std::shared_ptr<BackendInterface> const& backend,
-        std::shared_ptr<etl::LoadBalancer> const& balancer,
+        std::shared_ptr<LoadBalancerType> const& balancer,
         web::dosguard::DOSGuardInterface const& dosGuard,
         WorkQueue& workQueue,
-        Counters& counters,
+        CountersType& counters,
         std::shared_ptr<HandlerProvider const> const& handlerProvider
     )
     {
-        return std::make_shared<RPCEngine>(backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+        return std::make_shared<RPCEngine>(config, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
     }
 
     /**
@@ -131,8 +146,18 @@ public:
     Result
     buildResponse(web::Context const& ctx)
     {
-        if (forwardingProxy_.shouldForward(ctx))
+        if (forwardingProxy_.shouldForward(ctx)) {
+            // Disallow forwarding of the admin api, only user api is allowed for security reasons.
+            if (isAdminCmd(ctx.method, ctx.params))
+                return Result{Status{RippledError::rpcNO_PERMISSION}};
+
             return forwardingProxy_.forward(ctx);
+        }
+
+        if (not ctx.isAdmin and responseCache_) {
+            if (auto res = responseCache_->get(ctx.method); res.has_value())
+                return Result{std::move(res).value()};
+        }
 
         if (backend_->isTooBusy()) {
             LOG(log_.error()) << "Database is too busy. Rejecting request";
@@ -154,8 +179,11 @@ public:
 
             LOG(perfLog_.debug()) << ctx.tag() << " finish executing rpc `" << ctx.method << '`';
 
-            if (not v)
+            if (not v) {
                 notifyErrored(ctx.method);
+            } else if (not ctx.isAdmin and responseCache_) {
+                responseCache_->put(ctx.method, v.result->as_object());
+            }
 
             return Result{std::move(v)};
         } catch (data::DatabaseTimeout const& t) {

src/rpc/RPCHelpers.cpp

@@ -36,6 +36,7 @@
 #include <boost/json/array.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/parse.hpp>
+#include <boost/json/serialize.hpp>
 #include <boost/json/string.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
@@ -49,6 +50,7 @@
 #include <xrpl/basics/chrono.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/Zero.h>
+#include <xrpl/json/json_reader.h>
 #include <xrpl/json/json_value.h>
 #include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/Book.h>
@@ -79,6 +81,7 @@
 
 #include <algorithm>
 #include <array>
+#include <cassert>
 #include <chrono>
 #include <cstddef>
 #include <cstdint>
@@ -257,6 +260,7 @@ toExpandedJson(
     auto metaJson = toJson(*meta);
     insertDeliveredAmount(metaJson, txn, meta, blobs.date);
     insertDeliverMaxAlias(txnJson, apiVersion);
+    insertMPTIssuanceID(metaJson, txn, meta);
 
     if (nftEnabled == NFTokenjson::ENABLE) {
         Json::Value nftJson;
@@ -312,6 +316,67 @@ insertDeliveredAmount(
     return false;
 }
 
+/**
+ * @brief Get the delivered amount
+ *
+ * @param meta The metadata
+ * @return The mpt_issuance_id or std::nullopt if not available
+ */
+static std::optional<ripple::uint192>
+getMPTIssuanceID(std::shared_ptr<ripple::TxMeta const> const& meta)
+{
+    ripple::TxMeta const& transactionMeta = *meta;
+
+    for (ripple::STObject const& node : transactionMeta.getNodes()) {
+        if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN_ISSUANCE ||
+            node.getFName() != ripple::sfCreatedNode)
+            continue;
+
+        auto const& mptNode = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
+        return ripple::makeMptID(mptNode[ripple::sfSequence], mptNode[ripple::sfIssuer]);
+    }
+
+    return {};
+}
+
+/**
+ * @brief Check if transaction has a new MPToken created
+ *
+ * @param txn The transaction
+ * @param meta The metadata
+ * @return true if the transaction can have a mpt_issuance_id
+ */
+static bool
+canHaveMPTIssuanceID(std::shared_ptr<ripple::STTx const> const& txn, std::shared_ptr<ripple::TxMeta const> const& meta)
+{
+    if (txn->getTxnType() != ripple::ttMPTOKEN_ISSUANCE_CREATE)
+        return false;
+
+    if (meta->getResultTER() != ripple::tesSUCCESS)
+        return false;
+
+    return true;
+}
+
+bool
+insertMPTIssuanceID(
+    boost::json::object& metaJson,
+    std::shared_ptr<ripple::STTx const> const& txn,
+    std::shared_ptr<ripple::TxMeta const> const& meta
+)
+{
+    if (!canHaveMPTIssuanceID(txn, meta))
+        return false;
+
+    if (auto const id = getMPTIssuanceID(meta)) {
+        metaJson[JS(mpt_issuance_id)] = ripple::to_string(*id);
+        return true;
+    }
+
+    assert(false);
+    return false;
+}
+
 void
 insertDeliverMaxAlias(boost::json::object& txJson, std::uint32_t const apiVersion)
 {
@@ -428,8 +493,9 @@ ledgerHeaderFromRequest(std::shared_ptr<data::BackendInterface const> const& bac
             } else {
                 ledgerSequence = parseStringAsUInt(stringIndex);
             }
-        } else if (indexValue.is_int64())
+        } else if (indexValue.is_int64()) {
             ledgerSequence = indexValue.as_int64();
+        }
     } else {
         ledgerSequence = ctx.range.maxSequence;
     }
@@ -947,7 +1013,8 @@ accountHolds(
     auto const blob = backend.fetchLedgerObject(key, sequence, yield);
 
     if (!blob) {
-        amount.clear({currency, issuer});
+        amount.setIssue(ripple::Issue(currency, issuer));
+        amount.clear();
         return amount;
     }
 
@@ -955,7 +1022,8 @@ accountHolds(
     ripple::SLE const sle{it, key};
 
     if (zeroIfFrozen && isFrozen(backend, sequence, account, currency, issuer, yield)) {
-        amount.clear(ripple::Issue(currency, issuer));
+        amount.setIssue(ripple::Issue(currency, issuer));
+        amount.clear();
     } else {
         amount = sle.getFieldAmount(ripple::sfBalance);
         if (account > issuer) {
@@ -1273,6 +1341,31 @@ specifiesCurrentOrClosedLedger(boost::json::object const& request)
     return false;
 }
 
+bool
+isAdminCmd(std::string const& method, boost::json::object const& request)
+{
+    if (method == JS(ledger)) {
+        auto const requestStr = boost::json::serialize(request);
+        Json::Value jv;
+        Json::Reader{}.parse(requestStr, jv);
+        // rippled considers string/non-zero int/non-empty array/ non-empty json as true.
+        // Use rippled's API asBool to get the same result.
+        // https://github.com/XRPLF/rippled/issues/5119
+        auto const isFieldSet = [&jv](auto const field) { return jv.isMember(field) and jv[field].asBool(); };
+
+        // According to doc
+        // https://xrpl.org/docs/references/http-websocket-apis/public-api-methods/ledger-methods/ledger,
+        // full/accounts/type are admin only, but type only works when full/accounts are set, so we don't need to check
+        // type.
+        if (isFieldSet(JS(full)) or isFieldSet(JS(accounts)))
+            return true;
+    }
+
+    if (method == JS(feature) and request.contains(JS(vetoed)))
+        return true;
+    return false;
+}
+
 std::variant<ripple::uint256, Status>
 getNFTID(boost::json::object const& request)
 {

src/rpc/RPCHelpers.hpp

@@ -191,6 +191,21 @@ insertDeliveredAmount(
     uint32_t date
 );
 
+/**
+ * @brief Add "mpt_issuance_id" into MPTokenIssuanceCreate transaction json.
+ *
+ * @param metaJson The metadata json object to add "MPTokenIssuanceID"
+ * @param txn The transaction object
+ * @param meta The metadata object
+ * @return true if the "mpt_issuance_id" is added to the metadata json object
+ */
+bool
+insertMPTIssuanceID(
+    boost::json::object& metaJson,
+    std::shared_ptr<ripple::STTx const> const& txn,
+    std::shared_ptr<ripple::TxMeta const> const& meta
+);
+
 /**
  * @brief Convert STBase object to JSON
  *
@@ -557,6 +572,16 @@ parseIssue(boost::json::object const& issue);
 bool
 specifiesCurrentOrClosedLedger(boost::json::object const& request);
 
+/**
+ * @brief Check whether a request requires administrative privileges on rippled side.
+ *
+ * @param method The method name to check
+ * @param request The request to check
+ * @return true if the request requires ADMIN role
+ */
+bool
+isAdminCmd(std::string const& method, boost::json::object const& request);
+
 /**
  * @brief Get the NFTID from the request
  *

src/rpc/common/APIVersion.hpp

@@ -34,16 +34,13 @@ static constexpr uint32_t API_VERSION_DEFAULT = 1u;
 
 /**
  * @brief Minimum API version supported by this build
- *
- * Note: Clio does not natively support v1 and only supports v2 or newer.
- * However, Clio will forward all v1 requests to rippled for backward compatibility.
  */
 static constexpr uint32_t API_VERSION_MIN = 1u;
 
 /**
  * @brief Maximum API version supported by this build
  */
-static constexpr uint32_t API_VERSION_MAX = 2u;
+static constexpr uint32_t API_VERSION_MAX = 3u;
 
 /**
  * @brief A baseclass for API version helper

src/rpc/common/Validators.cpp

@@ -29,8 +29,10 @@
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
 #include <fmt/core.h>
+#include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/AccountID.h>
+#include <xrpl/protocol/Protocol.h>
 #include <xrpl/protocol/UintTypes.h>
 
 #include <charconv>
@@ -89,16 +91,19 @@ checkIsU32Numeric(std::string_view sv)
     return ec == std::errc();
 }
 
+CustomValidator CustomValidators::Uint160HexStringValidator =
+    CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
+        return makeHexStringValidator<ripple::uint160>(value, key);
+    }};
+
+CustomValidator CustomValidators::Uint192HexStringValidator =
+    CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
+        return makeHexStringValidator<ripple::uint192>(value, key);
+    }};
+
 CustomValidator CustomValidators::Uint256HexStringValidator =
     CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
-        if (!value.is_string())
-            return Error{Status{RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
-
-        ripple::uint256 ledgerHash;
-        if (!ledgerHash.parseHex(boost::json::value_to<std::string>(value)))
-            return Error{Status{RippledError::rpcINVALID_PARAMS, std::string(key) + "Malformed"}};
-
-        return MaybeError{};
+        return makeHexStringValidator<ripple::uint256>(value, key);
     }};
 
 CustomValidator CustomValidators::LedgerIndexValidator =
@@ -200,15 +205,13 @@ CustomValidator CustomValidators::SubscribeStreamValidator =
             "ledger", "transactions", "transactions_proposed", "book_changes", "manifests", "validations"
         };
 
-        static std::unordered_set<std::string> const reportingNotSupportStreams = {
-            "peer_status", "consensus", "server"
-        };
+        static std::unordered_set<std::string> const notSupportStreams = {"peer_status", "consensus", "server"};
         for (auto const& v : value.as_array()) {
             if (!v.is_string())
                 return Error{Status{RippledError::rpcINVALID_PARAMS, "streamNotString"}};
 
-            if (reportingNotSupportStreams.contains(boost::json::value_to<std::string>(v)))
-                return Error{Status{RippledError::rpcREPORTING_UNSUPPORTED}};
+            if (notSupportStreams.contains(boost::json::value_to<std::string>(v)))
+                return Error{Status{RippledError::rpcNOT_SUPPORTED}};
 
             if (not validStreams.contains(boost::json::value_to<std::string>(v)))
                 return Error{Status{RippledError::rpcSTREAM_MALFORMED}};
@@ -252,4 +255,79 @@ CustomValidator CustomValidators::CurrencyIssueValidator =
         return MaybeError{};
     }};
 
+CustomValidator CustomValidators::CredentialTypeValidator =
+    CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
+        if (not value.is_string())
+            return Error{Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, std::string(key) + " NotString"}};
+
+        auto const& credTypeHex = ripple::strViewUnHex(value.as_string());
+        if (!credTypeHex.has_value())
+            return Error{Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, std::string(key) + " NotHexString"}};
+
+        if (credTypeHex->empty())
+            return Error{Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, std::string(key) + " is empty"}};
+
+        if (credTypeHex->size() > ripple::maxCredentialTypeLength) {
+            return Error{
+                Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, std::string(key) + " greater than max length"}
+            };
+        }
+
+        return MaybeError{};
+    }};
+
+CustomValidator CustomValidators::AuthorizeCredentialValidator =
+    CustomValidator{[](boost::json::value const& value, std::string_view key) -> MaybeError {
+        if (not value.is_array())
+            return Error{Status{ClioError::rpcMALFORMED_REQUEST, std::string(key) + " not array"}};
+
+        auto const& authCred = value.as_array();
+        if (authCred.empty()) {
+            return Error{Status{
+                ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS,
+                fmt::format("Requires at least one element in authorized_credentials array.")
+            }};
+        }
+
+        if (authCred.size() > ripple::maxCredentialsArraySize) {
+            return Error{Status{
+                ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS,
+                fmt::format(
+                    "Max {} number of credentials in authorized_credentials array", ripple::maxCredentialsArraySize
+                )
+            }};
+        }
+
+        for (auto const& credObj : value.as_array()) {
+            if (!credObj.is_object()) {
+                return Error{Status{
+                    ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS,
+                    "authorized_credentials elements in array are not objects."
+                }};
+            }
+            auto const& obj = credObj.as_object();
+
+            if (!obj.contains("issuer")) {
+                return Error{
+                    Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, "Field 'Issuer' is required but missing."}
+                };
+            }
+
+            // don't want to change issuer error message to be about credentials
+            if (!IssuerValidator.verify(credObj, "issuer"))
+                return Error{Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, "issuer NotString"}};
+
+            if (!obj.contains("credential_type")) {
+                return Error{Status{
+                    ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, "Field 'CredentialType' is required but missing."
+                }};
+            }
+
+            if (auto const err = CredentialTypeValidator.verify(credObj, "credential_type"); !err)
+                return err;
+        }
+
+        return MaybeError{};
+    }};
+
 }  // namespace rpc::validation

src/rpc/common/Validators.hpp

@@ -27,15 +27,13 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <fmt/core.h>
+#include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/ErrorCodes.h>
 
 #include <concepts>
-#include <cstdint>
 #include <ctime>
 #include <functional>
 #include <initializer_list>
-#include <iomanip>
-#include <sstream>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -153,7 +151,7 @@ struct Type final {
     verify(boost::json::value const& value, std::string_view key) const
     {
         if (not value.is_object() or not value.as_object().contains(key.data()))
-            return {};  // ignore. field does not exist, let 'required' fail instead
+            return {};  // ignore. If field is supposed to exist, let 'required' fail instead
 
         auto const& res = value.as_object().at(key.data());
         auto const convertible = (checkType<Types>(res) || ...);
@@ -458,6 +456,21 @@ public:
 [[nodiscard]] bool
 checkIsU32Numeric(std::string_view sv);
 
+template <class HexType>
+    requires(std::is_same_v<HexType, ripple::uint160> || std::is_same_v<HexType, ripple::uint192> || std::is_same_v<HexType, ripple::uint256>)
+MaybeError
+makeHexStringValidator(boost::json::value const& value, std::string_view key)
+{
+    if (!value.is_string())
+        return Error{Status{RippledError::rpcINVALID_PARAMS, std::string(key) + "NotString"}};
+
+    HexType parsedInt;
+    if (!parsedInt.parseHex(value.as_string().c_str()))
+        return Error{Status{RippledError::rpcINVALID_PARAMS, std::string(key) + "Malformed"}};
+
+    return MaybeError{};
+}
+
 /**
  * @brief A group of custom validation functions
  */
@@ -492,6 +505,22 @@ struct CustomValidators final {
      */
     static CustomValidator AccountMarkerValidator;
 
+    /**
+     * @brief Provides a commonly used validator for uint160(AccountID) hex string.
+     *
+     * It must be a string and also a decodable hex.
+     * AccountID uses this validator.
+     */
+    static CustomValidator Uint160HexStringValidator;
+
+    /**
+     * @brief Provides a commonly used validator for uint192 hex string.
+     *
+     * It must be a string and also a decodable hex.
+     * MPTIssuanceID uses this validator.
+     */
+    static CustomValidator Uint192HexStringValidator;
+
     /**
      * @brief Provides a commonly used validator for uint256 hex string.
      *
@@ -528,6 +557,51 @@ struct CustomValidators final {
      * Used by amm_info.
      */
     static CustomValidator CurrencyIssueValidator;
+
+    /**
+     * @brief Provides a validator for validating authorized_credentials json array.
+     *
+     * Used by deposit_preauth.
+     */
+    static CustomValidator AuthorizeCredentialValidator;
+
+    /**
+     * @brief Provides a validator for validating credential_type.
+     *
+     * Used by AuthorizeCredentialValidator in deposit_preauth.
+     */
+    static CustomValidator CredentialTypeValidator;
+};
+
+/**
+ * @brief Validates that the elements of the array is of type Hex256 uint
+ */
+struct Hex256ItemType final {
+    /**
+     * @brief Validates given the prerequisite that the type of the json value is an array,
+     * verifies all values within the array is of uint256 hash
+     *
+     * @param value the value to verify
+     * @param key The key used to retrieve the tested value from the outer object
+     * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
+     */
+    [[nodiscard]] static MaybeError
+    verify(boost::json::value const& value, std::string_view key)
+    {
+        if (not value.is_object() or not value.as_object().contains(key.data()))
+            return {};  // ignore. If field is supposed to exist, let 'required' fail instead
+
+        auto const& res = value.as_object().at(key.data());
+
+        // loop through each item in the array and make sure it is uint256 hex string
+        for (auto const& elem : res.as_array()) {
+            ripple::uint256 num;
+            if (!elem.is_string() || !num.parseHex(elem.as_string())) {
+                return Error{Status{RippledError::rpcINVALID_PARAMS, "Item is not a valid uint256 type."}};
+            }
+        }
+        return {};
+    }
 };
 
 }  // namespace rpc::validation

src/rpc/common/impl/ForwardingProxy.hpp

@@ -60,10 +60,6 @@ public:
         if (ctx.method == "subscribe" || ctx.method == "unsubscribe")
             return false;
 
-        // Disallow forwarding of the admin api, only user api is allowed for security reasons.
-        if (ctx.method == "feature" and request.contains("vetoed"))
-            return false;
-
         if (handlerProvider_->isClioOnly(ctx.method))
             return false;
 
@@ -73,6 +69,9 @@ public:
         if (specifiesCurrentOrClosedLedger(request))
             return true;
 
+        if (isForcedForward(ctx))
+            return true;
+
         auto const checkAccountInfoForward = [&]() {
             return ctx.method == "account_info" and request.contains("queue") and request.at("queue").is_bool() and
                 request.at("queue").as_bool();
@@ -142,6 +141,14 @@ private:
     {
         return handlerProvider_->contains(method) || isProxied(method);
     }
+
+    bool
+    isForcedForward(web::Context const& ctx) const
+    {
+        static constexpr auto FORCE_FORWARD = "force_forward";
+        return ctx.isAdmin and ctx.params.contains(FORCE_FORWARD) and ctx.params.at(FORCE_FORWARD).is_bool() and
+            ctx.params.at(FORCE_FORWARD).as_bool();
+    }
 };
 
 }  // namespace rpc::impl

src/rpc/common/impl/HandlerProvider.cpp

@@ -45,6 +45,7 @@
 #include "rpc/handlers/LedgerEntry.hpp"
 #include "rpc/handlers/LedgerIndex.hpp"
 #include "rpc/handlers/LedgerRange.hpp"
+#include "rpc/handlers/MPTHolders.hpp"
 #include "rpc/handlers/NFTBuyOffers.hpp"
 #include "rpc/handlers/NFTHistory.hpp"
 #include "rpc/handlers/NFTInfo.hpp"
@@ -97,6 +98,7 @@ ProductionHandlerProvider::ProductionHandlerProvider(
           {"ledger_entry", {LedgerEntryHandler{backend}}},
           {"ledger_index", {LedgerIndexHandler{backend}, true}},  // clio only
           {"ledger_range", {LedgerRangeHandler{backend}}},
+          {"mpt_holders", {MPTHoldersHandler{backend}, true}},       // clio only
           {"nfts_by_issuer", {NFTsByIssuerHandler{backend}, true}},  // clio only
           {"nft_history", {NFTHistoryHandler{backend}, true}},       // clio only
           {"nft_buy_offers", {NFTBuyOffersHandler{backend}}},

src/rpc/handlers/AccountNFTs.cpp

@@ -78,10 +78,17 @@ AccountNFTsHandler::process(AccountNFTsHandler::Input input, Context const& ctx)
         input.marker ? ripple::uint256{input.marker->c_str()} : ripple::keylet::nftpage_max(*accountID).key;
     auto const blob = sharedPtrBackend_->fetchLedgerObject(pageKey, lgrInfo.seq, ctx.yield);
 
-    if (!blob)
+    if (!blob) {
+        if (input.marker.has_value())
+            return Error{Status{RippledError::rpcINVALID_PARAMS, "Marker field does not match any valid Page ID"}};
         return response;
+    }
 
     std::optional<ripple::SLE const> page{ripple::SLE{ripple::SerialIter{blob->data(), blob->size()}, pageKey}};
+
+    if (page->getType() != ripple::ltNFTOKEN_PAGE)
+        return Error{Status{RippledError::rpcINVALID_PARAMS, "Marker matches Page ID from another Account"}};
+
     auto numPages = 0u;
 
     while (page) {

src/rpc/handlers/AccountObjects.hpp

@@ -55,10 +55,6 @@ class AccountObjectsHandler {
     // dependencies
     std::shared_ptr<BackendInterface> sharedPtrBackend_;
 
-    // constants
-    static std::unordered_map<std::string, ripple::LedgerEntryType> const TYPES_MAP;
-    static std::unordered_set<std::string> const TYPES_KEYS;
-
 public:
     static auto constexpr LIMIT_MIN = 10;
     static auto constexpr LIMIT_MAX = 400;

src/rpc/handlers/AccountTx.hpp

@@ -39,7 +39,6 @@
 #include <xrpl/protocol/jss.h>
 
 #include <cstdint>
-#include <limits>
 #include <memory>
 #include <optional>
 #include <string>
@@ -57,8 +56,8 @@ class AccountTxHandler {
     std::shared_ptr<BackendInterface> sharedPtrBackend_;
 
 public:
-    // no max limit
     static auto constexpr LIMIT_MIN = 1;
+    static auto constexpr LIMIT_MAX = 1000;
     static auto constexpr LIMIT_DEFAULT = 200;
 
     /**
@@ -133,7 +132,7 @@ public:
             {JS(limit),
              validation::Type<uint32_t>{},
              validation::Min(1u),
-             modifiers::Clamp<int32_t>{LIMIT_MIN, std::numeric_limits<int32_t>::max()}},
+             modifiers::Clamp<int32_t>{LIMIT_MIN, LIMIT_MAX}},
             {JS(marker),
              meta::WithCustomError{
                  validation::Type<boost::json::object>{},

src/rpc/handlers/DepositAuthorized.cpp

@@ -19,25 +19,32 @@
 
 #include "rpc/handlers/DepositAuthorized.hpp"
 
+#include "rpc/CredentialHelpers.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/JS.hpp"
 #include "rpc/RPCHelpers.hpp"
 #include "rpc/common/Types.hpp"
+#include "util/Assert.hpp"
 
+#include <boost/json/array.hpp>
 #include <boost/json/conversion.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
+#include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/protocol/Indexes.h>
 #include <xrpl/protocol/LedgerFormats.h>
 #include <xrpl/protocol/LedgerHeader.h>
-#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/Protocol.h>
 #include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/jss.h>
 
+#include <memory>
 #include <string>
+#include <utility>
 #include <variant>
 
 namespace rpc {
@@ -71,26 +78,55 @@ DepositAuthorizedHandler::process(DepositAuthorizedHandler::Input input, Context
 
     Output response;
 
+    auto it = ripple::SerialIter{dstAccountLedgerObject->data(), dstAccountLedgerObject->size()};
+    auto const sleDest = ripple::SLE{it, dstKeylet};
+    bool const reqAuth = sleDest.isFlag(ripple::lsfDepositAuth) && (sourceAccountID != destinationAccountID);
+    auto const& creds = input.credentials;
+    bool const credentialsPresent = creds.has_value();
+
+    ripple::STArray authCreds;
+    if (credentialsPresent) {
+        if (creds.value().empty()) {
+            return Error{Status{RippledError::rpcINVALID_PARAMS, "credential array has no elements."}};
+        }
+        if (creds.value().size() > ripple::maxCredentialsArraySize) {
+            return Error{Status{RippledError::rpcINVALID_PARAMS, "credential array too long."}};
+        }
+        auto const credArray = credentials::fetchCredentialArray(
+            input.credentials, *sourceAccountID, *sharedPtrBackend_, lgrInfo, ctx.yield
+        );
+        if (!credArray.has_value())
+            return Error{std::move(credArray).error()};
+        authCreds = std::move(credArray).value();
+    }
+
+    // If the two accounts are the same OR if that flag is
+    // not set, then the deposit should be fine.
+    bool depositAuthorized = true;
+
+    if (reqAuth) {
+        ripple::uint256 hashKey;
+        if (credentialsPresent) {
+            auto const sortedAuthCreds = credentials::createAuthCredentials(authCreds);
+            ASSERT(
+                sortedAuthCreds.size() == authCreds.size(), "should already be checked above that there is no duplicate"
+            );
+
+            hashKey = ripple::keylet::depositPreauth(*destinationAccountID, sortedAuthCreds).key;
+        } else {
+            hashKey = ripple::keylet::depositPreauth(*destinationAccountID, *sourceAccountID).key;
+        }
+
+        depositAuthorized = sharedPtrBackend_->fetchLedgerObject(hashKey, lgrInfo.seq, ctx.yield).has_value();
+    }
+
     response.sourceAccount = input.sourceAccount;
     response.destinationAccount = input.destinationAccount;
     response.ledgerHash = ripple::strHex(lgrInfo.hash);
     response.ledgerIndex = lgrInfo.seq;
-
-    // If the two accounts are the same, then the deposit should be fine.
-    if (sourceAccountID != destinationAccountID) {
-        auto it = ripple::SerialIter{dstAccountLedgerObject->data(), dstAccountLedgerObject->size()};
-        auto sle = ripple::SLE{it, dstKeylet};
-
-        // Check destination for the DepositAuth flag.
-        // If that flag is not set then a deposit should be just fine.
-        if ((sle.getFieldU32(ripple::sfFlags) & ripple::lsfDepositAuth) != 0u) {
-            // See if a preauthorization entry is in the ledger.
-            auto const depositPreauthKeylet = ripple::keylet::depositPreauth(*destinationAccountID, *sourceAccountID);
-            auto const sleDepositAuth =
-                sharedPtrBackend_->fetchLedgerObject(depositPreauthKeylet.key, lgrInfo.seq, ctx.yield);
-            response.depositAuthorized = static_cast<bool>(sleDepositAuth);
-        }
-    }
+    response.depositAuthorized = depositAuthorized;
+    if (credentialsPresent)
+        response.credentials = input.credentials.value();
 
     return response;
 }
@@ -115,6 +151,10 @@ tag_invoke(boost::json::value_to_tag<DepositAuthorizedHandler::Input>, boost::js
         }
     }
 
+    if (jsonObject.contains(JS(credentials))) {
+        input.credentials = boost::json::value_to<boost::json::array>(jv.at(JS(credentials)));
+    }
+
     return input;
 }
 
@@ -127,8 +167,10 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, DepositAuthorize
         {JS(destination_account), output.destinationAccount},
         {JS(ledger_hash), output.ledgerHash},
         {JS(ledger_index), output.ledgerIndex},
-        {JS(validated), output.validated},
+        {JS(validated), output.validated}
     };
+    if (output.credentials)
+        jv.as_object()[JS(credentials)] = *output.credentials;
 }
 
 }  // namespace rpc

src/rpc/handlers/DepositAuthorized.hpp

@@ -25,8 +25,10 @@
 #include "rpc/common/Types.hpp"
 #include "rpc/common/Validators.hpp"
 
+#include <boost/json/array.hpp>
 #include <boost/json/conversion.hpp>
 #include <boost/json/value.hpp>
+#include <xrpl/protocol/STArray.h>
 #include <xrpl/protocol/jss.h>
 
 #include <cstdint>
@@ -59,6 +61,8 @@ public:
         std::string destinationAccount;
         std::string ledgerHash;
         uint32_t ledgerIndex{};
+        std::optional<boost::json::array> credentials;
+
         // validated should be sent via framework
         bool validated = true;
     };
@@ -71,6 +75,7 @@ public:
         std::string destinationAccount;
         std::optional<std::string> ledgerHash;
         std::optional<uint32_t> ledgerIndex;
+        std::optional<boost::json::array> credentials;
     };
 
     using Result = HandlerReturnType<Output>;
@@ -99,6 +104,7 @@ public:
             {JS(destination_account), validation::Required{}, validation::CustomValidators::AccountValidator},
             {JS(ledger_hash), validation::CustomValidators::Uint256HexStringValidator},
             {JS(ledger_index), validation::CustomValidators::LedgerIndexValidator},
+            {JS(credentials), validation::Type<boost::json::array>{}, validation::Hex256ItemType()}
         };
 
         return rpcSpec;

src/rpc/handlers/LedgerData.hpp

@@ -43,7 +43,6 @@
 #include <memory>
 #include <optional>
 #include <string>
-#include <unordered_set>
 
 namespace rpc {
 

src/rpc/handlers/LedgerEntry.cpp

@@ -19,6 +19,7 @@
 
 #include "rpc/handlers/LedgerEntry.hpp"
 
+#include "rpc/CredentialHelpers.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/JS.hpp"
 #include "rpc/RPCHelpers.hpp"
@@ -30,6 +31,8 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/json/json_value.h>
@@ -97,11 +100,30 @@ LedgerEntryHandler::process(LedgerEntryHandler::Input input, Context const& ctx)
         auto const owner = util::parseBase58Wrapper<ripple::AccountID>(
             boost::json::value_to<std::string>(input.depositPreauth->at(JS(owner)))
         );
-        auto const authorized = util::parseBase58Wrapper<ripple::AccountID>(
-            boost::json::value_to<std::string>(input.depositPreauth->at(JS(authorized)))
-        );
+        // Only one of authorize or authorized_credentials MUST exist;
+        if (input.depositPreauth->contains(JS(authorized)) ==
+            input.depositPreauth->contains(JS(authorized_credentials))) {
+            return Error{
+                Status{ClioError::rpcMALFORMED_REQUEST, "Must have one of authorized or authorized_credentials."}
+            };
+        }
 
-        key = ripple::keylet::depositPreauth(*owner, *authorized).key;
+        if (input.depositPreauth->contains(JS(authorized))) {
+            auto const authorized = util::parseBase58Wrapper<ripple::AccountID>(
+                boost::json::value_to<std::string>(input.depositPreauth->at(JS(authorized)))
+            );
+            key = ripple::keylet::depositPreauth(*owner, *authorized).key;
+        } else {
+            auto const authorizedCredentials = rpc::credentials::parseAuthorizeCredentials(
+                input.depositPreauth->at(JS(authorized_credentials)).as_array()
+            );
+
+            auto const authCreds = credentials::createAuthCredentials(authorizedCredentials);
+            if (authCreds.size() != authorizedCredentials.size())
+                return Error{Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, "duplicates in credentials."}};
+
+            key = ripple::keylet::depositPreauth(owner.value(), authCreds).key;
+        }
     } else if (input.ticket) {
         auto const id =
             util::parseBase58Wrapper<ripple::AccountID>(boost::json::value_to<std::string>(input.ticket->at(JS(account))
@@ -145,6 +167,18 @@ LedgerEntryHandler::process(LedgerEntryHandler::Input input, Context const& ctx)
         }
     } else if (input.oracleNode) {
         key = input.oracleNode.value();
+    } else if (input.credential) {
+        key = input.credential.value();
+    } else if (input.mptIssuance) {
+        auto const mptIssuanceID = ripple::uint192{std::string_view(*(input.mptIssuance))};
+        key = ripple::keylet::mptIssuance(mptIssuanceID).key;
+    } else if (input.mptoken) {
+        auto const holder =
+            ripple::parseBase58<ripple::AccountID>(boost::json::value_to<std::string>(input.mptoken->at(JS(account))));
+        auto const mptIssuanceID =
+            ripple::uint192{std::string_view(boost::json::value_to<std::string>(input.mptoken->at(JS(mpt_issuance_id))))
+            };
+        key = ripple::keylet::mptoken(mptIssuanceID, *holder).key;
     } else {
         // Must specify 1 of the following fields to indicate what type
         if (ctx.apiVersion == 1)
@@ -277,6 +311,8 @@ tag_invoke(boost::json::value_to_tag<LedgerEntryHandler::Input>, boost::json::va
         {JS(xchain_owned_create_account_claim_id), ripple::ltXCHAIN_OWNED_CREATE_ACCOUNT_CLAIM_ID},
         {JS(xchain_owned_claim_id), ripple::ltXCHAIN_OWNED_CLAIM_ID},
         {JS(oracle), ripple::ltORACLE},
+        {JS(credential), ripple::ltCREDENTIAL},
+        {JS(mptoken), ripple::ltMPTOKEN},
     };
 
     auto const parseBridgeFromJson = [](boost::json::value const& bridgeJson) {
@@ -302,6 +338,16 @@ tag_invoke(boost::json::value_to_tag<LedgerEntryHandler::Input>, boost::json::va
         return ripple::keylet::oracle(*account, documentId).key;
     };
 
+    auto const parseCredentialFromJson = [](boost::json::value const& json) {
+        auto const subject =
+            util::parseBase58Wrapper<ripple::AccountID>(boost::json::value_to<std::string>(json.at(JS(subject))));
+        auto const issuer =
+            util::parseBase58Wrapper<ripple::AccountID>(boost::json::value_to<std::string>(json.at(JS(issuer))));
+        auto const credType = ripple::strUnHex(boost::json::value_to<std::string>(json.at(JS(credential_type))));
+
+        return ripple::keylet::credential(*subject, *issuer, ripple::Slice(credType->data(), credType->size())).key;
+    };
+
     auto const indexFieldType =
         std::find_if(indexFieldTypeMap.begin(), indexFieldTypeMap.end(), [&jsonObject](auto const& pair) {
             auto const& [field, _] = pair;
@@ -317,6 +363,8 @@ tag_invoke(boost::json::value_to_tag<LedgerEntryHandler::Input>, boost::json::va
         input.accountRoot = boost::json::value_to<std::string>(jv.at(JS(account_root)));
     } else if (jsonObject.contains(JS(did))) {
         input.did = boost::json::value_to<std::string>(jv.at(JS(did)));
+    } else if (jsonObject.contains(JS(mpt_issuance))) {
+        input.mptIssuance = boost::json::value_to<std::string>(jv.at(JS(mpt_issuance)));
     }
     // no need to check if_object again, validator only allows string or object
     else if (jsonObject.contains(JS(directory))) {
@@ -348,6 +396,10 @@ tag_invoke(boost::json::value_to_tag<LedgerEntryHandler::Input>, boost::json::va
         );
     } else if (jsonObject.contains(JS(oracle))) {
         input.oracleNode = parseOracleFromJson(jv.at(JS(oracle)));
+    } else if (jsonObject.contains(JS(credential))) {
+        input.credential = parseCredentialFromJson(jv.at(JS(credential)));
+    } else if (jsonObject.contains(JS(mptoken))) {
+        input.mptoken = jv.at(JS(mptoken)).as_object();
     }
 
     if (jsonObject.contains("include_deleted"))

src/rpc/handlers/LedgerEntry.hpp

@@ -30,6 +30,7 @@
 #include "rpc/common/Validators.hpp"
 #include "util/AccountUtils.hpp"
 
+#include <boost/json/array.hpp>
 #include <boost/json/conversion.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
@@ -91,6 +92,8 @@ public:
         std::optional<std::string> accountRoot;
         // account id to address did object
         std::optional<std::string> did;
+        // mpt issuance id to address mptIssuance object
+        std::optional<std::string> mptIssuance;
         // TODO: extract into custom objects, remove json from Input
         std::optional<boost::json::object> directory;
         std::optional<boost::json::object> offer;
@@ -99,11 +102,13 @@ public:
         std::optional<boost::json::object> depositPreauth;
         std::optional<boost::json::object> ticket;
         std::optional<boost::json::object> amm;
+        std::optional<boost::json::object> mptoken;
         std::optional<ripple::STXChainBridge> bridge;
         std::optional<std::string> bridgeAccount;
         std::optional<uint32_t> chainClaimId;
         std::optional<uint32_t> createAccountClaimId;
         std::optional<ripple::uint256> oracleNode;
+        std::optional<ripple::uint256> credential;
         bool includeDeleted = false;
     };
 
@@ -194,7 +199,8 @@ public:
                       meta::WithCustomError{
                           validation::CustomValidators::AccountBase58Validator, Status(ClioError::rpcMALFORMED_OWNER)
                       }},
-                     {JS(authorized), validation::Required{}, validation::CustomValidators::AccountBase58Validator},
+                     {JS(authorized), validation::CustomValidators::AccountBase58Validator},
+                     {JS(authorized_credentials), validation::CustomValidators::AuthorizeCredentialValidator}
                  },
              }},
             {JS(directory),
@@ -315,6 +321,59 @@ public:
                   },
                   meta::WithCustomError{modifiers::ToNumber{}, Status(ClioError::rpcMALFORMED_ORACLE_DOCUMENT_ID)}},
              }}},
+            {JS(credential),
+             meta::WithCustomError{
+                 validation::Type<std::string, boost::json::object>{}, Status(ClioError::rpcMALFORMED_REQUEST)
+             },
+             meta::IfType<std::string>{
+                 meta::WithCustomError{malformedRequestHexStringValidator, Status(ClioError::rpcMALFORMED_ADDRESS)}
+             },
+             meta::IfType<boost::json::object>{meta::Section{
+                 {JS(subject),
+                  meta::WithCustomError{validation::Required{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                  meta::WithCustomError{
+                      validation::CustomValidators::AccountBase58Validator, Status(ClioError::rpcMALFORMED_ADDRESS)
+                  }},
+                 {JS(issuer),
+                  meta::WithCustomError{validation::Required{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                  meta::WithCustomError{
+                      validation::CustomValidators::AccountBase58Validator, Status(ClioError::rpcMALFORMED_ADDRESS)
+                  }},
+                 {
+                     JS(credential_type),
+                     meta::WithCustomError{validation::Required{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                     meta::WithCustomError{validation::Type<std::string>{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                 },
+             }}},
+            {JS(mpt_issuance),
+             meta::WithCustomError{
+                 validation::CustomValidators::Uint192HexStringValidator, Status(ClioError::rpcMALFORMED_REQUEST)
+             }},
+            {JS(mptoken),
+             meta::WithCustomError{
+                 validation::Type<std::string, boost::json::object>{}, Status(ClioError::rpcMALFORMED_REQUEST)
+             },
+             meta::IfType<std::string>{malformedRequestHexStringValidator},
+             meta::IfType<boost::json::object>{
+                 meta::Section{
+                     {
+                         JS(account),
+                         meta::WithCustomError{validation::Required{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                         meta::WithCustomError{
+                             validation::CustomValidators::AccountBase58Validator,
+                             Status(ClioError::rpcMALFORMED_ADDRESS)
+                         },
+                     },
+                     {
+                         JS(mpt_issuance_id),
+                         meta::WithCustomError{validation::Required{}, Status(ClioError::rpcMALFORMED_REQUEST)},
+                         meta::WithCustomError{
+                             validation::CustomValidators::Uint192HexStringValidator,
+                             Status(ClioError::rpcMALFORMED_REQUEST)
+                         },
+                     },
+                 },
+             }},
             {JS(ledger), check::Deprecated{}},
             {"include_deleted", validation::Type<bool>{}},
         };

src/rpc/handlers/MPTHolders.cpp

@@ -0,0 +1,139 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "rpc/handlers/MPTHolders.hpp"
+
+#include "rpc/Errors.hpp"
+#include "rpc/JS.hpp"
+#include "rpc/RPCHelpers.hpp"
+#include "rpc/common/Types.hpp"
+
+#include <boost/json/array.hpp>
+#include <boost/json/conversion.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
+#include <ripple/basics/base_uint.h>
+#include <ripple/basics/strHex.h>
+#include <ripple/protocol/AccountID.h>
+#include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/LedgerHeader.h>
+#include <ripple/protocol/jss.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+
+#include <optional>
+#include <string>
+#include <variant>
+
+using namespace ripple;
+
+namespace rpc {
+
+MPTHoldersHandler::Result
+MPTHoldersHandler::process(MPTHoldersHandler::Input input, Context const& ctx) const
+{
+    auto const range = sharedPtrBackend_->fetchLedgerRange();
+    auto const lgrInfoOrStatus = getLedgerHeaderFromHashOrSeq(
+        *sharedPtrBackend_, ctx.yield, input.ledgerHash, input.ledgerIndex, range->maxSequence
+    );
+    if (auto const status = std::get_if<Status>(&lgrInfoOrStatus))
+        return Error{*status};
+
+    auto const lgrInfo = std::get<LedgerInfo>(lgrInfoOrStatus);
+    auto const limit = input.limit.value_or(MPTHoldersHandler::LIMIT_DEFAULT);
+    auto const mptID = ripple::uint192{input.mptID.c_str()};
+
+    auto const issuanceLedgerObject =
+        sharedPtrBackend_->fetchLedgerObject(ripple::keylet::mptIssuance(mptID).key, lgrInfo.seq, ctx.yield);
+    if (!issuanceLedgerObject)
+        return Error{Status{RippledError::rpcOBJECT_NOT_FOUND, "objectNotFound"}};
+
+    std::optional<ripple::AccountID> cursor;
+    if (input.marker)
+        cursor = ripple::AccountID{input.marker->c_str()};
+
+    auto const dbResponse = sharedPtrBackend_->fetchMPTHolders(mptID, limit, cursor, lgrInfo.seq, ctx.yield);
+    auto output = MPTHoldersHandler::Output{};
+    output.mptID = to_string(mptID);
+    output.limit = limit;
+    output.ledgerIndex = lgrInfo.seq;
+
+    boost::json::array const mpts;
+    for (auto const& mpt : dbResponse.mptokens) {
+        ripple::STLedgerEntry const sle{ripple::SerialIter{mpt.data(), mpt.size()}, keylet::mptIssuance(mptID).key};
+        boost::json::object mptJson;
+
+        mptJson[JS(account)] = toBase58(sle[ripple::sfAccount]);
+        mptJson[JS(flags)] = sle.getFlags();
+        mptJson["mpt_amount"] =
+            toBoostJson(ripple::STUInt64{ripple::sfMPTAmount, sle[ripple::sfMPTAmount]}.getJson(JsonOptions::none));
+        mptJson["mptoken_index"] = ripple::to_string(ripple::keylet::mptoken(mptID, sle[ripple::sfAccount]).key);
+
+        output.mpts.push_back(mptJson);
+    }
+
+    if (dbResponse.cursor.has_value())
+        output.marker = strHex(*dbResponse.cursor);
+
+    return output;
+}
+
+void
+tag_invoke(boost::json::value_from_tag, boost::json::value& jv, MPTHoldersHandler::Output const& output)
+{
+    jv = {
+        {JS(mpt_issuance_id), output.mptID},
+        {JS(limit), output.limit},
+        {JS(ledger_index), output.ledgerIndex},
+        {"mptokens", output.mpts},
+        {JS(validated), output.validated},
+    };
+
+    if (output.marker.has_value())
+        jv.as_object()[JS(marker)] = *(output.marker);
+}
+
+MPTHoldersHandler::Input
+tag_invoke(boost::json::value_to_tag<MPTHoldersHandler::Input>, boost::json::value const& jv)
+{
+    auto const& jsonObject = jv.as_object();
+    MPTHoldersHandler::Input input;
+
+    input.mptID = jsonObject.at(JS(mpt_issuance_id)).as_string().c_str();
+
+    if (jsonObject.contains(JS(ledger_hash)))
+        input.ledgerHash = jsonObject.at(JS(ledger_hash)).as_string().c_str();
+
+    if (jsonObject.contains(JS(ledger_index))) {
+        if (!jsonObject.at(JS(ledger_index)).is_string()) {
+            input.ledgerIndex = jsonObject.at(JS(ledger_index)).as_int64();
+        } else if (jsonObject.at(JS(ledger_index)).as_string() != "validated") {
+            input.ledgerIndex = std::stoi(jsonObject.at(JS(ledger_index)).as_string().c_str());
+        }
+    }
+
+    if (jsonObject.contains(JS(limit)))
+        input.limit = jsonObject.at(JS(limit)).as_int64();
+
+    if (jsonObject.contains(JS(marker)))
+        input.marker = jsonObject.at(JS(marker)).as_string().c_str();
+
+    return input;
+}
+}  // namespace rpc

src/rpc/handlers/MPTHolders.hpp

@@ -0,0 +1,128 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "rpc/JS.hpp"
+#include "rpc/common/Modifiers.hpp"
+#include "rpc/common/Specs.hpp"
+#include "rpc/common/Types.hpp"
+#include "rpc/common/Validators.hpp"
+
+namespace rpc {
+
+/**
+ * @brief The mpt_holders command asks the Clio server for all holders of a particular MPTokenIssuance.
+ */
+class MPTHoldersHandler {
+    std::shared_ptr<BackendInterface> sharedPtrBackend_;
+
+public:
+    static auto constexpr LIMIT_MIN = 1;
+    static auto constexpr LIMIT_MAX = 100;
+    static auto constexpr LIMIT_DEFAULT = 50;
+
+    /**
+     * @brief A struct to hold the output data of the command
+     */
+    struct Output {
+        boost::json::array mpts;
+        uint32_t ledgerIndex;
+        std::string mptID;
+        bool validated = true;
+        uint32_t limit;
+        std::optional<std::string> marker;
+    };
+
+    /**
+     * @brief A struct to hold the input data for the command
+     */
+    struct Input {
+        std::string mptID;
+        std::optional<std::string> ledgerHash;
+        std::optional<uint32_t> ledgerIndex;
+        std::optional<std::string> marker;
+        std::optional<uint32_t> limit;
+    };
+
+    using Result = HandlerReturnType<Output>;
+
+    /**
+     * @brief Construct a new MPTHoldersHandler object
+     *
+     * @param sharedPtrBackend The backend to use
+     */
+    MPTHoldersHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
+    {
+    }
+
+    /**
+     * @brief Returns the API specification for the command
+     *
+     * @param apiVersion The api version to return the spec for
+     * @return The spec for the given apiVersion
+     */
+    static RpcSpecConstRef
+    spec([[maybe_unused]] uint32_t apiVersion)
+    {
+        static auto const rpcSpec = RpcSpec{
+            {JS(mpt_issuance_id), validation::Required{}, validation::CustomValidators::Uint192HexStringValidator},
+            {JS(ledger_hash), validation::CustomValidators::Uint256HexStringValidator},
+            {JS(ledger_index), validation::CustomValidators::LedgerIndexValidator},
+            {JS(limit),
+             validation::Type<uint32_t>{},
+             validation::Min(1u),
+             modifiers::Clamp<int32_t>{LIMIT_MIN, LIMIT_MAX}},
+            {JS(marker), validation::CustomValidators::Uint160HexStringValidator},
+        };
+
+        return rpcSpec;
+    }
+
+    /**
+     * @brief Process the MPTHolders command
+     *
+     * @param input The input data for the command
+     * @param ctx The context of the request
+     * @return The result of the operation
+     */
+    Result
+    process(Input input, Context const& ctx) const;
+
+private:
+    /**
+     * @brief Convert the Output to a JSON object
+     *
+     * @param [out] jv The JSON object to convert to
+     * @param output The output to convert
+     */
+    friend void
+    tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output);
+
+    /**
+     * @brief Convert a JSON object to Input type
+     *
+     * @param jv The JSON object to convert
+     * @return Input parsed from the JSON object
+     */
+    friend Input
+    tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
+};
+}  // namespace rpc

src/util/AccountUtils.hpp

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <xrpl/basics/StringUtilities.h>
+#include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/tokens.h>
 
 #include <cctype>

src/util/CMakeLists.txt

@@ -4,6 +4,7 @@ target_sources(
   clio_util
   PRIVATE build/Build.cpp
           config/Config.cpp
+          CoroutineGroup.cpp
           log/Logger.cpp
           prometheus/Http.cpp
           prometheus/Label.cpp
@@ -19,15 +20,19 @@ target_sources(
           requests/Types.cpp
           requests/WsConnection.cpp
           requests/impl/SslContext.cpp
+          ResponseExpirationCache.cpp
           SignalsHandler.cpp
           Taggable.cpp
           TerminationHandler.cpp
           TimeUtils.cpp
           TxUtils.cpp
           LedgerUtils.cpp
-          newconfig/ConfigDefinition.cpp
-          newconfig/ObjectView.cpp
+          newconfig/Array.cpp
           newconfig/ArrayView.cpp
+          newconfig/ConfigConstraints.cpp
+          newconfig/ConfigDefinition.cpp
+          newconfig/ConfigFileJson.cpp
+          newconfig/ObjectView.cpp
           newconfig/ValueView.cpp
 )
 

src/util/Concepts.hpp

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <algorithm>
+#include <array>
 #include <type_traits>
 
 namespace util {
@@ -29,4 +31,19 @@ namespace util {
 template <typename T>
 concept SomeNumberType = std::is_arithmetic_v<T> && !std::is_same_v<T, bool> && !std::is_const_v<T>;
 
+/**
+ * @brief Checks that the list of given values contains no duplicates
+ *
+ * @param values The list of values to check
+ * @returns true if no duplicates exist; false otherwise
+ */
+static consteval auto
+hasNoDuplicates(auto&&... values)
+{
+    auto store = std::array{values...};
+    auto end = store.end();
+    std::ranges::sort(store);
+    return (std::unique(std::begin(store), end) == end);
+}
+
 }  // namespace util

src/util/CoroutineGroup.cpp

@@ -0,0 +1,82 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/CoroutineGroup.hpp"
+
+#include "util/Assert.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/steady_timer.hpp>
+
+#include <cstddef>
+#include <functional>
+#include <optional>
+#include <utility>
+
+namespace util {
+
+CoroutineGroup::CoroutineGroup(boost::asio::yield_context yield, std::optional<int> maxChildren)
+    : timer_{yield.get_executor(), boost::asio::steady_timer::duration::max()}, maxChildren_{maxChildren}
+{
+}
+
+CoroutineGroup::~CoroutineGroup()
+{
+    ASSERT(childrenCounter_ == 0, "CoroutineGroup is destroyed without waiting for child coroutines to finish");
+}
+
+bool
+CoroutineGroup::canSpawn() const
+{
+    return not maxChildren_.has_value() or childrenCounter_ < *maxChildren_;
+}
+
+bool
+CoroutineGroup::spawn(boost::asio::yield_context yield, std::function<void(boost::asio::yield_context)> fn)
+{
+    if (not canSpawn())
+        return false;
+
+    ++childrenCounter_;
+    boost::asio::spawn(yield, [this, fn = std::move(fn)](boost::asio::yield_context yield) {
+        fn(yield);
+        --childrenCounter_;
+        if (childrenCounter_ == 0)
+            timer_.cancel();
+    });
+    return true;
+}
+
+void
+CoroutineGroup::asyncWait(boost::asio::yield_context yield)
+{
+    if (childrenCounter_ == 0)
+        return;
+
+    boost::system::error_code error;
+    timer_.async_wait(yield[error]);
+}
+
+size_t
+CoroutineGroup::size() const
+{
+    return childrenCounter_;
+}
+
+}  // namespace util

src/util/CoroutineGroup.hpp

@@ -0,0 +1,96 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/steady_timer.hpp>
+
+#include <cstddef>
+#include <functional>
+#include <optional>
+
+namespace util {
+
+/**
+ * @brief CoroutineGroup is a helper class to manage a group of coroutines. It allows to spawn multiple coroutines and
+ * wait for all of them to finish.
+ */
+class CoroutineGroup {
+    boost::asio::steady_timer timer_;
+    std::optional<int> maxChildren_;
+    int childrenCounter_{0};
+
+public:
+    /**
+     * @brief Construct a new Coroutine Group object
+     *
+     * @param yield The yield context to use for the internal timer
+     * @param maxChildren The maximum number of coroutines that can be spawned at the same time. If not provided, there
+     * is no limit
+     */
+    CoroutineGroup(boost::asio::yield_context yield, std::optional<int> maxChildren = std::nullopt);
+
+    /**
+     * @brief Destroy the Coroutine Group object
+     *
+     * @note asyncWait() must be called before the object is destroyed
+     */
+    ~CoroutineGroup();
+
+    /**
+     * @brief Check if a new coroutine can be spawned (i.e. there is space for a new coroutine in the group)
+     *
+     * @return true If a new coroutine can be spawned. false if the maximum number of coroutines has been reached
+     */
+    bool
+    canSpawn() const;
+
+    /**
+     * @brief Spawn a new coroutine in the group
+     *
+     * @param yield The yield context to use for the coroutine (it should be the same as the one used in the
+     * constructor)
+     * @param fn The function to execute
+     * @return true If the coroutine was spawned successfully. false if the maximum number of coroutines has been
+     * reached
+     */
+    bool
+    spawn(boost::asio::yield_context yield, std::function<void(boost::asio::yield_context)> fn);
+
+    /**
+     * @brief Wait for all the coroutines in the group to finish
+     *
+     * @note This method must be called before the object is destroyed
+     *
+     * @param yield The yield context to use for the internal timer
+     */
+    void
+    asyncWait(boost::asio::yield_context yield);
+
+    /**
+     * @brief Get the number of coroutines in the group
+     *
+     * @return size_t The number of coroutines in the group
+     */
+    size_t
+    size() const;
+};
+
+}  // namespace util

src/util/LedgerUtils.hpp

@@ -112,7 +112,10 @@ class LedgerTypes {
         ),
         LedgerTypeAttribute::AccountOwnedLedgerType(JS(did), ripple::ltDID),
         LedgerTypeAttribute::AccountOwnedLedgerType(JS(oracle), ripple::ltORACLE),
+        LedgerTypeAttribute::AccountOwnedLedgerType(JS(credential), ripple::ltCREDENTIAL),
         LedgerTypeAttribute::ChainLedgerType(JS(nunl), ripple::ltNEGATIVE_UNL),
+        LedgerTypeAttribute::DeletionBlockerLedgerType(JS(mpt_issuance), ripple::ltMPTOKEN_ISSUANCE),
+        LedgerTypeAttribute::DeletionBlockerLedgerType(JS(mptoken), ripple::ltMPTOKEN),
     };
 
 public:

src/util/ResponseExpirationCache.cpp

@@ -17,88 +17,56 @@
 */
 //==============================================================================
 
-#include "etl/impl/ForwardingCache.hpp"
+#include "util/ResponseExpirationCache.hpp"
 
 #include "util/Assert.hpp"
 
 #include <boost/json/object.hpp>
-#include <boost/json/value_to.hpp>
 
 #include <chrono>
 #include <mutex>
 #include <optional>
 #include <shared_mutex>
 #include <string>
-#include <unordered_set>
 #include <utility>
 
-namespace etl::impl {
-
-namespace {
-
-std::optional<std::string>
-getCommand(boost::json::object const& request)
-{
-    if (not request.contains("command")) {
-        return std::nullopt;
-    }
-    return boost::json::value_to<std::string>(request.at("command"));
-}
-
-}  // namespace
+namespace util {
 
 void
-CacheEntry::put(boost::json::object response)
+ResponseExpirationCache::Entry::put(boost::json::object response)
 {
     response_ = std::move(response);
     lastUpdated_ = std::chrono::steady_clock::now();
 }
 
 std::optional<boost::json::object>
-CacheEntry::get() const
+ResponseExpirationCache::Entry::get() const
 {
     return response_;
 }
 
 std::chrono::steady_clock::time_point
-CacheEntry::lastUpdated() const
+ResponseExpirationCache::Entry::lastUpdated() const
 {
     return lastUpdated_;
 }
 
 void
-CacheEntry::invalidate()
+ResponseExpirationCache::Entry::invalidate()
 {
     response_.reset();
 }
 
-std::unordered_set<std::string> const
-    ForwardingCache::CACHEABLE_COMMANDS{"server_info", "server_state", "server_definitions", "fee", "ledger_closed"};
-
-ForwardingCache::ForwardingCache(std::chrono::steady_clock::duration const cacheTimeout) : cacheTimeout_{cacheTimeout}
-{
-    for (auto const& command : CACHEABLE_COMMANDS) {
-        cache_.emplace(command, CacheEntry{});
-    }
-}
-
 bool
-ForwardingCache::shouldCache(boost::json::object const& request)
+ResponseExpirationCache::shouldCache(std::string const& cmd)
 {
-    auto const command = getCommand(request);
-    return command.has_value() and CACHEABLE_COMMANDS.contains(*command);
+    return cache_.contains(cmd);
 }
 
 std::optional<boost::json::object>
-ForwardingCache::get(boost::json::object const& request) const
+ResponseExpirationCache::get(std::string const& cmd) const
 {
-    auto const command = getCommand(request);
-
-    if (not command.has_value()) {
-        return std::nullopt;
-    }
-
-    auto it = cache_.find(*command);
+    auto it = cache_.find(cmd);
     if (it == cache_.end())
         return std::nullopt;
 
@@ -110,20 +78,19 @@ ForwardingCache::get(boost::json::object const& request) const
 }
 
 void
-ForwardingCache::put(boost::json::object const& request, boost::json::object const& response)
+ResponseExpirationCache::put(std::string const& cmd, boost::json::object const& response)
 {
-    auto const command = getCommand(request);
-    if (not command.has_value() or not shouldCache(request))
+    if (not shouldCache(cmd))
         return;
 
-    ASSERT(cache_.contains(*command), "Command is not in the cache: {}", *command);
+    ASSERT(cache_.contains(cmd), "Command is not in the cache: {}", cmd);
 
-    auto entry = cache_[*command].lock<std::unique_lock>();
+    auto entry = cache_[cmd].lock<std::unique_lock>();
     entry->put(response);
 }
 
 void
-ForwardingCache::invalidate()
+ResponseExpirationCache::invalidate()
 {
     for (auto& [_, entry] : cache_) {
         auto entryLock = entry.lock<std::unique_lock>();
@@ -131,4 +98,4 @@ ForwardingCache::invalidate()
     }
 }
 
-}  // namespace etl::impl
+}  // namespace util

src/util/ResponseExpirationCache.hpp

@@ -30,90 +30,92 @@
 #include <unordered_map>
 #include <unordered_set>
 
-namespace etl::impl {
+namespace util {
 
 /**
- * @brief A class to store a cache entry.
+ * @brief Cache of requests' responses with TTL support and configurable cachable commands
  */
-class CacheEntry {
-    std::chrono::steady_clock::time_point lastUpdated_;
-    std::optional<boost::json::object> response_;
-
-public:
+class ResponseExpirationCache {
     /**
-     * @brief Put a response into the cache
-     *
-     * @param response The response to store
+     * @brief A class to store a cache entry.
      */
-    void
-    put(boost::json::object response);
+    class Entry {
+        std::chrono::steady_clock::time_point lastUpdated_;
+        std::optional<boost::json::object> response_;
 
-    /**
-     * @brief Get the response from the cache
-     *
-     * @return The response
-     */
-    std::optional<boost::json::object>
-    get() const;
+    public:
+        /**
+         * @brief Put a response into the cache
+         *
+         * @param response The response to store
+         */
+        void
+        put(boost::json::object response);
 
-    /**
-     * @brief Get the last time the cache was updated
-     *
-     * @return The last time the cache was updated
-     */
-    std::chrono::steady_clock::time_point
-    lastUpdated() const;
+        /**
+         * @brief Get the response from the cache
+         *
+         * @return The response
+         */
+        std::optional<boost::json::object>
+        get() const;
 
-    /**
-     * @brief Invalidate the cache entry
-     */
-    void
-    invalidate();
-};
+        /**
+         * @brief Get the last time the cache was updated
+         *
+         * @return The last time the cache was updated
+         */
+        std::chrono::steady_clock::time_point
+        lastUpdated() const;
+
+        /**
+         * @brief Invalidate the cache entry
+         */
+        void
+        invalidate();
+    };
 
-/**
- * @brief A class to store a cache of forwarding responses
- */
-class ForwardingCache {
     std::chrono::steady_clock::duration cacheTimeout_;
-    std::unordered_map<std::string, util::Mutex<CacheEntry, std::shared_mutex>> cache_;
+    std::unordered_map<std::string, util::Mutex<Entry, std::shared_mutex>> cache_;
+
+    bool
+    shouldCache(std::string const& cmd);
 
 public:
-    static std::unordered_set<std::string> const CACHEABLE_COMMANDS;
-
     /**
-     * @brief Construct a new Forwarding Cache object
+     * @brief Construct a new Cache object
      *
      * @param cacheTimeout The time for cache entries to expire
+     * @param cmds The commands that should be cached
      */
-    ForwardingCache(std::chrono::steady_clock::duration cacheTimeout);
-
-    /**
-     * @brief Check if a request should be cached
-     *
-     * @param request The request to check
-     * @return true if the request should be cached and false otherwise
-     */
-    [[nodiscard]] static bool
-    shouldCache(boost::json::object const& request);
+    ResponseExpirationCache(
+        std::chrono::steady_clock::duration cacheTimeout,
+        std::unordered_set<std::string> const& cmds
+    )
+        : cacheTimeout_(cacheTimeout)
+    {
+        for (auto const& command : cmds) {
+            cache_.emplace(command, Entry{});
+        }
+    }
 
     /**
      * @brief Get a response from the cache
      *
-     * @param request The request to get the response for
+     * @param cmd The command to get the response for
      * @return The response if it exists or std::nullopt otherwise
      */
     [[nodiscard]] std::optional<boost::json::object>
-    get(boost::json::object const& request) const;
+    get(std::string const& cmd) const;
 
     /**
      * @brief Put a response into the cache if the request should be cached
      *
-     * @param request The request to store the response for
+     * @param cmd The command to store the response for
      * @param response The response to store
      */
     void
-    put(boost::json::object const& request, boost::json::object const& response);
+    put(std::string const& cmd, boost::json::object const& response);
 
     /**
      * @brief Invalidate all entries in the cache
@@ -121,5 +123,4 @@ public:
     void
     invalidate();
 };
-
-}  // namespace etl::impl
+}  // namespace util

src/util/Retry.cpp

@@ -57,10 +57,16 @@ Retry::Retry(RetryStrategyPtr strategy, boost::asio::strand<boost::asio::io_cont
 {
 }
 
+Retry::~Retry()
+{
+    *canceled_ = true;
+}
+
 void
 Retry::cancel()
 {
     timer_.cancel();
+    *canceled_ = true;
 }
 
 size_t

src/util/Retry.hpp

@@ -24,6 +24,7 @@
 #include <boost/asio/steady_timer.hpp>
 #include <boost/asio/strand.hpp>
 
+#include <atomic>
 #include <chrono>
 #include <cstddef>
 #include <memory>
@@ -80,6 +81,7 @@ class Retry {
     RetryStrategyPtr strategy_;
     boost::asio::steady_timer timer_;
     size_t attemptNumber_ = 0;
+    std::shared_ptr<std::atomic_bool> canceled_{std::make_shared<std::atomic_bool>(false)};
 
 public:
     /**
@@ -90,6 +92,11 @@ public:
      */
     Retry(RetryStrategyPtr strategy, boost::asio::strand<boost::asio::io_context::executor_type> strand);
 
+    /**
+     * @brief Destroy the Retry object
+     */
+    ~Retry();
+
     /**
      * @brief Schedule a retry
      *
@@ -100,15 +107,18 @@ public:
     void
     retry(Fn&& func)
     {
+        *canceled_ = false;
         timer_.expires_after(strategy_->getDelay());
         strategy_->increaseDelay();
-        timer_.async_wait([this, func = std::forward<Fn>(func)](boost::system::error_code const& ec) {
-            if (ec == boost::asio::error::operation_aborted) {
-                return;
+        timer_.async_wait(
+            [this, canceled = canceled_, func = std::forward<Fn>(func)](boost::system::error_code const& ec) {
+                if (ec == boost::asio::error::operation_aborted or *canceled) {
+                    return;
+                }
+                ++attemptNumber_;
+                func();
             }
-            ++attemptNumber_;
-            func();
-        });
+        );
     }
 
     /**

src/util/WithTimeout.hpp

@@ -0,0 +1,71 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/asio/associated_executor.hpp>
+#include <boost/asio/bind_cancellation_slot.hpp>
+#include <boost/asio/cancellation_signal.hpp>
+#include <boost/asio/cancellation_type.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/asio/steady_timer.hpp>
+#include <boost/system/detail/error_code.hpp>
+#include <boost/system/errc.hpp>
+
+#include <chrono>
+#include <ctime>
+#include <memory>
+
+namespace util {
+
+/**
+ * @brief Perform a coroutine operation with a timeout.
+ *
+ * @tparam Operation The operation type to perform. Must be a callable accepting yield context with bound cancellation
+ * token.
+ * @param operation The operation to perform.
+ * @param yield The yield context.
+ * @param timeout The timeout duration.
+ * @return The error code of the operation.
+ */
+template <typename Operation>
+boost::system::error_code
+withTimeout(Operation&& operation, boost::asio::yield_context yield, std::chrono::steady_clock::duration timeout)
+{
+    boost::system::error_code error;
+    auto operationCompleted = std::make_shared<bool>(false);
+    boost::asio::cancellation_signal cancellationSignal;
+    auto cyield = boost::asio::bind_cancellation_slot(cancellationSignal.slot(), yield[error]);
+
+    boost::asio::steady_timer timer{boost::asio::get_associated_executor(cyield), timeout};
+    timer.async_wait([&cancellationSignal, operationCompleted](boost::system::error_code errorCode) {
+        if (!errorCode and !*operationCompleted)
+            cancellationSignal.emit(boost::asio::cancellation_type::terminal);
+    });
+    operation(cyield);
+    *operationCompleted = true;
+
+    // Map error code to timeout
+    if (error == boost::system::errc::operation_canceled) {
+        return boost::system::errc::make_error_code(boost::system::errc::timed_out);
+    }
+    return error;
+}
+
+}  // namespace util

src/util/async/context/BasicExecutionContext.hpp

@@ -369,7 +369,7 @@ public:
      * @brief Block until all operations are completed
      */
     void
-    join() noexcept
+    join() const noexcept
     {
         context_.join();
     }

src/util/newconfig/Array.cpp

@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/newconfig/Array.hpp"
+
+#include "util/Assert.hpp"
+#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <cstddef>
+#include <optional>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace util::config {
+
+Array::Array(ConfigValue arg) : itemPattern_{std::move(arg)}
+{
+}
+
+std::optional<Error>
+Array::addValue(Value value, std::optional<std::string_view> key)
+{
+    auto const& configValPattern = itemPattern_;
+    auto const constraint = configValPattern.getConstraint();
+
+    auto newElem = constraint.has_value() ? ConfigValue{configValPattern.type()}.withConstraint(constraint->get())
+                                          : ConfigValue{configValPattern.type()};
+    if (auto const maybeError = newElem.setValue(value, key); maybeError.has_value())
+        return maybeError;
+    elements_.emplace_back(std::move(newElem));
+    return std::nullopt;
+}
+
+size_t
+Array::size() const
+{
+    return elements_.size();
+}
+
+ConfigValue const&
+Array::at(std::size_t idx) const
+{
+    ASSERT(idx < elements_.size(), "Index is out of scope");
+    return elements_[idx];
+}
+
+ConfigValue const&
+Array::getArrayPattern() const
+{
+    return itemPattern_;
+}
+
+std::vector<ConfigValue>::const_iterator
+Array::begin() const
+{
+    return elements_.begin();
+}
+
+std::vector<ConfigValue>::const_iterator
+Array::end() const
+{
+    return elements_.end();
+}
+
+}  // namespace util::config

src/util/newconfig/Array.hpp

@@ -19,47 +19,42 @@
 
 #pragma once
 
-#include "util/Assert.hpp"
 #include "util/newconfig/ConfigValue.hpp"
-#include "util/newconfig/ObjectView.hpp"
-#include "util/newconfig/ValueView.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
 
 #include <cstddef>
-#include <iterator>
-#include <type_traits>
-#include <utility>
+#include <optional>
+#include <string_view>
 #include <vector>
 
 namespace util::config {
 
 /**
- * @brief Array definition for Json/Yaml config
+ * @brief Array definition to store multiple values provided by the user from Json/Yaml
  *
  * Used in ClioConfigDefinition to represent multiple potential values (like whitelist)
+ * Is constructed with only 1 element which states which type/constraint must every element
+ * In the array satisfy
  */
 class Array {
 public:
     /**
-     * @brief Constructs an Array with the provided arguments
+     * @brief Constructs an Array with provided Arg
      *
-     * @tparam Args Types of the arguments
-     * @param args Arguments to initialize the elements of the Array
+     * @param arg Argument to set the type and constraint of ConfigValues in Array
      */
-    template <typename... Args>
-    constexpr Array(Args&&... args) : elements_{std::forward<Args>(args)...}
-    {
-    }
+    Array(ConfigValue arg);
 
     /**
      * @brief Add ConfigValues to Array class
      *
      * @param value The ConfigValue to add
+     * @param key optional string key to include that will show in error message
+     * @return optional error if adding config value to array fails. nullopt otherwise
      */
-    void
-    emplaceBack(ConfigValue value)
-    {
-        elements_.push_back(std::move(value));
-    }
+    std::optional<Error>
+    addValue(Value value, std::optional<std::string_view> key = std::nullopt);
 
     /**
      * @brief Returns the number of values stored in the Array
@@ -67,10 +62,7 @@ public:
      * @return Number of values stored in the Array
      */
     [[nodiscard]] size_t
-    size() const
-    {
-        return elements_.size();
-    }
+    size() const;
 
     /**
      * @brief Returns the ConfigValue at the specified index
@@ -79,13 +71,35 @@ public:
      * @return ConfigValue at the specified index
      */
     [[nodiscard]] ConfigValue const&
-    at(std::size_t idx) const
-    {
-        ASSERT(idx < elements_.size(), "index is out of scope");
-        return elements_[idx];
-    }
+    at(std::size_t idx) const;
+
+    /**
+     * @brief Returns the ConfigValue that defines the type/constraint every
+     * ConfigValue must follow in Array
+     *
+     * @return The item_pattern
+     */
+    [[nodiscard]] ConfigValue const&
+    getArrayPattern() const;
+
+    /**
+     * @brief Returns an iterator to the beginning of the ConfigValue vector.
+     *
+     * @return A constant iterator to the beginning of the vector.
+     */
+    [[nodiscard]] std::vector<ConfigValue>::const_iterator
+    begin() const;
+
+    /**
+     * @brief Returns an iterator to the end of the ConfigValue vector.
+     *
+     * @return A constant iterator to the end of the vector.
+     */
+    [[nodiscard]] std::vector<ConfigValue>::const_iterator
+    end() const;
 
 private:
+    ConfigValue itemPattern_;
     std::vector<ConfigValue> elements_;
 };
 

src/util/newconfig/ConfigConstraints.cpp

@@ -0,0 +1,103 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/newconfig/ConfigConstraints.hpp"
+
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <cstdint>
+#include <optional>
+#include <regex>
+#include <stdexcept>
+#include <string>
+#include <variant>
+
+namespace util::config {
+
+std::optional<Error>
+PortConstraint::checkTypeImpl(Value const& port) const
+{
+    if (!(std::holds_alternative<int64_t>(port) || std::holds_alternative<std::string>(port)))
+        return Error{"Port must be a string or integer"};
+    return std::nullopt;
+}
+
+std::optional<Error>
+PortConstraint::checkValueImpl(Value const& port) const
+{
+    uint32_t p = 0;
+    if (std::holds_alternative<std::string>(port)) {
+        try {
+            p = static_cast<uint32_t>(std::stoi(std::get<std::string>(port)));
+        } catch (std::invalid_argument const& e) {
+            return Error{"Port string must be an integer."};
+        }
+    } else {
+        p = static_cast<uint32_t>(std::get<int64_t>(port));
+    }
+    if (p >= portMin && p <= portMax)
+        return std::nullopt;
+    return Error{"Port does not satisfy the constraint bounds"};
+}
+
+std::optional<Error>
+ValidIPConstraint::checkTypeImpl(Value const& ip) const
+{
+    if (!std::holds_alternative<std::string>(ip))
+        return Error{"Ip value must be a string"};
+    return std::nullopt;
+}
+
+std::optional<Error>
+ValidIPConstraint::checkValueImpl(Value const& ip) const
+{
+    if (std::get<std::string>(ip) == "localhost")
+        return std::nullopt;
+
+    static std::regex const ipv4(
+        R"(^((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])$)"
+    );
+
+    static std::regex const ip_url(
+        R"(^((http|https):\/\/)?((([a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6})|(((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])\.){3}(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])))(:\d{1,5})?(\/[^\s]*)?$)"
+    );
+    if (std::regex_match(std::get<std::string>(ip), ipv4) || std::regex_match(std::get<std::string>(ip), ip_url))
+        return std::nullopt;
+
+    return Error{"Ip is not a valid ip address"};
+}
+
+std::optional<Error>
+PositiveDouble::checkTypeImpl(Value const& num) const
+{
+    if (!(std::holds_alternative<double>(num) || std::holds_alternative<int64_t>(num)))
+        return Error{"Double number must be of type int or double"};
+    return std::nullopt;
+}
+
+std::optional<Error>
+PositiveDouble::checkValueImpl(Value const& num) const
+{
+    if (std::get<double>(num) >= 0)
+        return std::nullopt;
+    return Error{"Double number must be greater than 0"};
+}
+
+}  // namespace util::config

src/util/newconfig/ConfigConstraints.hpp

@@ -0,0 +1,362 @@
+//------------------------------------------------------------------------------
+/*
+   This file is part of clio: https://github.com/XRPLF/clio
+   Copyright (c) 2024, the clio developers.
+
+   Permission to use, copy, modify, and distribute this software for any
+   purpose with or without fee is hereby granted, provided that the above
+   copyright notice and this permission notice appear in all copies.
+
+   THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+   WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+   MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+   ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+   WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+   ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "rpc/common/APIVersion.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <fmt/core.h>
+#include <fmt/format.h>
+
+#include <algorithm>
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <variant>
+
+namespace util::config {
+class ValueView;
+class ConfigValue;
+
+/**
+ * @brief specific values that are accepted for logger levels in config.
+ */
+static constexpr std::array<char const*, 7> LOG_LEVELS = {
+    "trace",
+    "debug",
+    "info",
+    "warning",
+    "error",
+    "fatal",
+    "count",
+};
+
+/**
+ * @brief specific values that are accepted for logger tag style in config.
+ */
+static constexpr std::array<char const*, 5> LOG_TAGS = {
+    "int",
+    "uint",
+    "null",
+    "none",
+    "uuid",
+};
+
+/**
+ * @brief specific values that are accepted for cache loading in config.
+ */
+static constexpr std::array<char const*, 3> LOAD_CACHE_MODE = {
+    "sync",
+    "async",
+    "none",
+};
+
+/**
+ * @brief specific values that are accepted for database type in config.
+ */
+static constexpr std::array<char const*, 1> DATABASE_TYPE = {"cassandra"};
+
+/**
+ * @brief An interface to enforce constraints on certain values within ClioConfigDefinition.
+ */
+class Constraint {
+public:
+    constexpr virtual ~Constraint() noexcept = default;
+
+    /**
+     * @brief Check if the value meets the specific constraint.
+     *
+     * @param val The value to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]]
+    std::optional<Error>
+    checkConstraint(Value const& val) const
+    {
+        if (auto const maybeError = checkTypeImpl(val); maybeError.has_value())
+            return maybeError;
+        return checkValueImpl(val);
+    }
+
+protected:
+    /**
+     * @brief Creates an error message for all constraints that must satisfy certain hard-coded values.
+     *
+     * @tparam arrSize, the size of the array of hardcoded values
+     * @param key The key to the value
+     * @param value The value the user provided
+     * @param arr The array with hard-coded values to add to error message
+     * @return The error message specifying what the value of key must be
+     */
+    template <std::size_t arrSize>
+    constexpr std::string
+    makeErrorMsg(std::string_view key, Value const& value, std::array<char const*, arrSize> arr) const
+    {
+        // Extract the value from the variant
+        auto const valueStr = std::visit([](auto const& v) { return fmt::format("{}", v); }, value);
+
+        // Create the error message
+        return fmt::format(
+            R"(You provided value "{}". Key "{}"'s value must be one of the following: {})",
+            valueStr,
+            key,
+            fmt::join(arr, ", ")
+        );
+    }
+
+    /**
+     * @brief Check if the value is of a correct type for the constraint.
+     *
+     * @param val The value type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    virtual std::optional<Error>
+    checkTypeImpl(Value const& val) const = 0;
+
+    /**
+     * @brief Check if the value is within the constraint.
+     *
+     * @param val The value type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    virtual std::optional<Error>
+    checkValueImpl(Value const& val) const = 0;
+};
+
+/**
+ * @brief A constraint to ensure the port number is within a valid range.
+ */
+class PortConstraint final : public Constraint {
+public:
+    constexpr ~PortConstraint() noexcept override = default;
+
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param port The type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& port) const override;
+
+    /**
+     * @brief Check if the value is within the constraint.
+     *
+     * @param port The value to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& port) const override;
+
+    static constexpr uint32_t portMin = 1;
+    static constexpr uint32_t portMax = 65535;
+};
+
+/**
+ * @brief A constraint to ensure the IP address is valid.
+ */
+class ValidIPConstraint final : public Constraint {
+public:
+    constexpr ~ValidIPConstraint() noexcept override = default;
+
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param ip The type to be checked.
+     * @return An optional Error if the constraint is not met, std::nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& ip) const override;
+
+    /**
+     * @brief Check if the value is within the constraint.
+     *
+     * @param ip The value to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& ip) const override;
+};
+
+/**
+ * @brief A constraint class to ensure the provided value is one of the specified values in an array.
+ *
+ * @tparam arrSize The size of the array containing the valid values for the constraint
+ */
+template <std::size_t arrSize>
+class OneOf final : public Constraint {
+public:
+    /**
+     * @brief Constructs a constraint where the value must be one of the values in the provided array.
+     *
+     * @param key The key of the ConfigValue that has this constraint
+     * @param arr The value that has this constraint must be of the values in arr
+     */
+    constexpr OneOf(std::string_view key, std::array<char const*, arrSize> arr) : key_{key}, arr_{arr}
+    {
+    }
+
+    constexpr ~OneOf() noexcept override = default;
+
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param val The type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& val) const override
+    {
+        if (!std::holds_alternative<std::string>(val))
+            return Error{fmt::format(R"(Key "{}"'s value must be a string)", key_)};
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Check if the value matches one of the value in the provided array
+     *
+     * @param val The value to check
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& val) const override
+    {
+        namespace rg = std::ranges;
+        auto const check = [&val](std::string_view name) { return std::get<std::string>(val) == name; };
+        if (rg::any_of(arr_, check))
+            return std::nullopt;
+
+        return Error{makeErrorMsg(key_, val, arr_)};
+    }
+
+    std::string_view key_;
+    std::array<char const*, arrSize> arr_;
+};
+
+/**
+ * @brief A constraint class to ensure an integer value is between two numbers (inclusive)
+ */
+template <typename numType>
+class NumberValueConstraint final : public Constraint {
+public:
+    /**
+     * @brief Constructs a constraint where the number must be between min_ and max_.
+     *
+     * @param min the minimum number it can be to satisfy this constraint
+     * @param max the maximum number it can be to satisfy this constraint
+     */
+    constexpr NumberValueConstraint(numType min, numType max) : min_{min}, max_{max}
+    {
+    }
+
+    constexpr ~NumberValueConstraint() noexcept override = default;
+
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param num The type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& num) const override
+    {
+        if (!std::holds_alternative<int64_t>(num))
+            return Error{"Number must be of type integer"};
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Check if the number is positive.
+     *
+     * @param num The number to check
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& num) const override
+    {
+        auto const numValue = std::get<int64_t>(num);
+        if (numValue >= static_cast<int64_t>(min_) && numValue <= static_cast<int64_t>(max_))
+            return std::nullopt;
+        return Error{fmt::format("Number must be between {} and {}", min_, max_)};
+    }
+
+    numType min_;
+    numType max_;
+};
+
+/**
+ * @brief A constraint to ensure a double number is positive
+ */
+class PositiveDouble final : public Constraint {
+public:
+    constexpr ~PositiveDouble() noexcept override = default;
+
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param num The type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& num) const override;
+
+    /**
+     * @brief Check if the number is positive.
+     *
+     * @param num The number to check
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& num) const override;
+};
+
+static constinit PortConstraint validatePort{};
+static constinit ValidIPConstraint validateIP{};
+
+static constinit OneOf validateChannelName{"channel", Logger::CHANNELS};
+static constinit OneOf validateLogLevelName{"log_level", LOG_LEVELS};
+static constinit OneOf validateCassandraName{"database.type", DATABASE_TYPE};
+static constinit OneOf validateLoadMode{"cache.load", LOAD_CACHE_MODE};
+static constinit OneOf validateLogTag{"log_tag_style", LOG_TAGS};
+
+static constinit PositiveDouble validatePositiveDouble{};
+
+static constinit NumberValueConstraint<uint16_t> validateUint16{
+    std::numeric_limits<uint16_t>::min(),
+    std::numeric_limits<uint16_t>::max()
+};
+static constinit NumberValueConstraint<uint32_t> validateUint32{
+    std::numeric_limits<uint32_t>::min(),
+    std::numeric_limits<uint32_t>::max()
+};
+static constinit NumberValueConstraint<uint32_t> validateApiVersion{rpc::API_VERSION_MIN, rpc::API_VERSION_MAX};
+
+}  // namespace util::config

src/util/newconfig/ConfigDefinition.cpp

@@ -20,10 +20,15 @@
 #include "util/newconfig/ConfigDefinition.hpp"
 
 #include "util/Assert.hpp"
+#include "util/OverloadSet.hpp"
 #include "util/newconfig/Array.hpp"
 #include "util/newconfig/ArrayView.hpp"
+#include "util/newconfig/ConfigConstraints.hpp"
+#include "util/newconfig/ConfigFileInterface.hpp"
 #include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Error.hpp"
 #include "util/newconfig/ObjectView.hpp"
+#include "util/newconfig/Types.hpp"
 #include "util/newconfig/ValueView.hpp"
 
 #include <fmt/core.h>
@@ -38,6 +43,7 @@
 #include <thread>
 #include <utility>
 #include <variant>
+#include <vector>
 
 namespace util::config {
 /**
@@ -47,62 +53,76 @@ namespace util::config {
  * without default values must be present in the user's config file.
  */
 static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
-    {{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
+    {{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra").withConstraint(validateCassandraName)},
      {"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("localhost")},
-     {"database.cassandra.port", ConfigValue{ConfigType::Integer}},
+     {"database.cassandra.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)},
      {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue("clio")},
      {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(3u)},
      {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.defaultValue("table_prefix")},
-     {"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
-     {"database.cassandra.max_read_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(100'000)},
+     {"database.cassandra.max_write_requests_outstanding",
+      ConfigValue{ConfigType::Integer}.defaultValue(10'000).withConstraint(validateUint32)},
+     {"database.cassandra.max_read_requests_outstanding",
+      ConfigValue{ConfigType::Integer}.defaultValue(100'000).withConstraint(validateUint32)},
      {"database.cassandra.threads",
-      ConfigValue{ConfigType::Integer}.defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))},
-     {"database.cassandra.core_connections_per_host", ConfigValue{ConfigType::Integer}.defaultValue(1)},
-     {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional()},
-     {"database.cassandra.write_batch_size", ConfigValue{ConfigType::Integer}.defaultValue(20)},
-     {"etl_source.[].ip", Array{ConfigValue{ConfigType::String}.optional()}},
-     {"etl_source.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().min(1).max(65535)}},
-     {"etl_source.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().min(1).max(65535)}},
-     {"forwarding.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0)},
-     {"forwarding.request_timeout", ConfigValue{ConfigType::Double}.defaultValue(10.0)},
+      ConfigValue{ConfigType::Integer}
+          .defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))
+          .withConstraint(validateUint32)},
+     {"database.cassandra.core_connections_per_host",
+      ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint16)},
+     {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint16)},
+     {"database.cassandra.write_batch_size",
+      ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint16)},
+     {"etl_source.[].ip", Array{ConfigValue{ConfigType::String}.withConstraint(validateIP)}},
+     {"etl_source.[].ws_port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}},
+     {"etl_source.[].grpc_port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}},
+     {"forwarding.cache_timeout",
+      ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)},
+     {"forwarding.request_timeout",
+      ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)},
      {"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}},
-     {"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000)},
-     {"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20)},
-     {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20)},
-     {"dos_guard.sweep_interval", ConfigValue{ConfigType::Double}.defaultValue(1.0)},
-     {"cache.peers.[].ip", Array{ConfigValue{ConfigType::String}}},
-     {"cache.peers.[].port", Array{ConfigValue{ConfigType::String}}},
-     {"server.ip", ConfigValue{ConfigType::String}},
-     {"server.port", ConfigValue{ConfigType::Integer}},
-     {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0)},
+     {"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000).withConstraint(validateUint32)},
+     {"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint32)},
+     {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint32)},
+     {"dos_guard.sweep_interval",
+      ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)},
+     {"cache.peers.[].ip", Array{ConfigValue{ConfigType::String}.withConstraint(validateIP)}},
+     {"cache.peers.[].port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}},
+     {"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)},
+     {"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)},
+     {"server.workers", ConfigValue{ConfigType::Integer}.withConstraint(validateUint32)},
+     {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)},
      {"server.local_admin", ConfigValue{ConfigType::Boolean}.optional()},
+     {"server.admin_password", ConfigValue{ConfigType::String}.optional()},
      {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
      {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)},
-     {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2)},
-     {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32)},
-     {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48)},
-     {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0)},
-     {"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0)},
-     {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512)},
-     {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async")},
-     {"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional()}},
-     {"log_channels.[].log_level", Array{ConfigValue{ConfigType::String}.optional()}},
-     {"log_level", ConfigValue{ConfigType::String}.defaultValue("info")},
+     {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateUint16)},
+     {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)},
+     {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)},
+     {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},
+     {"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)
+     },
+     {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)},
+     {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)},
+     {"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}},
+     {"log_channels.[].log_level",
+      Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}},
+     {"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)},
      {"log_format",
       ConfigValue{ConfigType::String}.defaultValue(
           R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)"
       )},
      {"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
      {"log_directory", ConfigValue{ConfigType::String}.optional()},
-     {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048)},
-     {"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024)},
-     {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12)},
-     {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("uint")},
-     {"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(2u)},
+     {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048u).withConstraint(validateUint32)},
+     {"log_directory_max_size",
+      ConfigValue{ConfigType::Integer}.defaultValue(50u * 1024u).withConstraint(validateUint32)},
+     {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)},
+     {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("uint").withConstraint(validateLogTag)},
+     {"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(2u).withConstraint(validateUint32)},
      {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
-     {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0)},
-     {"start_sequence", ConfigValue{ConfigType::String}.optional()},
-     {"finish_sequence", ConfigValue{ConfigType::String}.optional()},
+     {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)},
+     {"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
+     {"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
      {"ssl_cert_file", ConfigValue{ConfigType::String}.optional()},
      {"ssl_key_file", ConfigValue{ConfigType::String}.optional()},
      {"api_version.min", ConfigValue{ConfigType::Integer}},
@@ -113,7 +133,7 @@ ClioConfigDefinition::ClioConfigDefinition(std::initializer_list<KeyValuePair> p
 {
     for (auto const& [key, value] : pair) {
         if (key.contains("[]"))
-            ASSERT(std::holds_alternative<Array>(value), "Value must be array if key has \"[]\"");
+            ASSERT(std::holds_alternative<Array>(value), R"(Value must be array if key has "[]")");
         map_.insert({key, value});
     }
 }
@@ -206,4 +226,51 @@ ClioConfigDefinition::arraySize(std::string_view prefix) const
     std::unreachable();
 }
 
+std::optional<std::vector<Error>>
+ClioConfigDefinition::parse(ConfigFileInterface const& config)
+{
+    std::vector<Error> listOfErrors;
+    for (auto& [key, value] : map_) {
+        // if key doesn't exist in user config, makes sure it is marked as ".optional()" or has ".defaultValue()"" in
+        // ClioConfigDefitinion above
+        if (!config.containsKey(key)) {
+            if (std::holds_alternative<ConfigValue>(value)) {
+                if (!(std::get<ConfigValue>(value).isOptional() || std::get<ConfigValue>(value).hasValue()))
+                    listOfErrors.emplace_back(key, "key is required in user Config");
+            } else if (std::holds_alternative<Array>(value)) {
+                if (!(std::get<Array>(value).getArrayPattern().isOptional()))
+                    listOfErrors.emplace_back(key, "key is required in user Config");
+            }
+            continue;
+        }
+        ASSERT(
+            std::holds_alternative<ConfigValue>(value) || std::holds_alternative<Array>(value),
+            "Value must be of type ConfigValue or Array"
+        );
+        std::visit(
+            util::OverloadSet{// handle the case where the config value is a single element.
+                              // attempt to set the value from the configuration for the specified key.
+                              [&key, &config, &listOfErrors](ConfigValue& val) {
+                                  if (auto const maybeError = val.setValue(config.getValue(key), key);
+                                      maybeError.has_value())
+                                      listOfErrors.emplace_back(maybeError.value());
+                              },
+                              // handle the case where the config value is an array.
+                              // iterate over each provided value in the array and attempt to set it for the key.
+                              [&key, &config, &listOfErrors](Array& arr) {
+                                  for (auto const& val : config.getArray(key)) {
+                                      if (auto const maybeError = arr.addValue(val, key); maybeError.has_value())
+                                          listOfErrors.emplace_back(maybeError.value());
+                                  }
+                              }
+            },
+            value
+        );
+    }
+    if (!listOfErrors.empty())
+        return listOfErrors;
+
+    return std::nullopt;
+}
+
 }  // namespace util::config

src/util/newconfig/ConfigDefinition.hpp

@@ -24,7 +24,7 @@
 #include "util/newconfig/ConfigDescription.hpp"
 #include "util/newconfig/ConfigFileInterface.hpp"
 #include "util/newconfig/ConfigValue.hpp"
-#include "util/newconfig/Errors.hpp"
+#include "util/newconfig/Error.hpp"
 #include "util/newconfig/ObjectView.hpp"
 #include "util/newconfig/ValueView.hpp"
 
@@ -41,6 +41,7 @@
 #include <unordered_map>
 #include <utility>
 #include <variant>
+#include <vector>
 
 namespace util::config {
 
@@ -66,12 +67,13 @@ public:
     /**
      * @brief Parses the configuration file
      *
-     * Should also check that no extra configuration key/value pairs are present
+     * Also checks that no extra configuration key/value pairs are present. Adds to list of Errors
+     * if it does
      *
      * @param config The configuration file interface
-     * @return An optional Error object if parsing fails
+     * @return An optional vector of Error objects stating all the failures if parsing fails
      */
-    [[nodiscard]] std::optional<Error>
+    [[nodiscard]] std::optional<std::vector<Error>>
     parse(ConfigFileInterface const& config);
 
     /**
@@ -80,9 +82,9 @@ public:
      * Should only check for valid values, without populating
      *
      * @param config The configuration file interface
-     * @return An optional Error object if validation fails
+     * @return An optional vector of Error objects stating all the failures if validation fails
      */
-    [[nodiscard]] std::optional<Error>
+    [[nodiscard]] std::optional<std::vector<Error>>
     validate(ConfigFileInterface const& config) const;
 
     /**

src/util/newconfig/ConfigDescription.hpp

@@ -90,7 +90,9 @@ private:
         KV{"server.ip", "IP address of the Clio HTTP server."},
         KV{"server.port", "Port number of the Clio HTTP server."},
         KV{"server.max_queue_size", "Maximum size of the server's request queue."},
+        KV{"server.workers", "Maximum number of threads for server to run with."},
         KV{"server.local_admin", "Indicates if the server should run with admin privileges."},
+        KV{"server.admin_password", "Password for Clio admin-only APIs."},
         KV{"prometheus.enabled", "Enable or disable Prometheus metrics."},
         KV{"prometheus.compress_reply", "Enable or disable compression of Prometheus responses."},
         KV{"io_threads", "Number of I/O threads."},

src/util/newconfig/ConfigFileInterface.hpp

@@ -19,9 +19,8 @@
 
 #pragma once
 
-#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
 
-#include <optional>
 #include <string_view>
 #include <vector>
 
@@ -36,31 +35,33 @@ namespace util::config {
 class ConfigFileInterface {
 public:
     virtual ~ConfigFileInterface() = default;
-    /**
-     * @brief Parses the provided path of user clio configuration data
-     *
-     * @param filePath The path to the Clio Config data
-     */
-    virtual void
-    parse(std::string_view filePath) = 0;
 
     /**
-     * @brief Retrieves a configuration value.
+     * @brief Retrieves the value of configValue.
      *
-     * @param key The key of the configuration value.
-     * @return An optional containing the configuration value if found, otherwise std::nullopt.
+     * @param key The key of configuration.
+     * @return the value assosiated with key.
      */
-    virtual std::optional<ConfigValue>
+    virtual Value
     getValue(std::string_view key) const = 0;
 
     /**
      * @brief Retrieves an array of configuration values.
      *
      * @param key The key of the configuration array.
-     * @return An optional containing a vector of configuration values if found, otherwise std::nullopt.
+     * @return A vector of configuration values if found, otherwise std::nullopt.
      */
-    virtual std::optional<std::vector<ConfigValue>>
+    virtual std::vector<Value>
     getArray(std::string_view key) const = 0;
+
+    /**
+     * @brief Checks if key exist in configuration file.
+     *
+     * @param key The key to search for.
+     * @return true if key exists in configuration file, false otherwise.
+     */
+    virtual bool
+    containsKey(std::string_view key) const = 0;
 };
 
 }  // namespace util::config

src/util/newconfig/ConfigFileJson.cpp

@@ -0,0 +1,166 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/newconfig/ConfigFileJson.hpp"
+
+#include "util/Assert.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/parse_options.hpp>
+#include <boost/json/value.hpp>
+#include <fmt/core.h>
+
+#include <cstddef>
+#include <exception>
+#include <fstream>
+#include <ios>
+#include <iostream>
+#include <ostream>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace util::config {
+
+namespace {
+/**
+ * @brief Extracts the value from a JSON object and converts it into the corresponding type.
+ *
+ * @param jsonValue The JSON value to extract.
+ * @return A variant containing the same type corresponding to the extracted value.
+ */
+[[nodiscard]] Value
+extractJsonValue(boost::json::value const& jsonValue)
+{
+    if (jsonValue.is_int64()) {
+        return jsonValue.as_int64();
+    }
+    if (jsonValue.is_string()) {
+        return jsonValue.as_string().c_str();
+    }
+    if (jsonValue.is_bool()) {
+        return jsonValue.as_bool();
+    }
+    if (jsonValue.is_double()) {
+        return jsonValue.as_double();
+    }
+    ASSERT(false, "Json is not of type int, string, bool or double");
+    std::unreachable();
+}
+}  // namespace
+
+ConfigFileJson::ConfigFileJson(boost::json::object jsonObj)
+{
+    flattenJson(jsonObj, "");
+}
+
+std::expected<ConfigFileJson, Error>
+ConfigFileJson::make_ConfigFileJson(boost::filesystem::path configFilePath)
+{
+    try {
+        if (auto const in = std::ifstream(configFilePath.string(), std::ios::in | std::ios::binary); in) {
+            std::stringstream contents;
+            contents << in.rdbuf();
+            auto opts = boost::json::parse_options{};
+            opts.allow_comments = true;
+            auto const tempObj = boost::json::parse(contents.str(), {}, opts).as_object();
+            return ConfigFileJson{tempObj};
+        }
+        return std::unexpected<Error>(
+            Error{fmt::format("Could not open configuration file '{}'", configFilePath.string())}
+        );
+
+    } catch (std::exception const& e) {
+        return std::unexpected<Error>(Error{fmt::format(
+            "An error occurred while processing configuration file '{}': {}", configFilePath.string(), e.what()
+        )});
+    }
+}
+
+Value
+ConfigFileJson::getValue(std::string_view key) const
+{
+    auto const jsonValue = jsonObject_.at(key);
+    auto const value = extractJsonValue(jsonValue);
+    return value;
+}
+
+std::vector<Value>
+ConfigFileJson::getArray(std::string_view key) const
+{
+    ASSERT(jsonObject_.at(key).is_array(), "Key {} has value that is not an array", key);
+
+    std::vector<Value> configValues;
+    auto const arr = jsonObject_.at(key).as_array();
+
+    for (auto const& item : arr) {
+        auto const value = extractJsonValue(item);
+        configValues.emplace_back(value);
+    }
+    return configValues;
+}
+
+bool
+ConfigFileJson::containsKey(std::string_view key) const
+{
+    return jsonObject_.contains(key);
+}
+
+void
+ConfigFileJson::flattenJson(boost::json::object const& obj, std::string const& prefix)
+{
+    for (auto const& [key, value] : obj) {
+        std::string const fullKey = prefix.empty() ? std::string(key) : fmt::format("{}.{}", prefix, std::string(key));
+
+        // In ClioConfigDefinition, value must be a primitive or array
+        if (value.is_object()) {
+            flattenJson(value.as_object(), fullKey);
+        } else if (value.is_array()) {
+            auto const& arr = value.as_array();
+            for (std::size_t i = 0; i < arr.size(); ++i) {
+                std::string const arrayPrefix = fullKey + ".[]";
+                if (arr[i].is_object()) {
+                    flattenJson(arr[i].as_object(), arrayPrefix);
+                } else {
+                    jsonObject_[arrayPrefix] = arr;
+                }
+            }
+        } else {
+            // if "[]" is present in key, then value must be an array instead of primitive
+            if (fullKey.contains(".[]") && !jsonObject_.contains(fullKey)) {
+                boost::json::array newArray;
+                newArray.emplace_back(value);
+                jsonObject_[fullKey] = newArray;
+            } else if (fullKey.contains(".[]") && jsonObject_.contains(fullKey)) {
+                jsonObject_[fullKey].as_array().emplace_back(value);
+            } else {
+                jsonObject_[fullKey] = value;
+            }
+        }
+    }
+}
+
+}  // namespace util::config

src/util/newconfig/ConfigFileJson.hpp

@@ -0,0 +1,100 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ConfigFileInterface.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <boost/filesystem/path.hpp>
+#include <boost/json/object.hpp>
+
+#include <expected>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace util::config {
+
+/** @brief Json representation of config */
+class ConfigFileJson final : public ConfigFileInterface {
+public:
+    /**
+     * @brief Construct a new ConfigJson object and stores the values from
+     * user's config into a json object.
+     *
+     * @param jsonObj the Json object to parse; represents user's config
+     */
+    ConfigFileJson(boost::json::object jsonObj);
+
+    /**
+     * @brief Retrieves a configuration value by its key.
+     *
+     * @param key The key of the configuration value to retrieve.
+     * @return A variant containing the same type corresponding to the extracted value.
+     */
+    [[nodiscard]] Value
+    getValue(std::string_view key) const override;
+
+    /**
+     * @brief Retrieves an array of configuration values by its key.
+     *
+     * @param key The key of the configuration array to retrieve.
+     * @return A vector of variants holding the config values specified by user.
+     */
+    [[nodiscard]] std::vector<Value>
+    getArray(std::string_view key) const override;
+
+    /**
+     * @brief Checks if the configuration contains a specific key.
+     *
+     * @param key The key to check for.
+     * @return True if the key exists, false otherwise.
+     */
+    [[nodiscard]] bool
+    containsKey(std::string_view key) const override;
+
+    /**
+     * @brief Creates a new ConfigFileJson by parsing the provided JSON file and
+     * stores the values in the object.
+     *
+     * @param configFilePath The path to the JSON file to be parsed.
+     * @return A ConfigFileJson object if parsing user file is successful. Error otherwise
+     */
+    [[nodiscard]] static std::expected<ConfigFileJson, Error>
+    make_ConfigFileJson(boost::filesystem::path configFilePath);
+
+private:
+    /**
+     * @brief Recursive function to flatten a JSON object into the same structure as the Clio Config.
+     *
+     * The keys will end up having the same naming convensions in Clio Config.
+     * Other than the keys specified in user Config file, no new keys are created.
+     *
+     * @param obj The JSON object to flatten.
+     * @param prefix The prefix to use for the keys in the flattened object.
+     */
+    void
+    flattenJson(boost::json::object const& obj, std::string const& prefix);
+
+    boost::json::object jsonObject_;
+};
+
+}  // namespace util::config

src/util/newconfig/ConfigFileYaml.hpp

@@ -0,0 +1,49 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ConfigFileInterface.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <boost/filesystem/path.hpp>
+
+#include <string_view>
+#include <vector>
+
+// TODO: implement when we support yaml
+
+namespace util::config {
+
+/** @brief Yaml representation of config */
+class ConfigFileYaml final : public ConfigFileInterface {
+public:
+    ConfigFileYaml() = default;
+
+    Value
+    getValue(std::string_view key) const override;
+
+    std::vector<Value>
+    getArray(std::string_view key) const override;
+
+    bool
+    containsKey(std::string_view key) const override;
+};
+
+}  // namespace util::config

src/util/newconfig/ConfigValue.hpp

@@ -20,43 +20,23 @@
 #pragma once
 
 #include "util/Assert.hpp"
-#include "util/UnsupportedType.hpp"
+#include "util/OverloadSet.hpp"
+#include "util/newconfig/ConfigConstraints.hpp"
+#include "util/newconfig/Error.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <fmt/core.h>
 
 #include <cstddef>
 #include <cstdint>
+#include <functional>
 #include <optional>
 #include <string>
-#include <type_traits>
+#include <string_view>
 #include <variant>
 
 namespace util::config {
 
-/** @brief Custom clio config types */
-enum class ConfigType { Integer, String, Double, Boolean };
-
-/**
- * @brief Get the corresponding clio config type
- *
- * @tparam Type The type to get the corresponding ConfigType for
- * @return The corresponding ConfigType
- */
-template <typename Type>
-constexpr ConfigType
-getType()
-{
-    if constexpr (std::is_same_v<Type, int64_t>) {
-        return ConfigType::Integer;
-    } else if constexpr (std::is_same_v<Type, std::string>) {
-        return ConfigType::String;
-    } else if constexpr (std::is_same_v<Type, double>) {
-        return ConfigType::Double;
-    } else if constexpr (std::is_same_v<Type, bool>) {
-        return ConfigType::Boolean;
-    } else {
-        static_assert(util::Unsupported<Type>, "Wrong config type");
-    }
-}
-
 /**
  * @brief Represents the config values for Json/Yaml config
  *
@@ -65,8 +45,6 @@ getType()
  */
 class ConfigValue {
 public:
-    using Type = std::variant<int64_t, std::string, bool, double>;
-
     /**
      * @brief Constructor initializing with the config type
      *
@@ -83,12 +61,92 @@ public:
      * @return Reference to this ConfigValue
      */
     [[nodiscard]] ConfigValue&
-    defaultValue(Type value)
+    defaultValue(Value value)
     {
-        setValue(value);
+        auto const err = checkTypeConsistency(type_, value);
+        ASSERT(!err.has_value(), "{}", err->error);
+        value_ = value;
         return *this;
     }
 
+    /**
+     * @brief Sets the value current ConfigValue given by the User's defined value
+     *
+     * @param value The value to set
+     * @param key The Config key associated with the value. Optional to include; Used for debugging message to user.
+     * @return optional Error if user tries to set a value of wrong type or not within a constraint
+     */
+    [[nodiscard]] std::optional<Error>
+    setValue(Value value, std::optional<std::string_view> key = std::nullopt)
+    {
+        auto err = checkTypeConsistency(type_, value);
+        if (err.has_value()) {
+            if (key.has_value())
+                err->error = fmt::format("{} {}", key.value(), err->error);
+            return err;
+        }
+
+        if (cons_.has_value()) {
+            auto constraintCheck = cons_->get().checkConstraint(value);
+            if (constraintCheck.has_value()) {
+                if (key.has_value())
+                    constraintCheck->error = fmt::format("{} {}", key.value(), constraintCheck->error);
+                return constraintCheck;
+            }
+        }
+        value_ = value;
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Assigns a constraint to the ConfigValue.
+     *
+     * This method associates a specific constraint with the ConfigValue.
+     * If the ConfigValue already holds a value, the method will check whether
+     * the value satisfies the given constraint. If the constraint is not satisfied,
+     * an assertion failure will occur with a detailed error message.
+     *
+     * @param cons The constraint to be applied to the ConfigValue.
+     * @return A reference to the modified ConfigValue object.
+     */
+    [[nodiscard]] constexpr ConfigValue&
+    withConstraint(Constraint const& cons)
+    {
+        cons_ = std::reference_wrapper<Constraint const>(cons);
+        ASSERT(cons_.has_value(), "Constraint must be defined");
+
+        if (value_.has_value()) {
+            auto const& temp = cons_.value().get();
+            auto const& result = temp.checkConstraint(value_.value());
+            if (result.has_value()) {
+                // useful for specifying clear Error message
+                std::string type;
+                std::visit(
+                    util::OverloadSet{
+                        [&type](bool tmp) { type = fmt::format("bool {}", tmp); },
+                        [&type](std::string const& tmp) { type = fmt::format("string {}", tmp); },
+                        [&type](double tmp) { type = fmt::format("double {}", tmp); },
+                        [&type](int64_t tmp) { type = fmt::format("int {}", tmp); }
+                    },
+                    value_.value()
+                );
+                ASSERT(false, "Value {} ConfigValue does not satisfy the set Constraint", type);
+            }
+        }
+        return *this;
+    }
+
+    /**
+     * @brief Retrieves the constraint associated with this ConfigValue, if any.
+     *
+     * @return An optional reference to the associated Constraint.
+     */
+    [[nodiscard]] std::optional<std::reference_wrapper<Constraint const>>
+    getConstraint() const
+    {
+        return cons_;
+    }
+
     /**
      * @brief Gets the config type
      *
@@ -100,32 +158,6 @@ public:
         return type_;
     }
 
-    /**
-     * @brief Sets the minimum value for the config
-     *
-     * @param min The minimum value
-     * @return Reference to this ConfigValue
-     */
-    [[nodiscard]] constexpr ConfigValue&
-    min(std::uint32_t min)
-    {
-        min_ = min;
-        return *this;
-    }
-
-    /**
-     * @brief Sets the maximum value for the config
-     *
-     * @param max The maximum value
-     * @return Reference to this ConfigValue
-     */
-    [[nodiscard]] constexpr ConfigValue&
-    max(std::uint32_t max)
-    {
-        max_ = max;
-        return *this;
-    }
-
     /**
      * @brief Sets the config value as optional, meaning the user doesn't have to provide the value in their config
      *
@@ -165,7 +197,7 @@ public:
      *
      * @return Config Value
      */
-    [[nodiscard]] Type const&
+    [[nodiscard]] Value const&
     getValue() const
     {
         return value_.value();
@@ -178,39 +210,28 @@ private:
      * @param type The config type
      * @param value The config value
      */
-    static void
-    checkTypeConsistency(ConfigType type, Type value)
+    static std::optional<Error>
+    checkTypeConsistency(ConfigType type, Value value)
     {
-        if (std::holds_alternative<std::string>(value)) {
-            ASSERT(type == ConfigType::String, "Value does not match type string");
-        } else if (std::holds_alternative<bool>(value)) {
-            ASSERT(type == ConfigType::Boolean, "Value does not match type boolean");
-        } else if (std::holds_alternative<double>(value)) {
-            ASSERT(type == ConfigType::Double, "Value does not match type double");
-        } else if (std::holds_alternative<int64_t>(value)) {
-            ASSERT(type == ConfigType::Integer, "Value does not match type integer");
+        if (type == ConfigType::String && !std::holds_alternative<std::string>(value)) {
+            return Error{"value does not match type string"};
         }
-    }
-
-    /**
-     * @brief Sets the value for the config
-     *
-     * @param value The value to set
-     * @return The value that was set
-     */
-    Type
-    setValue(Type value)
-    {
-        checkTypeConsistency(type_, value);
-        value_ = value;
-        return value;
+        if (type == ConfigType::Boolean && !std::holds_alternative<bool>(value)) {
+            return Error{"value does not match type boolean"};
+        }
+        if (type == ConfigType::Double && !std::holds_alternative<double>(value)) {
+            return Error{"value does not match type double"};
+        }
+        if (type == ConfigType::Integer && !std::holds_alternative<int64_t>(value)) {
+            return Error{"value does not match type integer"};
+        }
+        return std::nullopt;
     }
 
     ConfigType type_{};
     bool optional_{false};
-    std::optional<Type> value_;
-    std::optional<std::uint32_t> min_;
-    std::optional<std::uint32_t> max_;
+    std::optional<Value> value_;
+    std::optional<std::reference_wrapper<Constraint const>> cons_;
 };
 
 }  // namespace util::config

src/util/newconfig/Error.hpp

@@ -0,0 +1,57 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <fmt/core.h>
+
+#include <string>
+#include <string_view>
+#include <utility>
+
+namespace util::config {
+
+/** @brief Displays the different errors when parsing user config */
+struct Error {
+    /**
+     * @brief Constructs an Error with a custom error message.
+     *
+     * @param err the error message to display to users.
+     */
+    Error(std::string err) : error{std::move(err)}
+    {
+    }
+
+    /**
+     * @brief Constructs an Error with a custom error message.
+     *
+     * @param key the key associated with the error.
+     * @param err the error message to display to users.
+     */
+    Error(std::string_view key, std::string_view err)
+        : error{
+              fmt::format("{} {}", key, err),
+          }
+    {
+    }
+
+    std::string error;
+};
+
+}  // namespace util::config

src/util/newconfig/Types.hpp

@@ -0,0 +1,60 @@
+//------------------------------------------------------------------------------
+/*
+   This file is part of clio: https://github.com/XRPLF/clio
+   Copyright (c) 2024, the clio developers.
+
+   Permission to use, copy, modify, and distribute this software for any
+   purpose with or without fee is hereby granted, provided that the above
+   copyright notice and this permission notice appear in all copies.
+
+   THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+   WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+   MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+   ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+   WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+   ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/UnsupportedType.hpp"
+
+#include <cstdint>
+#include <string>
+#include <type_traits>
+#include <variant>
+
+namespace util::config {
+
+/** @brief Custom clio config types */
+enum class ConfigType { Integer, String, Double, Boolean };
+
+/** @brief Represents the supported Config Values */
+using Value = std::variant<int64_t, std::string, bool, double>;
+
+/**
+ * @brief Get the corresponding clio config type
+ *
+ * @tparam Type The type to get the corresponding ConfigType for
+ * @return The corresponding ConfigType
+ */
+template <typename Type>
+constexpr ConfigType
+getType()
+{
+    if constexpr (std::is_same_v<Type, int64_t>) {
+        return ConfigType::Integer;
+    } else if constexpr (std::is_same_v<Type, std::string>) {
+        return ConfigType::String;
+    } else if constexpr (std::is_same_v<Type, double>) {
+        return ConfigType::Double;
+    } else if constexpr (std::is_same_v<Type, bool>) {
+        return ConfigType::Boolean;
+    } else {
+        static_assert(util::Unsupported<Type>, "Wrong config type");
+    }
+}
+
+}  // namespace util::config

src/util/newconfig/ValueView.cpp

@@ -21,6 +21,7 @@
 
 #include "util/Assert.hpp"
 #include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
 
 #include <cstdint>
 #include <string>
@@ -55,9 +56,9 @@ double
 ValueView::asDouble() const
 {
     if (configVal_.get().hasValue()) {
-        if (type() == ConfigType::Double) {
+        if (type() == ConfigType::Double)
             return std::get<double>(configVal_.get().getValue());
-        }
+
         if (type() == ConfigType::Integer)
             return static_cast<double>(std::get<int64_t>(configVal_.get().getValue()));
     }

src/util/newconfig/ValueView.hpp

@@ -21,6 +21,7 @@
 
 #include "util/Assert.hpp"
 #include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
 
 #include <fmt/core.h>
 
@@ -84,7 +85,7 @@ public:
                 return static_cast<T>(val);
             }
         }
-        ASSERT(false, "Value view is not of any Int type");
+        ASSERT(false, "Value view is not of Int type");
         return 0;
     }
 

src/util/requests/impl/WsConnectionImpl.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "util/WithTimeout.hpp"
 #include "util/requests/Types.hpp"
 #include "util/requests/WsConnection.hpp"
 
@@ -39,8 +40,10 @@
 #include <boost/beast/websocket/stream_base.hpp>
 #include <boost/system/errc.hpp>
 
+#include <atomic>
 #include <chrono>
 #include <expected>
+#include <memory>
 #include <optional>
 #include <string>
 #include <utility>
@@ -65,15 +68,13 @@ public:
 
         auto operation = [&](auto&& token) { ws_.async_read(buffer, token); };
         if (timeout) {
-            withTimeout(operation, yield[errorCode], *timeout);
+            errorCode = util::withTimeout(operation, yield[errorCode], *timeout);
         } else {
             operation(yield[errorCode]);
         }
 
-        if (errorCode) {
-            errorCode = mapError(errorCode);
+        if (errorCode)
             return std::unexpected{RequestError{"Read error", errorCode}};
-        }
 
         return boost::beast::buffers_to_string(std::move(buffer).data());
     }
@@ -88,15 +89,13 @@ public:
         boost::beast::error_code errorCode;
         auto operation = [&](auto&& token) { ws_.async_write(boost::asio::buffer(message), token); };
         if (timeout) {
-            withTimeout(operation, yield[errorCode], *timeout);
+            errorCode = util::withTimeout(operation, yield, *timeout);
         } else {
             operation(yield[errorCode]);
         }
 
-        if (errorCode) {
-            errorCode = mapError(errorCode);
+        if (errorCode)
             return RequestError{"Write error", errorCode};
-        }
 
         return std::nullopt;
     }
@@ -117,31 +116,6 @@ public:
             return RequestError{"Close error", errorCode};
         return std::nullopt;
     }
-
-private:
-    template <typename Operation>
-    static void
-    withTimeout(Operation&& operation, boost::asio::yield_context yield, std::chrono::steady_clock::duration timeout)
-    {
-        boost::asio::cancellation_signal cancellationSignal;
-        auto cyield = boost::asio::bind_cancellation_slot(cancellationSignal.slot(), yield);
-
-        boost::asio::steady_timer timer{boost::asio::get_associated_executor(cyield), timeout};
-        timer.async_wait([&cancellationSignal](boost::system::error_code errorCode) {
-            if (!errorCode)
-                cancellationSignal.emit(boost::asio::cancellation_type::terminal);
-        });
-        operation(cyield);
-    }
-
-    static boost::system::error_code
-    mapError(boost::system::error_code const ec)
-    {
-        if (ec == boost::system::errc::operation_canceled) {
-            return boost::system::errc::make_error_code(boost::system::errc::timed_out);
-        }
-        return ec;
-    }
 };
 
 using PlainWsConnection = WsConnectionImpl<boost::beast::websocket::stream<boost::beast::tcp_stream>>;

src/web/CMakeLists.txt

@@ -3,13 +3,17 @@ add_library(clio_web)
 target_sources(
   clio_web
   PRIVATE Resolver.cpp
-          Server.cpp
           dosguard/DOSGuard.cpp
           dosguard/IntervalSweepHandler.cpp
           dosguard/WhitelistHandler.cpp
           impl/AdminVerificationStrategy.cpp
-          impl/ServerSslContext.cpp
+          ng/Connection.cpp
+          ng/impl/ConnectionHandler.cpp
+          ng/impl/ServerSslContext.cpp
+          ng/impl/WsConnection.cpp
           ng/Server.cpp
+          ng/Request.cpp
+          ng/Response.cpp
 )
 
 target_link_libraries(clio_web PUBLIC clio_util)

src/web/HttpSession.hpp

@@ -30,6 +30,7 @@
 #include <boost/beast/core/flat_buffer.hpp>
 #include <boost/beast/core/tcp_stream.hpp>
 
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <string>
@@ -52,6 +53,7 @@ class HttpSession : public impl::HttpBase<HttpSession, HandlerType>,
                     public std::enable_shared_from_this<HttpSession<HandlerType>> {
     boost::beast::tcp_stream stream_;
     std::reference_wrapper<util::TagDecoratorFactory const> tagFactory_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -64,6 +66,7 @@ public:
      * @param dosGuard The denial of service guard to use
      * @param handler The server handler to use
      * @param buffer Buffer with initial data received from the peer
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     explicit HttpSession(
         tcp::socket&& socket,
@@ -72,7 +75,8 @@ public:
         std::reference_wrapper<util::TagDecoratorFactory const> tagFactory,
         std::reference_wrapper<dosguard::DOSGuardInterface> dosGuard,
         std::shared_ptr<HandlerType> const& handler,
-        boost::beast::flat_buffer buffer
+        boost::beast::flat_buffer buffer,
+        std::uint32_t maxWsSendingQueueSize
     )
         : impl::HttpBase<HttpSession, HandlerType>(
               ip,
@@ -84,6 +88,7 @@ public:
           )
         , stream_(std::move(socket))
         , tagFactory_(tagFactory)
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
     }
 
@@ -128,7 +133,8 @@ public:
             this->handler_,
             std::move(this->buffer_),
             std::move(this->req_),
-            ConnectionBase::isAdmin()
+            ConnectionBase::isAdmin(),
+            maxWsSendingQueueSize_
         )
             ->run();
     }

src/web/PlainWsSession.hpp

@@ -62,7 +62,8 @@ public:
      * @param dosGuard The denial of service guard to use
      * @param handler The server handler to use
      * @param buffer Buffer with initial data received from the peer
-     * @param isAdmin Whether the connection has admin privileges
+     * @param isAdmin Whether the connection has admin privileges,
+     * @param maxSendingQueueSize The maximum size of the sending queue for websocket
      */
     explicit PlainWsSession(
         boost::asio::ip::tcp::socket&& socket,
@@ -71,9 +72,17 @@ public:
         std::reference_wrapper<dosguard::DOSGuardInterface> dosGuard,
         std::shared_ptr<HandlerType> const& handler,
         boost::beast::flat_buffer&& buffer,
-        bool isAdmin
+        bool isAdmin,
+        std::uint32_t maxSendingQueueSize
     )
-        : impl::WsBase<PlainWsSession, HandlerType>(ip, tagFactory, dosGuard, handler, std::move(buffer))
+        : impl::WsBase<PlainWsSession, HandlerType>(
+              ip,
+              tagFactory,
+              dosGuard,
+              handler,
+              std::move(buffer),
+              maxSendingQueueSize
+          )
         , ws_(std::move(socket))
     {
         ConnectionBase::isAdmin_ = isAdmin;  // NOLINT(cppcoreguidelines-prefer-member-initializer)
@@ -107,6 +116,7 @@ class WsUpgrader : public std::enable_shared_from_this<WsUpgrader<HandlerType>>
     std::string ip_;
     std::shared_ptr<HandlerType> const handler_;
     bool isAdmin_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -120,6 +130,7 @@ public:
      * @param buffer Buffer with initial data received from the peer. Ownership is transferred
      * @param request The request. Ownership is transferred
      * @param isAdmin Whether the connection has admin privileges
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     WsUpgrader(
         boost::beast::tcp_stream&& stream,
@@ -129,7 +140,8 @@ public:
         std::shared_ptr<HandlerType> const& handler,
         boost::beast::flat_buffer&& buffer,
         http::request<http::string_body> request,
-        bool isAdmin
+        bool isAdmin,
+        std::uint32_t maxWsSendingQueueSize
     )
         : http_(std::move(stream))
         , buffer_(std::move(buffer))
@@ -139,6 +151,7 @@ public:
         , ip_(std::move(ip))
         , handler_(handler)
         , isAdmin_(isAdmin)
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
     }
 
@@ -175,7 +188,14 @@ private:
         boost::beast::get_lowest_layer(http_).expires_never();
 
         std::make_shared<PlainWsSession<HandlerType>>(
-            http_.release_socket(), ip_, tagFactory_, dosGuard_, handler_, std::move(buffer_), isAdmin_
+            http_.release_socket(),
+            ip_,
+            tagFactory_,
+            dosGuard_,
+            handler_,
+            std::move(buffer_),
+            isAdmin_,
+            maxWsSendingQueueSize_
         )
             ->run(std::move(req_));
     }

src/web/Server.cpp

@@ -1,48 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    This file is part of clio: https://github.com/XRPLF/clio
-    Copyright (c) 2024, the clio developers.
-
-    Permission to use, copy, modify, and distribute this software for any
-    purpose with or without fee is hereby granted, provided that the above
-    copyright notice and this permission notice appear in all copies.
-
-    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
-    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
-    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-//==============================================================================
-
-#include "web/Server.hpp"
-
-#include "util/config/Config.hpp"
-
-#include <boost/asio/ssl/context.hpp>
-
-#include <optional>
-#include <string>
-
-namespace web {
-
-std::expected<std::optional<boost::asio::ssl::context>, std::string>
-makeServerSslContext(util::Config const& config)
-{
-    bool const configHasCertFile = config.contains("ssl_cert_file");
-    bool const configHasKeyFile = config.contains("ssl_key_file");
-
-    if (configHasCertFile != configHasKeyFile)
-        return std::unexpected{"Config entries 'ssl_cert_file' and 'ssl_key_file' must be set or unset together."};
-
-    if (not configHasCertFile)
-        return std::nullopt;
-
-    auto const certFilename = config.value<std::string>("ssl_cert_file");
-    auto const keyFilename = config.value<std::string>("ssl_key_file");
-
-    return impl::makeServerSslContext(certFilename, keyFilename);
-}
-}  // namespace web

src/web/Server.hpp

@@ -24,8 +24,8 @@
 #include "web/HttpSession.hpp"
 #include "web/SslHttpSession.hpp"
 #include "web/dosguard/DOSGuardInterface.hpp"
-#include "web/impl/ServerSslContext.hpp"
 #include "web/interface/Concepts.hpp"
+#include "web/ng/impl/ServerSslContext.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/ip/address.hpp>
@@ -41,6 +41,7 @@
 #include <fmt/core.h>
 
 #include <chrono>
+#include <cstdint>
 #include <exception>
 #include <functional>
 #include <memory>
@@ -59,15 +60,6 @@
  */
 namespace web {
 
-/**
- * @brief A helper function to create a server SSL context.
- *
- * @param config The config to create the context
- * @return Optional SSL context or error message if any
- */
-std::expected<std::optional<boost::asio::ssl::context>, std::string>
-makeServerSslContext(util::Config const& config);
-
 /**
  * @brief The Detector class to detect if the connection is a ssl or not.
  *
@@ -79,10 +71,8 @@ makeServerSslContext(util::Config const& config);
  * @tparam HandlerType The executor to handle the requests
  */
 template <
-    template <typename>
-    class PlainSessionType,
-    template <typename>
-    class SslSessionType,
+    template <typename> class PlainSessionType,
+    template <typename> class SslSessionType,
     SomeServerHandler HandlerType>
 class Detector : public std::enable_shared_from_this<Detector<PlainSessionType, SslSessionType, HandlerType>> {
     using std::enable_shared_from_this<Detector<PlainSessionType, SslSessionType, HandlerType>>::shared_from_this;
@@ -95,6 +85,7 @@ class Detector : public std::enable_shared_from_this<Detector<PlainSessionType,
     std::shared_ptr<HandlerType> const handler_;
     boost::beast::flat_buffer buffer_;
     std::shared_ptr<impl::AdminVerificationStrategy> const adminVerification_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -106,6 +97,7 @@ public:
      * @param dosGuard The denial of service guard to use
      * @param handler The server handler to use
      * @param adminVerification The admin verification strategy to use
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     Detector(
         tcp::socket&& socket,
@@ -113,7 +105,8 @@ public:
         std::reference_wrapper<util::TagDecoratorFactory const> tagFactory,
         std::reference_wrapper<dosguard::DOSGuardInterface> dosGuard,
         std::shared_ptr<HandlerType> handler,
-        std::shared_ptr<impl::AdminVerificationStrategy> adminVerification
+        std::shared_ptr<impl::AdminVerificationStrategy> adminVerification,
+        std::uint32_t maxWsSendingQueueSize
     )
         : stream_(std::move(socket))
         , ctx_(ctx)
@@ -121,6 +114,7 @@ public:
         , dosGuard_(dosGuard)
         , handler_(std::move(handler))
         , adminVerification_(std::move(adminVerification))
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
     }
 
@@ -178,14 +172,22 @@ public:
                 tagFactory_,
                 dosGuard_,
                 handler_,
-                std::move(buffer_)
+                std::move(buffer_),
+                maxWsSendingQueueSize_
             )
                 ->run();
             return;
         }
 
         std::make_shared<PlainSessionType<HandlerType>>(
-            stream_.release_socket(), ip, adminVerification_, tagFactory_, dosGuard_, handler_, std::move(buffer_)
+            stream_.release_socket(),
+            ip,
+            adminVerification_,
+            tagFactory_,
+            dosGuard_,
+            handler_,
+            std::move(buffer_),
+            maxWsSendingQueueSize_
         )
             ->run();
     }
@@ -201,10 +203,8 @@ public:
  * @tparam HandlerType The handler to process the request and return response.
  */
 template <
-    template <typename>
-    class PlainSessionType,
-    template <typename>
-    class SslSessionType,
+    template <typename> class PlainSessionType,
+    template <typename> class SslSessionType,
     SomeServerHandler HandlerType>
 class Server : public std::enable_shared_from_this<Server<PlainSessionType, SslSessionType, HandlerType>> {
     using std::enable_shared_from_this<Server<PlainSessionType, SslSessionType, HandlerType>>::shared_from_this;
@@ -217,6 +217,7 @@ class Server : public std::enable_shared_from_this<Server<PlainSessionType, SslS
     std::shared_ptr<HandlerType> handler_;
     tcp::acceptor acceptor_;
     std::shared_ptr<impl::AdminVerificationStrategy> adminVerification_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -229,6 +230,7 @@ public:
      * @param dosGuard The denial of service guard to use
      * @param handler The server handler to use
      * @param adminPassword The optional password to verify admin role in requests
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     Server(
         boost::asio::io_context& ioc,
@@ -237,7 +239,8 @@ public:
         util::TagDecoratorFactory tagFactory,
         dosguard::DOSGuardInterface& dosGuard,
         std::shared_ptr<HandlerType> handler,
-        std::optional<std::string> adminPassword
+        std::optional<std::string> adminPassword,
+        std::uint32_t maxWsSendingQueueSize
     )
         : ioc_(std::ref(ioc))
         , ctx_(std::move(ctx))
@@ -246,6 +249,7 @@ public:
         , handler_(std::move(handler))
         , acceptor_(boost::asio::make_strand(ioc))
         , adminVerification_(impl::make_AdminVerificationStrategy(std::move(adminPassword)))
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
         boost::beast::error_code ec;
 
@@ -299,7 +303,13 @@ private:
                 ctx_ ? std::optional<std::reference_wrapper<boost::asio::ssl::context>>{ctx_.value()} : std::nullopt;
 
             std::make_shared<Detector<PlainSessionType, SslSessionType, HandlerType>>(
-                std::move(socket), ctxRef, std::cref(tagFactory_), dosGuard_, handler_, adminVerification_
+                std::move(socket),
+                ctxRef,
+                std::cref(tagFactory_),
+                dosGuard_,
+                handler_,
+                adminVerification_,
+                maxWsSendingQueueSize_
             )
                 ->run();
         }
@@ -333,7 +343,7 @@ make_HttpServer(
 {
     static util::Logger const log{"WebServer"};
 
-    auto expectedSslContext = makeServerSslContext(config);
+    auto expectedSslContext = ng::impl::makeServerSslContext(config);
     if (not expectedSslContext) {
         LOG(log.error()) << "Failed to create SSL context: " << expectedSslContext.error();
         return nullptr;
@@ -361,6 +371,10 @@ make_HttpServer(
         throw std::logic_error("Admin config error, one method must be specified to authorize admin.");
     }
 
+    // If the transactions number is 200 per ledger, A client which subscribes everything will send 400+ feeds for
+    // each ledger. we allow user delay 3 ledgers by default
+    auto const maxWsSendingQueueSize = serverConfig.valueOr("ws_max_sending_queue_size", 1500);
+
     auto server = std::make_shared<HttpServer<HandlerType>>(
         ioc,
         std::move(expectedSslContext).value(),
@@ -368,7 +382,8 @@ make_HttpServer(
         util::TagDecoratorFactory(config),
         dosGuard,
         handler,
-        std::move(adminPassword)
+        std::move(adminPassword),
+        maxWsSendingQueueSize
     );
 
     server->run();

src/web/SslHttpSession.hpp

@@ -37,6 +37,7 @@
 
 #include <chrono>
 #include <cstddef>
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <string>
@@ -59,6 +60,7 @@ class SslHttpSession : public impl::HttpBase<SslHttpSession, HandlerType>,
                        public std::enable_shared_from_this<SslHttpSession<HandlerType>> {
     boost::beast::ssl_stream<boost::beast::tcp_stream> stream_;
     std::reference_wrapper<util::TagDecoratorFactory const> tagFactory_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -72,6 +74,7 @@ public:
      * @param dosGuard The denial of service guard to use
      * @param handler The server handler to use
      * @param buffer Buffer with initial data received from the peer
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     explicit SslHttpSession(
         tcp::socket&& socket,
@@ -81,7 +84,8 @@ public:
         std::reference_wrapper<util::TagDecoratorFactory const> tagFactory,
         std::reference_wrapper<dosguard::DOSGuardInterface> dosGuard,
         std::shared_ptr<HandlerType> const& handler,
-        boost::beast::flat_buffer buffer
+        boost::beast::flat_buffer buffer,
+        std::uint32_t maxWsSendingQueueSize
     )
         : impl::HttpBase<SslHttpSession, HandlerType>(
               ip,
@@ -93,6 +97,7 @@ public:
           )
         , stream_(std::move(socket), ctx)
         , tagFactory_(tagFactory)
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
     }
 
@@ -173,7 +178,8 @@ public:
             this->handler_,
             std::move(this->buffer_),
             std::move(this->req_),
-            ConnectionBase::isAdmin()
+            ConnectionBase::isAdmin(),
+            maxWsSendingQueueSize_
         )
             ->run();
     }

src/web/SslWsSession.hpp

@@ -36,6 +36,7 @@
 #include <boost/optional/optional.hpp>
 
 #include <chrono>
+#include <cstdint>
 #include <functional>
 #include <memory>
 #include <string>
@@ -64,6 +65,7 @@ public:
      * @param handler The server handler to use
      * @param buffer Buffer with initial data received from the peer
      * @param isAdmin Whether the connection has admin privileges
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     explicit SslWsSession(
         boost::beast::ssl_stream<boost::beast::tcp_stream>&& stream,
@@ -72,9 +74,17 @@ public:
         std::reference_wrapper<dosguard::DOSGuardInterface> dosGuard,
         std::shared_ptr<HandlerType> const& handler,
         boost::beast::flat_buffer&& buffer,
-        bool isAdmin
+        bool isAdmin,
+        std::uint32_t maxWsSendingQueueSize
     )
-        : impl::WsBase<SslWsSession, HandlerType>(ip, tagFactory, dosGuard, handler, std::move(buffer))
+        : impl::WsBase<SslWsSession, HandlerType>(
+              ip,
+              tagFactory,
+              dosGuard,
+              handler,
+              std::move(buffer),
+              maxWsSendingQueueSize
+          )
         , ws_(std::move(stream))
     {
         ConnectionBase::isAdmin_ = isAdmin;  // NOLINT(cppcoreguidelines-prefer-member-initializer)
@@ -106,6 +116,7 @@ class SslWsUpgrader : public std::enable_shared_from_this<SslWsUpgrader<HandlerT
     std::shared_ptr<HandlerType> const handler_;
     http::request<http::string_body> req_;
     bool isAdmin_;
+    std::uint32_t maxWsSendingQueueSize_;
 
 public:
     /**
@@ -119,6 +130,7 @@ public:
      * @param buffer Buffer with initial data received from the peer. Ownership is transferred
      * @param request The request. Ownership is transferred
      * @param isAdmin Whether the connection has admin privileges
+     * @param maxWsSendingQueueSize The maximum size of the sending queue for websocket
      */
     SslWsUpgrader(
         boost::beast::ssl_stream<boost::beast::tcp_stream> stream,
@@ -128,7 +140,8 @@ public:
         std::shared_ptr<HandlerType> handler,
         boost::beast::flat_buffer&& buffer,
         http::request<http::string_body> request,
-        bool isAdmin
+        bool isAdmin,
+        std::uint32_t maxWsSendingQueueSize
     )
         : https_(std::move(stream))
         , buffer_(std::move(buffer))
@@ -138,6 +151,7 @@ public:
         , handler_(std::move(handler))
         , req_(std::move(request))
         , isAdmin_(isAdmin)
+        , maxWsSendingQueueSize_(maxWsSendingQueueSize)
     {
     }
 
@@ -179,7 +193,14 @@ private:
         boost::beast::get_lowest_layer(https_).expires_never();
 
         std::make_shared<SslWsSession<HandlerType>>(
-            std::move(https_), ip_, tagFactory_, dosGuard_, handler_, std::move(buffer_), isAdmin_
+            std::move(https_),
+            ip_,
+            tagFactory_,
+            dosGuard_,
+            handler_,
+            std::move(buffer_),
+            isAdmin_,
+            maxWsSendingQueueSize_
         )
             ->run(std::move(req_));
     }