feat: Forwarding metrics (#2128)

Port of #2096 and #2103 into `release/2.4.1`.

.github/actions/prepare_runner/action.yml

@@ -11,9 +11,35 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       run: |
-          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates
+          brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates
           echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
 
+    - name: Install CMake 3.31.6 on mac
+      if: ${{ runner.os == 'macOS' }}
+      shell: bash
+      run: |
+          # Uninstall any existing cmake
+          brew uninstall cmake --ignore-dependencies || true
+          
+          # Download specific cmake formula
+          FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
+          FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
+          
+          mkdir -p /tmp/homebrew-formula
+          curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
+          
+          # Verify the downloaded formula
+          ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1)
+          if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then
+            echo "Error: Formula checksum mismatch"
+            echo "Expected: $FORMULA_EXPECTED_SHA256"
+            echo "Actual: $ACTUAL_SHA256"
+            exit 1
+          fi
+          
+          # Install cmake from the specific formula with force flag
+          brew install --force /tmp/homebrew-formula/cmake.rb
+
     - name: Fix git permissions on Linux
       if: ${{ runner.os == 'Linux' }}
       shell: bash

src/app/ClioApplication.cpp

@@ -27,6 +27,8 @@
 #include "etl/ETLService.hpp"
 #include "etl/LoadBalancer.hpp"
 #include "etl/NetworkValidatedLedgers.hpp"
+#include "etlng/LoadBalancer.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManager.hpp"
 #include "migration/MigrationInspectorFactory.hpp"
 #include "rpc/Counters.hpp"
@@ -42,6 +44,7 @@
 #include "web/Server.hpp"
 #include "web/dosguard/DOSGuard.hpp"
 #include "web/dosguard/IntervalSweepHandler.hpp"
+#include "web/dosguard/Weights.hpp"
 #include "web/dosguard/WhitelistHandler.hpp"
 #include "web/ng/RPCServerHandler.hpp"
 #include "web/ng/Server.hpp"
@@ -101,7 +104,8 @@ ClioApplication::run(bool const useNgWebServer)
 
     // Rate limiter, to prevent abuse
     auto whitelistHandler = web::dosguard::WhitelistHandler{config_};
-    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler};
+    auto const dosguardWeights = web::dosguard::Weights::make(config_);
+    auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler, dosguardWeights};
     auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
     auto cache = data::LedgerCache{};
 
@@ -130,7 +134,12 @@ ClioApplication::run(bool const useNgWebServer)
     // ETL uses the balancer to extract data.
     // The server uses the balancer to forward RPCs to a rippled node.
     // The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
-    auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+    auto balancer = [&] -> std::shared_ptr<etlng::LoadBalancerInterface> {
+        if (config_.get<bool>("__ng_etl"))
+            return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+
+        return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+    }();
 
     // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
     auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
@@ -142,12 +151,12 @@ ClioApplication::run(bool const useNgWebServer)
         config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
     );
 
-    using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
+    using RPCEngineType = rpc::RPCEngine<rpc::Counters>;
     auto const rpcEngine =
         RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
 
     if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
-        web::ng::RPCServerHandler<RPCEngineType, etl::ETLService> handler{config_, backend, rpcEngine, etl};
+        web::ng::RPCServerHandler<RPCEngineType> handler{config_, backend, rpcEngine, etl, dosGuard};
 
         auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
         if (not expectedAdminVerifier.has_value()) {
@@ -165,7 +174,7 @@ ClioApplication::run(bool const useNgWebServer)
 
         httpServer->onGet("/metrics", MetricsHandler{adminVerifier});
         httpServer->onGet("/health", HealthCheckHandler{});
-        auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard};
+        auto requestHandler = RequestHandler{adminVerifier, handler};
         httpServer->onPost("/", requestHandler);
         httpServer->onWs(std::move(requestHandler));
 
@@ -188,8 +197,7 @@ ClioApplication::run(bool const useNgWebServer)
     }
 
     // Init the web server
-    auto handler =
-        std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
+    auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(config_, backend, rpcEngine, etl, dosGuard);
 
     auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler);
 

src/app/Stopper.hpp

@@ -20,8 +20,8 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
-#include "etl/ETLService.hpp"
-#include "etl/LoadBalancer.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/CoroutineGroup.hpp"
 #include "util/log/Logger.hpp"
@@ -74,15 +74,12 @@ public:
      * @param ioc The io_context to stop.
      * @return The callback to be called on application stop.
      */
-    template <
-        web::ng::SomeServer ServerType,
-        etl::SomeLoadBalancer LoadBalancerType,
-        etl::SomeETLService ETLServiceType>
+    template <web::ng::SomeServer ServerType>
     static std::function<void(boost::asio::yield_context)>
     makeOnStopCallback(
         ServerType& server,
-        LoadBalancerType& balancer,
-        ETLServiceType& etl,
+        etlng::LoadBalancerInterface& balancer,
+        etlng::ETLServiceInterface& etl,
         feed::SubscriptionManagerInterface& subscriptions,
         data::BackendInterface& backend,
         boost::asio::io_context& ioc

src/app/WebHandlers.hpp

@@ -147,7 +147,6 @@ class RequestHandler {
     util::Logger webServerLog_{"WebServer"};
     std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
     std::reference_wrapper<RpcHandlerType> rpcHandler_;
-    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
 
 public:
     /**
@@ -155,14 +154,9 @@ public:
      *
      * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
      * @param rpcHandler The RPC handler to use for handling the request.
-     * @param dosguard The DOSGuardInterface to use for checking the connection.
      */
-    RequestHandler(
-        std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
-        RpcHandlerType& rpcHandler,
-        web::dosguard::DOSGuardInterface& dosguard
-    )
-        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard)
+    RequestHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier, RpcHandlerType& rpcHandler)
+        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler)
     {
     }
 
@@ -183,21 +177,6 @@ public:
         boost::asio::yield_context yield
     )
     {
-        if (not dosguard_.get().request(connectionMetadata.ip())) {
-            auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN);
-
-            if (not request.isHttp()) {
-                try {
-                    auto requestJson = boost::json::parse(request.message());
-                    if (requestJson.is_object() && requestJson.as_object().contains("id"))
-                        error["id"] = requestJson.as_object().at("id");
-                    error["request"] = request.message();
-                } catch (std::exception const&) {
-                    error["request"] = request.message();
-                }
-            }
-            return web::ng::Response{boost::beast::http::status::service_unavailable, error, request};
-        }
         LOG(webServerLog_.info()) << connectionMetadata.tag()
                                   << "Received request from ip = " << connectionMetadata.ip()
                                   << " - posting to WorkQueue";
@@ -207,20 +186,7 @@ public:
         });
 
         try {
-            auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
-
-            if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) {
-                auto jsonResponse = boost::json::parse(response.message()).as_object();
-                jsonResponse["warning"] = "load";
-                if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) {
-                    jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit));
-                } else {
-                    jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)};
-                }
-                response.setMessage(jsonResponse);
-            }
-
-            return response;
+            return rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
         } catch (std::exception const&) {
             return web::ng::Response{
                 boost::beast::http::status::internal_server_error,

src/data/BackendInterface.hpp

@@ -154,7 +154,7 @@ public:
     }
     virtual ~BackendInterface() = default;
 
-    // TODO: Remove this hack once old ETL is removed.
+    // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
     // Cache should not be exposed thru BackendInterface
 
     /**
@@ -648,6 +648,14 @@ public:
     virtual void
     writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
 
+    /**
+     * @brief Write a new account transaction.
+     *
+     * @param record An object representing the account transaction
+     */
+    virtual void
+    writeAccountTransaction(AccountTransactionsData record) = 0;
+
     /**
      * @brief Write NFTs transactions.
      *

src/data/CassandraBackend.hpp

@@ -46,6 +46,7 @@
 #include <xrpl/protocol/LedgerHeader.h>
 #include <xrpl/protocol/nft.h>
 
+#include <algorithm>
 #include <atomic>
 #include <chrono>
 #include <cstddef>
@@ -906,30 +907,42 @@ public:
         statements.reserve(data.size() * 10);  // assume 10 transactions avg
 
         for (auto& record : data) {
-            std::transform(
-                std::begin(record.accounts),
-                std::end(record.accounts),
-                std::back_inserter(statements),
-                [this, &record](auto&& account) {
-                    return schema_->insertAccountTx.bind(
-                        std::forward<decltype(account)>(account),
-                        std::make_tuple(record.ledgerSequence, record.transactionIndex),
-                        record.txHash
-                    );
-                }
-            );
+            std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
+                return schema_->insertAccountTx.bind(
+                    std::forward<decltype(account)>(account),
+                    std::make_tuple(record.ledgerSequence, record.transactionIndex),
+                    record.txHash
+                );
+            });
         }
 
         executor_.write(std::move(statements));
     }
 
+    void
+    writeAccountTransaction(AccountTransactionsData record) override
+    {
+        std::vector<Statement> statements;
+        statements.reserve(record.accounts.size());
+
+        std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
+            return schema_->insertAccountTx.bind(
+                std::forward<decltype(account)>(account),
+                std::make_tuple(record.ledgerSequence, record.transactionIndex),
+                record.txHash
+            );
+        });
+
+        executor_.write(std::move(statements));
+    }
+
     void
     writeNFTTransactions(std::vector<NFTTransactionsData> const& data) override
     {
         std::vector<Statement> statements;
         statements.reserve(data.size());
 
-        std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
+        std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) {
             return schema_->insertNFTTx.bind(
                 record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
             );
@@ -999,7 +1012,7 @@ public:
         std::vector<Statement> statements;
         statements.reserve(data.size());
         for (auto [mptId, holder] : data)
-            statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
+            statements.push_back(schema_->insertMPTHolder.bind(mptId, holder));
 
         executor_.write(std::move(statements));
     }

src/data/DBHelpers.hpp

@@ -54,7 +54,7 @@ struct AccountTransactionsData {
      * @param meta The transaction metadata
      * @param txHash The transaction hash
      */
-    AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash)
+    AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash)
         : accounts(meta.getAffectedAccounts())
         , ledgerSequence(meta.getLgrSeq())
         , transactionIndex(meta.getIndex())

src/data/LedgerCache.cpp

@@ -20,6 +20,7 @@
 #include "data/LedgerCache.hpp"
 
 #include "data/Types.hpp"
+#include "etlng/Models.hpp"
 #include "util/Assert.hpp"
 
 #include <xrpl/basics/base_uint.h>
@@ -87,6 +88,42 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
     }
 }
 
+void
+LedgerCache::update(std::vector<etlng::model::Object> const& objs, uint32_t seq)
+{
+    if (disabled_)
+        return;
+
+    std::scoped_lock const lck{mtx_};
+    if (seq > latestSeq_) {
+        ASSERT(
+            seq == latestSeq_ + 1 || latestSeq_ == 0,
+            "New sequence must be either next or first. seq = {}, latestSeq_ = {}",
+            seq,
+            latestSeq_
+        );
+        latestSeq_ = seq;
+    }
+
+    deleted_.clear();  // previous update's deletes no longer needed
+
+    for (auto const& obj : objs) {
+        if (!obj.data.empty()) {
+            auto& e = map_[obj.key];
+            if (seq > e.seq)
+                e = {.seq = seq, .blob = obj.data};
+        } else {
+            if (map_.contains(obj.key))
+                deleted_[obj.key] = map_[obj.key];
+
+            map_.erase(obj.key);
+            if (!full_)
+                deletes_.insert(obj.key);
+        }
+    }
+    cv_.notify_all();
+}
+
 std::optional<LedgerObject>
 LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
 {
@@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
     return {e->second.blob};
 }
 
+std::optional<Blob>
+LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const
+{
+    if (disabled_)
+        return std::nullopt;
+
+    std::shared_lock const lck{mtx_};
+    if (seq > latestSeq_)
+        return std::nullopt;
+
+    ++objectReqCounter_.get();
+
+    auto e = deleted_.find(key);
+    if (e == deleted_.end())
+        return std::nullopt;
+
+    if (seq < e->second.seq)
+        return std::nullopt;
+
+    ++objectHitCounter_.get();
+    return {e->second.blob};
+}
+
 void
 LedgerCache::setDisabled()
 {

src/data/LedgerCache.hpp

@@ -21,6 +21,7 @@
 
 #include "data/LedgerCacheInterface.hpp"
 #include "data/Types.hpp"
+#include "etlng/Models.hpp"
 #include "util/prometheus/Bool.hpp"
 #include "util/prometheus/Counter.hpp"
 #include "util/prometheus/Label.hpp"
@@ -29,7 +30,6 @@
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/hardened_hash.h>
 
-#include <atomic>
 #include <condition_variable>
 #include <cstddef>
 #include <cstdint>
@@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface {
     )};
 
     std::map<ripple::uint256, CacheEntry> map_;
+    std::map<ripple::uint256, CacheEntry> deleted_;
 
     mutable std::shared_mutex mtx_;
     std::condition_variable_any cv_;
@@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface {
 
 public:
     void
-    update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) override;
+    update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground) override;
+
+    void
+    update(std::vector<etlng::model::Object> const& objs, uint32_t seq) override;
 
     std::optional<Blob>
     get(ripple::uint256 const& key, uint32_t seq) const override;
 
+    std::optional<Blob>
+    getDeleted(ripple::uint256 const& key, uint32_t seq) const override;
+
     std::optional<LedgerObject>
     getSuccessor(ripple::uint256 const& key, uint32_t seq) const override;
 

src/data/LedgerCacheInterface.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "data/Types.hpp"
+#include "etlng/Models.hpp"
 
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/hardened_hash.h>
@@ -55,6 +56,15 @@ public:
     virtual void
     update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) = 0;
 
+    /**
+     * @brief Update the cache with new ledger objects.
+     *
+     * @param objs The ledger objects to update cache with
+     * @param seq The sequence to update cache for
+     */
+    virtual void
+    update(std::vector<etlng::model::Object> const& objs, uint32_t seq) = 0;
+
     /**
      * @brief Fetch a cached object by its key and sequence number.
      *
@@ -65,6 +75,16 @@ public:
     virtual std::optional<Blob>
     get(ripple::uint256 const& key, uint32_t seq) const = 0;
 
+    /**
+     * @brief Fetch a recently deleted object by its key and sequence number.
+     *
+     * @param key The key to fetch for
+     * @param seq The sequence to fetch for
+     * @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned
+     */
+    virtual std::optional<Blob>
+    getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0;
+
     /**
      * @brief Gets a cached successor.
      *

src/etl/ETLService.cpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "etl/CorruptionDetector.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/Constants.hpp"
@@ -43,6 +44,7 @@
 #include <vector>
 
 namespace etl {
+
 // Database must be populated when this starts
 std::optional<uint32_t>
 ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
@@ -265,7 +267,7 @@ ETLService::ETLService(
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
-    std::shared_ptr<LoadBalancerType> balancer,
+    std::shared_ptr<etlng::LoadBalancerInterface> balancer,
     std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
 )
     : backend_(backend)

src/etl/ETLService.hpp

@@ -32,7 +32,12 @@
 #include "etl/impl/LedgerLoader.hpp"
 #include "etl/impl/LedgerPublisher.hpp"
 #include "etl/impl/Transformer.hpp"
+#include "etlng/ETLService.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancer.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
+#include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/io_context.hpp>
@@ -41,7 +46,6 @@
 #include <org/xrpl/rpc/v1/get_ledger.pb.h>
 #include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <concepts>
 #include <cstddef>
 #include <cstdint>
 #include <memory>
@@ -81,14 +85,13 @@ concept SomeETLService = std::derived_from<T, ETLServiceTag>;
  * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
  * to writing and from writing to monitoring, based on the activity of other processes running on different machines.
  */
-class ETLService : public ETLServiceTag {
+class ETLService : public etlng::ETLServiceInterface, ETLServiceTag {
     // TODO: make these template parameters in ETLService
-    using LoadBalancerType = LoadBalancer;
     using DataPipeType = etl::impl::ExtractionDataPipe<org::xrpl::rpc::v1::GetLedgerResponse>;
     using CacheLoaderType = etl::CacheLoader<>;
-    using LedgerFetcherType = etl::impl::LedgerFetcher<LoadBalancerType>;
+    using LedgerFetcherType = etl::impl::LedgerFetcher;
     using ExtractorType = etl::impl::Extractor<DataPipeType, LedgerFetcherType>;
-    using LedgerLoaderType = etl::impl::LedgerLoader<LoadBalancerType, LedgerFetcherType>;
+    using LedgerLoaderType = etl::impl::LedgerLoader<LedgerFetcherType>;
     using LedgerPublisherType = etl::impl::LedgerPublisher;
     using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler;
     using TransformerType =
@@ -97,7 +100,7 @@ class ETLService : public ETLServiceTag {
     util::Logger log_{"ETL"};
 
     std::shared_ptr<BackendInterface> backend_;
-    std::shared_ptr<LoadBalancerType> loadBalancer_;
+    std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
     std::shared_ptr<NetworkValidatedLedgersInterface> networkValidatedLedgers_;
 
     std::uint32_t extractorThreads_ = 1;
@@ -132,7 +135,7 @@ public:
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
-        std::shared_ptr<LoadBalancerType> balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> balancer,
         std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
     );
 
@@ -154,20 +157,37 @@ public:
      * @param ledgers The network validated ledgers datastructure
      * @return A shared pointer to a new instance of ETLService
      */
-    static std::shared_ptr<ETLService>
+    static std::shared_ptr<etlng::ETLServiceInterface>
     makeETLService(
         util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
-        std::shared_ptr<LoadBalancerType> balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> balancer,
         std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
     )
     {
-        auto etl = std::make_shared<ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
-        etl->run();
+        std::shared_ptr<etlng::ETLServiceInterface> ret;
 
-        return etl;
+        if (config.get<bool>("__ng_etl")) {
+            ASSERT(
+                std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer),
+                "LoadBalancer type must be etlng::LoadBalancer"
+            );
+            ret = std::make_shared<etlng::ETLService>(config, backend, subscriptions, balancer, ledgers);
+        } else {
+            ASSERT(
+                std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer"
+            );
+            ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
+        }
+
+        // inject networkID into subscriptions, as transaction feed require it to inject CTID in response
+        if (auto const state = ret->getETLState(); state)
+            subscriptions->setNetworkID(state->networkID);
+
+        ret->run();
+        return ret;
     }
 
     /**
@@ -184,7 +204,7 @@ public:
      * @note This method blocks until the ETL service has stopped.
      */
     void
-    stop()
+    stop() override
     {
         LOG(log_.info()) << "Stop called";
 
@@ -203,7 +223,7 @@ public:
      * @return Time passed since last ledger close
      */
     std::uint32_t
-    lastCloseAgeSeconds() const
+    lastCloseAgeSeconds() const override
     {
         return ledgerPublisher_.lastCloseAgeSeconds();
     }
@@ -214,7 +234,7 @@ public:
      * @return true if currently amendment blocked; false otherwise
      */
     bool
-    isAmendmentBlocked() const
+    isAmendmentBlocked() const override
     {
         return state_.isAmendmentBlocked;
     }
@@ -225,7 +245,7 @@ public:
      * @return true if corruption of DB was detected and cache was stopped.
      */
     bool
-    isCorruptionDetected() const
+    isCorruptionDetected() const override
     {
         return state_.isCorruptionDetected;
     }
@@ -236,7 +256,7 @@ public:
      * @return The state of ETL as a JSON object
      */
     boost::json::object
-    getInfo() const
+    getInfo() const override
     {
         boost::json::object result;
 
@@ -254,11 +274,17 @@ public:
      * @return The etl nodes' state, nullopt if etl nodes are not connected
      */
     std::optional<etl::ETLState>
-    getETLState() const noexcept
+    getETLState() const noexcept override
     {
         return loadBalancer_->getETLState();
     }
 
+    /**
+     * @brief Start all components to run ETL service.
+     */
+    void
+    run() override;
+
 private:
     /**
      * @brief Run the ETL pipeline.
@@ -325,12 +351,6 @@ private:
         return numMarkers_;
     }
 
-    /**
-     * @brief Start all components to run ETL service.
-     */
-    void
-    run();
-
     /**
      * @brief Spawn the worker thread and start monitoring.
      */

src/etl/ETLState.cpp

@@ -40,7 +40,7 @@ tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv)
     if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) {
         auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object();
         if (rippledInfo.contains(JS(network_id)))
-            state.networkID.emplace(boost::json::value_to<int64_t>(rippledInfo.at(JS(network_id))));
+            state.networkID = boost::json::value_to<int64_t>(rippledInfo.at(JS(network_id)));
     }
 
     return state;

src/etl/ETLState.hpp

@@ -38,7 +38,12 @@ namespace etl {
  * @brief This class is responsible for fetching and storing the state of the ETL information, such as the network id
  */
 struct ETLState {
-    std::optional<uint32_t> networkID;
+    /*
+     * NOTE: Rippled NetworkID: Mainnet = 0; Testnet = 1; Devnet = 2
+     * However, if rippled is running on neither of these (ie. standalone mode) rippled will default to 0, but
+     * is not included in the stateOpt response. Must manually add it here.
+     */
+    uint32_t networkID{0};
 
     /**
      * @brief Fetch the ETL state from the rippled server

src/etl/LoadBalancer.cpp

@@ -23,16 +23,20 @@
 #include "etl/ETLState.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
 #include "util/CoroutineGroup.hpp"
+#include "util/Profiler.hpp"
 #include "util/Random.hpp"
 #include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ArrayView.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
 #include "util/newconfig/ObjectView.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
@@ -56,10 +60,11 @@
 #include <vector>
 
 using namespace util::config;
+using util::prometheus::Labels;
 
 namespace etl {
 
-std::shared_ptr<LoadBalancer>
+std::shared_ptr<etlng::LoadBalancerInterface>
 LoadBalancer::makeLoadBalancer(
     ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
@@ -82,6 +87,34 @@ LoadBalancer::LoadBalancer(
     std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
     SourceFactory sourceFactory
 )
+    : forwardingCounters_{
+          .successDuration = PrometheusService::counterInt(
+              "forwarding_duration_milliseconds_counter",
+              Labels({util::prometheus::Label{"status", "success"}}),
+              "The duration of processing successful forwarded requests"
+          ),
+          .failDuration = PrometheusService::counterInt(
+              "forwarding_duration_milliseconds_counter",
+              Labels({util::prometheus::Label{"status", "fail"}}),
+              "The duration of processing failed forwarded requests"
+          ),
+          .retries = PrometheusService::counterInt(
+              "forwarding_retries_counter",
+              Labels(),
+              "The number of retries before a forwarded request was successful. Initial attempt excluded"
+          ),
+          .cacheHit = PrometheusService::counterInt(
+              "forwarding_cache_hit_counter",
+              Labels(),
+              "The number of requests that we served from the cache"
+          ),
+          .cacheMiss = PrometheusService::counterInt(
+              "forwarding_cache_miss_counter",
+              Labels(),
+              "The number of requests that were not served from the cache"
+          )
+      }
+
 {
     auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
     if (forwardingCacheTimeout > 0.f) {
@@ -141,12 +174,11 @@ LoadBalancer::LoadBalancer(
         if (!stateOpt) {
             LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
                              << " Please check the configuration and network";
-        } else if (etlState_ && etlState_->networkID && stateOpt->networkID &&
-                   etlState_->networkID != stateOpt->networkID) {
+        } else if (etlState_ && etlState_->networkID != stateOpt->networkID) {
             checkOnETLFailure(fmt::format(
                 "ETL sources must be on the same network. Source network id = {} does not match others network id = {}",
-                *(stateOpt->networkID),
-                *(etlState_->networkID)
+                stateOpt->networkID,
+                etlState_->networkID
             ));
         } else {
             etlState_ = stateOpt;
@@ -175,12 +207,12 @@ LoadBalancer::~LoadBalancer()
 }
 
 std::vector<std::string>
-LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly, std::chrono::steady_clock::duration retryAfter)
+LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter)
 {
     std::vector<std::string> response;
     execute(
-        [this, &response, &sequence, cacheOnly](auto& source) {
-            auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
+        [this, &response, &sequence](auto& source) {
+            auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_);
 
             if (!res) {
                 LOG(log_.error()) << "Failed to download initial ledger."
@@ -241,9 +273,11 @@ LoadBalancer::forwardToRippled(
     auto const cmd = boost::json::value_to<std::string>(request.at("command"));
     if (forwardingCache_) {
         if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
+            ++forwardingCounters_.cacheHit.get();
             return std::move(cachedResponse).value();
         }
     }
+    ++forwardingCounters_.cacheMiss.get();
 
     ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
     std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
@@ -255,11 +289,15 @@ LoadBalancer::forwardToRippled(
     std::optional<boost::json::object> response;
     rpc::ClioError error = rpc::ClioError::EtlConnectionError;
     while (numAttempts < sources_.size()) {
-        auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
+        auto [res, duration] =
+            util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
         if (res) {
+            forwardingCounters_.successDuration.get() += duration;
             response = std::move(res).value();
             break;
         }
+        forwardingCounters_.failDuration.get() += duration;
+        ++forwardingCounters_.retries.get();
         error = std::max(error, res.error());  // Choose the best result between all sources
 
         sourceIdx = (sourceIdx + 1) % sources_.size();

src/etl/LoadBalancer.hpp

@@ -23,12 +23,16 @@
 #include "etl/ETLState.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
+#include "util/Assert.hpp"
 #include "util/Mutex.hpp"
 #include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Counter.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/asio/io_context.hpp>
@@ -44,10 +48,12 @@
 #include <concepts>
 #include <cstdint>
 #include <expected>
+#include <functional>
 #include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
+#include <utility>
 #include <vector>
 
 namespace etl {
@@ -69,7 +75,7 @@ concept SomeLoadBalancer = std::derived_from<T, LoadBalancerTag>;
  * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
  * allows requests for ledger data to be load balanced across all possible ETL sources.
  */
-class LoadBalancer : public LoadBalancerTag {
+class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag {
 public:
     using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
     using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
@@ -88,6 +94,14 @@ private:
     std::uint32_t downloadRanges_ =
         kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
 
+    struct ForwardingCounters {
+        std::reference_wrapper<util::prometheus::CounterInt> successDuration;
+        std::reference_wrapper<util::prometheus::CounterInt> failDuration;
+        std::reference_wrapper<util::prometheus::CounterInt> retries;
+        std::reference_wrapper<util::prometheus::CounterInt> cacheHit;
+        std::reference_wrapper<util::prometheus::CounterInt> cacheMiss;
+    } forwardingCounters_;
+
     // Using mutext instead of atomic_bool because choosing a new source to
     // forward messages should be done with a mutual exclusion otherwise there will be a race condition
     util::Mutex<bool> hasForwardingSource_{false};
@@ -133,7 +147,7 @@ public:
      * @param sourceFactory A factory function to create a source
      * @return A shared pointer to a new instance of LoadBalancer
      */
-    static std::shared_ptr<LoadBalancer>
+    static std::shared_ptr<LoadBalancerInterface>
     makeLoadBalancer(
         util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
@@ -150,16 +164,32 @@ public:
      * @note This function will retry indefinitely until the ledger is downloaded.
      *
      * @param sequence Sequence of ledger to download
-     * @param cacheOnly Whether to only write to cache and not to the DB; defaults to false
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return A std::vector<std::string> The ledger data
+     */
+    std::vector<std::string>
+    loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2})
+        override;
+
+    /**
+     * @brief Load the initial ledger, writing data to the queue.
+     * @note This function will retry indefinitely until the ledger is downloaded.
+     *
+     * @param sequence Sequence of ledger to download
+     * @param observer The observer to notify of progress
      * @param retryAfter Time to wait between retries (2 seconds by default)
      * @return A std::vector<std::string> The ledger data
      */
     std::vector<std::string>
     loadInitialLedger(
-        uint32_t sequence,
-        bool cacheOnly = false,
-        std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
-    );
+        [[maybe_unused]] uint32_t sequence,
+        [[maybe_unused]] etlng::InitialLoadObserverInterface& observer,
+        [[maybe_unused]] std::chrono::steady_clock::duration retryAfter
+    ) override
+    {
+        ASSERT(false, "Not available for old ETL");
+        std::unreachable();
+    }
 
     /**
      * @brief Fetch data for a specific ledger.
@@ -180,7 +210,7 @@ public:
         bool getObjects,
         bool getObjectNeighbors,
         std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
-    );
+    ) override;
 
     /**
      * @brief Represent the state of this load balancer as a JSON object
@@ -188,7 +218,7 @@ public:
      * @return JSON representation of the state of this load balancer.
      */
     boost::json::value
-    toJson() const;
+    toJson() const override;
 
     /**
      * @brief Forward a JSON RPC request to a randomly selected rippled node.
@@ -205,14 +235,14 @@ public:
         std::optional<std::string> const& clientIp,
         bool isAdmin,
         boost::asio::yield_context yield
-    );
+    ) override;
 
     /**
      * @brief Return state of ETL nodes.
      * @return ETL state, nullopt if etl nodes not available
      */
     std::optional<ETLState>
-    getETLState() noexcept;
+    getETLState() noexcept override;
 
     /**
      * @brief Stop the load balancer. This will stop all subscription sources.
@@ -221,7 +251,7 @@ public:
      * @param yield The coroutine context
      */
     void
-    stop(boost::asio::yield_context yield);
+    stop(boost::asio::yield_context yield) override;
 
 private:
     /**

src/etl/NetworkValidatedLedgersInterface.hpp

@@ -26,6 +26,7 @@
 
 #include <cstdint>
 #include <optional>
+
 namespace etl {
 
 /**

src/etl/Source.hpp

@@ -23,8 +23,6 @@
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
-#include "util/log/Logger.hpp"
-#include "util/newconfig/ConfigDefinition.hpp"
 #include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
@@ -130,11 +128,10 @@ public:
      *
      * @param sequence Sequence of the ledger to download
      * @param numMarkers Number of markers to generate for async calls
-     * @param cacheOnly Only insert into cache, not the DB; defaults to false
      * @return A std::pair of the data and a bool indicating whether the download was successful
      */
     virtual std::pair<std::vector<std::string>, bool>
-    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0;
+    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) = 0;
 
     /**
      * @brief Forward a request to rippled.

src/etl/impl/GrpcSource.cpp

@@ -98,7 +98,7 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb
 }
 
 std::pair<std::vector<std::string>, bool>
-GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers, bool const cacheOnly)
+GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers)
 {
     if (!stub_)
         return {{}, false};
@@ -130,7 +130,7 @@ GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers
 
         LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix();
 
-        auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
+        auto result = ptr->process(stub_, cq, *backend_, abort);
         if (result != etl::impl::AsyncCallData::CallStatus::MORE) {
             ++numFinished;
             LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished;

src/etl/impl/GrpcSource.hpp

@@ -60,11 +60,10 @@ public:
      *
      * @param sequence Sequence of the ledger to download
      * @param numMarkers Number of markers to generate for async calls
-     * @param cacheOnly Only insert into cache, not the DB; defaults to false
      * @return A std::pair of the data and a bool indicating whether the download was successful
      */
     std::pair<std::vector<std::string>, bool>
-    loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly = false);
+    loadInitialLedger(uint32_t sequence, uint32_t numMarkers);
 };
 
 }  // namespace etl::impl

src/etl/impl/LedgerFetcher.hpp

@@ -20,6 +20,8 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "util/log/Logger.hpp"
 
 #include <grpcpp/grpcpp.h>
@@ -34,22 +36,18 @@ namespace etl::impl {
 /**
  * @brief GRPC Ledger data fetcher
  */
-template <typename LoadBalancerType>
-class LedgerFetcher {
-public:
-    using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
-
+class LedgerFetcher : public LedgerFetcherInterface {
 private:
     util::Logger log_{"ETL"};
 
     std::shared_ptr<BackendInterface> backend_;
-    std::shared_ptr<LoadBalancerType> loadBalancer_;
+    std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
 
 public:
     /**
      * @brief Create an instance of the fetcher
      */
-    LedgerFetcher(std::shared_ptr<BackendInterface> backend, std::shared_ptr<LoadBalancerType> balancer)
+    LedgerFetcher(std::shared_ptr<BackendInterface> backend, std::shared_ptr<etlng::LoadBalancerInterface> balancer)
         : backend_(std::move(backend)), loadBalancer_(std::move(balancer))
     {
     }
@@ -64,7 +62,7 @@ public:
      * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
      */
     [[nodiscard]] OptionalGetLedgerResponseType
-    fetchData(uint32_t sequence)
+    fetchData(uint32_t sequence) override
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
 
@@ -84,7 +82,7 @@ public:
      * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
      */
     [[nodiscard]] OptionalGetLedgerResponseType
-    fetchDataAndDiff(uint32_t sequence)
+    fetchDataAndDiff(uint32_t sequence) override
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
 

src/etl/impl/LedgerLoader.hpp

@@ -26,6 +26,7 @@
 #include "etl/NFTHelpers.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/LedgerUtils.hpp"
 #include "util/Profiler.hpp"
@@ -65,18 +66,18 @@ namespace etl::impl {
 /**
  * @brief Loads ledger data into the DB
  */
-template <typename LoadBalancerType, typename LedgerFetcherType>
+template <typename LedgerFetcherType>
 class LedgerLoader {
 public:
-    using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType;
-    using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
-    using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType;
+    using GetLedgerResponseType = etlng::LoadBalancerInterface::GetLedgerResponseType;
+    using OptionalGetLedgerResponseType = etlng::LoadBalancerInterface::OptionalGetLedgerResponseType;
+    using RawLedgerObjectType = etlng::LoadBalancerInterface::RawLedgerObjectType;
 
 private:
     util::Logger log_{"ETL"};
 
     std::shared_ptr<BackendInterface> backend_;
-    std::shared_ptr<LoadBalancerType> loadBalancer_;
+    std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
     std::reference_wrapper<LedgerFetcherType> fetcher_;
     std::reference_wrapper<SystemState const> state_;  // shared state for ETL
 
@@ -86,7 +87,7 @@ public:
      */
     LedgerLoader(
         std::shared_ptr<BackendInterface> backend,
-        std::shared_ptr<LoadBalancerType> balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> balancer,
         LedgerFetcherType& fetcher,
         SystemState const& state
     )
@@ -121,7 +122,7 @@ public:
 
             LOG(log_.trace()) << "Inserting transaction = " << sttx.getTransactionID();
 
-            ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
+            ripple::TxMeta const txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
 
             auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx);
             result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end());

src/etl/impl/SourceImpl.hpp

@@ -202,9 +202,9 @@ public:
      * @return A std::pair of the data and a bool indicating whether the download was successful
      */
     std::pair<std::vector<std::string>, bool>
-    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final
+    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) final
     {
-        return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly);
+        return grpcSource_.loadInitialLedger(sequence, numMarkers);
     }
 
     /**

src/etlng/CMakeLists.txt

@@ -2,13 +2,20 @@ add_library(clio_etlng)
 
 target_sources(
   clio_etlng
-  PRIVATE impl/AmendmentBlockHandler.cpp
+  PRIVATE LoadBalancer.cpp
+          Source.cpp
+          impl/AmendmentBlockHandler.cpp
           impl/AsyncGrpcCall.cpp
           impl/Extraction.cpp
           impl/GrpcSource.cpp
+          impl/ForwardingSource.cpp
           impl/Loading.cpp
           impl/Monitor.cpp
           impl/TaskManager.cpp
+          impl/ext/Cache.cpp
+          impl/ext/Core.cpp
+          impl/ext/NFT.cpp
+          impl/ext/Successor.cpp
 )
 
 target_link_libraries(clio_etlng PUBLIC clio_data)

src/etlng/ETLService.hpp

@@ -0,0 +1,283 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "data/LedgerCache.hpp"
+#include "data/Types.hpp"
+#include "etl/CacheLoader.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etl/SystemState.hpp"
+#include "etl/impl/AmendmentBlockHandler.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etl/impl/LedgerPublisher.hpp"
+#include "etlng/AmendmentBlockHandlerInterface.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/ExtractorInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/impl/AmendmentBlockHandler.hpp"
+#include "etlng/impl/Extraction.hpp"
+#include "etlng/impl/Loading.hpp"
+#include "etlng/impl/Monitor.hpp"
+#include "etlng/impl/Registry.hpp"
+#include "etlng/impl/Scheduling.hpp"
+#include "etlng/impl/TaskManager.hpp"
+#include "etlng/impl/ext/Cache.hpp"
+#include "etlng/impl/ext/Core.hpp"
+#include "etlng/impl/ext/NFT.hpp"
+#include "etlng/impl/ext/Successor.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "util/Assert.hpp"
+#include "util/Profiler.hpp"
+#include "util/async/context/BasicExecutionContext.hpp"
+#include "util/config/Config.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <boost/json/object.hpp>
+#include <fmt/core.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxFormats.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <chrono>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+#include <utility>
+
+namespace etlng {
+
+/**
+ * @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the
+ * databases.
+ *
+ * Usually, multiple different processes share access to the same network accessible databases, in which case only one
+ * such process is performing ETL and writing to the database. The other processes simply monitor the database for new
+ * ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the
+ * ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer.
+ *
+ * If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and
+ * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
+ * to writing and from writing to monitoring, based on the activity of other processes running on different machines.
+ */
+class ETLService : public ETLServiceInterface {
+    util::Logger log_{"ETL"};
+
+    std::shared_ptr<BackendInterface> backend_;
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
+    std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers_;
+    std::shared_ptr<etl::CacheLoader<>> cacheLoader_;
+
+    std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
+    std::shared_ptr<ExtractorInterface> extractor_;
+
+    etl::SystemState state_;
+    util::async::CoroExecutionContext ctx_{8};
+
+    std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
+    std::shared_ptr<impl::Loader> loader_;
+
+    std::optional<util::async::CoroExecutionContext::Operation<void>> mainLoop_;
+
+public:
+    /**
+     * @brief Create an instance of ETLService.
+     *
+     * @param config The configuration to use
+     * @param backend BackendInterface implementation
+     * @param subscriptions Subscription manager
+     * @param balancer Load balancer to use
+     * @param ledgers The network validated ledgers datastructure
+     */
+    ETLService(
+        util::config::ClioConfigDefinition const& config,
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+        std::shared_ptr<etlng::LoadBalancerInterface> balancer,
+        std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers
+    )
+        : backend_(std::move(backend))
+        , subscriptions_(std::move(subscriptions))
+        , balancer_(std::move(balancer))
+        , ledgers_(std::move(ledgers))
+        , cacheLoader_(std::make_shared<etl::CacheLoader<>>(config, backend_, backend_->cache()))
+        , fetcher_(std::make_shared<etl::impl::LedgerFetcher>(backend_, balancer_))
+        , extractor_(std::make_shared<impl::Extractor>(fetcher_))
+        , amendmentBlockHandler_(std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx_, state_))
+        , loader_(std::make_shared<impl::Loader>(
+              backend_,
+              fetcher_,
+              impl::makeRegistry(
+                  impl::CacheExt{backend_->cache()},
+                  impl::CoreExt{backend_},
+                  impl::SuccessorExt{backend_, backend_->cache()},
+                  impl::NFTExt{backend_}
+              ),
+              amendmentBlockHandler_
+          ))
+    {
+        LOG(log_.info()) << "Creating ETLng...";
+    }
+
+    ~ETLService() override
+    {
+        LOG(log_.debug()) << "Stopping ETLng";
+    }
+
+    void
+    run() override
+    {
+        LOG(log_.info()) << "run() in ETLng...";
+
+        mainLoop_.emplace(ctx_.execute([this] {
+            auto const rng = loadInitialLedgerIfNeeded();
+
+            LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
+            std::optional<uint32_t> mostRecentValidated = ledgers_->getMostRecent();
+
+            if (not mostRecentValidated) {
+                LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
+                                    "Exiting monitor loop";
+                return;
+            }
+
+            ASSERT(rng.has_value(), "Ledger range can't be null");
+            auto const nextSequence = rng->maxSequence + 1;
+
+            LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
+
+            auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence}
+                                                 // impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000},
+                                                 // TODO lift limit and start with rng.minSeq
+            );
+
+            auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_);
+
+            // TODO: figure out this: std::make_shared<impl::Monitor>(backend_, ledgers_, nextSequence)
+            man.run({});  // TODO: needs to be interruptable and fill out settings
+        }));
+    }
+
+    void
+    stop() override
+    {
+        LOG(log_.info()) << "Stop called";
+        // TODO: stop the service correctly
+    }
+
+    boost::json::object
+    getInfo() const override
+    {
+        // TODO
+        return {{"ok", true}};
+    }
+
+    bool
+    isAmendmentBlocked() const override
+    {
+        // TODO
+        return false;
+    }
+
+    bool
+    isCorruptionDetected() const override
+    {
+        // TODO
+        return false;
+    }
+
+    std::optional<etl::ETLState>
+    getETLState() const override
+    {
+        // TODO
+        return std::nullopt;
+    }
+
+    std::uint32_t
+    lastCloseAgeSeconds() const override
+    {
+        // TODO
+        return 0;
+    }
+
+private:
+    // TODO: this better be std::expected
+    std::optional<data::LedgerRange>
+    loadInitialLedgerIfNeeded()
+    {
+        if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) {
+            LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
+
+            try {
+                LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
+                if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
+                    auto const seq = *mostRecentValidated;
+                    LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
+
+                    auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
+                        return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
+                            // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
+                            data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_);
+
+                            // TODO: this should be interruptable for graceful shutdown
+                            return loader_->loadInitialLedger(data);
+                        });
+                    });
+
+                    LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
+                    LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
+
+                    if (ledger.has_value())
+                        return backend_->hardFetchLedgerRangeNoThrow();
+
+                    LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
+                } else {
+                    LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
+                                        "Exiting monitor loop";
+                }
+            } catch (std::runtime_error const& e) {
+                LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
+                amendmentBlockHandler_->notifyAmendmentBlocked();
+            }
+        } else {
+            LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
+            cacheLoader_->load(rng->maxSequence);
+
+            return rng;
+        }
+
+        return std::nullopt;
+    }
+};
+}  // namespace etlng

src/etlng/ETLServiceInterface.hpp

@@ -0,0 +1,92 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/ETLState.hpp"
+
+#include <boost/json/object.hpp>
+
+#include <cstdint>
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief This is a base class for any ETL service implementations.
+ * @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the
+ * databases.
+ */
+struct ETLServiceInterface {
+    virtual ~ETLServiceInterface() = default;
+
+    /**
+     * @brief Start all components to run ETL service.
+     */
+    virtual void
+    run() = 0;
+
+    /**
+     * @brief Stop the ETL service.
+     * @note This method blocks until the ETL service has stopped.
+     */
+    virtual void
+    stop() = 0;
+
+    /**
+     * @brief Get state of ETL as a JSON object
+     *
+     * @return The state of ETL as a JSON object
+     */
+    [[nodiscard]] virtual boost::json::object
+    getInfo() const = 0;
+
+    /**
+     * @brief Check for the amendment blocked state.
+     *
+     * @return true if currently amendment blocked; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    isAmendmentBlocked() const = 0;
+
+    /**
+     * @brief Check whether Clio detected DB corruptions.
+     *
+     * @return true if corruption of DB was detected and cache was stopped.
+     */
+    [[nodiscard]] virtual bool
+    isCorruptionDetected() const = 0;
+
+    /**
+     * @brief Get the etl nodes' state
+     * @return The etl nodes' state, nullopt if etl nodes are not connected
+     */
+    [[nodiscard]] virtual std::optional<etl::ETLState>
+    getETLState() const = 0;
+
+    /**
+     * @brief Get time passed since last ledger close, in seconds.
+     *
+     * @return Time passed since last ledger close
+     */
+    [[nodiscard]] virtual std::uint32_t
+    lastCloseAgeSeconds() const = 0;
+};
+
+}  // namespace etlng

src/etlng/LoadBalancer.cpp

@@ -0,0 +1,370 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/LoadBalancer.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/Source.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
+#include "util/Assert.hpp"
+#include "util/CoroutineGroup.hpp"
+#include "util/Random.hpp"
+#include "util/ResponseExpirationCache.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ArrayView.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
+#include <fmt/core.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <expected>
+#include <memory>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+using namespace util::config;
+
+namespace etlng {
+
+std::shared_ptr<LoadBalancerInterface>
+LoadBalancer::makeLoadBalancer(
+    ClioConfigDefinition const& config,
+    boost::asio::io_context& ioc,
+    std::shared_ptr<BackendInterface> backend,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+    SourceFactory sourceFactory
+)
+{
+    return std::make_shared<LoadBalancer>(
+        config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory)
+    );
+}
+
+LoadBalancer::LoadBalancer(
+    ClioConfigDefinition const& config,
+    boost::asio::io_context& ioc,
+    std::shared_ptr<BackendInterface> backend,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+    SourceFactory sourceFactory
+)
+{
+    auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
+    if (forwardingCacheTimeout > 0.f) {
+        forwardingCache_ = util::ResponseExpirationCache{
+            util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout),
+            {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
+        };
+    }
+
+    auto const numMarkers = config.getValueView("num_markers");
+    if (numMarkers.hasValue()) {
+        auto const value = numMarkers.asIntType<uint32_t>();
+        downloadRanges_ = value;
+    } else if (backend->fetchLedgerRange()) {
+        downloadRanges_ = 4;
+    }
+
+    auto const allowNoEtl = config.get<bool>("allow_no_etl");
+
+    auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) {
+        LOG(log_.warn()) << log;
+
+        if (!allowNoEtl) {
+            LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources.";
+            throw std::logic_error("ETL configuration error.");
+        }
+    };
+
+    auto const forwardingTimeout =
+        ClioConfigDefinition::toMilliseconds(config.get<float>("forwarding.request_timeout"));
+    auto const etlArray = config.getArray("etl_sources");
+    for (auto it = etlArray.begin<ObjectView>(); it != etlArray.end<ObjectView>(); ++it) {
+        auto source = sourceFactory(
+            *it,
+            ioc,
+            subscriptions,
+            validatedLedgers,
+            forwardingTimeout,
+            [this]() {
+                if (not hasForwardingSource_.lock().get())
+                    chooseForwardingSource();
+            },
+            [this](bool wasForwarding) {
+                if (wasForwarding)
+                    chooseForwardingSource();
+            },
+            [this]() {
+                if (forwardingCache_.has_value())
+                    forwardingCache_->invalidate();
+            }
+        );
+
+        // checking etl node validity
+        auto const stateOpt = etl::ETLState::fetchETLStateFromSource(*source);
+
+        if (!stateOpt) {
+            LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
+                             << " Please check the configuration and network";
+        } else if (etlState_ && etlState_->networkID != stateOpt->networkID) {
+            checkOnETLFailure(fmt::format(
+                "ETL sources must be on the same network. Source network id = {} does not match others network id = {}",
+                stateOpt->networkID,
+                etlState_->networkID
+            ));
+        } else {
+            etlState_ = stateOpt;
+        }
+
+        sources_.push_back(std::move(source));
+        LOG(log_.info()) << "Added etl source - " << sources_.back()->toString();
+    }
+
+    if (!etlState_)
+        checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network");
+
+    if (sources_.empty())
+        checkOnETLFailure("No ETL sources configured. Please check the configuration");
+
+    // This is made separate from source creation to prevent UB in case one of the sources will call
+    // chooseForwardingSource while we are still filling the sources_ vector
+    for (auto const& source : sources_) {
+        source->run();
+    }
+}
+
+LoadBalancer::~LoadBalancer()
+{
+    sources_.clear();
+}
+
+std::vector<std::string>
+LoadBalancer::loadInitialLedger(
+    uint32_t sequence,
+    etlng::InitialLoadObserverInterface& loadObserver,
+    std::chrono::steady_clock::duration retryAfter
+)
+{
+    std::vector<std::string> response;
+    execute(
+        [this, &response, &sequence, &loadObserver](auto& source) {
+            auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, loadObserver);
+
+            if (!res) {
+                LOG(log_.error()) << "Failed to download initial ledger."
+                                  << " Sequence = " << sequence << " source = " << source->toString();
+            } else {
+                response = std::move(data);
+            }
+
+            return res;
+        },
+        sequence,
+        retryAfter
+    );
+    return response;
+}
+
+LoadBalancer::OptionalGetLedgerResponseType
+LoadBalancer::fetchLedger(
+    uint32_t ledgerSequence,
+    bool getObjects,
+    bool getObjectNeighbors,
+    std::chrono::steady_clock::duration retryAfter
+)
+{
+    GetLedgerResponseType response;
+    execute(
+        [&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) {
+            auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
+            response = std::move(data);
+            if (status.ok() && response.validated()) {
+                LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence
+                                << " from source = " << source->toString();
+                return true;
+            }
+
+            LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString()
+                            << ", error_code: " << status.error_code() << ", error_msg: " << status.error_message()
+                            << ", source = " << source->toString();
+            return false;
+        },
+        ledgerSequence,
+        retryAfter
+    );
+    return response;
+}
+
+std::expected<boost::json::object, rpc::ClioError>
+LoadBalancer::forwardToRippled(
+    boost::json::object const& request,
+    std::optional<std::string> const& clientIp,
+    bool isAdmin,
+    boost::asio::yield_context yield
+)
+{
+    if (not request.contains("command"))
+        return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
+
+    auto const cmd = boost::json::value_to<std::string>(request.at("command"));
+    if (forwardingCache_) {
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
+            return std::move(cachedResponse).value();
+        }
+    }
+
+    ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
+    std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
+
+    auto numAttempts = 0u;
+
+    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
+
+    std::optional<boost::json::object> response;
+    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
+    while (numAttempts < sources_.size()) {
+        auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
+        if (res) {
+            response = std::move(res).value();
+            break;
+        }
+        error = std::max(error, res.error());  // Choose the best result between all sources
+
+        sourceIdx = (sourceIdx + 1) % sources_.size();
+        ++numAttempts;
+    }
+
+    if (response) {
+        if (forwardingCache_ and not response->contains("error"))
+            forwardingCache_->put(cmd, *response);
+        return std::move(response).value();
+    }
+
+    return std::unexpected{error};
+}
+
+boost::json::value
+LoadBalancer::toJson() const
+{
+    boost::json::array ret;
+    for (auto& src : sources_)
+        ret.push_back(src->toJson());
+
+    return ret;
+}
+
+template <typename Func>
+void
+LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
+{
+    ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
+    size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
+
+    size_t numAttempts = 0;
+
+    while (true) {
+        auto& source = sources_[sourceIdx];
+
+        LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence
+                          << " - source = " << source->toString();
+        // Originally, it was (source->hasLedger(ledgerSequence) || true)
+        /* Sometimes rippled has ledger but doesn't actually know. However,
+        but this does NOT happen in the normal case and is safe to remove
+        This || true is only needed when loading full history standalone */
+        if (source->hasLedger(ledgerSequence)) {
+            bool const res = f(source);
+            if (res) {
+                LOG(log_.debug()) << "Successfully executed func at source = " << source->toString()
+                                  << " - ledger sequence = " << ledgerSequence;
+                break;
+            }
+
+            LOG(log_.warn()) << "Failed to execute func at source = " << source->toString()
+                             << " - ledger sequence = " << ledgerSequence;
+        } else {
+            LOG(log_.warn()) << "Ledger not present at source = " << source->toString()
+                             << " - ledger sequence = " << ledgerSequence;
+        }
+        sourceIdx = (sourceIdx + 1) % sources_.size();
+        numAttempts++;
+        if (numAttempts % sources_.size() == 0) {
+            LOG(log_.info()) << "Ledger sequence " << ledgerSequence
+                             << " is not yet available from any configured sources. Sleeping and trying again";
+            std::this_thread::sleep_for(retryAfter);
+        }
+    }
+}
+
+std::optional<etl::ETLState>
+LoadBalancer::getETLState() noexcept
+{
+    if (!etlState_) {
+        // retry ETLState fetch
+        etlState_ = etl::ETLState::fetchETLStateFromSource(*this);
+    }
+    return etlState_;
+}
+
+void
+LoadBalancer::stop(boost::asio::yield_context yield)
+{
+    util::CoroutineGroup group{yield};
+    std::ranges::for_each(sources_, [&group, yield](auto& source) {
+        group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); });
+    });
+    group.asyncWait(yield);
+}
+
+void
+LoadBalancer::chooseForwardingSource()
+{
+    LOG(log_.info()) << "Choosing a new source to forward subscriptions";
+    auto hasForwardingSourceLock = hasForwardingSource_.lock();
+    hasForwardingSourceLock.get() = false;
+    for (auto& source : sources_) {
+        if (not hasForwardingSourceLock.get() and source->isConnected()) {
+            source->setForwarding(true);
+            hasForwardingSourceLock.get() = true;
+        } else {
+            source->setForwarding(false);
+        }
+    }
+}
+
+}  // namespace etlng

src/etlng/LoadBalancer.hpp

@@ -0,0 +1,271 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etl/ETLState.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
+#include "etlng/Source.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
+#include "util/Assert.hpp"
+#include "util/Mutex.hpp"
+#include "util/ResponseExpirationCache.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <boost/asio.hpp>
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
+#include <grpcpp/grpcpp.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <chrono>
+#include <cstdint>
+#include <expected>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief A tag class to help identify LoadBalancer in templated code.
+ */
+struct LoadBalancerTag {
+    virtual ~LoadBalancerTag() = default;
+};
+
+template <typename T>
+concept SomeLoadBalancer = std::derived_from<T, LoadBalancerTag>;
+
+/**
+ * @brief This class is used to manage connections to transaction processing processes.
+ *
+ * This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of
+ * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
+ * allows requests for ledger data to be load balanced across all possible ETL sources.
+ */
+class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag {
+public:
+    using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+private:
+    static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16;
+
+    util::Logger log_{"ETL"};
+    // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
+    std::optional<util::ResponseExpirationCache> forwardingCache_;
+    std::optional<std::string> forwardingXUserValue_;
+
+    std::vector<SourcePtr> sources_;
+    std::optional<etl::ETLState> etlState_;
+    std::uint32_t downloadRanges_ =
+        kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
+
+    // Using mutext instead of atomic_bool because choosing a new source to
+    // forward messages should be done with a mutual exclusion otherwise there will be a race condition
+    util::Mutex<bool> hasForwardingSource_{false};
+
+public:
+    /**
+     * @brief Value for the X-User header when forwarding admin requests
+     */
+    static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin";
+
+    /**
+     * @brief Value for the X-User header when forwarding user requests
+     */
+    static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user";
+
+    /**
+     * @brief Create an instance of the load balancer.
+     *
+     * @param config The configuration to use
+     * @param ioc The io_context to run on
+     * @param backend BackendInterface implementation
+     * @param subscriptions Subscription manager
+     * @param validatedLedgers The network validated ledgers datastructure
+     * @param sourceFactory A factory function to create a source
+     */
+    LoadBalancer(
+        util::config::ClioConfigDefinition const& config,
+        boost::asio::io_context& ioc,
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+        std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+        SourceFactory sourceFactory = makeSource
+    );
+
+    /**
+     * @brief A factory function for the load balancer.
+     *
+     * @param config The configuration to use
+     * @param ioc The io_context to run on
+     * @param backend BackendInterface implementation
+     * @param subscriptions Subscription manager
+     * @param validatedLedgers The network validated ledgers datastructure
+     * @param sourceFactory A factory function to create a source
+     * @return A shared pointer to a new instance of LoadBalancer
+     */
+    static std::shared_ptr<LoadBalancerInterface>
+    makeLoadBalancer(
+        util::config::ClioConfigDefinition const& config,
+        boost::asio::io_context& ioc,
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+        std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+        SourceFactory sourceFactory = makeSource
+    );
+
+    ~LoadBalancer() override;
+
+    /**
+     * @brief Load the initial ledger, writing data to the queue.
+     * @note This function will retry indefinitely until the ledger is downloaded.
+     *
+     * @param sequence Sequence of ledger to download
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return A std::vector<std::string> The ledger data
+     */
+    std::vector<std::string>
+    loadInitialLedger(
+        [[maybe_unused]] uint32_t sequence,
+        [[maybe_unused]] std::chrono::steady_clock::duration retryAfter
+    ) override
+    {
+        ASSERT(false, "Not available for new ETL");
+        std::unreachable();
+    };
+
+    /**
+     * @brief Load the initial ledger, writing data to the queue.
+     * @note This function will retry indefinitely until the ledger is downloaded.
+     *
+     * @param sequence Sequence of ledger to download
+     * @param observer The observer to notify of progress
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return A std::vector<std::string> The ledger data
+     */
+    std::vector<std::string>
+    loadInitialLedger(
+        uint32_t sequence,
+        etlng::InitialLoadObserverInterface& observer,
+        std::chrono::steady_clock::duration retryAfter
+    ) override;
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param ledgerSequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one
+     * @param getObjectNeighbors Whether to request object neighbors
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return The extracted data, if extraction was successful. If the ledger was found
+     * in the database or the server is shutting down, the optional will be empty
+     */
+    OptionalGetLedgerResponseType
+    fetchLedger(
+        uint32_t ledgerSequence,
+        bool getObjects,
+        bool getObjectNeighbors,
+        std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
+    ) override;
+
+    /**
+     * @brief Represent the state of this load balancer as a JSON object
+     *
+     * @return JSON representation of the state of this load balancer.
+     */
+    boost::json::value
+    toJson() const override;
+
+    /**
+     * @brief Forward a JSON RPC request to a randomly selected rippled node.
+     *
+     * @param request JSON-RPC request to forward
+     * @param clientIp The IP address of the peer, if known
+     * @param isAdmin Whether the request is from an admin
+     * @param yield The coroutine context
+     * @return Response received from rippled node as JSON object on success or error on failure
+     */
+    std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& clientIp,
+        bool isAdmin,
+        boost::asio::yield_context yield
+    ) override;
+
+    /**
+     * @brief Return state of ETL nodes.
+     * @return ETL state, nullopt if etl nodes not available
+     */
+    std::optional<etl::ETLState>
+    getETLState() noexcept override;
+
+    /**
+     * @brief Stop the load balancer. This will stop all subscription sources.
+     * @note This function will asynchronously wait for all sources to stop.
+     *
+     * @param yield The coroutine context
+     */
+    void
+    stop(boost::asio::yield_context yield) override;
+
+private:
+    /**
+     * @brief Execute a function on a randomly selected source.
+     *
+     * @note f is a function that takes an Source as an argument and returns a bool.
+     * Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another
+     * randomly chosen Source is used. The process repeats until f returns true.
+     *
+     * @param f Function to execute. This function takes the ETL source as an argument, and returns a bool
+     * @param ledgerSequence f is executed for each Source that has this ledger
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * server is shutting down
+     */
+    template <typename Func>
+    void
+    execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2});
+
+    /**
+     * @brief Choose a new source to forward requests
+     */
+    void
+    chooseForwardingSource();
+};
+
+}  // namespace etlng

src/etlng/LoadBalancerInterface.hpp

@@ -129,6 +129,15 @@ public:
      */
     virtual std::optional<etl::ETLState>
     getETLState() noexcept = 0;
+
+    /**
+     * @brief Stop the load balancer. This will stop all subscription sources.
+     * @note This function will asynchronously wait for all sources to stop.
+     *
+     * @param yield The coroutine context
+     */
+    virtual void
+    stop(boost::asio::yield_context yield) = 0;
 };
 
 }  // namespace etlng

src/etlng/LoaderInterface.hpp

@@ -45,7 +45,7 @@ struct LoaderInterface {
      * @param data The data to load
      * @return Optional ledger header
      */
-    virtual std::optional<ripple::LedgerHeader>
+    [[nodiscard]] virtual std::optional<ripple::LedgerHeader>
     loadInitialLedger(model::LedgerData const& data) = 0;
 };
 

src/etlng/Source.cpp

@@ -0,0 +1,73 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/Source.hpp"
+
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etl/impl/ForwardingSource.hpp"
+#include "etl/impl/SubscriptionSource.hpp"
+#include "etlng/impl/GrpcSource.hpp"
+#include "etlng/impl/SourceImpl.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/io_context.hpp>
+
+#include <chrono>
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace etlng {
+
+SourcePtr
+makeSource(
+    util::config::ObjectView const& config,
+    boost::asio::io_context& ioc,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
+    SourceBase::OnConnectHook onConnect,
+    SourceBase::OnDisconnectHook onDisconnect,
+    SourceBase::OnLedgerClosedHook onLedgerClosed
+)
+{
+    auto const ip = config.get<std::string>("ip");
+    auto const wsPort = config.get<std::string>("ws_port");
+    auto const grpcPort = config.get<std::string>("grpc_port");
+
+    etl::impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout};
+    impl::GrpcSource grpcSource{ip, grpcPort};
+    auto subscriptionSource = std::make_unique<etl::impl::SubscriptionSource>(
+        ioc,
+        ip,
+        wsPort,
+        std::move(validatedLedgers),
+        std::move(subscriptions),
+        std::move(onConnect),
+        std::move(onDisconnect),
+        std::move(onLedgerClosed)
+    );
+
+    return std::make_unique<impl::SourceImpl<>>(
+        ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource)
+    );
+}
+
+}  // namespace etlng

src/etlng/Source.hpp

@@ -0,0 +1,194 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+
+#include <chrono>
+#include <cstdint>
+#include <expected>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief Provides an implementation of a ETL source
+ */
+class SourceBase {
+public:
+    using OnConnectHook = std::function<void()>;
+    using OnDisconnectHook = std::function<void(bool)>;
+    using OnLedgerClosedHook = std::function<void()>;
+
+    virtual ~SourceBase() = default;
+
+    /**
+     * @brief Run subscriptions loop of the source
+     */
+    virtual void
+    run() = 0;
+
+    /**
+     * @brief Stop Source.
+     * @note This method will asynchronously wait for source to be stopped.
+     *
+     * @param yield The coroutine context.
+     */
+    virtual void
+    stop(boost::asio::yield_context yield) = 0;
+
+    /**
+     * @brief Check if source is connected
+     *
+     * @return true if source is connected; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    isConnected() const = 0;
+
+    /**
+     * @brief Set the forwarding state of the source.
+     *
+     * @param isForwarding Whether to forward or not
+     */
+    virtual void
+    setForwarding(bool isForwarding) = 0;
+
+    /**
+     * @brief Represent the source as a JSON object
+     *
+     * @return JSON representation of the source
+     */
+    [[nodiscard]] virtual boost::json::object
+    toJson() const = 0;
+
+    /** @return String representation of the source (for debug) */
+    [[nodiscard]] virtual std::string
+    toString() const = 0;
+
+    /**
+     * @brief Check if ledger is known by this source.
+     *
+     * @param sequence The ledger sequence to check
+     * @return true if ledger is in the range of this source; false otherwise
+     */
+    [[nodiscard]] virtual bool
+    hasLedger(uint32_t sequence) const = 0;
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param sequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
+     * @param getObjectNeighbors Whether to request object neighbors; defaults to false
+     * @return A std::pair of the response status and the response itself
+     */
+    [[nodiscard]] virtual std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+    fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0;
+
+    /**
+     * @brief Download a ledger in full.
+     *
+     * @param sequence Sequence of the ledger to download
+     * @param numMarkers Number of markers to generate for async calls
+     * @param loader InitialLoadObserverInterface implementation
+     * @return A std::pair of the data and a bool indicating whether the download was successful
+     */
+    virtual std::pair<std::vector<std::string>, bool>
+    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) = 0;
+
+    /**
+     * @brief Forward a request to rippled.
+     *
+     * @param request The request to forward
+     * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Value of the X-User header
+     * @param yield The coroutine context
+     * @return Response on success or error on failure
+     */
+    [[nodiscard]] virtual std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
+        boost::asio::yield_context yield
+    ) const = 0;
+};
+
+using SourcePtr = std::unique_ptr<SourceBase>;
+
+using SourceFactory = std::function<SourcePtr(
+    util::config::ObjectView const& config,
+    boost::asio::io_context& ioc,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
+    SourceBase::OnConnectHook onConnect,
+    SourceBase::OnDisconnectHook onDisconnect,
+    SourceBase::OnLedgerClosedHook onLedgerClosed
+)>;
+
+/**
+ * @brief Create a source
+ *
+ * @param config The configuration to use
+ * @param ioc The io_context to run on
+ * @param subscriptions Subscription manager
+ * @param validatedLedgers The network validated ledgers data structure
+ * @param forwardingTimeout The timeout for forwarding to rippled
+ * @param onConnect The hook to call on connect
+ * @param onDisconnect The hook to call on disconnect
+ * @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set
+ * as forwarding.
+ * @return The created source
+ */
+[[nodiscard]] SourcePtr
+makeSource(
+    util::config::ObjectView const& config,
+    boost::asio::io_context& ioc,
+    std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
+    std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
+    std::chrono::steady_clock::duration forwardingTimeout,
+    SourceBase::OnConnectHook onConnect,
+    SourceBase::OnDisconnectHook onDisconnect,
+    SourceBase::OnLedgerClosedHook onLedgerClosed
+);
+
+}  // namespace etlng

src/etlng/impl/ForwardingSource.cpp

@@ -0,0 +1,116 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/ForwardingSource.hpp"
+
+#include "rpc/Errors.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/beast/http/field.hpp>
+#include <boost/beast/version.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/serialize.hpp>
+#include <fmt/core.h>
+
+#include <chrono>
+#include <exception>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <utility>
+
+namespace etlng::impl {
+
+ForwardingSource::ForwardingSource(
+    std::string ip,
+    std::string wsPort,
+    std::chrono::steady_clock::duration forwardingTimeout,
+    std::chrono::steady_clock::duration connTimeout
+)
+    : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort))
+    , connectionBuilder_(std::move(ip), std::move(wsPort))
+    , forwardingTimeout_{forwardingTimeout}
+{
+    connectionBuilder_.setConnectionTimeout(connTimeout)
+        .addHeader(
+            {boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)}
+        );
+}
+
+std::expected<boost::json::object, rpc::ClioError>
+ForwardingSource::forwardToRippled(
+    boost::json::object const& request,
+    std::optional<std::string> const& forwardToRippledClientIp,
+    std::string_view xUserValue,
+    boost::asio::yield_context yield
+) const
+{
+    auto connectionBuilder = connectionBuilder_;
+    if (forwardToRippledClientIp) {
+        connectionBuilder.addHeader(
+            {boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)}
+        );
+    }
+
+    connectionBuilder.addHeader({"X-User", std::string{xUserValue}});
+
+    auto expectedConnection = connectionBuilder.connect(yield);
+    if (not expectedConnection) {
+        LOG(log_.debug()) << "Couldn't connect to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::EtlConnectionError};
+    }
+    auto& connection = expectedConnection.value();
+
+    auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_);
+    if (writeError) {
+        LOG(log_.debug()) << "Error sending request to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::EtlRequestError};
+    }
+
+    auto response = connection->read(yield, forwardingTimeout_);
+    if (not response) {
+        if (auto errorCode = response.error().errorCode();
+            errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) {
+            LOG(log_.debug()) << "Request to rippled timed out";
+            return std::unexpected{rpc::ClioError::EtlRequestTimeout};
+        }
+        LOG(log_.debug()) << "Error sending request to rippled to forward request.";
+        return std::unexpected{rpc::ClioError::EtlRequestError};
+    }
+
+    boost::json::value parsedResponse;
+    try {
+        parsedResponse = boost::json::parse(*response);
+        if (not parsedResponse.is_object())
+            throw std::runtime_error("response is not an object");
+    } catch (std::exception const& e) {
+        LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
+        return std::unexpected{rpc::ClioError::EtlInvalidResponse};
+    }
+
+    auto responseObject = parsedResponse.as_object();
+    responseObject["forwarded"] = true;
+
+    return responseObject;
+}
+
+}  // namespace etlng::impl

src/etlng/impl/ForwardingSource.hpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "rpc/Errors.hpp"
+#include "util/log/Logger.hpp"
+#include "util/requests/WsConnection.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+
+#include <chrono>
+#include <expected>
+#include <optional>
+#include <string>
+#include <string_view>
+
+namespace etlng::impl {
+
+class ForwardingSource {
+    util::Logger log_;
+    util::requests::WsConnectionBuilder connectionBuilder_;
+    std::chrono::steady_clock::duration forwardingTimeout_;
+
+    static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3};
+
+public:
+    ForwardingSource(
+        std::string ip,
+        std::string wsPort,
+        std::chrono::steady_clock::duration forwardingTimeout,
+        std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT
+    );
+
+    /**
+     * @brief Forward a request to rippled.
+     *
+     * @param request The request to forward
+     * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Optional value for X-User header
+     * @param yield The coroutine context
+     * @return Response on success or error on failure
+     */
+    std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
+        boost::asio::yield_context yield
+    ) const;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Loading.hpp

@@ -39,7 +39,6 @@
 #include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/TxMeta.h>
 
-#include <cstddef>
 #include <cstdint>
 #include <memory>
 #include <optional>

src/etlng/impl/Registry.hpp

@@ -24,7 +24,6 @@
 
 #include <xrpl/protocol/TxFormats.h>
 
-#include <concepts>
 #include <cstdint>
 #include <string>
 #include <tuple>
@@ -81,7 +80,7 @@ concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or
 
 template <typename T>
 concept NoTwoOfKind = not(HasLedgerDataHook<T> and HasTransactionHook<T>) and
-    not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and not(HasInitialDataHook<T> and HasObjectHook<T>) and
+    not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and
     not(HasInitialObjectsHook<T> and HasInitialObjectHook<T>);
 
 template <typename T>
@@ -216,4 +215,10 @@ public:
     }
 };
 
+static auto
+makeRegistry(auto&&... exts)
+{
+    return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(std::forward<decltype(exts)>(exts)...);
+}
+
 }  // namespace etlng::impl

src/etlng/impl/SourceImpl.hpp

@@ -0,0 +1,232 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/impl/ForwardingSource.hpp"
+#include "etl/impl/SubscriptionSource.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/Source.hpp"
+#include "etlng/impl/GrpcSource.hpp"
+#include "rpc/Errors.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+
+#include <chrono>
+#include <cstdint>
+#include <expected>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+/**
+ * @brief Provides an implementation of a ETL source
+ *
+ * @tparam GrpcSourceType The type of the gRPC source
+ * @tparam SubscriptionSourceTypePtr The type of the subscription source
+ * @tparam ForwardingSourceType The type of the forwarding source
+ */
+template <
+    typename GrpcSourceType = GrpcSource,
+    typename SubscriptionSourceTypePtr = std::unique_ptr<etl::impl::SubscriptionSource>,
+    typename ForwardingSourceType = etl::impl::ForwardingSource>
+class SourceImpl : public SourceBase {
+    std::string ip_;
+    std::string wsPort_;
+    std::string grpcPort_;
+
+    GrpcSourceType grpcSource_;
+    SubscriptionSourceTypePtr subscriptionSource_;
+    ForwardingSourceType forwardingSource_;
+
+public:
+    /**
+     * @brief Construct a new SourceImpl object
+     *
+     * @param ip The IP of the source
+     * @param wsPort The web socket port of the source
+     * @param grpcPort The gRPC port of the source
+     * @param grpcSource The gRPC source
+     * @param subscriptionSource The subscription source
+     * @param forwardingSource The forwarding source
+     */
+    template <typename SomeGrpcSourceType, typename SomeForwardingSourceType>
+        requires std::is_same_v<GrpcSourceType, SomeGrpcSourceType> and
+                     std::is_same_v<ForwardingSourceType, SomeForwardingSourceType>
+    SourceImpl(
+        std::string ip,
+        std::string wsPort,
+        std::string grpcPort,
+        SomeGrpcSourceType&& grpcSource,
+        SubscriptionSourceTypePtr subscriptionSource,
+        SomeForwardingSourceType&& forwardingSource
+    )
+        : ip_(std::move(ip))
+        , wsPort_(std::move(wsPort))
+        , grpcPort_(std::move(grpcPort))
+        , grpcSource_(std::forward<SomeGrpcSourceType>(grpcSource))
+        , subscriptionSource_(std::move(subscriptionSource))
+        , forwardingSource_(std::forward<SomeForwardingSourceType>(forwardingSource))
+    {
+    }
+
+    /**
+     * @brief Run subscriptions loop of the source
+     */
+    void
+    run() final
+    {
+        subscriptionSource_->run();
+    }
+
+    void
+    stop(boost::asio::yield_context yield) final
+    {
+        subscriptionSource_->stop(yield);
+    }
+
+    /**
+     * @brief Check if source is connected
+     *
+     * @return true if source is connected; false otherwise
+     */
+    bool
+    isConnected() const final
+    {
+        return subscriptionSource_->isConnected();
+    }
+
+    /**
+     * @brief Set the forwarding state of the source.
+     *
+     * @param isForwarding Whether to forward or not
+     */
+    void
+    setForwarding(bool isForwarding) final
+    {
+        subscriptionSource_->setForwarding(isForwarding);
+    }
+
+    /**
+     * @brief Represent the source as a JSON object
+     *
+     * @return JSON representation of the source
+     */
+    boost::json::object
+    toJson() const final
+    {
+        boost::json::object res;
+
+        res["validated_range"] = subscriptionSource_->validatedRange();
+        res["is_connected"] = std::to_string(static_cast<int>(subscriptionSource_->isConnected()));
+        res["ip"] = ip_;
+        res["ws_port"] = wsPort_;
+        res["grpc_port"] = grpcPort_;
+
+        auto last = subscriptionSource_->lastMessageTime();
+        if (last.time_since_epoch().count() != 0) {
+            res["last_msg_age_seconds"] = std::to_string(
+                std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - last).count()
+            );
+        }
+
+        return res;
+    }
+
+    /** @return String representation of the source (for debug) */
+    std::string
+    toString() const final
+    {
+        return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ +
+            ", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}";
+    }
+
+    /**
+     * @brief Check if ledger is known by this source.
+     *
+     * @param sequence The ledger sequence to check
+     * @return true if ledger is in the range of this source; false otherwise
+     */
+    bool
+    hasLedger(uint32_t sequence) const final
+    {
+        return subscriptionSource_->hasLedger(sequence);
+    }
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param sequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
+     * @param getObjectNeighbors Whether to request object neighbors; defaults to false
+     * @return A std::pair of the response status and the response itself
+     */
+    std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+    fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final
+    {
+        return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors);
+    }
+
+    /**
+     * @brief Download a ledger in full.
+     *
+     * @param sequence Sequence of the ledger to download
+     * @param numMarkers Number of markers to generate for async calls
+     * @param loader InitialLoadObserverInterface implementation
+     * @return A std::pair of the data and a bool indicating whether the download was successful
+     */
+    std::pair<std::vector<std::string>, bool>
+    loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) final
+    {
+        return grpcSource_.loadInitialLedger(sequence, numMarkers, loader);
+    }
+
+    /**
+     * @brief Forward a request to rippled.
+     *
+     * @param request The request to forward
+     * @param forwardToRippledClientIp IP of the client forwarding this request if known
+     * @param xUserValue Optional value of the X-User header
+     * @param yield The coroutine context
+     * @return Response or ClioError
+     */
+    std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
+        boost::asio::yield_context yield
+    ) const final
+    {
+        return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield);
+    }
+};
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Cache.cpp

@@ -0,0 +1,59 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/ext/Cache.hpp"
+
+#include "data/LedgerCacheInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache)
+{
+}
+
+void
+CacheExt::onLedgerData(model::LedgerData const& data) const
+{
+    cache_.get().update(data.objects, data.seq);
+    LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size();
+}
+
+void
+CacheExt::onInitialData(model::LedgerData const& data) const
+{
+    LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size();
+    cache_.get().update(data.objects, data.seq);
+    cache_.get().setFull();
+}
+
+void
+CacheExt::onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey)
+    const
+{
+    LOG(log_.trace()) << "got initial objects cnt = " << objs.size();
+    cache_.get().update(objs, seq);
+}
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Cache.hpp

@@ -0,0 +1,51 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/LedgerCacheInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <functional>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+class CacheExt {
+    std::reference_wrapper<data::LedgerCacheInterface> cache_;
+
+    util::Logger log_{"ETL"};
+
+public:
+    CacheExt(data::LedgerCacheInterface& cache);
+
+    void
+    onLedgerData(model::LedgerData const& data) const;
+
+    void
+    onInitialData(model::LedgerData const& data) const;
+
+    void
+    onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey) const;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Core.cpp

@@ -0,0 +1,83 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/ext/Core.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <utility>
+
+namespace etlng::impl {
+
+CoreExt::CoreExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
+{
+}
+
+void
+CoreExt::onLedgerData(model::LedgerData const& data) const
+{
+    LOG(log_.debug()) << "Loading ledger data for " << data.seq;
+    backend_->writeLedger(data.header, auto{data.rawHeader});
+    insertTransactions(data);
+}
+
+void
+CoreExt::onInitialData(model::LedgerData const& data) const
+{
+    LOG(log_.info()) << "Loading initial ledger data for " << data.seq;
+    backend_->writeLedger(data.header, auto{data.rawHeader});
+    insertTransactions(data);
+}
+
+void
+CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const
+{
+    LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq;
+    backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
+}
+
+void
+CoreExt::onObject(uint32_t seq, model::Object const& obj) const
+{
+    LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq;
+    backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
+}
+
+void
+CoreExt::insertTransactions(model::LedgerData const& data) const
+{
+    for (auto const& txn : data.transactions) {
+        LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID();
+
+        backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()});
+        backend_->writeTransaction(
+            auto{txn.key},
+            data.seq,
+            data.header.closeTime.time_since_epoch().count(),  // This is why we can't use 'onTransaction'
+            auto{txn.raw},
+            auto{txn.metaRaw}
+        );
+    }
+}
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Core.hpp

@@ -0,0 +1,58 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/basics/base_uint.h>
+
+#include <cstdint>
+#include <memory>
+
+namespace etlng::impl {
+
+class CoreExt {
+    std::shared_ptr<BackendInterface> backend_;
+
+    util::Logger log_{"ETL"};
+
+public:
+    CoreExt(std::shared_ptr<BackendInterface> backend);
+
+    void
+    onLedgerData(model::LedgerData const& data) const;
+
+    void
+    onInitialData(model::LedgerData const& data) const;
+
+    void
+    onInitialObject(uint32_t seq, model::Object const& obj) const;
+
+    void
+    onObject(uint32_t seq, model::Object const& obj) const;
+
+private:
+    void
+    insertTransactions(model::LedgerData const& data) const;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/ext/NFT.cpp

@@ -0,0 +1,77 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/ext/NFT.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "etl/NFTHelpers.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <memory>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+NFTExt::NFTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
+{
+}
+
+void
+NFTExt::onLedgerData(model::LedgerData const& data) const
+{
+    writeNFTs(data);
+}
+
+void
+NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const
+{
+    LOG(log_.trace()) << "got initial object with key = " << obj.key;
+    backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw));
+}
+
+void
+NFTExt::onInitialData(model::LedgerData const& data) const
+{
+    LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
+    writeNFTs(data);
+}
+
+void
+NFTExt::writeNFTs(model::LedgerData const& data) const
+{
+    std::vector<NFTsData> nfts;
+    std::vector<NFTTransactionsData> nftTxs;
+
+    for (auto const& tx : data.transactions) {
+        auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx);
+        nftTxs.insert(nftTxs.end(), txs.begin(), txs.end());
+        if (maybeNFT)
+            nfts.push_back(*maybeNFT);
+    }
+
+    // This is uniqued so that we only write latest modification (as in previous implementation)
+    backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts));
+    backend_->writeNFTTransactions(nftTxs);
+}
+
+}  // namespace etlng::impl

src/etlng/impl/ext/NFT.hpp

@@ -0,0 +1,52 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <cstdint>
+#include <memory>
+
+namespace etlng::impl {
+
+class NFTExt {
+    std::shared_ptr<BackendInterface> backend_;
+    util::Logger log_{"ETL"};
+
+public:
+    NFTExt(std::shared_ptr<BackendInterface> backend);
+
+    void
+    onLedgerData(model::LedgerData const& data) const;
+
+    void
+    onInitialObject(uint32_t seq, model::Object const& obj) const;
+
+    void
+    onInitialData(model::LedgerData const& data) const;
+
+private:
+    void
+    writeNFTs(model::LedgerData const& data) const;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Successor.cpp

@@ -0,0 +1,222 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/ext/Successor.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/LedgerCacheInterface.hpp"
+#include "data/Types.hpp"
+#include "etlng/Models.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+SuccessorExt::SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache)
+    : backend_(std::move(backend)), cache_(cache)
+{
+}
+
+void
+SuccessorExt::onInitialData(model::LedgerData const& data) const
+{
+    ASSERT(cache_.get().isFull(), "Cache must be full at this point");
+    ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load");
+    ASSERT(data.objects.empty(), "Should not have objects from initial data");
+    writeSuccessors(data.seq);
+    writeEdgeKeys(data.seq, data.edgeKeys.value());
+}
+
+void
+SuccessorExt::onInitialObjects(
+    uint32_t seq,
+    [[maybe_unused]] std::vector<model::Object> const& objs,
+    std::string lastKey
+) const
+{
+    for (auto const& obj : objs) {
+        if (!lastKey.empty())
+            backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw});
+        lastKey = obj.keyRaw;
+    }
+}
+
+void
+SuccessorExt::onLedgerData(model::LedgerData const& data) const
+{
+    namespace vs = std::views;
+
+    LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size()
+                     << "; got successors = " << data.successors.has_value() << "; cache is "
+                     << (cache_.get().isFull() ? "FULL" : "Not full");
+
+    auto filteredObjects = data.objects  //
+        | vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; });
+
+    if (data.successors.has_value()) {
+        for (auto const& successor : data.successors.value())
+            writeIncludedSuccessor(data.seq, successor);
+
+        for (auto const& obj : filteredObjects)
+            writeIncludedSuccessor(data.seq, obj);
+    } else {
+        if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq)
+            throw std::logic_error("Cache is not full, but object neighbors were not included");
+
+        for (auto const& obj : filteredObjects)
+            updateSuccessorFromCache(data.seq, obj);
+    }
+}
+
+void
+SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const
+{
+    auto firstBook = succ.firstBook;
+    if (firstBook.empty())
+        firstBook = uint256ToString(data::kLAST_KEY);
+
+    backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook));
+}
+
+void
+SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const
+{
+    ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object");
+
+    // TODO: perhaps make these optionals inside of obj and move value_or here
+    auto pred = obj.predecessor;
+    auto succ = obj.successor;
+
+    if (obj.type == model::Object::ModType::Deleted) {
+        backend_->writeSuccessor(std::move(pred), seq, std::move(succ));
+    } else if (obj.type == model::Object::ModType::Created) {
+        backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw});
+        backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ));
+    }
+}
+
+void
+SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const
+{
+    auto const lb =
+        cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}});
+    auto const ub =
+        cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}});
+
+    auto checkBookBase = false;
+    auto const isDeleted = obj.data.empty();
+
+    if (isDeleted) {
+        backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key));
+    } else {
+        backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key));
+        backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key));
+    }
+
+    if (isDeleted) {
+        auto const old = cache_.get().getDeleted(obj.key, seq - 1);
+        ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key));
+
+        checkBookBase = isBookDir(obj.key, *old);
+    } else {
+        checkBookBase = isBookDir(obj.key, obj.data);
+    }
+
+    if (checkBookBase) {
+        auto const current = cache_.get().get(obj.key, seq);
+        auto const bookBase = getBookBase(obj.key);
+
+        if (isDeleted and not current.has_value()) {
+            updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase);
+        } else if (current.has_value()) {
+            auto const successor = cache_.get().getSuccessor(bookBase, seq);
+            ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq);
+
+            if (successor->key == obj.key) {
+                updateBookSuccessor(successor, seq, bookBase);
+            }
+        }
+    }
+}
+
+void
+SuccessorExt::updateBookSuccessor(
+    std::optional<data::LedgerObject> const& maybeSuccessor,
+    auto seq,
+    ripple::uint256 const& bookBase
+) const
+{
+    if (maybeSuccessor.has_value()) {
+        backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key));
+    } else {
+        backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY));
+    }
+}
+
+void
+SuccessorExt::writeSuccessors(uint32_t seq) const
+{
+    ripple::uint256 prev = data::kFIRST_KEY;
+    while (auto cur = cache_.get().getSuccessor(prev, seq)) {
+        if (prev == data::kFIRST_KEY)
+            backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key));
+
+        if (isBookDir(cur->key, cur->blob)) {
+            auto base = getBookBase(cur->key);
+
+            // make sure the base is not an actual object
+            if (not cache_.get().get(base, seq)) {
+                auto succ = cache_.get().getSuccessor(base, seq);
+                ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
+
+                if (succ->key == cur->key)
+                    backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key));
+            }
+        }
+
+        prev = cur->key;
+    }
+
+    backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY));
+}
+
+void
+SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const
+{
+    for (auto const& key : edgeKeys) {
+        auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq);
+        if (succ)
+            backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key));
+    }
+}
+
+}  // namespace etlng::impl

src/etlng/impl/ext/Successor.hpp

@@ -0,0 +1,82 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "data/LedgerCacheInterface.hpp"
+#include "data/Types.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+class SuccessorExt {
+    std::shared_ptr<BackendInterface> backend_;
+    std::reference_wrapper<data::LedgerCacheInterface> cache_;
+
+    util::Logger log_{"ETL"};
+
+public:
+    SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache);
+
+    void
+    onInitialData(model::LedgerData const& data) const;
+
+    void
+    onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector<model::Object> const& objs, std::string lastKey) const;
+
+    void
+    onLedgerData(model::LedgerData const& data) const;
+
+private:
+    void
+    writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const;
+
+    void
+    writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const;
+
+    void
+    updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const;
+
+    void
+    updateBookSuccessor(
+        std::optional<data::LedgerObject> const& maybeSuccessor,
+        auto seq,
+        ripple::uint256 const& bookBase
+    ) const;
+
+    void
+    writeSuccessors(uint32_t seq) const;
+
+    void
+    writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const;
+};
+
+}  // namespace etlng::impl

src/feed/SubscriptionManager.cpp

@@ -191,7 +191,7 @@ SubscriptionManager::unsubBook(ripple::Book const& book, SubscriberSharedPtr con
 void
 SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo)
 {
-    transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_);
+    transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_, networkID_);
 }
 
 boost::json::object
@@ -210,4 +210,16 @@ SubscriptionManager::report() const
     };
 }
 
+void
+SubscriptionManager::setNetworkID(uint32_t const networkID)
+{
+    networkID_ = networkID;
+}
+
+uint32_t
+SubscriptionManager::getNetworkID() const
+{
+    return networkID_;
+}
+
 }  // namespace feed

src/feed/SubscriptionManager.hpp

@@ -69,6 +69,7 @@ class SubscriptionManager : public SubscriptionManagerInterface {
     impl::BookChangesFeed bookChangesFeed_;
     impl::TransactionFeed transactionFeed_;
     impl::ProposedTransactionFeed proposedTransactionFeed_;
+    uint32_t networkID_{0};
 
 public:
     /**
@@ -332,6 +333,21 @@ public:
      */
     boost::json::object
     report() const final;
+
+    /**
+     * @brief Set the networkID.
+     * @param networkID The network id to set.
+     */
+    void
+    setNetworkID(uint32_t networkID) final;
+
+    /**
+     * @brief Get the networkID.
+     *
+     * @return The network id.
+     */
+    uint32_t
+    getNetworkID() const final;
 };
 
 }  // namespace feed

src/feed/SubscriptionManagerInterface.hpp

@@ -244,10 +244,25 @@ public:
     /**
      * @brief Get the number of subscribers.
      *
-     * @return The report of the number of subscribers
+     * @return The report of the number of subscribers.
      */
     virtual boost::json::object
     report() const = 0;
+
+    /**
+     * @brief Set the networkID.
+     * @param networkID The network id to set.
+     */
+    virtual void
+    setNetworkID(uint32_t networkID) = 0;
+
+    /**
+     * @brief Get the networkID.
+     *
+     * @return The network id.
+     */
+    virtual uint32_t
+    getNetworkID() const = 0;
 };
 
 }  // namespace feed

src/feed/impl/TransactionFeed.cpp

@@ -25,6 +25,7 @@
 #include "feed/Types.hpp"
 #include "rpc/JS.hpp"
 #include "rpc/RPCHelpers.hpp"
+#include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
 
 #include <boost/asio/spawn.hpp>
@@ -176,7 +177,8 @@ TransactionFeed::pub(
     data::TransactionAndMetadata const& txMeta,
     ripple::LedgerHeader const& lgrInfo,
     std::shared_ptr<data::BackendInterface const> const& backend,
-    std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter
+    std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
+    uint32_t const networkID
 )
 {
     auto [tx, meta] = rpc::deserializeTxPlusMeta(txMeta, lgrInfo.seq);
@@ -205,6 +207,15 @@ TransactionFeed::pub(
         rpc::insertDeliverMaxAlias(pubObj[txKey].as_object(), version);
         rpc::insertMPTIssuanceID(pubObj[JS(meta)].as_object(), tx, meta);
 
+        auto const& metaObj = pubObj[JS(meta)];
+        ASSERT(metaObj.is_object(), "meta must be an obj in rippled and clio");
+        if (metaObj.as_object().contains("TransactionIndex") && metaObj.as_object().at("TransactionIndex").is_int64()) {
+            if (auto const& ctid =
+                    rpc::encodeCTID(lgrInfo.seq, metaObj.as_object().at("TransactionIndex").as_int64(), networkID);
+                ctid)
+                pubObj[JS(ctid)] = ctid.value();
+        }
+
         pubObj[JS(type)] = "transaction";
         pubObj[JS(validated)] = true;
         pubObj[JS(status)] = "closed";

src/feed/impl/TransactionFeed.hpp

@@ -182,12 +182,14 @@ public:
      * @param txMeta The transaction and metadata.
      * @param lgrInfo The ledger header.
      * @param backend The backend.
+     * @param networkID The network ID.
      */
     void
     pub(data::TransactionAndMetadata const& txMeta,
         ripple::LedgerHeader const& lgrInfo,
         std::shared_ptr<data::BackendInterface const> const& backend,
-        std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter);
+        std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
+        uint32_t networkID);
 
     /**
      * @brief Get the number of subscribers of the transaction feed.

src/rpc/CMakeLists.txt

@@ -1,3 +1,8 @@
+# Have to use RPCCenter as a separate library since it is used in util
+add_library(clio_rpc_center)
+target_sources(clio_rpc_center PRIVATE RPCCenter.cpp)
+target_include_directories(clio_rpc_center PUBLIC "${CMAKE_SOURCE_DIR}/src")
+
 add_library(clio_rpc)
 
 target_sources(

src/rpc/RPCCenter.cpp

@@ -0,0 +1,112 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "rpc/RPCCenter.hpp"
+
+#include <string_view>
+#include <unordered_set>
+
+namespace rpc {
+
+namespace {
+
+std::unordered_set<std::string_view> const&
+handledRpcs()
+{
+    static std::unordered_set<std::string_view> kHANDLED_RPCS = {
+        "account_channels",
+        "account_currencies",
+        "account_info",
+        "account_lines",
+        "account_nfts",
+        "account_objects",
+        "account_offers",
+        "account_tx",
+        "amm_info",
+        "book_changes",
+        "book_offers",
+        "deposit_authorized",
+        "feature",
+        "gateway_balances",
+        "get_aggregate_price",
+        "ledger",
+        "ledger_data",
+        "ledger_entry",
+        "ledger_index",
+        "ledger_range",
+        "mpt_holders",
+        "nfts_by_issuer",
+        "nft_history",
+        "nft_buy_offers",
+        "nft_info",
+        "nft_sell_offers",
+        "noripple_check",
+        "ping",
+        "random",
+        "server_info",
+        "transaction_entry",
+        "tx",
+        "subscribe",
+        "unsubscribe",
+        "version",
+    };
+    return kHANDLED_RPCS;
+}
+
+std::unordered_set<std::string_view> const&
+forwardedRpcs()
+{
+    static std::unordered_set<std::string_view> const kFORWARDED_RPCS = {
+        "server_definitions",
+        "server_state",
+        "submit",
+        "submit_multisigned",
+        "fee",
+        "ledger_closed",
+        "ledger_current",
+        "ripple_path_find",
+        "manifest",
+        "channel_authorize",
+        "channel_verify",
+        "simulate",
+    };
+    return kFORWARDED_RPCS;
+}
+
+}  // namespace
+
+bool
+RPCCenter::isRpcName(std::string_view s)
+{
+    return isHandled(s) || isForwarded(s);
+}
+
+bool
+RPCCenter::isHandled(std::string_view s)
+{
+    return handledRpcs().contains(s);
+}
+
+bool
+RPCCenter::isForwarded(std::string_view s)
+{
+    return forwardedRpcs().contains(s);
+}
+
+}  // namespace rpc

src/rpc/RPCCenter.hpp

@@ -0,0 +1,61 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <string_view>
+
+namespace rpc {
+
+/**
+ * @brief Registry of RPC commands supported by Clio
+ *
+ * The RPCCenter maintains lists of RPC commands that can be handled locally
+ * and those that need to be forwarded to rippled.
+ */
+struct RPCCenter {
+    /**
+     * @brief Checks if a string is a valid RPC command name
+     *
+     * @param s The string to check
+     * @return true if the string is a recognized RPC name, false otherwise
+     */
+    static bool
+    isRpcName(std::string_view s);
+
+    /**
+     * @brief Checks if a string is a RPC command handled by Clio without forwarding to rippled
+     *
+     * @param s The string to check
+     * @return true if the string is a handled RPC command, false otherwise
+     */
+    static bool
+    isHandled(std::string_view s);
+
+    /**
+     * @brief Checks if a string is a RPC command that will be forwarded to rippled
+     *
+     * @param s The string to check
+     * @return true if the string is a forwarded RPC command, false otherwise
+     */
+    static bool
+    isForwarded(std::string_view s);
+};
+
+}  // namespace rpc

src/rpc/RPCEngine.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/RPCHelpers.hpp"
 #include "rpc/WorkQueue.hpp"
@@ -55,7 +56,7 @@ namespace rpc {
 /**
  * @brief The RPC engine that ties all RPC-related functionality together.
  */
-template <typename LoadBalancerType, typename CountersType>
+template <typename CountersType>
 class RPCEngine {
     util::Logger perfLog_{"Performance"};
     util::Logger log_{"RPC"};
@@ -67,7 +68,7 @@ class RPCEngine {
 
     std::shared_ptr<HandlerProvider const> handlerProvider_;
 
-    impl::ForwardingProxy<LoadBalancerType, CountersType, HandlerProvider> forwardingProxy_;
+    impl::ForwardingProxy<CountersType, HandlerProvider> forwardingProxy_;
 
     std::optional<util::ResponseExpirationCache> responseCache_;
 
@@ -86,7 +87,7 @@ public:
     RPCEngine(
         util::config::ClioConfigDefinition const& config,
         std::shared_ptr<BackendInterface> const& backend,
-        std::shared_ptr<LoadBalancerType> const& balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
         web::dosguard::DOSGuardInterface const& dosGuard,
         WorkQueue& workQueue,
         CountersType& counters,
@@ -128,7 +129,7 @@ public:
     makeRPCEngine(
         util::config::ClioConfigDefinition const& config,
         std::shared_ptr<BackendInterface> const& backend,
-        std::shared_ptr<LoadBalancerType> const& balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
         web::dosguard::DOSGuardInterface const& dosGuard,
         WorkQueue& workQueue,
         CountersType& counters,

src/rpc/RPCHelpers.cpp

@@ -290,7 +290,10 @@ std::optional<std::string>
 encodeCTID(uint32_t ledgerSeq, uint16_t txnIndex, uint16_t networkId) noexcept
 {
     static constexpr uint32_t kMAX_LEDGER_SEQ = 0x0FFF'FFFF;
-    if (ledgerSeq > kMAX_LEDGER_SEQ)
+    static constexpr uint32_t kMAX_TXN_INDEX = 0xFFFF;
+    static constexpr uint32_t kMAX_NETWORK_ID = 0xFFFF;
+
+    if (ledgerSeq > kMAX_LEDGER_SEQ || txnIndex > kMAX_TXN_INDEX || networkId > kMAX_NETWORK_ID)
         return {};
 
     static constexpr uint64_t kCTID_PREFIX = 0xC000'0000;

src/rpc/RPCHelpers.hpp

@@ -30,6 +30,7 @@
 #include "rpc/Errors.hpp"
 #include "rpc/common/Types.hpp"
 #include "util/JsonUtils.hpp"
+#include "util/Taggable.hpp"
 #include "util/log/Logger.hpp"
 #include "web/Context.hpp"
 
@@ -744,12 +745,13 @@ decodeCTID(T const ctid) noexcept
  * @brief Log the duration of the request processing
  *
  * @tparam T The type of the duration
- * @param ctx The context of the request
+ * @param request The request to log
+ * @param tag The tag of the context of the request
  * @param dur The duration to log
  */
-template <typename T>
+template <typename DurationType>
 void
-logDuration(web::Context const& ctx, T const& dur)
+logDuration(boost::json::object const& request, util::BaseTagDecorator const& tag, DurationType const& dur)
 {
     using boost::json::serialize;
 
@@ -759,15 +761,15 @@ logDuration(web::Context const& ctx, T const& dur)
     auto const millis = std::chrono::duration_cast<std::chrono::milliseconds>(dur).count();
     auto const seconds = std::chrono::duration_cast<std::chrono::seconds>(dur).count();
     auto const msg = fmt::format(
-        "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(ctx.params))
+        "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(request))
     );
 
     if (seconds > kDURATION_ERROR_THRESHOLD_SECONDS) {
-        LOG(log.error()) << ctx.tag() << msg;
+        LOG(log.error()) << tag << msg;
     } else if (seconds > 1) {
-        LOG(log.warn()) << ctx.tag() << msg;
+        LOG(log.warn()) << tag << msg;
     } else
-        LOG(log.info()) << ctx.tag() << msg;
+        LOG(log.info()) << tag << msg;
 }
 
 /**

src/rpc/common/impl/ForwardingProxy.hpp

@@ -19,7 +19,9 @@
 
 #pragma once
 
+#include "etlng/LoadBalancerInterface.hpp"
 #include "rpc/Errors.hpp"
+#include "rpc/RPCCenter.hpp"
 #include "rpc/RPCHelpers.hpp"
 #include "rpc/common/Types.hpp"
 #include "util/log/Logger.hpp"
@@ -31,20 +33,21 @@
 #include <memory>
 #include <string>
 #include <unordered_set>
+#include <utility>
 
 namespace rpc::impl {
 
-template <typename LoadBalancerType, typename CountersType, typename HandlerProviderType>
+template <typename CountersType, typename HandlerProviderType>
 class ForwardingProxy {
     util::Logger log_{"RPC"};
 
-    std::shared_ptr<LoadBalancerType> balancer_;
+    std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
     std::reference_wrapper<CountersType> counters_;
     std::shared_ptr<HandlerProviderType const> handlerProvider_;
 
 public:
     ForwardingProxy(
-        std::shared_ptr<LoadBalancerType> const& balancer,
+        std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
         CountersType& counters,
         std::shared_ptr<HandlerProviderType const> const& handlerProvider
     )
@@ -104,22 +107,7 @@ public:
     bool
     isProxied(std::string const& method) const
     {
-        static std::unordered_set<std::string> const kPROXIED_COMMANDS{
-            "server_definitions",
-            "server_state",
-            "submit",
-            "submit_multisigned",
-            "fee",
-            "ledger_closed",
-            "ledger_current",
-            "ripple_path_find",
-            "manifest",
-            "channel_authorize",
-            "channel_verify",
-            "simulate",
-        };
-
-        return kPROXIED_COMMANDS.contains(method);
+        return RPCCenter::isForwarded(method);
     }
 
 private:

src/rpc/common/impl/HandlerProvider.cpp

@@ -21,7 +21,8 @@
 
 #include "data/AmendmentCenterInterface.hpp"
 #include "data/BackendInterface.hpp"
-#include "etl/ETLService.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Counters.hpp"
 #include "rpc/common/AnyHandler.hpp"
@@ -65,6 +66,7 @@
 #include <memory>
 #include <optional>
 #include <string>
+#include <unordered_set>
 
 namespace rpc::impl {
 
@@ -72,8 +74,8 @@ ProductionHandlerProvider::ProductionHandlerProvider(
     util::config::ClioConfigDefinition const& config,
     std::shared_ptr<BackendInterface> const& backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptionManager,
-    std::shared_ptr<etl::LoadBalancer> const& balancer,
-    std::shared_ptr<etl::ETLService const> const& etl,
+    std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
+    std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
     std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
     Counters const& counters
 )
@@ -85,7 +87,7 @@ ProductionHandlerProvider::ProductionHandlerProvider(
           {"account_nfts", {.handler = AccountNFTsHandler{backend}}},
           {"account_objects", {.handler = AccountObjectsHandler{backend}}},
           {"account_offers", {.handler = AccountOffersHandler{backend}}},
-          {"account_tx", {.handler = AccountTxHandler{backend}}},
+          {"account_tx", {.handler = AccountTxHandler{backend, etl}}},
           {"amm_info", {.handler = AMMInfoHandler{backend, amendmentCenter}}},
           {"book_changes", {.handler = BookChangesHandler{backend}}},
           {"book_offers", {.handler = BookOffersHandler{backend, amendmentCenter}}},
@@ -138,4 +140,13 @@ ProductionHandlerProvider::isClioOnly(std::string const& command) const
     return handlerMap_.contains(command) && handlerMap_.at(command).isClioOnly;
 }
 
+std::unordered_set<std::string>
+ProductionHandlerProvider::handlerNames() const
+{
+    std::unordered_set<std::string> result;
+    for (auto const& [name, handler] : handlerMap_)
+        result.insert(name);
+    return result;
+}
+
 }  // namespace rpc::impl

src/rpc/common/impl/HandlerProvider.hpp

@@ -21,6 +21,8 @@
 
 #include "data/AmendmentCenterInterface.hpp"
 #include "data/BackendInterface.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/common/AnyHandler.hpp"
 #include "rpc/common/HandlerProvider.hpp"
@@ -31,11 +33,8 @@
 #include <optional>
 #include <string>
 #include <unordered_map>
+#include <unordered_set>
 
-namespace etl {
-class ETLService;
-class LoadBalancer;
-}  // namespace etl
 namespace rpc {
 class Counters;
 }  // namespace rpc
@@ -55,8 +54,8 @@ public:
         util::config::ClioConfigDefinition const& config,
         std::shared_ptr<BackendInterface> const& backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptionManager,
-        std::shared_ptr<etl::LoadBalancer> const& balancer,
-        std::shared_ptr<etl::ETLService const> const& etl,
+        std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
         std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
         Counters const& counters
     );
@@ -69,6 +68,9 @@ public:
 
     bool
     isClioOnly(std::string const& command) const override;
+
+    std::unordered_set<std::string>
+    handlerNames() const;
 };
 
 }  // namespace rpc::impl

src/rpc/handlers/AccountLines.cpp

@@ -86,8 +86,9 @@ AccountLinesHandler::addLine(
     bool const lineNoRipplePeer = (flags & (not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple)) != 0u;
     bool const lineFreeze = (flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u;
     bool const lineFreezePeer = (flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u;
-    bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u;
-    bool const lineDeepFreezePeer = (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u;
+    bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u;
+    bool const lineDeepFreezePeer =
+        (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u;
 
     ripple::STAmount const& saBalance = balance;
     ripple::STAmount const& saLimit = lineLimit;
@@ -102,6 +103,12 @@ AccountLinesHandler::addLine(
     line.qualityIn = lineQualityIn;
     line.qualityOut = lineQualityOut;
 
+    if (lineNoRipple)
+        line.noRipple = true;
+
+    if (lineNoRipplePeer)
+        line.noRipplePeer = true;
+
     if (lineAuth)
         line.authorized = true;
 
@@ -120,8 +127,6 @@ AccountLinesHandler::addLine(
     if (lineDeepFreezePeer)
         line.deepFreezePeer = true;
 
-    line.noRipple = lineNoRipple;
-    line.noRipplePeer = lineNoRipplePeer;
     lines.push_back(line);
 }
 
@@ -257,8 +262,11 @@ tag_invoke(
         {JS(quality_out), line.qualityOut},
     };
 
-    obj[JS(no_ripple)] = line.noRipple;
-    obj[JS(no_ripple_peer)] = line.noRipplePeer;
+    if (line.noRipple)
+        obj[JS(no_ripple)] = *(line.noRipple);
+
+    if (line.noRipplePeer)
+        obj[JS(no_ripple_peer)] = *(line.noRipplePeer);
 
     if (line.authorized)
         obj[JS(authorized)] = *(line.authorized);

src/rpc/handlers/AccountLines.hpp

@@ -70,8 +70,8 @@ public:
         std::string limitPeer;
         uint32_t qualityIn{};
         uint32_t qualityOut{};
-        bool noRipple{};
-        bool noRipplePeer{};
+        std::optional<bool> noRipple;
+        std::optional<bool> noRipplePeer;
         std::optional<bool> authorized;
         std::optional<bool> peerAuthorized;
         std::optional<bool> freeze;

src/rpc/handlers/AccountTx.cpp

@@ -161,6 +161,18 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) con
                 auto const txKey = ctx.apiVersion < 2u ? JS(tx) : JS(tx_json);
                 obj[JS(meta)] = std::move(meta);
                 obj[txKey] = std::move(txn);
+
+                // Put CTID into tx or tx_json
+                if (obj[JS(meta)].as_object().contains("TransactionIndex")) {
+                    auto networkID = 0u;
+                    if (auto const& etlState = etl_->getETLState(); etlState.has_value())
+                        networkID = etlState->networkID;
+
+                    auto const txnIdx = obj[JS(meta)].as_object().at("TransactionIndex").as_int64();
+                    if (auto const& ctid = rpc::encodeCTID(txnPlusMeta.ledgerSequence, txnIdx, networkID); ctid)
+                        obj[txKey].as_object()[JS(ctid)] = ctid.value();
+                }
+
                 obj[txKey].as_object()[JS(date)] = txnPlusMeta.date;
                 obj[txKey].as_object()[JS(ledger_index)] = txnPlusMeta.ledgerSequence;
 

src/rpc/handlers/AccountTx.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
+#include "etlng/ETLServiceInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/JS.hpp"
 #include "rpc/common/JsonBool.hpp"
@@ -55,6 +56,7 @@ namespace rpc {
 class AccountTxHandler {
     util::Logger log_{"RPC"};
     std::shared_ptr<BackendInterface> sharedPtrBackend_;
+    std::shared_ptr<etlng::ETLServiceInterface const> etl_;
 
 public:
     static constexpr auto kLIMIT_MIN = 1;
@@ -109,8 +111,13 @@ public:
      * @brief Construct a new AccountTxHandler object
      *
      * @param sharedPtrBackend The backend to use
+     * @param etl The ETL service to use
      */
-    AccountTxHandler(std::shared_ptr<BackendInterface> const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend)
+    AccountTxHandler(
+        std::shared_ptr<BackendInterface> const& sharedPtrBackend,
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl
+    )
+        : sharedPtrBackend_(sharedPtrBackend), etl_{etl}
     {
     }
 
@@ -130,6 +137,7 @@ public:
             {JS(ledger_index), validation::CustomValidators::ledgerIndexValidator},
             {JS(ledger_index_min), validation::Type<int32_t>{}},
             {JS(ledger_index_max), validation::Type<int32_t>{}},
+            {JS(ctid), validation::Type<std::string>{}},
             {JS(limit),
              validation::Type<uint32_t>{},
              validation::Min(1u),

src/rpc/handlers/ServerInfo.hpp

@@ -21,6 +21,8 @@
 
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
+#include "etlng/ETLServiceInterface.hpp"
+#include "etlng/LoadBalancerInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/JS.hpp"
@@ -49,10 +51,6 @@
 #include <optional>
 #include <string>
 
-namespace etl {
-class ETLService;
-class LoadBalancer;
-}  // namespace etl
 namespace rpc {
 class Counters;
 }  // namespace rpc
@@ -62,18 +60,16 @@ namespace rpc {
 /**
  * @brief Contains common functionality for handling the `server_info` command
  *
- * @tparam LoadBalancerType The type of the load balancer
- * @tparam ETLServiceType The type of the ETL service
  * @tparam CountersType The type of the counters
  */
-template <typename LoadBalancerType, typename ETLServiceType, typename CountersType>
+template <typename CountersType>
 class BaseServerInfoHandler {
     static constexpr auto kBACKEND_COUNTERS_KEY = "backend_counters";
 
     std::shared_ptr<BackendInterface> backend_;
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
-    std::shared_ptr<LoadBalancerType> balancer_;
-    std::shared_ptr<ETLServiceType const> etl_;
+    std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
+    std::shared_ptr<etlng::ETLServiceInterface const> etl_;
     std::reference_wrapper<CountersType const> counters_;
 
 public:
@@ -158,8 +154,8 @@ public:
     BaseServerInfoHandler(
         std::shared_ptr<BackendInterface> const& backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptions,
-        std::shared_ptr<LoadBalancerType> const& balancer,
-        std::shared_ptr<ETLServiceType const> const& etl,
+        std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
         CountersType const& counters
     )
         : backend_(backend)
@@ -352,6 +348,6 @@ private:
  *
  * For more details see: https://xrpl.org/server_info-clio.html
  */
-using ServerInfoHandler = BaseServerInfoHandler<etl::LoadBalancer, etl::ETLService, Counters>;
+using ServerInfoHandler = BaseServerInfoHandler<Counters>;
 
 }  // namespace rpc

src/rpc/handlers/Tx.hpp

@@ -21,7 +21,7 @@
 
 #include "data/BackendInterface.hpp"
 #include "data/Types.hpp"
-#include "etl/ETLService.hpp"
+#include "etlng/ETLServiceInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/JS.hpp"
 #include "rpc/RPCHelpers.hpp"
@@ -37,7 +37,9 @@
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
 #include <boost/json/value_to.hpp>
+#include <fmt/core.h>
 #include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/chrono.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/protocol/ErrorCodes.h>
 #include <xrpl/protocol/LedgerHeader.h>
@@ -52,14 +54,13 @@
 namespace rpc {
 
 /**
- * @brief Contains common functionality for handling the `tx` command
+ * @brief The tx method retrieves information on a single transaction, by its identifying hash.
  *
- * @tparam ETLServiceType The type of the ETL service to use
+ * For more details see: https://xrpl.org/tx.html
  */
-template <typename ETLServiceType>
-class BaseTxHandler {
+class TxHandler {
     std::shared_ptr<BackendInterface> sharedPtrBackend_;
-    std::shared_ptr<ETLServiceType const> etl_;
+    std::shared_ptr<etlng::ETLServiceInterface const> etl_;
 
 public:
     /**
@@ -95,14 +96,14 @@ public:
     using Result = HandlerReturnType<Output>;
 
     /**
-     * @brief Construct a new BaseTxHandler object
+     * @brief Construct a new TxHandler object
      *
      * @param sharedPtrBackend The backend to use
      * @param etl The ETL service to use
      */
-    BaseTxHandler(
+    TxHandler(
         std::shared_ptr<BackendInterface> const& sharedPtrBackend,
-        std::shared_ptr<ETLServiceType const> const& etl
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl
     )
         : sharedPtrBackend_(sharedPtrBackend), etl_(etl)
     {
@@ -183,7 +184,7 @@ public:
             dbResponse = sharedPtrBackend_->fetchTransaction(ripple::uint256{input.transaction->c_str()}, ctx.yield);
         }
 
-        auto output = BaseTxHandler::Output{.apiVersion = ctx.apiVersion};
+        auto output = TxHandler::Output{.apiVersion = ctx.apiVersion};
 
         if (!dbResponse) {
             if (rangeSupplied && input.transaction)  // ranges not for ctid
@@ -214,17 +215,15 @@ public:
             // input.transaction might be not available, get hash via tx object
             if (txn.contains(JS(hash)))
                 output.hash = txn.at(JS(hash)).as_string();
+        }
 
-            // append ctid here to mimic rippled 1.12 behavior: return ctid even binary=true
-            // rippled will change it in the future, ctid should be part of tx json which not available in binary
-            // mode
-            auto const txnIdx = boost::json::value_to<uint64_t>(meta.at("TransactionIndex"));
-            if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId &&
-                *currentNetId <= 0xFFFFU) {
-                output.ctid = rpc::encodeCTID(
-                    dbResponse->ledgerSequence, static_cast<uint16_t>(txnIdx), static_cast<uint16_t>(*currentNetId)
-                );
-            }
+        // append ctid here to mimic rippled behavior
+        auto const txnIdx = boost::json::value_to<uint64_t>(meta.at("TransactionIndex"));
+        if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId &&
+            *currentNetId <= 0xFFFFU) {
+            output.ctid = rpc::encodeCTID(
+                dbResponse->ledgerSequence, static_cast<uint16_t>(txnIdx), static_cast<uint16_t>(*currentNetId)
+            );
         }
 
         output.date = dbResponse->date;
@@ -281,12 +280,10 @@ private:
             if (output.tx) {
                 obj[JS(tx_json)] = *output.tx;
                 obj[JS(tx_json)].as_object()[JS(date)] = output.date;
+                if (output.ctid)
+                    obj[JS(tx_json)].as_object()[JS(ctid)] = *output.ctid;
+
                 obj[JS(tx_json)].as_object()[JS(ledger_index)] = output.ledgerIndex;
-                // move ctid from tx_json to root
-                if (obj[JS(tx_json)].as_object().contains(JS(ctid))) {
-                    obj[JS(ctid)] = obj[JS(tx_json)].as_object()[JS(ctid)];
-                    obj[JS(tx_json)].as_object().erase(JS(ctid));
-                }
                 // move hash from tx_json to root
                 if (obj[JS(tx_json)].as_object().contains(JS(hash))) {
                     obj[JS(hash)] = obj[JS(tx_json)].as_object()[JS(hash)];
@@ -320,7 +317,7 @@ private:
     friend Input
     tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv)
     {
-        auto input = BaseTxHandler::Input{};
+        auto input = TxHandler::Input{};
         auto const& jsonObject = jv.as_object();
 
         if (jsonObject.contains(JS(transaction)))
@@ -344,10 +341,4 @@ private:
     }
 };
 
-/**
- * @brief The tx method retrieves information on a single transaction, by its identifying hash.
- *
- * For more details see: https://xrpl.org/tx.html
- */
-using TxHandler = BaseTxHandler<etl::ETLService>;
 }  // namespace rpc

src/util/CMakeLists.txt

@@ -24,6 +24,7 @@ target_sources(
           ResponseExpirationCache.cpp
           SignalsHandler.cpp
           StopHelper.cpp
+          StringHash.cpp
           Taggable.cpp
           TerminationHandler.cpp
           TimeUtils.cpp
@@ -41,7 +42,7 @@ target_sources(
 
 # This must be above the target_link_libraries call otherwise backtrace doesn't work
 if ("${san}" STREQUAL "")
-  target_link_libraries(clio_util PUBLIC Boost::stacktrace_backtrace dl libbacktrace::libbacktrace)
+  target_link_libraries(clio_util PUBLIC Boost::stacktrace_backtrace dl libbacktrace::libbacktrace clio_rpc_center)
 endif ()
 
 target_link_libraries(

src/util/StringHash.cpp

@@ -0,0 +1,46 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/StringHash.hpp"
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+namespace util {
+
+size_t
+StringHash::operator()(char const* str) const
+{
+    return hash_type{}(str);
+}
+
+size_t
+StringHash::operator()(std::string_view str) const
+{
+    return hash_type{}(str);
+}
+
+size_t
+StringHash::operator()(std::string const& str) const
+{
+    return hash_type{}(str);
+}
+
+}  // namespace util

src/util/StringHash.hpp

@@ -0,0 +1,65 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <cstddef>
+#include <functional>
+#include <string>
+#include <string_view>
+
+namespace util {
+
+/**
+ * @brief A string hash functor that provides transparent hash operations for various string types.
+ *
+ * This hash functor can be used with unordered containers to enable heterogeneous lookups
+ * for different string-like types without unnecessary conversions. It supports C-style strings,
+ * string views, and standard strings.
+ */
+struct StringHash {
+    using hash_type = std::hash<std::string_view>;
+    using is_transparent = void;  ///< Enables heterogeneous lookup
+
+    /**
+     * @brief Computes the hash of a C-style string.
+     * @param str Null-terminated C-style string to hash
+     * @return Size_t hash value
+     */
+    std::size_t
+    operator()(char const* str) const;
+
+    /**
+     * @brief Computes the hash of a string_view.
+     * @param str String view to hash
+     * @return Size_t hash value
+     */
+    std::size_t
+    operator()(std::string_view str) const;
+
+    /**
+     * @brief Computes the hash of a standard string.
+     * @param str String to hash
+     * @return Size_t hash value
+     */
+    std::size_t
+    operator()(std::string const& str) const;
+};
+
+}  // namespace util

src/util/newconfig/ConfigConstraints.cpp

@@ -19,6 +19,7 @@
 
 #include "util/newconfig/ConfigConstraints.hpp"
 
+#include "rpc/RPCCenter.hpp"
 #include "util/newconfig/Error.hpp"
 #include "util/newconfig/Types.hpp"
 
@@ -103,4 +104,22 @@ PositiveDouble::checkValueImpl(Value const& num) const
     return Error{"Double number must be greater than or equal to 0"};
 }
 
+std::optional<Error>
+RpcNameConstraint::checkTypeImpl(Value const& value) const
+{
+    if (not std::holds_alternative<std::string>(value))
+        return Error{"RPC command name must be a string"};
+    return std::nullopt;
+}
+
+std::optional<Error>
+RpcNameConstraint::checkValueImpl(Value const& value) const
+{
+    auto const str = std::get<std::string>(value);
+    if (not rpc::RPCCenter::isRpcName(str))
+        return Error{"Invalid RPC command name"};
+
+    return std::nullopt;
+}
+
 }  // namespace util::config

src/util/newconfig/ConfigConstraints.hpp

@@ -421,6 +421,41 @@ private:
     }
 };
 
+/**
+ * @brief A constraint to ensure the value is a valid RPC command name.
+ */
+class RpcNameConstraint final : public Constraint {
+private:
+    /**
+     * @brief Check if the type of the value is correct for this specific constraint.
+     *
+     * @param value The type to be checked
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkTypeImpl(Value const& value) const override;
+
+    /**
+     * @brief Check if the value is a valid RPC command name.
+     *
+     * @param value The value to check
+     * @return An Error object if the constraint is not met, nullopt otherwise
+     */
+    [[nodiscard]] std::optional<Error>
+    checkValueImpl(Value const& value) const override;
+
+    /**
+     * @brief Prints to the output stream for this specific constraint.
+     *
+     * @param stream The output stream
+     */
+    void
+    print(std::ostream& stream) const override
+    {
+        stream << "Checks whether provided RPC name is valid";
+    }
+};
+
 static constinit PortConstraint gValidatePort{};
 static constinit ValidIPConstraint gValidateIp{};
 
@@ -448,6 +483,8 @@ static constinit NumberValueConstraint<uint32_t> gValidateUint32{
     std::numeric_limits<uint32_t>::min(),
     std::numeric_limits<uint32_t>::max()
 };
+static constinit NumberValueConstraint<uint32_t> gValidateNonNegativeUint32{0, std::numeric_limits<uint32_t>::max()};
 static constinit NumberValueConstraint<uint32_t> gValidateApiVersion{rpc::kAPI_VERSION_MIN, rpc::kAPI_VERSION_MAX};
 
+static constinit RpcNameConstraint gRpcNameConstraint{};
 }  // namespace util::config

src/util/newconfig/ConfigDefinition.hpp

@@ -293,7 +293,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{
      {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
 
      {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
-
+     {"__ng_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
      {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}},
      {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},
      {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},
@@ -314,6 +314,15 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{
      {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(gValidateUint32)},
      {"dos_guard.sweep_interval",
       ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(gValidatePositiveDouble)},
+     {"dos_guard.__ng_default_weight",
+      ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(gValidateNonNegativeUint32)},
+     {"dos_guard.__ng_weights.[].method", Array{ConfigValue{ConfigType::String}.withConstraint(gRpcNameConstraint)}},
+     {"dos_guard.__ng_weights.[].weight",
+      Array{ConfigValue{ConfigType::Integer}.withConstraint(gValidateNonNegativeUint32)}},
+     {"dos_guard.__ng_weights.[].weight_ledger_current",
+      Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}},
+     {"dos_guard.__ng_weights.[].weight_ledger_validated",
+      Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}},
 
      {"workers",
       ConfigValue{ConfigType::Integer}

src/web/CMakeLists.txt

@@ -5,12 +5,12 @@ target_sources(
   PRIVATE AdminVerificationStrategy.cpp
           dosguard/DOSGuard.cpp
           dosguard/IntervalSweepHandler.cpp
+          dosguard/Weights.cpp
           dosguard/WhitelistHandler.cpp
           ng/Connection.cpp
           ng/impl/ErrorHandling.cpp
           ng/impl/ConnectionHandler.cpp
           ng/impl/ServerSslContext.cpp
-          ng/impl/WsConnection.cpp
           ng/Request.cpp
           ng/Response.cpp
           ng/Server.cpp

src/web/RPCServerHandler.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
+#include "etlng/ETLServiceInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/Factories.hpp"
 #include "rpc/JS.hpp"
@@ -30,6 +31,7 @@
 #include "util/Taggable.hpp"
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
 #include "web/impl/ErrorHandling.hpp"
 #include "web/interface/ConnectionBase.hpp"
 
@@ -58,13 +60,14 @@ namespace web {
  *
  * Note: see @ref web::SomeServerHandler concept
  */
-template <typename RPCEngineType, typename ETLType>
+template <typename RPCEngineType>
 class RPCServerHandler {
     std::shared_ptr<BackendInterface const> const backend_;
     std::shared_ptr<RPCEngineType> const rpcEngine_;
-    std::shared_ptr<ETLType const> const etl_;
+    std::shared_ptr<etlng::ETLServiceInterface const> const etl_;
     util::TagDecoratorFactory const tagFactory_;
     rpc::impl::ProductionAPIVersionParser apiVersionParser_;  // can be injected if needed
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
 
     util::Logger log_{"RPC"};
     util::Logger perfLog_{"Performance"};
@@ -77,18 +80,21 @@ public:
      * @param backend The backend to use
      * @param rpcEngine The RPC engine to use
      * @param etl The ETL to use
+     * @param dosguard The DOS guard service to use for request rate limiting
      */
     RPCServerHandler(
         util::config::ClioConfigDefinition const& config,
         std::shared_ptr<BackendInterface const> const& backend,
         std::shared_ptr<RPCEngineType> const& rpcEngine,
-        std::shared_ptr<ETLType const> const& etl
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
+        web::dosguard::DOSGuardInterface& dosguard
     )
         : backend_(backend)
         , rpcEngine_(rpcEngine)
         , etl_(etl)
         , tagFactory_(config)
         , apiVersionParser_(config.getObject("api_version"))
+        , dosguard_(dosguard)
     {
     }
 
@@ -101,6 +107,11 @@ public:
     void
     operator()(std::string const& request, std::shared_ptr<web::ConnectionBase> const& connection)
     {
+        if (not dosguard_.get().isOk(connection->clientIp)) {
+            connection->sendSlowDown(request);
+            return;
+        }
+
         try {
             auto req = boost::json::parse(request).as_object();
             LOG(perfLog_.debug()) << connection->tag() << "Adding to work queue";
@@ -108,6 +119,11 @@ public:
             if (not connection->upgraded and shouldReplaceParams(req))
                 req[JS(params)] = boost::json::array({boost::json::object{}});
 
+            if (not dosguard_.get().request(connection->clientIp, req)) {
+                connection->sendSlowDown(request);
+                return;
+            }
+
             if (!rpcEngine_->post(
                     [this, request = std::move(req), connection](boost::asio::yield_context yield) mutable {
                         handleRequest(yield, std::move(request), connection);
@@ -195,8 +211,8 @@ private:
 
             auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); });
 
-            auto us = std::chrono::duration<int, std::milli>(timeDiff);
-            rpc::logDuration(*context, us);
+            auto const us = std::chrono::duration<int, std::milli>(timeDiff);
+            rpc::logDuration(request, context->tag(), us);
 
             boost::json::object response;
 

src/web/dosguard/DOSGuard.cpp

@@ -24,8 +24,11 @@
 #include "util/newconfig/ArrayView.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
 #include "util/newconfig/ValueView.hpp"
+#include "web/dosguard/WeightsInterface.hpp"
 #include "web/dosguard/WhitelistHandlerInterface.hpp"
 
+#include <boost/json/object.hpp>
+
 #include <cstdint>
 #include <functional>
 #include <mutex>
@@ -37,8 +40,13 @@ using namespace util::config;
 
 namespace web::dosguard {
 
-DOSGuard::DOSGuard(ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler)
+DOSGuard::DOSGuard(
+    ClioConfigDefinition const& config,
+    WhitelistHandlerInterface const& whitelistHandler,
+    WeightsInterface const& weights
+)
     : whitelistHandler_{std::cref(whitelistHandler)}
+    , weights_(weights)
     , maxFetches_{config.get<uint32_t>("dos_guard.max_fetches")}
     , maxConnCount_{config.get<uint32_t>("dos_guard.max_connections")}
     , maxRequestCount_{config.get<uint32_t>("dos_guard.max_requests")}
@@ -59,8 +67,8 @@ DOSGuard::isOk(std::string const& ip) const noexcept
 
     {
         auto lock = mtx_.lock<std::scoped_lock>();
-        if (lock->ipState.find(ip) != lock->ipState.end()) {
-            auto [transferredByte, requests] = lock->ipState.at(ip);
+        if (auto const it = lock->ipState.find(ip); it != lock->ipState.end()) {
+            auto const [transferredByte, requests] = it->second;
             if (transferredByte > maxFetches_ || requests > maxRequestCount_) {
                 LOG(log_.warn()) << "Dosguard: Client surpassed the rate limit. ip = " << ip
                                  << " Transfered Byte: " << transferredByte << "; Requests: " << requests;
@@ -115,14 +123,16 @@ DOSGuard::add(std::string const& ip, uint32_t numObjects) noexcept
 }
 
 [[maybe_unused]] bool
-DOSGuard::request(std::string const& ip) noexcept
+DOSGuard::request(std::string const& ip, boost::json::object const& request)
 {
     if (whitelistHandler_.get().isWhiteListed(ip))
         return true;
 
+    auto const weight = weights_.get().requestWeight(request);
+
     {
         auto lock = mtx_.lock<std::scoped_lock>();
-        lock->ipState[ip].requestsCount++;
+        lock->ipState[ip].requestsCount += weight;
     }
 
     return isOk(ip);

src/web/dosguard/DOSGuard.hpp

@@ -23,10 +23,12 @@
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
 #include "web/dosguard/DOSGuardInterface.hpp"
+#include "web/dosguard/WeightsInterface.hpp"
 #include "web/dosguard/WhitelistHandlerInterface.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/iterator/transform_iterator.hpp>
+#include <boost/json/object.hpp>
 #include <boost/system/error_code.hpp>
 
 #include <cstdint>
@@ -59,6 +61,7 @@ class DOSGuard : public DOSGuardInterface {
     util::Mutex<State> mtx_;
 
     std::reference_wrapper<WhitelistHandlerInterface const> whitelistHandler_;
+    std::reference_wrapper<WeightsInterface const> weights_;
 
     std::uint32_t const maxFetches_;
     std::uint32_t const maxConnCount_;
@@ -71,8 +74,13 @@ public:
      *
      * @param config Clio config
      * @param whitelistHandler Whitelist handler that checks whitelist for IP addresses
+     * @param weights API methods weights
      */
-    DOSGuard(util::config::ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler);
+    DOSGuard(
+        util::config::ClioConfigDefinition const& config,
+        WhitelistHandlerInterface const& whitelistHandler,
+        WeightsInterface const& weights
+    );
 
     /**
      * @brief Check whether an ip address is in the whitelist or not.
@@ -133,11 +141,12 @@ public:
      * returned otherwise.
      *
      * @param ip
+     * @param request The request as json object
      * @return true
      * @return false
      */
     [[maybe_unused]] bool
-    request(std::string const& ip) noexcept override;
+    request(std::string const& ip, boost::json::object const& request) override;
 
     /**
      * @brief Instantly clears all fetch counters added by @see add(std::string const&, uint32_t).

src/web/dosguard/DOSGuardInterface.hpp

@@ -19,6 +19,8 @@
 
 #pragma once
 
+#include <boost/json/object.hpp>
+
 #include <cstdint>
 #include <string>
 #include <string_view>
@@ -99,12 +101,13 @@ public:
      *
      *
      * @param ip
+     * @param request The request as json object
      * @return If the total sums up to a value equal or larger than maxRequestCount_
      * the operation is no longer allowed and false is returned; true is
      * returned otherwise.
      */
     [[maybe_unused]] virtual bool
-    request(std::string const& ip) noexcept = 0;
+    request(std::string const& ip, boost::json::object const& request) = 0;
 };
 
 }  // namespace web::dosguard

src/web/dosguard/Weights.cpp

@@ -0,0 +1,106 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "web/dosguard/Weights.hpp"
+
+#include "rpc/JS.hpp"
+#include "util/Assert.hpp"
+#include "util/newconfig/ArrayView.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <boost/json/object.hpp>
+#include <xrpl/protocol/jss.h>
+
+#include <cstddef>
+#include <iterator>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <utility>
+
+namespace web::dosguard {
+
+Weights::Weights(size_t defaultWeight, std::unordered_map<std::string, Entry> weights)
+    : defaultWeight_(defaultWeight), weights_(std::move_iterator(weights.begin()), std::move_iterator(weights.end()))
+{
+}
+
+Weights
+Weights::make(util::config::ClioConfigDefinition const& config)
+{
+    std::unordered_map<std::string, Weights::Entry> weights;
+    auto const configWeights = config.getArray("dos_guard.__ng_weights");
+    for (size_t i = 0; i < configWeights.size(); ++i) {
+        auto const w = configWeights.objectAt(i);
+        Weights::Entry const entry{
+            .weight = w.get<size_t>("weight"),
+            .weightLedgerCurrent = w.maybeValue<size_t>("weight_ledger_current"),
+            .weightLedgerValidated = w.maybeValue<size_t>("weight_ledger_validated"),
+        };
+        weights.emplace(w.get<std::string>("method"), entry);
+    }
+    return Weights{config.get<size_t>("dos_guard.__ng_default_weight"), std::move(weights)};
+}
+
+size_t
+Weights::requestWeight(boost::json::object const& request) const
+{
+    if (not((request.contains(JS(method)) and request.at(JS(method)).is_string()) or
+            (request.contains(JS(command)) and request.at(JS(command)).is_string()))) {
+        return defaultWeight_;
+    }
+
+    std::string_view cmd =
+        request.contains(JS(method)) ? request.at(JS(method)).as_string() : request.at(JS(command)).as_string();
+
+    auto it = weights_.find(cmd);
+    if (it == weights_.end()) {
+        return defaultWeight_;
+    }
+
+    auto const& entry = it->second;
+
+    boost::json::value const* ledgerIndex = nullptr;
+    if (request.contains(JS(ledger_index))) {
+        ledgerIndex = &request.at(JS(ledger_index));
+    } else if (request.contains(JS(params))) {
+        ASSERT(
+            request.at(JS(params)).is_array() and not request.at(JS(params)).as_array().empty() and
+                request.at(JS(params)).as_array().at(0).is_object(),
+            "params should be [{{}}]"
+        );
+        if (auto const& params = request.at(JS(params)).as_array().at(0).as_object();
+            params.contains(JS(ledger_index))) {
+            ledgerIndex = &params.at(JS(ledger_index));
+        }
+    }
+
+    if (ledgerIndex != nullptr and ledgerIndex->is_string()) {
+        auto const& ledgerIndexString = ledgerIndex->as_string();
+        if (ledgerIndexString == JS(validated)) {
+            return entry.weightLedgerValidated.value_or(entry.weight);
+        }
+        if (ledgerIndexString == JS(current)) {
+            return entry.weightLedgerCurrent.value_or(entry.weight);
+        }
+    }
+    return entry.weight;
+}
+
+}  // namespace web::dosguard

src/web/dosguard/Weights.hpp

@@ -0,0 +1,88 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/StringHash.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "web/dosguard/WeightsInterface.hpp"
+
+#include <boost/json/object.hpp>
+
+#include <cstddef>
+#include <functional>
+#include <optional>
+#include <string>
+#include <unordered_map>
+
+namespace web::dosguard {
+
+/**
+ * @brief Implementation of WeightsInterface that manages command weights for DosGuard.
+ *
+ * This class provides a mechanism to assign different weights to API commands
+ * for the purpose of DOS protection calculations. Commands can have specific weights,
+ * or fall back to a default weight.
+ */
+class Weights : public WeightsInterface {
+public:
+    /**
+     * @brief Structure representing weight configuration for a command.
+     *
+     * Contains the base weight and optional specialized weights for different ledger specifications.
+     */
+    struct Entry {
+        size_t weight;
+        std::optional<size_t> weightLedgerCurrent;
+        std::optional<size_t> weightLedgerValidated;
+    };
+
+private:
+    size_t defaultWeight_;
+    std::unordered_map<std::string, Entry, util::StringHash, std::equal_to<>> weights_;
+
+public:
+    /**
+     * @brief Construct a new Weights object
+     *
+     * @param defaultWeight The default weight to use when a command-specific weight is not defined
+     * @param weights Map of command names to their specific weights
+     */
+    Weights(size_t defaultWeight, std::unordered_map<std::string, Entry> weights);
+
+    /**
+     * @brief Create a Weights object from configuration
+     *
+     * @param config The application configuration
+     * @return Weights instance initialized with values from configuration
+     */
+    static Weights
+    make(util::config::ClioConfigDefinition const& config);
+
+    /**
+     * @brief Get the weight assigned to a specific command
+     *
+     * @param request Json request
+     * @return size_t The weight value (specific weight if defined, otherwise default weight)
+     */
+    size_t
+    requestWeight(boost::json::object const& request) const override;
+};
+
+}  // namespace web::dosguard

src/web/dosguard/WeightsInterface.hpp

@@ -0,0 +1,48 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/json/object.hpp>
+
+#include <cstddef>
+
+namespace web::dosguard {
+
+/**
+ * @brief Interface for determining request weights in DOS protection.
+ *
+ * This interface defines the contract for classes that calculate weights for incoming
+ * requests, which is used for DOS protection mechanisms.
+ */
+class WeightsInterface {
+public:
+    virtual ~WeightsInterface() = default;
+
+    /**
+     * @brief Calculate the weight of a request.
+     *
+     * @param request The JSON object representing the request
+     * @return The calculated weight of the request
+     */
+    virtual size_t
+    requestWeight(boost::json::object const& request) const = 0;
+};
+
+}  // namespace web::dosguard

src/web/impl/HttpBase.hpp

@@ -240,19 +240,7 @@ public:
             return sender_(httpResponse(http::status::bad_request, "text/html", "Expected a POST request"));
         }
 
-        // to avoid overwhelm work queue, the request limit check should be
-        // before posting to queue the web socket creation will be guarded via
-        // connection limit
-        if (!dosGuard_.get().request(clientIp)) {
-            // TODO: this looks like it could be useful to count too in the future
-            return sender_(httpResponse(
-                http::status::service_unavailable,
-                "text/plain",
-                boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN))
-            ));
-        }
-
-        LOG(log_.info()) << tag() << "Received request from ip = " << clientIp << " - posting to WorkQueue";
+        LOG(log_.info()) << tag() << "Received request from ip = " << clientIp;
 
         try {
             (*handler_)(req_.body(), derived().shared_from_this());
@@ -265,6 +253,16 @@ public:
         }
     }
 
+    void
+    sendSlowDown(std::string const&) override
+    {
+        sender_(httpResponse(
+            http::status::service_unavailable,
+            "text/plain",
+            boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN))
+        ));
+    }
+
     /**
      * @brief Send a response to the client
      * The message length will be added to the DOSGuard, if the limit is reached, a warning will be added to the

src/web/impl/WsBase.hpp

@@ -164,6 +164,12 @@ public:
         doWrite();
     }
 
+    void
+    sendSlowDown(std::string const& request) override
+    {
+        sendError(rpc::RippledError::rpcSLOW_DOWN, request);
+    }
+
     /**
      * @brief Send a message to the client
      * @param msg The message to send, it will keep the string alive until it is sent. It is useful when we have
@@ -173,7 +179,8 @@ public:
     void
     send(std::shared_ptr<std::string> msg) override
     {
-        boost::asio::dispatch(
+        // Note: post used instead of dispatch to guarantee async behavior of wsFail and maybeSendNext
+        boost::asio::post(
             derived().ws().get_executor(),
             [this, self = derived().shared_from_this(), msg = std::move(msg)]() {
                 if (messages_.size() > maxSendingQueueSize_) {
@@ -279,36 +286,33 @@ public:
 
         LOG(perfLog_.info()) << tag() << "Received request from ip = " << this->clientIp;
 
-        auto sendError = [this](auto error, std::string&& requestStr) {
-            auto e = rpc::makeError(error);
-
-            try {
-                auto request = boost::json::parse(requestStr);
-                if (request.is_object() && request.as_object().contains("id"))
-                    e["id"] = request.as_object().at("id");
-                e["request"] = std::move(request);
-            } catch (std::exception const&) {
-                e["request"] = std::move(requestStr);
-            }
-
-            this->send(std::make_shared<std::string>(boost::json::serialize(e)));
-        };
-
         std::string requestStr{static_cast<char const*>(buffer_.data().data()), buffer_.size()};
 
-        // dosGuard served request++ and check ip address
-        if (!dosGuard_.get().request(clientIp)) {
-            // TODO: could be useful to count in counters in the future too
-            sendError(rpc::RippledError::rpcSLOW_DOWN, std::move(requestStr));
-        } else {
-            try {
-                (*handler_)(requestStr, shared_from_this());
-            } catch (std::exception const&) {
-                sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr));
-            }
+        try {
+            (*handler_)(requestStr, shared_from_this());
+        } catch (std::exception const&) {
+            sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr));
         }
 
         doRead();
     }
+
+private:
+    void
+    sendError(rpc::RippledError error, std::string requestStr)
+    {
+        auto e = rpc::makeError(error);
+
+        try {
+            auto request = boost::json::parse(requestStr);
+            if (request.is_object() && request.as_object().contains("id"))
+                e["id"] = request.as_object().at("id");
+            e["request"] = std::move(request);
+        } catch (std::exception const&) {
+            e["request"] = std::move(requestStr);
+        }
+
+        this->send(std::make_shared<std::string>(boost::json::serialize(e)));
+    }
 };
 }  // namespace web::impl

src/web/interface/ConnectionBase.hpp

@@ -82,6 +82,13 @@ public:
         throw std::logic_error("web server can not send the shared payload");
     }
 
+    /**
+     * @brief Send a "slow down" error response to the client.
+     *
+     * @param request The original request that triggered the rate limiting
+     */
+    virtual void
+    sendSlowDown(std::string const& request) = 0;
     /**
      * @brief Get the subscription context for this connection.
      *

src/web/ng/RPCServerHandler.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "data/BackendInterface.hpp"
+#include "etlng/ETLServiceInterface.hpp"
 #include "rpc/Errors.hpp"
 #include "rpc/Factories.hpp"
 #include "rpc/JS.hpp"
@@ -32,6 +33,7 @@
 #include "util/Taggable.hpp"
 #include "util/log/Logger.hpp"
 #include "web/SubscriptionContextInterface.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
 #include "web/ng/Connection.hpp"
 #include "web/ng/Request.hpp"
 #include "web/ng/Response.hpp"
@@ -64,11 +66,12 @@ namespace web::ng {
  *
  * Note: see @ref web::SomeServerHandler concept
  */
-template <typename RPCEngineType, typename ETLType>
+template <typename RPCEngineType>
 class RPCServerHandler {
     std::shared_ptr<BackendInterface const> const backend_;
     std::shared_ptr<RPCEngineType> const rpcEngine_;
-    std::shared_ptr<ETLType const> const etl_;
+    std::shared_ptr<etlng::ETLServiceInterface const> const etl_;
+    std::reference_wrapper<dosguard::DOSGuardInterface> dosguard_;
     util::TagDecoratorFactory const tagFactory_;
     rpc::impl::ProductionAPIVersionParser apiVersionParser_;  // can be injected if needed
 
@@ -83,16 +86,19 @@ public:
      * @param backend The backend to use
      * @param rpcEngine The RPC engine to use
      * @param etl The ETL to use
+     * @param dosguard The DOS guard service to use for request rate limiting
      */
     RPCServerHandler(
         util::config::ClioConfigDefinition const& config,
         std::shared_ptr<BackendInterface const> const& backend,
         std::shared_ptr<RPCEngineType> const& rpcEngine,
-        std::shared_ptr<ETLType const> const& etl
+        std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
+        dosguard::DOSGuardInterface& dosguard
     )
         : backend_(backend)
         , rpcEngine_(rpcEngine)
         , etl_(etl)
+        , dosguard_(dosguard)
         , tagFactory_(config)
         , apiVersionParser_(config.getObject("api_version"))
     {
@@ -115,6 +121,10 @@ public:
         boost::asio::yield_context yield
     )
     {
+        if (not dosguard_.get().isOk(connectionMetadata.ip())) {
+            return makeSlowDownResponse(request, std::nullopt);
+        }
+
         std::optional<Response> response;
         util::CoroutineGroup coroutineGroup{yield, 1};
         auto const onTaskComplete = coroutineGroup.registerForeign(yield);
@@ -141,18 +151,23 @@ public:
                         }
                     } else {
                         auto parsedObject = std::move(parsedRequest).as_object();
-                        LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue";
 
-                        if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject))
-                            parsedObject[JS(params)] = boost::json::array({boost::json::object{}});
+                        if (not dosguard_.get().request(connectionMetadata.ip(), parsedObject)) {
+                            response = makeSlowDownResponse(request, parsedObject);
+                        } else {
+                            LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue";
 
-                        response = handleRequest(
-                            innerYield,
-                            request,
-                            std::move(parsedObject),
-                            connectionMetadata,
-                            std::move(subscriptionContext)
-                        );
+                            if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject))
+                                parsedObject[JS(params)] = boost::json::array({boost::json::object{}});
+
+                            response = handleRequest(
+                                innerYield,
+                                request,
+                                std::move(parsedObject),
+                                connectionMetadata,
+                                std::move(subscriptionContext)
+                            );
+                        }
                     }
                 } catch (std::exception const& ex) {
                     LOG(perfLog_.error()) << connectionMetadata.tag() << "Caught exception: " << ex.what();
@@ -176,6 +191,11 @@ public:
         // Put the coroutine to sleep until the foreign task is done
         coroutineGroup.asyncWait(yield);
         ASSERT(response.has_value(), "Woke up coroutine without setting response");
+
+        if (not dosguard_.get().add(connectionMetadata.ip(), response->message().size())) {
+            response->setMessage(makeLoadWarning(*response));
+        }
+
         return std::move(response).value();
     }
 
@@ -240,7 +260,7 @@ private:
             auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); });
 
             auto us = std::chrono::duration<int, std::milli>(timeDiff);
-            rpc::logDuration(*context, us);
+            rpc::logDuration(request, context->tag(), us);
 
             boost::json::object response;
 
@@ -315,6 +335,39 @@ private:
         }
     }
 
+    static Response
+    makeSlowDownResponse(Request const& request, std::optional<boost::json::value> requestJson)
+    {
+        auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN);
+
+        if (not request.isHttp()) {
+            try {
+                if (not requestJson.has_value()) {
+                    requestJson = boost::json::parse(request.message());
+                }
+                if (requestJson->is_object() && requestJson->as_object().contains("id"))
+                    error["id"] = requestJson->as_object().at("id");
+                error["request"] = request.message();
+            } catch (std::exception const&) {
+                error["request"] = request.message();
+            }
+        }
+        return web::ng::Response{boost::beast::http::status::service_unavailable, error, request};
+    }
+
+    static boost::json::object
+    makeLoadWarning(Response const& response)
+    {
+        auto jsonResponse = boost::json::parse(response.message()).as_object();
+        jsonResponse["warning"] = "load";
+        if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) {
+            jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit));
+        } else {
+            jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)};
+        }
+        return jsonResponse;
+    }
+
     bool
     shouldReplaceParams(boost::json::object const& req) const
     {

src/web/ng/Server.cpp

@@ -46,6 +46,7 @@
 #include <boost/system/system_error.hpp>
 #include <fmt/core.h>
 
+#include <chrono>
 #include <cstddef>
 #include <functional>
 #include <memory>
@@ -136,13 +137,19 @@ makeConnection(
         if (not sslContext.has_value())
             return std::unexpected{"Error creating a connection: SSL is not supported by this server"};
 
-        connection = std::make_unique<impl::SslHttpConnection>(
+        auto sslConnection = std::make_unique<impl::SslHttpConnection>(
             std::move(sslDetectionResult.socket),
             std::move(ip),
             std::move(sslDetectionResult.buffer),
             *sslContext,
             tagDecoratorFactory
         );
+        sslConnection->setTimeout(std::chrono::seconds{10});
+        auto const maybeError = sslConnection->sslHandshake(yield);
+        if (maybeError.has_value())
+            return std::unexpected{fmt::format("SSL handshake error: {}", maybeError->message())};
+
+        connection = std::move(sslConnection);
     } else {
         connection = std::make_unique<impl::PlainHttpConnection>(
             std::move(sslDetectionResult.socket),
@@ -164,7 +171,6 @@ makeConnection(
 std::expected<ConnectionPtr, std::string>
 tryUpgradeConnection(
     impl::UpgradableConnectionPtr connection,
-    std::optional<boost::asio::ssl::context>& sslContext,
     util::TagDecoratorFactory& tagDecoratorFactory,
     boost::asio::yield_context yield
 )
@@ -177,7 +183,7 @@ tryUpgradeConnection(
     }
 
     if (*expectedIsUpgrade) {
-        auto expectedUpgradedConnection = connection->upgrade(sslContext, tagDecoratorFactory, yield);
+        auto expectedUpgradedConnection = connection->upgrade(tagDecoratorFactory, yield);
         if (expectedUpgradedConnection.has_value())
             return std::move(expectedUpgradedConnection).value();
 
@@ -316,8 +322,7 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield
         return;
     }
 
-    auto connection =
-        tryUpgradeConnection(std::move(connectionExpected).value(), sslContext_, tagDecoratorFactory_, yield);
+    auto connection = tryUpgradeConnection(std::move(connectionExpected).value(), tagDecoratorFactory_, yield);
     if (not connection.has_value()) {
         LOG(log_.info()) << connection.error();
         return;

src/web/ng/impl/ConnectionHandler.cpp

@@ -86,24 +86,6 @@ handleWsRequest(
 
 }  // namespace
 
-size_t
-ConnectionHandler::StringHash::operator()(char const* str) const
-{
-    return hash_type{}(str);
-}
-
-size_t
-ConnectionHandler::StringHash::operator()(std::string_view str) const
-{
-    return hash_type{}(str);
-}
-
-size_t
-ConnectionHandler::StringHash::operator()(std::string const& str) const
-{
-    return hash_type{}(str);
-}
-
 ConnectionHandler::ConnectionHandler(
     ProcessingPolicy processingPolicy,
     std::optional<size_t> maxParallelRequests,

src/web/ng/impl/ConnectionHandler.hpp

@@ -20,6 +20,7 @@
 #pragma once
 
 #include "util/StopHelper.hpp"
+#include "util/StringHash.hpp"
 #include "util/Taggable.hpp"
 #include "util/log/Logger.hpp"
 #include "util/prometheus/Gauge.hpp"
@@ -44,7 +45,6 @@
 #include <memory>
 #include <optional>
 #include <string>
-#include <string_view>
 #include <unordered_map>
 
 namespace web::ng::impl {
@@ -52,20 +52,7 @@ namespace web::ng::impl {
 class ConnectionHandler {
 public:
     using OnDisconnectHook = std::function<void(Connection const&)>;
-
-    struct StringHash {
-        using hash_type = std::hash<std::string_view>;
-        using is_transparent = void;
-
-        std::size_t
-        operator()(char const* str) const;
-        std::size_t
-        operator()(std::string_view str) const;
-        std::size_t
-        operator()(std::string const& str) const;
-    };
-
-    using TargetToHandlerMap = std::unordered_map<std::string, MessageHandler, StringHash, std::equal_to<>>;
+    using TargetToHandlerMap = std::unordered_map<std::string, MessageHandler, util::StringHash, std::equal_to<>>;
 
 private:
     util::Logger log_{"WebServer"};

src/web/ng/impl/HttpConnection.hpp

@@ -28,10 +28,12 @@
 #include "web/ng/impl/Concepts.hpp"
 #include "web/ng/impl/WsConnection.hpp"
 
+#include <boost/asio/buffer.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/asio/ssl/context.hpp>
 #include <boost/asio/ssl/stream.hpp>
+#include <boost/asio/ssl/stream_base.hpp>
 #include <boost/beast/core/basic_stream.hpp>
 #include <boost/beast/core/error.hpp>
 #include <boost/beast/core/flat_buffer.hpp>
@@ -57,11 +59,7 @@ public:
     isUpgradeRequested(boost::asio::yield_context yield) = 0;
 
     virtual std::expected<ConnectionPtr, Error>
-    upgrade(
-        std::optional<boost::asio::ssl::context>& sslContext,
-        util::TagDecoratorFactory const& tagDecoratorFactory,
-        boost::asio::yield_context yield
-    ) = 0;
+    upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) = 0;
 
     virtual std::optional<Error>
     sendRaw(
@@ -104,6 +102,22 @@ public:
     {
     }
 
+    std::optional<Error>
+    sslHandshake(boost::asio::yield_context yield)
+        requires IsSslTcpStream<StreamType>
+    {
+        boost::system::error_code error;
+        boost::beast::get_lowest_layer(stream_).expires_after(timeout_);
+        auto const bytesUsed =
+            stream_.async_handshake(boost::asio::ssl::stream_base::server, buffer_.cdata(), yield[error]);
+        if (error)
+            return error;
+
+        buffer_.consume(bytesUsed);
+
+        return std::nullopt;
+    }
+
     bool
     wasUpgraded() const override
     {
@@ -183,35 +197,18 @@ public:
     }
 
     std::expected<ConnectionPtr, Error>
-    upgrade(
-        [[maybe_unused]] std::optional<boost::asio::ssl::context>& sslContext,
-        util::TagDecoratorFactory const& tagDecoratorFactory,
-        boost::asio::yield_context yield
-    ) override
+    upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) override
     {
         ASSERT(request_.has_value(), "Request must be present to upgrade the connection");
 
-        if constexpr (IsSslTcpStream<StreamType>) {
-            ASSERT(sslContext.has_value(), "SSL context must be present to upgrade the connection");
-            return makeSslWsConnection(
-                boost::beast::get_lowest_layer(stream_).release_socket(),
-                std::move(ip_),
-                std::move(buffer_),
-                std::move(request_).value(),
-                sslContext.value(),
-                tagDecoratorFactory,
-                yield
-            );
-        } else {
-            return makePlainWsConnection(
-                stream_.release_socket(),
-                std::move(ip_),
-                std::move(buffer_),
-                std::move(request_).value(),
-                tagDecoratorFactory,
-                yield
-            );
-        }
+        return makeWsConnection(
+            std::move(stream_),
+            std::move(ip_),
+            std::move(buffer_),
+            std::move(request_).value(),
+            tagDecoratorFactory,
+            yield
+        );
     }
 
 private:

src/web/ng/impl/WsConnection.cpp

@@ -1,77 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    This file is part of clio: https://github.com/XRPLF/clio
-    Copyright (c) 2024, the clio developers.
-
-    Permission to use, copy, modify, and distribute this software for any
-    purpose with or without fee is hereby granted, provided that the above
-    copyright notice and this permission notice appear in all copies.
-
-    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
-    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
-    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-//==============================================================================
-
-#include "web/ng/impl/WsConnection.hpp"
-
-#include "util/Taggable.hpp"
-#include "web/ng/Error.hpp"
-
-#include <boost/asio/ip/tcp.hpp>
-#include <boost/asio/spawn.hpp>
-#include <boost/asio/ssl/context.hpp>
-#include <boost/beast/core/flat_buffer.hpp>
-#include <boost/beast/http/message.hpp>
-#include <boost/beast/http/string_body.hpp>
-
-#include <memory>
-#include <string>
-#include <utility>
-
-namespace web::ng::impl {
-
-std::expected<std::unique_ptr<PlainWsConnection>, Error>
-makePlainWsConnection(
-    boost::asio::ip::tcp::socket socket,
-    std::string ip,
-    boost::beast::flat_buffer buffer,
-    boost::beast::http::request<boost::beast::http::string_body> request,
-    util::TagDecoratorFactory const& tagDecoratorFactory,
-    boost::asio::yield_context yield
-)
-{
-    auto connection = std::make_unique<PlainWsConnection>(
-        std::move(socket), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
-    );
-    auto maybeError = connection->performHandshake(yield);
-    if (maybeError.has_value())
-        return std::unexpected{maybeError.value()};
-    return connection;
-}
-
-std::expected<std::unique_ptr<SslWsConnection>, Error>
-makeSslWsConnection(
-    boost::asio::ip::tcp::socket socket,
-    std::string ip,
-    boost::beast::flat_buffer buffer,
-    boost::beast::http::request<boost::beast::http::string_body> request,
-    boost::asio::ssl::context& sslContext,
-    util::TagDecoratorFactory const& tagDecoratorFactory,
-    boost::asio::yield_context yield
-)
-{
-    auto connection = std::make_unique<SslWsConnection>(
-        std::move(socket), std::move(ip), std::move(buffer), sslContext, std::move(request), tagDecoratorFactory
-    );
-    auto maybeError = connection->performHandshake(yield);
-    if (maybeError.has_value())
-        return std::unexpected{maybeError.value()};
-    return connection;
-}
-
-}  // namespace web::ng::impl

src/web/ng/impl/WsConnection.hpp

@@ -68,31 +68,14 @@ class WsConnection : public WsConnectionBase {
 
 public:
     WsConnection(
-        boost::asio::ip::tcp::socket socket,
+        StreamType&& stream,
         std::string ip,
         boost::beast::flat_buffer buffer,
         boost::beast::http::request<boost::beast::http::string_body> initialRequest,
         util::TagDecoratorFactory const& tagDecoratorFactory
     )
-        requires IsTcpStream<StreamType>
         : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
-        , stream_(std::move(socket))
-        , initialRequest_(std::move(initialRequest))
-    {
-        setupWsStream();
-    }
-
-    WsConnection(
-        boost::asio::ip::tcp::socket socket,
-        std::string ip,
-        boost::beast::flat_buffer buffer,
-        boost::asio::ssl::context& sslContext,
-        boost::beast::http::request<boost::beast::http::string_body> initialRequest,
-        util::TagDecoratorFactory const& tagDecoratorFactory
-    )
-        requires IsSslTcpStream<StreamType>
-        : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
-        , stream_(std::move(socket), sslContext)
+        , stream_(std::move(stream))
         , initialRequest_(std::move(initialRequest))
     {
         setupWsStream();
@@ -189,25 +172,24 @@ private:
 using PlainWsConnection = WsConnection<boost::beast::tcp_stream>;
 using SslWsConnection = WsConnection<boost::asio::ssl::stream<boost::beast::tcp_stream>>;
 
-std::expected<std::unique_ptr<PlainWsConnection>, Error>
-makePlainWsConnection(
-    boost::asio::ip::tcp::socket socket,
+template <typename StreamType>
+std::expected<std::unique_ptr<WsConnection<StreamType>>, Error>
+makeWsConnection(
+    StreamType&& stream,
     std::string ip,
     boost::beast::flat_buffer buffer,
     boost::beast::http::request<boost::beast::http::string_body> request,
     util::TagDecoratorFactory const& tagDecoratorFactory,
     boost::asio::yield_context yield
-);
-
-std::expected<std::unique_ptr<SslWsConnection>, Error>
-makeSslWsConnection(
-    boost::asio::ip::tcp::socket socket,
-    std::string ip,
-    boost::beast::flat_buffer buffer,
-    boost::beast::http::request<boost::beast::http::string_body> request,
-    boost::asio::ssl::context& sslContext,
-    util::TagDecoratorFactory const& tagDecoratorFactory,
-    boost::asio::yield_context yield
-);
+)
+{
+    auto connection = std::make_unique<WsConnection<StreamType>>(
+        std::forward<StreamType>(stream), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
+    );
+    auto maybeError = connection->performHandshake(yield);
+    if (maybeError.has_value())
+        return std::unexpected{maybeError.value()};
+    return connection;
+}
 
 }  // namespace web::ng::impl

tests/common/util/BinaryTestObject.cpp

@@ -19,134 +19,31 @@
 
 #include "util/BinaryTestObject.hpp"
 
+#include "data/DBHelpers.hpp"
 #include "etlng/Models.hpp"
 #include "etlng/impl/Extraction.hpp"
 #include "util/StringUtils.hpp"
+#include "util/TestObject.hpp"
 
 #include <gtest/gtest.h>
 #include <org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/protocol/AccountID.h>
 #include <xrpl/protocol/STTx.h>
 #include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/TxFormats.h>
 #include <xrpl/protocol/TxMeta.h>
 
+#include <optional>
 #include <string>
 #include <utility>
 
 namespace {
 
 constinit auto const kSEQ = 30;
-
-constinit auto const kTXN_HEX =
-    "1200192200000008240011CC9B201B001F71D6202A0000000168400000"
-    "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
-    "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
-    "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
-    "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
-    "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
-    "DE3538F248662FC73C";
-
-constinit auto const kTXN_META =
-    "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
-    "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
-    "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
-    "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
-    "0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
-    "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
-    "C200000028751868747470733A2F2F677265677765697362726F642E63"
-    "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
-    "9808B6B90000001D751868747470733A2F2F677265677765697362726F"
-    "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
-    "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
-    "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
-    "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
-    "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
-    "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
-    "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
-    "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
-    "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
-    "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
-    "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
-    "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
-    "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
-    "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
-    "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
-    "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
-    "00001F751868747470733A2F2F677265677765697362726F642E636F6D"
-    "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
-    "5DAE00000014751868747470733A2F2F677265677765697362726F642E"
-    "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
-    "3CCE1462A500000009751868747470733A2F2F67726567776569736272"
-    "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
-    "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
-    "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
-    "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
-    "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
-    "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
-    "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
-    "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
-    "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
-    "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
-    "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
-    "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
-    "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
-    "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
-    "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
-    "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
-    "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
-    "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
-    "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
-    "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
-    "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
-    "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
-    "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
-    "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
-    "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
-    "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
-    "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
-    "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
-    "00000029751868747470733A2F2F677265677765697362726F642E636F"
-    "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
-    "EE87B80000001E751868747470733A2F2F677265677765697362726F64"
-    "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
-    "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
-    "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
-    "48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
-    "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
-    "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
-    "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
-    "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
-    "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
-    "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
-    "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
-    "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
-    "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
-    "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
-    "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
-    "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
-    "0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
-    "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
-    "B100000015751868747470733A2F2F677265677765697362726F642E63"
-    "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
-    "E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
-    "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
-    "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
-    "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
-    "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
-    "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
-    "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
-    "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
-    "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
-    "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
-    "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
-    "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
-    "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
-    "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
-    "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
-    "2FC73CE1E1F1031000";
-
 constinit auto const kRAW_HEADER =
     "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
     "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
@@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER =
 namespace util {
 
 std::pair<std::string, std::string>
-createNftTxAndMetaBlobs()
+createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr)
 {
-    return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)};
+    return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)};
 }
 
 std::pair<ripple::STTx, ripple::TxMeta>
-createNftTxAndMeta()
+createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr)
 {
     ripple::uint256 hash;
-    EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
+    EXPECT_TRUE(hash.parseHex(hashStr));
 
-    auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs();
+    auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr);
 
     ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
     return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}};
 }
 
 etlng::model::Transaction
-createTransaction(ripple::TxType type)
+createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr)
 {
-    auto const [sttx, meta] = createNftTxAndMeta();
+    auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr);
     return {
         .raw = "",
         .metaRaw = "",
@@ -192,10 +89,9 @@ createTransaction(ripple::TxType type)
 }
 
 etlng::model::Object
-createObject()
+createObject(etlng::model::Object::ModType modType, std::string key)
 {
     // random object taken from initial ledger load
-    static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
     static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
     static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
     static constinit auto const kOBJ_BLOB =
@@ -205,12 +101,62 @@ createObject()
         "8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
 
     return {
-        .key = {},
-        .keyRaw = hexStringToBinaryString(kOBJ_KEY),
-        .data = {},
-        .dataRaw = hexStringToBinaryString(kOBJ_BLOB),
+        .key = binaryStringToUint256(hexStringToBinaryString(key)),
+        .keyRaw = hexStringToBinaryString(key),
+        .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
+        .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
         .successor = hexStringToBinaryString(kOBJ_SUCC),
         .predecessor = hexStringToBinaryString(kOBJ_PRED),
+        .type = modType,
+    };
+}
+
+etlng::model::Object
+createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key)
+{
+    // random object taken from initial ledger load
+    static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
+    static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
+    static constinit auto const kOBJ_BLOB =
+        "11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE"
+        "3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000"
+        "0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3";
+
+    return {
+        .key = binaryStringToUint256(hexStringToBinaryString(key)),
+        .keyRaw = hexStringToBinaryString(key),
+        .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
+        .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
+        .successor = hexStringToBinaryString(kOBJ_SUCC),
+        .predecessor = hexStringToBinaryString(kOBJ_PRED),
+        .type = modType,
+    };
+}
+
+etlng::model::Object
+createObjectWithTwoNFTs()
+{
+    std::string const url1 = "abcd1";
+    std::string const url2 = "abcd2";
+    ripple::Blob const uri1Blob(url1.begin(), url1.end());
+    ripple::Blob const uri2Blob(url2.begin(), url2.end());
+
+    constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q";
+    constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000";
+    constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA";
+
+    auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt);
+    auto const serializerNftPage = nftPage.getSerializer();
+
+    auto const account = getAccountIdWithString(kACCOUNT);
+    return {
+        .key = {},
+        .keyRaw = std::string(reinterpret_cast<char const*>(account.data()), ripple::AccountID::size()),
+        .data = {},
+        .dataRaw =
+            std::string(static_cast<char const*>(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()),
+        .successor = "",
+        .predecessor = "",
         .type = etlng::model::Object::ModType::Created,
     };
 }
@@ -219,8 +165,10 @@ etlng::model::BookSuccessor
 createSuccessor()
 {
     return {
-        .firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
-        .bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
+        .firstBook =
+            uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}),
+        .bookBase =
+            uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}),
     };
 }
 

tests/common/util/BinaryTestObject.hpp

@@ -32,17 +32,149 @@
 
 namespace util {
 
+static constexpr auto kDEFAULT_TXN_HEX =
+    "1200192200000008240011CC9B201B001F71D6202A0000000168400000"
+    "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
+    "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
+    "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
+    "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
+    "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
+    "DE3538F248662FC73C";
+
+static constexpr auto kDEFAULT_TXN_META =
+    "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
+    "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
+    "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
+    "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
+    "0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
+    "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
+    "C200000028751868747470733A2F2F677265677765697362726F642E63"
+    "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
+    "9808B6B90000001D751868747470733A2F2F677265677765697362726F"
+    "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
+    "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
+    "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
+    "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
+    "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
+    "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
+    "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
+    "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
+    "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
+    "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
+    "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
+    "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
+    "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
+    "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
+    "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
+    "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
+    "00001F751868747470733A2F2F677265677765697362726F642E636F6D"
+    "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
+    "5DAE00000014751868747470733A2F2F677265677765697362726F642E"
+    "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
+    "3CCE1462A500000009751868747470733A2F2F67726567776569736272"
+    "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
+    "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
+    "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
+    "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
+    "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
+    "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
+    "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
+    "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
+    "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
+    "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
+    "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
+    "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
+    "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
+    "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
+    "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
+    "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
+    "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
+    "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
+    "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
+    "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
+    "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
+    "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
+    "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
+    "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
+    "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
+    "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
+    "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
+    "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
+    "00000029751868747470733A2F2F677265677765697362726F642E636F"
+    "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
+    "EE87B80000001E751868747470733A2F2F677265677765697362726F64"
+    "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
+    "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
+    "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
+    "48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
+    "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
+    "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
+    "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
+    "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
+    "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
+    "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
+    "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
+    "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
+    "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
+    "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
+    "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
+    "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
+    "0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
+    "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
+    "B100000015751868747470733A2F2F677265677765697362726F642E63"
+    "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
+    "E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
+    "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
+    "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
+    "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
+    "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
+    "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
+    "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
+    "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
+    "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
+    "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
+    "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
+    "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
+    "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
+    "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
+    "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
+    "2FC73CE1E1F1031000";
+
+static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E";
+static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
+
 [[maybe_unused, nodiscard]] std::pair<std::string, std::string>
-createNftTxAndMetaBlobs();
+createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX);
 
 [[maybe_unused, nodiscard]] std::pair<ripple::STTx, ripple::TxMeta>
-createNftTxAndMeta();
+createNftTxAndMeta(
+    std::string hashStr = kDEFAULT_HASH,
+    std::string metaStr = kDEFAULT_TXN_META,
+    std::string txnStr = kDEFAULT_TXN_HEX
+);
 
 [[maybe_unused, nodiscard]] etlng::model::Transaction
-createTransaction(ripple::TxType type);
+createTransaction(
+    ripple::TxType type,
+    std::string hashStr = kDEFAULT_HASH,
+    std::string metaStr = kDEFAULT_TXN_META,
+    std::string txnStr = kDEFAULT_TXN_HEX
+);
 
 [[maybe_unused, nodiscard]] etlng::model::Object
-createObject();
+createObject(
+    etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
+    std::string key = kDEFAULT_OBJ_KEY
+);
+
+[[maybe_unused, nodiscard]] etlng::model::Object
+createObjectWithBookBase(
+    etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
+    std::string key = kDEFAULT_OBJ_KEY
+);
+
+[[maybe_unused, nodiscard]] etlng::model::Object
+createObjectWithTwoNFTs();
 
 [[maybe_unused, nodiscard]] etlng::model::BookSuccessor
 createSuccessor();

tests/common/util/MockBackend.hpp

@@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface {
 
     MOCK_METHOD(void, writeAccountTransactions, (std::vector<AccountTransactionsData>), (override));
 
+    MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override));
+
     MOCK_METHOD(void, writeNFTTransactions, (std::vector<NFTTransactionsData> const&), (override));
 
     MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override));

tests/common/util/MockETLService.hpp

@@ -20,21 +20,21 @@
 #pragma once
 
 #include "etl/ETLState.hpp"
+#include "etlng/ETLServiceInterface.hpp"
 
 #include <boost/json.hpp>
 #include <boost/json/object.hpp>
 #include <gmock/gmock.h>
 
-#include <chrono>
 #include <cstdint>
 #include <optional>
 
-struct MockETLService {
-    MOCK_METHOD(boost::json::object, getInfo, (), (const));
-    MOCK_METHOD(std::chrono::time_point<std::chrono::system_clock>, getLastPublish, (), (const));
-    MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const));
-    MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const));
-    MOCK_METHOD(bool, isAmendmentBlocked, (), (const));
-    MOCK_METHOD(bool, isCorruptionDetected, (), (const));
-    MOCK_METHOD(std::optional<etl::ETLState>, getETLState, (), (const));
+struct MockETLService : etlng::ETLServiceInterface {
+    MOCK_METHOD(void, run, (), (override));
+    MOCK_METHOD(void, stop, (), (override));
+    MOCK_METHOD(boost::json::object, getInfo, (), (const, override));
+    MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const, override));
+    MOCK_METHOD(bool, isAmendmentBlocked, (), (const, override));
+    MOCK_METHOD(bool, isCorruptionDetected, (), (const, override));
+    MOCK_METHOD(std::optional<etl::ETLState>, getETLState, (), (const, override));
 };

tests/common/util/MockLedgerCache.hpp

@@ -21,6 +21,7 @@
 
 #include "data/LedgerCacheInterface.hpp"
 #include "data/Types.hpp"
+#include "etlng/Models.hpp"
 
 #include <gmock/gmock.h>
 #include <xrpl/basics/base_uint.h>
@@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface {
 
     MOCK_METHOD(std::optional<data::Blob>, get, (ripple::uint256 const& a, uint32_t b), (const, override));
 
+    MOCK_METHOD(void, update, (std::vector<etlng::model::Object> const&, uint32_t), (override));
+
+    MOCK_METHOD(std::optional<data::Blob>, getDeleted, (ripple::uint256 const&, uint32_t), (const, override));
+
     MOCK_METHOD(
         std::optional<data::LedgerObject>,
         getSuccessor,

tests/common/util/MockLoadBalancer.hpp

@@ -38,22 +38,6 @@
 #include <string>
 #include <vector>
 
-struct MockLoadBalancer {
-    using RawLedgerObjectType = FakeLedgerObject;
-
-    MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ());
-    MOCK_METHOD(std::optional<FakeFetchResponse>, fetchLedger, (uint32_t, bool, bool), ());
-    MOCK_METHOD(boost::json::value, toJson, (), (const));
-
-    using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
-    MOCK_METHOD(
-        ForwardToRippledReturnType,
-        forwardToRippled,
-        (boost::json::object const&, std::optional<std::string> const&, bool, boost::asio::yield_context),
-        (const)
-    );
-};
-
 struct MockNgLoadBalancer : etlng::LoadBalancerInterface {
     using RawLedgerObjectType = FakeLedgerObject;
 
@@ -85,4 +69,7 @@ struct MockNgLoadBalancer : etlng::LoadBalancerInterface {
         (boost::json::object const&, std::optional<std::string> const&, bool, boost::asio::yield_context),
         (override)
     );
+    MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
 };
+
+using MockLoadBalancer = MockNgLoadBalancer;

tests/common/util/MockSource.hpp

@@ -60,7 +60,7 @@ struct MockSource : etl::SourceBase {
         (uint32_t, bool, bool),
         (override)
     );
-    MOCK_METHOD((std::pair<std::vector<std::string>, bool>), loadInitialLedger, (uint32_t, uint32_t, bool), (override));
+    MOCK_METHOD((std::pair<std::vector<std::string>, bool>), loadInitialLedger, (uint32_t, uint32_t), (override));
 
     using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
     MOCK_METHOD(
@@ -132,9 +132,9 @@ public:
     }
 
     std::pair<std::vector<std::string>, bool>
-    loadInitialLedger(uint32_t sequence, uint32_t maxLedger, bool getObjects) override
+    loadInitialLedger(uint32_t sequence, uint32_t maxLedger) override
     {
-        return mock_->loadInitialLedger(sequence, maxLedger, getObjects);
+        return mock_->loadInitialLedger(sequence, maxLedger);
     }
 
     std::expected<boost::json::object, rpc::ClioError>

tests/common/util/MockSourceNg.hpp

@@ -0,0 +1,245 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2023, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+#pragma once
+
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/Source.hpp"
+#include "feed/SubscriptionManagerInterface.hpp"
+#include "rpc/Errors.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/uuid/uuid.hpp>
+#include <gmock/gmock.h>
+#include <grpcpp/support/status.h>
+#include <gtest/gtest.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <expected>
+#include <memory>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+struct MockSourceNg : etlng::SourceBase {
+    MOCK_METHOD(void, run, (), (override));
+    MOCK_METHOD(void, stop, (boost::asio::yield_context), (override));
+    MOCK_METHOD(bool, isConnected, (), (const, override));
+    MOCK_METHOD(void, setForwarding, (bool), (override));
+    MOCK_METHOD(boost::json::object, toJson, (), (const, override));
+    MOCK_METHOD(std::string, toString, (), (const, override));
+    MOCK_METHOD(bool, hasLedger, (uint32_t), (const, override));
+    MOCK_METHOD(
+        (std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>),
+        fetchLedger,
+        (uint32_t, bool, bool),
+        (override)
+    );
+    MOCK_METHOD(
+        (std::pair<std::vector<std::string>, bool>),
+        loadInitialLedger,
+        (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&),
+        (override)
+    );
+
+    using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
+    MOCK_METHOD(
+        ForwardToRippledReturnType,
+        forwardToRippled,
+        (boost::json::object const&, std::optional<std::string> const&, std::string_view, boost::asio::yield_context),
+        (const, override)
+    );
+};
+
+template <template <typename> typename MockType>
+using MockSourceNgPtr = std::shared_ptr<MockType<MockSourceNg>>;
+
+template <template <typename> typename MockType>
+class MockSourceNgWrapper : public etlng::SourceBase {
+    MockSourceNgPtr<MockType> mock_;
+
+public:
+    MockSourceNgWrapper(MockSourceNgPtr<MockType> mockData) : mock_(std::move(mockData))
+    {
+    }
+
+    void
+    run() override
+    {
+        mock_->run();
+    }
+
+    void
+    stop(boost::asio::yield_context yield) override
+    {
+        mock_->stop(yield);
+    }
+
+    bool
+    isConnected() const override
+    {
+        return mock_->isConnected();
+    }
+
+    void
+    setForwarding(bool isForwarding) override
+    {
+        mock_->setForwarding(isForwarding);
+    }
+
+    boost::json::object
+    toJson() const override
+    {
+        return mock_->toJson();
+    }
+
+    std::string
+    toString() const override
+    {
+        return mock_->toString();
+    }
+
+    bool
+    hasLedger(uint32_t sequence) const override
+    {
+        return mock_->hasLedger(sequence);
+    }
+
+    std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+    fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors) override
+    {
+        return mock_->fetchLedger(sequence, getObjects, getObjectNeighbors);
+    }
+
+    std::pair<std::vector<std::string>, bool>
+    loadInitialLedger(uint32_t sequence, uint32_t maxLedger, etlng::InitialLoadObserverInterface& observer) override
+    {
+        return mock_->loadInitialLedger(sequence, maxLedger, observer);
+    }
+
+    std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& forwardToRippledClientIp,
+        std::string_view xUserValue,
+        boost::asio::yield_context yield
+    ) const override
+    {
+        return mock_->forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield);
+    }
+};
+
+struct MockSourceNgCallbacks {
+    etlng::SourceBase::OnDisconnectHook onDisconnect;
+    etlng::SourceBase::OnConnectHook onConnect;
+    etlng::SourceBase::OnLedgerClosedHook onLedgerClosed;
+};
+
+template <template <typename> typename MockType>
+struct MockSourceNgData {
+    MockSourceNgPtr<MockType> source = std::make_shared<MockType<MockSourceNg>>();
+    std::optional<MockSourceNgCallbacks> callbacks;
+};
+
+template <template <typename> typename MockType = testing::NiceMock>
+class MockSourceNgFactoryImpl {
+    std::vector<MockSourceNgData<MockType>> mockData_;
+
+public:
+    MockSourceNgFactoryImpl(size_t numSources)
+    {
+        setSourcesNumber(numSources);
+
+        ON_CALL(*this, makeSource)
+            .WillByDefault([this](
+                               util::config::ObjectView const&,
+                               boost::asio::io_context&,
+                               std::shared_ptr<feed::SubscriptionManagerInterface>,
+                               std::shared_ptr<etl::NetworkValidatedLedgersInterface>,
+                               std::chrono::steady_clock::duration,
+                               etlng::SourceBase::OnConnectHook onConnect,
+                               etlng::SourceBase::OnDisconnectHook onDisconnect,
+                               etlng::SourceBase::OnLedgerClosedHook onLedgerClosed
+                           ) {
+                auto it = std::ranges::find_if(mockData_, [](auto const& d) { return not d.callbacks.has_value(); });
+                [&]() { ASSERT_NE(it, mockData_.end()) << "Make source called more than expected"; }();
+                it->callbacks = MockSourceNgCallbacks{
+                    .onDisconnect = std::move(onDisconnect),
+                    .onConnect = std::move(onConnect),
+                    .onLedgerClosed = std::move(onLedgerClosed)
+                };
+
+                return std::make_unique<MockSourceNgWrapper<MockType>>(it->source);
+            });
+    }
+
+    void
+    setSourcesNumber(size_t numSources)
+    {
+        mockData_.clear();
+        mockData_.reserve(numSources);
+        std::ranges::generate_n(std::back_inserter(mockData_), numSources, [] { return MockSourceNgData<MockType>{}; });
+    }
+
+    template <typename... Args>
+    etlng::SourcePtr
+    operator()(Args&&... args)
+    {
+        return makeSource(std::forward<Args>(args)...);
+    }
+
+    MOCK_METHOD(
+        etlng::SourcePtr,
+        makeSource,
+        (util::config::ObjectView const&,
+         boost::asio::io_context&,
+         std::shared_ptr<feed::SubscriptionManagerInterface>,
+         std::shared_ptr<etl::NetworkValidatedLedgersInterface>,
+         std::chrono::steady_clock::duration,
+         etlng::SourceBase::OnConnectHook,
+         etlng::SourceBase::OnDisconnectHook,
+         etlng::SourceBase::OnLedgerClosedHook)
+    );
+
+    MockType<MockSourceNg>&
+    sourceAt(size_t index)
+    {
+        return *mockData_.at(index).source;
+    }
+
+    MockSourceNgCallbacks&
+    callbacksAt(size_t index)
+    {
+        auto& callbacks = mockData_.at(index).callbacks;
+        [&]() { ASSERT_TRUE(callbacks.has_value()) << "Callbacks not set"; }();
+        return *callbacks;
+    }
+};
+
+using MockSourceNgFactory = testing::NiceMock<MockSourceNgFactoryImpl<>>;
+using StrictMockSourceNgFactory = testing::StrictMock<MockSourceNgFactoryImpl<testing::StrictMock>>;

tests/common/util/MockSubscriptionManager.hpp

@@ -103,6 +103,10 @@ struct MockSubscriptionManager : feed::SubscriptionManagerInterface {
 
     MOCK_METHOD(boost::json::object, report, (), (const, override));
 
+    MOCK_METHOD(void, setNetworkID, (uint32_t), (override));
+
+    MOCK_METHOD(uint32_t, getNetworkID, (), (const, override));
+
     MOCK_METHOD(void, stop, (), (override));
 };
 

tests/common/util/NameGenerator.hpp

@@ -19,8 +19,6 @@
 
 #pragma once
 
-#include <string>
-
 namespace tests::util {
 
 static auto const kNAME_GENERATOR = [](auto const& info) { return info.param.testName; };

tests/common/web/dosguard/DOSGuardMock.hpp

@@ -21,6 +21,7 @@
 
 #include "web/dosguard/DOSGuardInterface.hpp"
 
+#include <boost/json/object.hpp>
 #include <gmock/gmock.h>
 
 #include <cstdint>
@@ -33,7 +34,7 @@ struct DOSGuardMockImpl : web::dosguard::DOSGuardInterface {
     MOCK_METHOD(void, increment, (std::string const& ip), (noexcept, override));
     MOCK_METHOD(void, decrement, (std::string const& ip), (noexcept, override));
     MOCK_METHOD(bool, add, (std::string const& ip, uint32_t size), (noexcept, override));
-    MOCK_METHOD(bool, request, (std::string const& ip), (noexcept, override));
+    MOCK_METHOD(bool, request, (std::string const& ip, boost::json::object const& request), (override));
     MOCK_METHOD(void, clear, (), (noexcept, override));
 };