ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 20:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: ETLng integration (#1986)

Browse Source 
For #1594
This commit is contained in:
Alex Kremer
2025-04-04 15:52:22 +01:00
committed by GitHub
parent 6896a2545a
commit 1d011cf8d9
57 changed files with 3473 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/app/ClioApplication.cpp
View File

@@ -27,6 +27,8 @@
#include "etl/ETLService.hpp"
#include "etl/LoadBalancer.hpp"
#include "etl/NetworkValidatedLedgers.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManager.hpp"
#include "migration/MigrationInspectorFactory.hpp"
#include "rpc/Counters.hpp"
@@ -130,7 +132,12 @@ ClioApplication::run(bool const useNgWebServer)
// ETL uses the balancer to extract data.
// The server uses the balancer to forward RPCs to a rippled node.
// The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
auto balancer = [&] -> std::shared_ptr<etlng::LoadBalancerInterface> {
if (config_.get<bool>("__ng_etl"))
return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
}();
// ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
@@ -142,12 +149,12 @@ ClioApplication::run(bool const useNgWebServer)
config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
);
using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
using RPCEngineType = rpc::RPCEngine<rpc::Counters>;
auto const rpcEngine =
RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
web::ng::RPCServerHandler<RPCEngineType, etl::ETLService> handler{config_, backend, rpcEngine, etl};
web::ng::RPCServerHandler<RPCEngineType> handler{config_, backend, rpcEngine, etl};
auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
if (not expectedAdminVerifier.has_value()) {
@@ -188,8 +195,7 @@ ClioApplication::run(bool const useNgWebServer)
}
// Init the web server
auto handler =
std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(config_, backend, rpcEngine, etl);
auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler);

13
src/app/Stopper.hpp
View File

@@ -20,8 +20,8 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "etl/ETLService.hpp"
#include "etl/LoadBalancer.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/CoroutineGroup.hpp"
#include "util/log/Logger.hpp"
@@ -74,15 +74,12 @@ public:
* @param ioc The io_context to stop.
* @return The callback to be called on application stop.
*/
template <
web::ng::SomeServer ServerType,
etl::SomeLoadBalancer LoadBalancerType,
etl::SomeETLService ETLServiceType>
template <web::ng::SomeServer ServerType>
static std::function<void(boost::asio::yield_context)>
makeOnStopCallback(
ServerType& server,
LoadBalancerType& balancer,
ETLServiceType& etl,
etlng::LoadBalancerInterface& balancer,
etlng::ETLServiceInterface& etl,
feed::SubscriptionManagerInterface& subscriptions,
data::BackendInterface& backend,
boost::asio::io_context& ioc

4
src/etl/ETLService.cpp
View File

@@ -22,6 +22,7 @@
#include "data/BackendInterface.hpp"
#include "etl/CorruptionDetector.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/Constants.hpp"
@@ -43,6 +44,7 @@
#include <vector>
namespace etl {
// Database must be populated when this starts
std::optional<uint32_t>
ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
@@ -265,7 +267,7 @@ ETLService::ETLService(
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
)
: backend_(backend)

64
src/etl/ETLService.hpp
View File

@@ -32,7 +32,12 @@
#include "etl/impl/LedgerLoader.hpp"
#include "etl/impl/LedgerPublisher.hpp"
#include "etl/impl/Transformer.hpp"
#include "etlng/ETLService.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/io_context.hpp>
@@ -41,7 +46,6 @@
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <concepts>
#include <cstddef>
#include <cstdint>
#include <memory>
@@ -81,14 +85,13 @@ concept SomeETLService = std::derived_from<T, ETLServiceTag>;
* the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
* to writing and from writing to monitoring, based on the activity of other processes running on different machines.
*/
class ETLService : public ETLServiceTag {
class ETLService : public etlng::ETLServiceInterface, ETLServiceTag {
// TODO: make these template parameters in ETLService
using LoadBalancerType = LoadBalancer;
using DataPipeType = etl::impl::ExtractionDataPipe<org::xrpl::rpc::v1::GetLedgerResponse>;
using CacheLoaderType = etl::CacheLoader<>;
using LedgerFetcherType = etl::impl::LedgerFetcher<LoadBalancerType>;
using LedgerFetcherType = etl::impl::LedgerFetcher;
using ExtractorType = etl::impl::Extractor<DataPipeType, LedgerFetcherType>;
using LedgerLoaderType = etl::impl::LedgerLoader<LoadBalancerType, LedgerFetcherType>;
using LedgerLoaderType = etl::impl::LedgerLoader<LedgerFetcherType>;
using LedgerPublisherType = etl::impl::LedgerPublisher;
using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler;
using TransformerType =
@@ -97,7 +100,7 @@ class ETLService : public ETLServiceTag {
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
std::shared_ptr<NetworkValidatedLedgersInterface> networkValidatedLedgers_;
std::uint32_t extractorThreads_ = 1;
@@ -132,7 +135,7 @@ public:
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
);
@@ -154,20 +157,33 @@ public:
* @param ledgers The network validated ledgers datastructure
* @return A shared pointer to a new instance of ETLService
*/
static std::shared_ptr<ETLService>
static std::shared_ptr<etlng::ETLServiceInterface>
makeETLService(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<NetworkValidatedLedgersInterface> ledgers
)
{
auto etl = std::make_shared<ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
etl->run();
std::shared_ptr<etlng::ETLServiceInterface> ret;
return etl;
if (config.get<bool>("__ng_etl")) {
ASSERT(
std::dynamic_pointer_cast<etlng::LoadBalancer>(balancer),
"LoadBalancer type must be etlng::LoadBalancer"
);
ret = std::make_shared<etlng::ETLService>(config, backend, subscriptions, balancer, ledgers);
} else {
ASSERT(
std::dynamic_pointer_cast<etl::LoadBalancer>(balancer), "LoadBalancer type must be etl::LoadBalancer"
);
ret = std::make_shared<etl::ETLService>(config, ioc, backend, subscriptions, balancer, ledgers);
}
ret->run();
return ret;
}
/**
@@ -184,7 +200,7 @@ public:
* @note This method blocks until the ETL service has stopped.
*/
void
stop()
stop() override
{
LOG(log_.info()) << "Stop called";
@@ -203,7 +219,7 @@ public:
* @return Time passed since last ledger close
*/
std::uint32_t
lastCloseAgeSeconds() const
lastCloseAgeSeconds() const override
{
return ledgerPublisher_.lastCloseAgeSeconds();
}
@@ -214,7 +230,7 @@ public:
* @return true if currently amendment blocked; false otherwise
*/
bool
isAmendmentBlocked() const
isAmendmentBlocked() const override
{
return state_.isAmendmentBlocked;
}
@@ -225,7 +241,7 @@ public:
* @return true if corruption of DB was detected and cache was stopped.
*/
bool
isCorruptionDetected() const
isCorruptionDetected() const override
{
return state_.isCorruptionDetected;
}
@@ -236,7 +252,7 @@ public:
* @return The state of ETL as a JSON object
*/
boost::json::object
getInfo() const
getInfo() const override
{
boost::json::object result;
@@ -254,11 +270,17 @@ public:
* @return The etl nodes' state, nullopt if etl nodes are not connected
*/
std::optional<etl::ETLState>
getETLState() const noexcept
getETLState() const noexcept override
{
return loadBalancer_->getETLState();
}
/**
* @brief Start all components to run ETL service.
*/
void
run() override;
private:
/**
* @brief Run the ETL pipeline.
@@ -325,12 +347,6 @@ private:
return numMarkers_;
}
/**
* @brief Start all components to run ETL service.
*/
void
run();
/**
* @brief Spawn the worker thread and start monitoring.
*/

9
src/etl/LoadBalancer.cpp
View File

@@ -23,6 +23,7 @@
#include "etl/ETLState.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/Source.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
@@ -59,7 +60,7 @@ using namespace util::config;
namespace etl {
std::shared_ptr<LoadBalancer>
std::shared_ptr<etlng::LoadBalancerInterface>
LoadBalancer::makeLoadBalancer(
ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
@@ -175,12 +176,12 @@ LoadBalancer::~LoadBalancer()
}
std::vector<std::string>
LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly, std::chrono::steady_clock::duration retryAfter)
LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter)
{
std::vector<std::string> response;
execute(
[this, &response, &sequence, cacheOnly](auto& source) {
auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly);
[this, &response, &sequence](auto& source) {
auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_);
if (!res) {
LOG(log_.error()) << "Failed to download initial ledger."

45
src/etl/LoadBalancer.hpp
View File

@@ -23,8 +23,11 @@
#include "etl/ETLState.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/Source.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/ResponseExpirationCache.hpp"
#include "util/log/Logger.hpp"
@@ -41,13 +44,13 @@
#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <chrono>
#include <concepts>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace etl {
@@ -69,7 +72,7 @@ concept SomeLoadBalancer = std::derived_from<T, LoadBalancerTag>;
* which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
* allows requests for ledger data to be load balanced across all possible ETL sources.
*/
class LoadBalancer : public LoadBalancerTag {
class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag {
public:
using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
@@ -133,7 +136,7 @@ public:
* @param sourceFactory A factory function to create a source
* @return A shared pointer to a new instance of LoadBalancer
*/
static std::shared_ptr<LoadBalancer>
static std::shared_ptr<LoadBalancerInterface>
makeLoadBalancer(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
@@ -150,16 +153,32 @@ public:
* @note This function will retry indefinitely until the ledger is downloaded.
*
* @param sequence Sequence of ledger to download
* @param cacheOnly Whether to only write to cache and not to the DB; defaults to false
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::vector<std::string> The ledger data
*/
std::vector<std::string>
loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2})
override;
/**
* @brief Load the initial ledger, writing data to the queue.
* @note This function will retry indefinitely until the ledger is downloaded.
*
* @param sequence Sequence of ledger to download
* @param observer The observer to notify of progress
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::vector<std::string> The ledger data
*/
std::vector<std::string>
loadInitialLedger(
uint32_t sequence,
bool cacheOnly = false,
std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
);
[[maybe_unused]] uint32_t sequence,
[[maybe_unused]] etlng::InitialLoadObserverInterface& observer,
[[maybe_unused]] std::chrono::steady_clock::duration retryAfter
) override
{
ASSERT(false, "Not available for old ETL");
std::unreachable();
}
/**
* @brief Fetch data for a specific ledger.
@@ -180,7 +199,7 @@ public:
bool getObjects,
bool getObjectNeighbors,
std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
);
) override;
/**
* @brief Represent the state of this load balancer as a JSON object
@@ -188,7 +207,7 @@ public:
* @return JSON representation of the state of this load balancer.
*/
boost::json::value
toJson() const;
toJson() const override;
/**
* @brief Forward a JSON RPC request to a randomly selected rippled node.
@@ -205,14 +224,14 @@ public:
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
);
) override;
/**
* @brief Return state of ETL nodes.
* @return ETL state, nullopt if etl nodes not available
*/
std::optional<ETLState>
getETLState() noexcept;
getETLState() noexcept override;
/**
* @brief Stop the load balancer. This will stop all subscription sources.
@@ -221,7 +240,7 @@ public:
* @param yield The coroutine context
*/
void
stop(boost::asio::yield_context yield);
stop(boost::asio::yield_context yield) override;
private:
/**

1
src/etl/NetworkValidatedLedgersInterface.hpp
View File

@@ -26,6 +26,7 @@
#include <cstdint>
#include <optional>
namespace etl {
/**

5
src/etl/Source.hpp
View File

@@ -23,8 +23,6 @@
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/io_context.hpp>
@@ -130,11 +128,10 @@ public:
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param cacheOnly Only insert into cache, not the DB; defaults to false
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
virtual std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0;
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) = 0;
/**
* @brief Forward a request to rippled.

4
src/etl/impl/GrpcSource.cpp
View File

@@ -98,7 +98,7 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb
}
std::pair<std::vector<std::string>, bool>
GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers, bool const cacheOnly)
GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers)
{
if (!stub_)
return {{}, false};
@@ -130,7 +130,7 @@ GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers
LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix();
auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
auto result = ptr->process(stub_, cq, *backend_, abort);
if (result != etl::impl::AsyncCallData::CallStatus::MORE) {
++numFinished;
LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished;

3
src/etl/impl/GrpcSource.hpp
View File

@@ -60,11 +60,10 @@ public:
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param cacheOnly Only insert into cache, not the DB; defaults to false
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly = false);
loadInitialLedger(uint32_t sequence, uint32_t numMarkers);
};
} // namespace etl::impl

16
src/etl/impl/LedgerFetcher.hpp
View File

@@ -20,6 +20,8 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "util/log/Logger.hpp"
#include <grpcpp/grpcpp.h>
@@ -34,22 +36,18 @@ namespace etl::impl {
/**
* @brief GRPC Ledger data fetcher
*/
template <typename LoadBalancerType>
class LedgerFetcher {
public:
using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
class LedgerFetcher : public LedgerFetcherInterface {
private:
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
public:
/**
* @brief Create an instance of the fetcher
*/
LedgerFetcher(std::shared_ptr<BackendInterface> backend, std::shared_ptr<LoadBalancerType> balancer)
LedgerFetcher(std::shared_ptr<BackendInterface> backend, std::shared_ptr<etlng::LoadBalancerInterface> balancer)
: backend_(std::move(backend)), loadBalancer_(std::move(balancer))
{
}
@@ -64,7 +62,7 @@ public:
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
*/
[[nodiscard]] OptionalGetLedgerResponseType
fetchData(uint32_t sequence)
fetchData(uint32_t sequence) override
{
LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
@@ -84,7 +82,7 @@ public:
* @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
*/
[[nodiscard]] OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t sequence)
fetchDataAndDiff(uint32_t sequence) override
{
LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;

13
src/etl/impl/LedgerLoader.hpp
View File

@@ -26,6 +26,7 @@
#include "etl/NFTHelpers.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "util/Assert.hpp"
#include "util/LedgerUtils.hpp"
#include "util/Profiler.hpp"
@@ -65,18 +66,18 @@ namespace etl::impl {
/**
* @brief Loads ledger data into the DB
*/
template <typename LoadBalancerType, typename LedgerFetcherType>
template <typename LedgerFetcherType>
class LedgerLoader {
public:
using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType;
using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType;
using GetLedgerResponseType = etlng::LoadBalancerInterface::GetLedgerResponseType;
using OptionalGetLedgerResponseType = etlng::LoadBalancerInterface::OptionalGetLedgerResponseType;
using RawLedgerObjectType = etlng::LoadBalancerInterface::RawLedgerObjectType;
private:
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
std::shared_ptr<etlng::LoadBalancerInterface> loadBalancer_;
std::reference_wrapper<LedgerFetcherType> fetcher_;
std::reference_wrapper<SystemState const> state_; // shared state for ETL
@@ -86,7 +87,7 @@ public:
*/
LedgerLoader(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
LedgerFetcherType& fetcher,
SystemState const& state
)

4
src/etl/impl/SourceImpl.hpp
View File

@@ -202,9 +202,9 @@ public:
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) final
{
return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly);
return grpcSource_.loadInitialLedger(sequence, numMarkers);
}
/**

5
src/etlng/CMakeLists.txt
View File

@@ -2,10 +2,13 @@ add_library(clio_etlng)
target_sources(
clio_etlng
PRIVATE impl/AmendmentBlockHandler.cpp
PRIVATE LoadBalancer.cpp
Source.cpp
impl/AmendmentBlockHandler.cpp
impl/AsyncGrpcCall.cpp
impl/Extraction.cpp
impl/GrpcSource.cpp
impl/ForwardingSource.cpp
impl/Loading.cpp
impl/Monitor.cpp
impl/TaskManager.cpp

283
src/etlng/ETLService.hpp Normal file
View File

@@ -0,0 +1,283 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include "etl/CacheLoader.hpp"
#include "etl/ETLState.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etl/impl/LedgerPublisher.hpp"
#include "etlng/AmendmentBlockHandlerInterface.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etlng/impl/Extraction.hpp"
#include "etlng/impl/Loading.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etlng/impl/Registry.hpp"
#include "etlng/impl/Scheduling.hpp"
#include "etlng/impl/TaskManager.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/Assert.hpp"
#include "util/Profiler.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include "util/config/Config.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include <boost/json/object.hpp>
#include <fmt/core.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <chrono>
#include <cstdint>
#include <memory>
#include <optional>
#include <ranges>
#include <stdexcept>
#include <string>
#include <tuple>
#include <utility>
namespace etlng {
/**
* @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the
* databases.
*
* Usually, multiple different processes share access to the same network accessible databases, in which case only one
* such process is performing ETL and writing to the database. The other processes simply monitor the database for new
* ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the
* ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer.
*
* If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and
* the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
* to writing and from writing to monitoring, based on the activity of other processes running on different machines.
*/
class ETLService : public ETLServiceInterface {
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers_;
std::shared_ptr<etl::CacheLoader<>> cacheLoader_;
std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
std::shared_ptr<ExtractorInterface> extractor_;
etl::SystemState state_;
util::async::CoroExecutionContext ctx_{8};
std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
std::shared_ptr<impl::Loader> loader_;
std::optional<util::async::CoroExecutionContext::Operation<void>> mainLoop_;
public:
/**
* @brief Create an instance of ETLService.
*
* @param config The configuration to use
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param balancer Load balancer to use
* @param ledgers The network validated ledgers datastructure
*/
ETLService(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etlng::LoadBalancerInterface> balancer,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers
)
: backend_(std::move(backend))
, subscriptions_(std::move(subscriptions))
, balancer_(std::move(balancer))
, ledgers_(std::move(ledgers))
, cacheLoader_(std::make_shared<etl::CacheLoader<>>(config, backend_, backend_->cache()))
, fetcher_(std::make_shared<etl::impl::LedgerFetcher>(backend_, balancer_))
, extractor_(std::make_shared<impl::Extractor>(fetcher_))
, amendmentBlockHandler_(std::make_shared<etlng::impl::AmendmentBlockHandler>(ctx_, state_))
, loader_(std::make_shared<impl::Loader>(
backend_,
fetcher_,
impl::makeRegistry(
impl::CacheExt{backend_->cache()},
impl::CoreExt{backend_},
impl::SuccessorExt{backend_, backend_->cache()},
impl::NFTExt{backend_}
),
amendmentBlockHandler_
))
{
LOG(log_.info()) << "Creating ETLng...";
}
~ETLService() override
{
LOG(log_.debug()) << "Stopping ETLng";
}
void
run() override
{
LOG(log_.info()) << "run() in ETLng...";
mainLoop_.emplace(ctx_.execute([this] {
auto const rng = loadInitialLedgerIfNeeded();
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
std::optional<uint32_t> mostRecentValidated = ledgers_->getMostRecent();
if (not mostRecentValidated) {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
return;
}
ASSERT(rng.has_value(), "Ledger range can't be null");
auto const nextSequence = rng->maxSequence + 1;
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence}
// impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000},
// TODO lift limit and start with rng.minSeq
);
auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_);
// TODO: figure out this: std::make_shared<impl::Monitor>(backend_, ledgers_, nextSequence)
man.run({}); // TODO: needs to be interruptable and fill out settings
}));
}
void
stop() override
{
LOG(log_.info()) << "Stop called";
// TODO: stop the service correctly
}
boost::json::object
getInfo() const override
{
// TODO
return {{"ok", true}};
}
bool
isAmendmentBlocked() const override
{
// TODO
return false;
}
bool
isCorruptionDetected() const override
{
// TODO
return false;
}
std::optional<etl::ETLState>
getETLState() const override
{
// TODO
return std::nullopt;
}
std::uint32_t
lastCloseAgeSeconds() const override
{
// TODO
return 0;
}
private:
// TODO: this better be std::expected
std::optional<data::LedgerRange>
loadInitialLedgerIfNeeded()
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) {
LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
try {
LOG(log_.info()) << "Waiting for next ledger to be validated by network...";
if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) {
auto const seq = *mostRecentValidated;
LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... ";
auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) {
// TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_);
// TODO: this should be interruptable for graceful shutdown
return loader_->loadInitialLedger(data);
});
});
LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
if (ledger.has_value())
return backend_->hardFetchLedgerRangeNoThrow();
LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop";
} else {
LOG(log_.info()) << "The wait for the next validated ledger has been aborted. "
"Exiting monitor loop";
}
} catch (std::runtime_error const& e) {
LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
amendmentBlockHandler_->notifyAmendmentBlocked();
}
} else {
LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
cacheLoader_->load(rng->maxSequence);
return rng;
}
return std::nullopt;
}
};
} // namespace etlng

92
src/etlng/ETLServiceInterface.hpp Normal file
View File

@@ -0,0 +1,92 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etl/ETLState.hpp"
#include <boost/json/object.hpp>
#include <cstdint>
#include <optional>
namespace etlng {
/**
* @brief This is a base class for any ETL service implementations.
* @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the
* databases.
*/
struct ETLServiceInterface {
virtual ~ETLServiceInterface() = default;
/**
* @brief Start all components to run ETL service.
*/
virtual void
run() = 0;
/**
* @brief Stop the ETL service.
* @note This method blocks until the ETL service has stopped.
*/
virtual void
stop() = 0;
/**
* @brief Get state of ETL as a JSON object
*
* @return The state of ETL as a JSON object
*/
[[nodiscard]] virtual boost::json::object
getInfo() const = 0;
/**
* @brief Check for the amendment blocked state.
*
* @return true if currently amendment blocked; false otherwise
*/
[[nodiscard]] virtual bool
isAmendmentBlocked() const = 0;
/**
* @brief Check whether Clio detected DB corruptions.
*
* @return true if corruption of DB was detected and cache was stopped.
*/
[[nodiscard]] virtual bool
isCorruptionDetected() const = 0;
/**
* @brief Get the etl nodes' state
* @return The etl nodes' state, nullopt if etl nodes are not connected
*/
[[nodiscard]] virtual std::optional<etl::ETLState>
getETLState() const = 0;
/**
* @brief Get time passed since last ledger close, in seconds.
*
* @return Time passed since last ledger close
*/
[[nodiscard]] virtual std::uint32_t
lastCloseAgeSeconds() const = 0;
};
} // namespace etlng

371
src/etlng/LoadBalancer.cpp Normal file
View File

@@ -0,0 +1,371 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/LoadBalancer.hpp"
#include "data/BackendInterface.hpp"
#include "etl/ETLState.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/Source.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
#include "util/CoroutineGroup.hpp"
#include "util/Random.hpp"
#include "util/ResponseExpirationCache.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ArrayView.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/array.hpp>
#include <boost/json/object.hpp>
#include <boost/json/value.hpp>
#include <boost/json/value_to.hpp>
#include <fmt/core.h>
#include <algorithm>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <vector>
using namespace util::config;
namespace etlng {
std::shared_ptr<LoadBalancerInterface>
LoadBalancer::makeLoadBalancer(
ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
{
return std::make_shared<LoadBalancer>(
config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory)
);
}
LoadBalancer::LoadBalancer(
ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory
)
{
auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
if (forwardingCacheTimeout > 0.f) {
forwardingCache_ = util::ResponseExpirationCache{
util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout),
{"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
};
}
auto const numMarkers = config.getValueView("num_markers");
if (numMarkers.hasValue()) {
auto const value = numMarkers.asIntType<uint32_t>();
downloadRanges_ = value;
} else if (backend->fetchLedgerRange()) {
downloadRanges_ = 4;
}
auto const allowNoEtl = config.get<bool>("allow_no_etl");
auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) {
LOG(log_.warn()) << log;
if (!allowNoEtl) {
LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources.";
throw std::logic_error("ETL configuration error.");
}
};
auto const forwardingTimeout =
ClioConfigDefinition::toMilliseconds(config.get<float>("forwarding.request_timeout"));
auto const etlArray = config.getArray("etl_sources");
for (auto it = etlArray.begin<ObjectView>(); it != etlArray.end<ObjectView>(); ++it) {
auto source = sourceFactory(
*it,
ioc,
subscriptions,
validatedLedgers,
forwardingTimeout,
[this]() {
if (not hasForwardingSource_.lock().get())
chooseForwardingSource();
},
[this](bool wasForwarding) {
if (wasForwarding)
chooseForwardingSource();
},
[this]() {
if (forwardingCache_.has_value())
forwardingCache_->invalidate();
}
);
// checking etl node validity
auto const stateOpt = etl::ETLState::fetchETLStateFromSource(*source);
if (!stateOpt) {
LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString()
<< " Please check the configuration and network";
} else if (etlState_ && etlState_->networkID && stateOpt->networkID &&
etlState_->networkID != stateOpt->networkID) {
checkOnETLFailure(fmt::format(
"ETL sources must be on the same network. Source network id = {} does not match others network id = {}",
*(stateOpt->networkID),
*(etlState_->networkID)
));
} else {
etlState_ = stateOpt;
}
sources_.push_back(std::move(source));
LOG(log_.info()) << "Added etl source - " << sources_.back()->toString();
}
if (!etlState_)
checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network");
if (sources_.empty())
checkOnETLFailure("No ETL sources configured. Please check the configuration");
// This is made separate from source creation to prevent UB in case one of the sources will call
// chooseForwardingSource while we are still filling the sources_ vector
for (auto const& source : sources_) {
source->run();
}
}
LoadBalancer::~LoadBalancer()
{
sources_.clear();
}
std::vector<std::string>
LoadBalancer::loadInitialLedger(
uint32_t sequence,
etlng::InitialLoadObserverInterface& loadObserver,
std::chrono::steady_clock::duration retryAfter
)
{
std::vector<std::string> response;
execute(
[this, &response, &sequence, &loadObserver](auto& source) {
auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, loadObserver);
if (!res) {
LOG(log_.error()) << "Failed to download initial ledger."
<< " Sequence = " << sequence << " source = " << source->toString();
} else {
response = std::move(data);
}
return res;
},
sequence,
retryAfter
);
return response;
}
LoadBalancer::OptionalGetLedgerResponseType
LoadBalancer::fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors,
std::chrono::steady_clock::duration retryAfter
)
{
GetLedgerResponseType response;
execute(
[&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) {
auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
response = std::move(data);
if (status.ok() && response.validated()) {
LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence
<< " from source = " << source->toString();
return true;
}
LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString()
<< ", error_code: " << status.error_code() << ", error_msg: " << status.error_message()
<< ", source = " << source->toString();
return false;
},
ledgerSequence,
retryAfter
);
return response;
}
std::expected<boost::json::object, rpc::ClioError>
LoadBalancer::forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
)
{
if (not request.contains("command"))
return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
auto const cmd = boost::json::value_to<std::string>(request.at("command"));
if (forwardingCache_) {
if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
return std::move(cachedResponse).value();
}
}
ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests.");
std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
auto numAttempts = 0u;
auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
if (res) {
response = std::move(res).value();
break;
}
error = std::max(error, res.error()); // Choose the best result between all sources
sourceIdx = (sourceIdx + 1) % sources_.size();
++numAttempts;
}
if (response) {
if (forwardingCache_ and not response->contains("error"))
forwardingCache_->put(cmd, *response);
return std::move(response).value();
}
return std::unexpected{error};
}
boost::json::value
LoadBalancer::toJson() const
{
boost::json::array ret;
for (auto& src : sources_)
ret.push_back(src->toJson());
return ret;
}
template <typename Func>
void
LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
{
ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1);
size_t numAttempts = 0;
while (true) {
auto& source = sources_[sourceIdx];
LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence
<< " - source = " << source->toString();
// Originally, it was (source->hasLedger(ledgerSequence) || true)
/* Sometimes rippled has ledger but doesn't actually know. However,
but this does NOT happen in the normal case and is safe to remove
This || true is only needed when loading full history standalone */
if (source->hasLedger(ledgerSequence)) {
bool const res = f(source);
if (res) {
LOG(log_.debug()) << "Successfully executed func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
break;
}
LOG(log_.warn()) << "Failed to execute func at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
} else {
LOG(log_.warn()) << "Ledger not present at source = " << source->toString()
<< " - ledger sequence = " << ledgerSequence;
}
sourceIdx = (sourceIdx + 1) % sources_.size();
numAttempts++;
if (numAttempts % sources_.size() == 0) {
LOG(log_.info()) << "Ledger sequence " << ledgerSequence
<< " is not yet available from any configured sources. Sleeping and trying again";
std::this_thread::sleep_for(retryAfter);
}
}
}
std::optional<etl::ETLState>
LoadBalancer::getETLState() noexcept
{
if (!etlState_) {
// retry ETLState fetch
etlState_ = etl::ETLState::fetchETLStateFromSource(*this);
}
return etlState_;
}
void
LoadBalancer::stop(boost::asio::yield_context yield)
{
util::CoroutineGroup group{yield};
std::ranges::for_each(sources_, [&group, yield](auto& source) {
group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); });
});
group.asyncWait(yield);
}
void
LoadBalancer::chooseForwardingSource()
{
LOG(log_.info()) << "Choosing a new source to forward subscriptions";
auto hasForwardingSourceLock = hasForwardingSource_.lock();
hasForwardingSourceLock.get() = false;
for (auto& source : sources_) {
if (not hasForwardingSourceLock.get() and source->isConnected()) {
source->setForwarding(true);
hasForwardingSourceLock.get() = true;
} else {
source->setForwarding(false);
}
}
}
} // namespace etlng

271
src/etlng/LoadBalancer.hpp Normal file
View File

@@ -0,0 +1,271 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etl/ETLState.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/Source.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/Assert.hpp"
#include "util/Mutex.hpp"
#include "util/ResponseExpirationCache.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include <boost/asio.hpp>
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/json/value.hpp>
#include <grpcpp/grpcpp.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace etlng {
/**
* @brief A tag class to help identify LoadBalancer in templated code.
*/
struct LoadBalancerTag {
virtual ~LoadBalancerTag() = default;
};
template <typename T>
concept SomeLoadBalancer = std::derived_from<T, LoadBalancerTag>;
/**
* @brief This class is used to manage connections to transaction processing processes.
*
* This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of
* which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
* allows requests for ledger data to be load balanced across all possible ETL sources.
*/
class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag {
public:
using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
private:
static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16;
util::Logger log_{"ETL"};
// Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
std::optional<util::ResponseExpirationCache> forwardingCache_;
std::optional<std::string> forwardingXUserValue_;
std::vector<SourcePtr> sources_;
std::optional<etl::ETLState> etlState_;
std::uint32_t downloadRanges_ =
kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
// Using mutext instead of atomic_bool because choosing a new source to
// forward messages should be done with a mutual exclusion otherwise there will be a race condition
util::Mutex<bool> hasForwardingSource_{false};
public:
/**
* @brief Value for the X-User header when forwarding admin requests
*/
static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin";
/**
* @brief Value for the X-User header when forwarding user requests
*/
static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user";
/**
* @brief Create an instance of the load balancer.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param sourceFactory A factory function to create a source
*/
LoadBalancer(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);
/**
* @brief A factory function for the load balancer.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param sourceFactory A factory function to create a source
* @return A shared pointer to a new instance of LoadBalancer
*/
static std::shared_ptr<LoadBalancerInterface>
makeLoadBalancer(
util::config::ClioConfigDefinition const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
SourceFactory sourceFactory = makeSource
);
~LoadBalancer() override;
/**
* @brief Load the initial ledger, writing data to the queue.
* @note This function will retry indefinitely until the ledger is downloaded.
*
* @param sequence Sequence of ledger to download
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::vector<std::string> The ledger data
*/
std::vector<std::string>
loadInitialLedger(
[[maybe_unused]] uint32_t sequence,
[[maybe_unused]] std::chrono::steady_clock::duration retryAfter
) override
{
ASSERT(false, "Not available for new ETL");
std::unreachable();
};
/**
* @brief Load the initial ledger, writing data to the queue.
* @note This function will retry indefinitely until the ledger is downloaded.
*
* @param sequence Sequence of ledger to download
* @param observer The observer to notify of progress
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::vector<std::string> The ledger data
*/
std::vector<std::string>
loadInitialLedger(
uint32_t sequence,
etlng::InitialLoadObserverInterface& observer,
std::chrono::steady_clock::duration retryAfter
) override;
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
*
* @param ledgerSequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one
* @param getObjectNeighbors Whether to request object neighbors
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return The extracted data, if extraction was successful. If the ledger was found
* in the database or the server is shutting down, the optional will be empty
*/
OptionalGetLedgerResponseType
fetchLedger(
uint32_t ledgerSequence,
bool getObjects,
bool getObjectNeighbors,
std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
) override;
/**
* @brief Represent the state of this load balancer as a JSON object
*
* @return JSON representation of the state of this load balancer.
*/
boost::json::value
toJson() const override;
/**
* @brief Forward a JSON RPC request to a randomly selected rippled node.
*
* @param request JSON-RPC request to forward
* @param clientIp The IP address of the peer, if known
* @param isAdmin Whether the request is from an admin
* @param yield The coroutine context
* @return Response received from rippled node as JSON object on success or error on failure
*/
std::expected<boost::json::object, rpc::ClioError>
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& clientIp,
bool isAdmin,
boost::asio::yield_context yield
) override;
/**
* @brief Return state of ETL nodes.
* @return ETL state, nullopt if etl nodes not available
*/
std::optional<etl::ETLState>
getETLState() noexcept override;
/**
* @brief Stop the load balancer. This will stop all subscription sources.
* @note This function will asynchronously wait for all sources to stop.
*
* @param yield The coroutine context
*/
void
stop(boost::asio::yield_context yield) override;
private:
/**
* @brief Execute a function on a randomly selected source.
*
* @note f is a function that takes an Source as an argument and returns a bool.
* Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another
* randomly chosen Source is used. The process repeats until f returns true.
*
* @param f Function to execute. This function takes the ETL source as an argument, and returns a bool
* @param ledgerSequence f is executed for each Source that has this ledger
* @param retryAfter Time to wait between retries (2 seconds by default)
* server is shutting down
*/
template <typename Func>
void
execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2});
/**
* @brief Choose a new source to forward requests
*/
void
chooseForwardingSource();
};
} // namespace etlng

9
src/etlng/LoadBalancerInterface.hpp
View File

@@ -129,6 +129,15 @@ public:
*/
virtual std::optional<etl::ETLState>
getETLState() noexcept = 0;
/**
* @brief Stop the load balancer. This will stop all subscription sources.
* @note This function will asynchronously wait for all sources to stop.
*
* @param yield The coroutine context
*/
virtual void
stop(boost::asio::yield_context yield) = 0;
};
} // namespace etlng

2
src/etlng/LoaderInterface.hpp
View File

@@ -45,7 +45,7 @@ struct LoaderInterface {
* @param data The data to load
* @return Optional ledger header
*/
virtual std::optional<ripple::LedgerHeader>
[[nodiscard]] virtual std::optional<ripple::LedgerHeader>
loadInitialLedger(model::LedgerData const& data) = 0;
};

73
src/etlng/Source.cpp Normal file
View File

@@ -0,0 +1,73 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Source.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/impl/ForwardingSource.hpp"
#include "etl/impl/SubscriptionSource.hpp"
#include "etlng/impl/GrpcSource.hpp"
#include "etlng/impl/SourceImpl.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/io_context.hpp>
#include <chrono>
#include <memory>
#include <string>
#include <utility>
namespace etlng {
SourcePtr
makeSource(
util::config::ObjectView const& config,
boost::asio::io_context& ioc,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
std::chrono::steady_clock::duration forwardingTimeout,
SourceBase::OnConnectHook onConnect,
SourceBase::OnDisconnectHook onDisconnect,
SourceBase::OnLedgerClosedHook onLedgerClosed
)
{
auto const ip = config.get<std::string>("ip");
auto const wsPort = config.get<std::string>("ws_port");
auto const grpcPort = config.get<std::string>("grpc_port");
etl::impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout};
impl::GrpcSource grpcSource{ip, grpcPort};
auto subscriptionSource = std::make_unique<etl::impl::SubscriptionSource>(
ioc,
ip,
wsPort,
std::move(validatedLedgers),
std::move(subscriptions),
std::move(onConnect),
std::move(onDisconnect),
std::move(onLedgerClosed)
);
return std::make_unique<impl::SourceImpl<>>(
ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource)
);
}
} // namespace etlng

194
src/etlng/Source.hpp Normal file
View File

@@ -0,0 +1,194 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/uuid/uuid.hpp>
#include <grpcpp/support/status.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace etlng {
/**
* @brief Provides an implementation of a ETL source
*/
class SourceBase {
public:
using OnConnectHook = std::function<void()>;
using OnDisconnectHook = std::function<void(bool)>;
using OnLedgerClosedHook = std::function<void()>;
virtual ~SourceBase() = default;
/**
* @brief Run subscriptions loop of the source
*/
virtual void
run() = 0;
/**
* @brief Stop Source.
* @note This method will asynchronously wait for source to be stopped.
*
* @param yield The coroutine context.
*/
virtual void
stop(boost::asio::yield_context yield) = 0;
/**
* @brief Check if source is connected
*
* @return true if source is connected; false otherwise
*/
[[nodiscard]] virtual bool
isConnected() const = 0;
/**
* @brief Set the forwarding state of the source.
*
* @param isForwarding Whether to forward or not
*/
virtual void
setForwarding(bool isForwarding) = 0;
/**
* @brief Represent the source as a JSON object
*
* @return JSON representation of the source
*/
[[nodiscard]] virtual boost::json::object
toJson() const = 0;
/** @return String representation of the source (for debug) */
[[nodiscard]] virtual std::string
toString() const = 0;
/**
* @brief Check if ledger is known by this source.
*
* @param sequence The ledger sequence to check
* @return true if ledger is in the range of this source; false otherwise
*/
[[nodiscard]] virtual bool
hasLedger(uint32_t sequence) const = 0;
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
*
* @param sequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
* @return A std::pair of the response status and the response itself
*/
[[nodiscard]] virtual std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0;
/**
* @brief Download a ledger in full.
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param loader InitialLoadObserverInterface implementation
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
virtual std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) = 0;
/**
* @brief Forward a request to rippled.
*
* @param request The request to forward
* @param forwardToRippledClientIp IP of the client forwarding this request if known
* @param xUserValue Value of the X-User header
* @param yield The coroutine context
* @return Response on success or error on failure
*/
[[nodiscard]] virtual std::expected<boost::json::object, rpc::ClioError>
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledClientIp,
std::string_view xUserValue,
boost::asio::yield_context yield
) const = 0;
};
using SourcePtr = std::unique_ptr<SourceBase>;
using SourceFactory = std::function<SourcePtr(
util::config::ObjectView const& config,
boost::asio::io_context& ioc,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
std::chrono::steady_clock::duration forwardingTimeout,
SourceBase::OnConnectHook onConnect,
SourceBase::OnDisconnectHook onDisconnect,
SourceBase::OnLedgerClosedHook onLedgerClosed
)>;
/**
* @brief Create a source
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers data structure
* @param forwardingTimeout The timeout for forwarding to rippled
* @param onConnect The hook to call on connect
* @param onDisconnect The hook to call on disconnect
* @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set
* as forwarding.
* @return The created source
*/
[[nodiscard]] SourcePtr
makeSource(
util::config::ObjectView const& config,
boost::asio::io_context& ioc,
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
std::chrono::steady_clock::duration forwardingTimeout,
SourceBase::OnConnectHook onConnect,
SourceBase::OnDisconnectHook onDisconnect,
SourceBase::OnLedgerClosedHook onLedgerClosed
);
} // namespace etlng

116
src/etlng/impl/ForwardingSource.cpp Normal file
View File

@@ -0,0 +1,116 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ForwardingSource.hpp"
#include "rpc/Errors.hpp"
#include "util/log/Logger.hpp"
#include <boost/asio/spawn.hpp>
#include <boost/beast/http/field.hpp>
#include <boost/beast/version.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/serialize.hpp>
#include <fmt/core.h>
#include <chrono>
#include <exception>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
namespace etlng::impl {
ForwardingSource::ForwardingSource(
std::string ip,
std::string wsPort,
std::chrono::steady_clock::duration forwardingTimeout,
std::chrono::steady_clock::duration connTimeout
)
: log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort))
, connectionBuilder_(std::move(ip), std::move(wsPort))
, forwardingTimeout_{forwardingTimeout}
{
connectionBuilder_.setConnectionTimeout(connTimeout)
.addHeader(
{boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)}
);
}
std::expected<boost::json::object, rpc::ClioError>
ForwardingSource::forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledClientIp,
std::string_view xUserValue,
boost::asio::yield_context yield
) const
{
auto connectionBuilder = connectionBuilder_;
if (forwardToRippledClientIp) {
connectionBuilder.addHeader(
{boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)}
);
}
connectionBuilder.addHeader({"X-User", std::string{xUserValue}});
auto expectedConnection = connectionBuilder.connect(yield);
if (not expectedConnection) {
LOG(log_.debug()) << "Couldn't connect to rippled to forward request.";
return std::unexpected{rpc::ClioError::EtlConnectionError};
}
auto& connection = expectedConnection.value();
auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_);
if (writeError) {
LOG(log_.debug()) << "Error sending request to rippled to forward request.";
return std::unexpected{rpc::ClioError::EtlRequestError};
}
auto response = connection->read(yield, forwardingTimeout_);
if (not response) {
if (auto errorCode = response.error().errorCode();
errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) {
LOG(log_.debug()) << "Request to rippled timed out";
return std::unexpected{rpc::ClioError::EtlRequestTimeout};
}
LOG(log_.debug()) << "Error sending request to rippled to forward request.";
return std::unexpected{rpc::ClioError::EtlRequestError};
}
boost::json::value parsedResponse;
try {
parsedResponse = boost::json::parse(*response);
if (not parsedResponse.is_object())
throw std::runtime_error("response is not an object");
} catch (std::exception const& e) {
LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
return std::unexpected{rpc::ClioError::EtlInvalidResponse};
}
auto responseObject = parsedResponse.as_object();
responseObject["forwarded"] = true;
return responseObject;
}
} // namespace etlng::impl

70
src/etlng/impl/ForwardingSource.hpp Normal file
View File

@@ -0,0 +1,70 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "rpc/Errors.hpp"
#include "util/log/Logger.hpp"
#include "util/requests/WsConnection.hpp"
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <chrono>
#include <expected>
#include <optional>
#include <string>
#include <string_view>
namespace etlng::impl {
class ForwardingSource {
util::Logger log_;
util::requests::WsConnectionBuilder connectionBuilder_;
std::chrono::steady_clock::duration forwardingTimeout_;
static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3};
public:
ForwardingSource(
std::string ip,
std::string wsPort,
std::chrono::steady_clock::duration forwardingTimeout,
std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT
);
/**
* @brief Forward a request to rippled.
*
* @param request The request to forward
* @param forwardToRippledClientIp IP of the client forwarding this request if known
* @param xUserValue Optional value for X-User header
* @param yield The coroutine context
* @return Response on success or error on failure
*/
std::expected<boost::json::object, rpc::ClioError>
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledClientIp,
std::string_view xUserValue,
boost::asio::yield_context yield
) const;
};
} // namespace etlng::impl

1
src/etlng/impl/Loading.hpp
View File

@@ -39,7 +39,6 @@
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxMeta.h>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <optional>

9
src/etlng/impl/Registry.hpp
View File

@@ -24,7 +24,6 @@
#include <xrpl/protocol/TxFormats.h>
#include <concepts>
#include <cstdint>
#include <string>
#include <tuple>
@@ -81,7 +80,7 @@ concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or
template <typename T>
concept NoTwoOfKind = not(HasLedgerDataHook<T> and HasTransactionHook<T>) and
not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and not(HasInitialDataHook<T> and HasObjectHook<T>) and
not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and
not(HasInitialObjectsHook<T> and HasInitialObjectHook<T>);
template <typename T>
@@ -216,4 +215,10 @@ public:
}
};
static auto
makeRegistry(auto&&... exts)
{
return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(std::forward<decltype(exts)>(exts)...);
}
} // namespace etlng::impl

232
src/etlng/impl/SourceImpl.hpp Normal file
View File

@@ -0,0 +1,232 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etl/impl/ForwardingSource.hpp"
#include "etl/impl/SubscriptionSource.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/Source.hpp"
#include "etlng/impl/GrpcSource.hpp"
#include "rpc/Errors.hpp"
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <grpcpp/support/status.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace etlng::impl {
/**
* @brief Provides an implementation of a ETL source
*
* @tparam GrpcSourceType The type of the gRPC source
* @tparam SubscriptionSourceTypePtr The type of the subscription source
* @tparam ForwardingSourceType The type of the forwarding source
*/
template <
typename GrpcSourceType = GrpcSource,
typename SubscriptionSourceTypePtr = std::unique_ptr<etl::impl::SubscriptionSource>,
typename ForwardingSourceType = etl::impl::ForwardingSource>
class SourceImpl : public SourceBase {
std::string ip_;
std::string wsPort_;
std::string grpcPort_;
GrpcSourceType grpcSource_;
SubscriptionSourceTypePtr subscriptionSource_;
ForwardingSourceType forwardingSource_;
public:
/**
* @brief Construct a new SourceImpl object
*
* @param ip The IP of the source
* @param wsPort The web socket port of the source
* @param grpcPort The gRPC port of the source
* @param grpcSource The gRPC source
* @param subscriptionSource The subscription source
* @param forwardingSource The forwarding source
*/
template <typename SomeGrpcSourceType, typename SomeForwardingSourceType>
requires std::is_same_v<GrpcSourceType, SomeGrpcSourceType> and
std::is_same_v<ForwardingSourceType, SomeForwardingSourceType>
SourceImpl(
std::string ip,
std::string wsPort,
std::string grpcPort,
SomeGrpcSourceType&& grpcSource,
SubscriptionSourceTypePtr subscriptionSource,
SomeForwardingSourceType&& forwardingSource
)
: ip_(std::move(ip))
, wsPort_(std::move(wsPort))
, grpcPort_(std::move(grpcPort))
, grpcSource_(std::forward<SomeGrpcSourceType>(grpcSource))
, subscriptionSource_(std::move(subscriptionSource))
, forwardingSource_(std::forward<SomeForwardingSourceType>(forwardingSource))
{
}
/**
* @brief Run subscriptions loop of the source
*/
void
run() final
{
subscriptionSource_->run();
}
void
stop(boost::asio::yield_context yield) final
{
subscriptionSource_->stop(yield);
}
/**
* @brief Check if source is connected
*
* @return true if source is connected; false otherwise
*/
bool
isConnected() const final
{
return subscriptionSource_->isConnected();
}
/**
* @brief Set the forwarding state of the source.
*
* @param isForwarding Whether to forward or not
*/
void
setForwarding(bool isForwarding) final
{
subscriptionSource_->setForwarding(isForwarding);
}
/**
* @brief Represent the source as a JSON object
*
* @return JSON representation of the source
*/
boost::json::object
toJson() const final
{
boost::json::object res;
res["validated_range"] = subscriptionSource_->validatedRange();
res["is_connected"] = std::to_string(static_cast<int>(subscriptionSource_->isConnected()));
res["ip"] = ip_;
res["ws_port"] = wsPort_;
res["grpc_port"] = grpcPort_;
auto last = subscriptionSource_->lastMessageTime();
if (last.time_since_epoch().count() != 0) {
res["last_msg_age_seconds"] = std::to_string(
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - last).count()
);
}
return res;
}
/** @return String representation of the source (for debug) */
std::string
toString() const final
{
return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ +
", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}";
}
/**
* @brief Check if ledger is known by this source.
*
* @param sequence The ledger sequence to check
* @return true if ledger is in the range of this source; false otherwise
*/
bool
hasLedger(uint32_t sequence) const final
{
return subscriptionSource_->hasLedger(sequence);
}
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
*
* @param sequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
* @return A std::pair of the response status and the response itself
*/
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final
{
return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors);
}
/**
* @brief Download a ledger in full.
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param loader InitialLoadObserverInterface implementation
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) final
{
return grpcSource_.loadInitialLedger(sequence, numMarkers, loader);
}
/**
* @brief Forward a request to rippled.
*
* @param request The request to forward
* @param forwardToRippledClientIp IP of the client forwarding this request if known
* @param xUserValue Optional value of the X-User header
* @param yield The coroutine context
* @return Response or ClioError
*/
std::expected<boost::json::object, rpc::ClioError>
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledClientIp,
std::string_view xUserValue,
boost::asio::yield_context yield
) const final
{
return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield);
}
};
} // namespace etlng::impl

4
src/etlng/impl/ext/NFT.hpp
View File

@@ -20,15 +20,11 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/NFTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace etlng::impl {

9
src/rpc/RPCEngine.hpp
View File

@@ -20,6 +20,7 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/RPCHelpers.hpp"
#include "rpc/WorkQueue.hpp"
@@ -55,7 +56,7 @@ namespace rpc {
/**
* @brief The RPC engine that ties all RPC-related functionality together.
*/
template <typename LoadBalancerType, typename CountersType>
template <typename CountersType>
class RPCEngine {
util::Logger perfLog_{"Performance"};
util::Logger log_{"RPC"};
@@ -67,7 +68,7 @@ class RPCEngine {
std::shared_ptr<HandlerProvider const> handlerProvider_;
impl::ForwardingProxy<LoadBalancerType, CountersType, HandlerProvider> forwardingProxy_;
impl::ForwardingProxy<CountersType, HandlerProvider> forwardingProxy_;
std::optional<util::ResponseExpirationCache> responseCache_;
@@ -86,7 +87,7 @@ public:
RPCEngine(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<LoadBalancerType> const& balancer,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
web::dosguard::DOSGuardInterface const& dosGuard,
WorkQueue& workQueue,
CountersType& counters,
@@ -128,7 +129,7 @@ public:
makeRPCEngine(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<LoadBalancerType> const& balancer,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
web::dosguard::DOSGuardInterface const& dosGuard,
WorkQueue& workQueue,
CountersType& counters,

8
src/rpc/common/impl/ForwardingProxy.hpp
View File

@@ -19,6 +19,7 @@
#pragma once
#include "etlng/LoadBalancerInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/RPCHelpers.hpp"
#include "rpc/common/Types.hpp"
@@ -31,20 +32,21 @@
#include <memory>
#include <string>
#include <unordered_set>
#include <utility>
namespace rpc::impl {
template <typename LoadBalancerType, typename CountersType, typename HandlerProviderType>
template <typename CountersType, typename HandlerProviderType>
class ForwardingProxy {
util::Logger log_{"RPC"};
std::shared_ptr<LoadBalancerType> balancer_;
std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
std::reference_wrapper<CountersType> counters_;
std::shared_ptr<HandlerProviderType const> handlerProvider_;
public:
ForwardingProxy(
std::shared_ptr<LoadBalancerType> const& balancer,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
CountersType& counters,
std::shared_ptr<HandlerProviderType const> const& handlerProvider
)

7
src/rpc/common/impl/HandlerProvider.cpp
View File

@@ -21,7 +21,8 @@
#include "data/AmendmentCenterInterface.hpp"
#include "data/BackendInterface.hpp"
#include "etl/ETLService.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Counters.hpp"
#include "rpc/common/AnyHandler.hpp"
@@ -72,8 +73,8 @@ ProductionHandlerProvider::ProductionHandlerProvider(
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptionManager,
std::shared_ptr<etl::LoadBalancer> const& balancer,
std::shared_ptr<etl::ETLService const> const& etl,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
Counters const& counters
)

10
src/rpc/common/impl/HandlerProvider.hpp
View File

@@ -21,6 +21,8 @@
#include "data/AmendmentCenterInterface.hpp"
#include "data/BackendInterface.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/common/AnyHandler.hpp"
#include "rpc/common/HandlerProvider.hpp"
@@ -32,10 +34,6 @@
#include <string>
#include <unordered_map>
namespace etl {
class ETLService;
class LoadBalancer;
} // namespace etl
namespace rpc {
class Counters;
} // namespace rpc
@@ -55,8 +53,8 @@ public:
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptionManager,
std::shared_ptr<etl::LoadBalancer> const& balancer,
std::shared_ptr<etl::ETLService const> const& etl,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
std::shared_ptr<data::AmendmentCenterInterface const> const& amendmentCenter,
Counters const& counters
);

20
src/rpc/handlers/ServerInfo.hpp
View File

@@ -21,6 +21,8 @@
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/JS.hpp"
@@ -49,10 +51,6 @@
#include <optional>
#include <string>
namespace etl {
class ETLService;
class LoadBalancer;
} // namespace etl
namespace rpc {
class Counters;
} // namespace rpc
@@ -62,18 +60,16 @@ namespace rpc {
/**
* @brief Contains common functionality for handling the `server_info` command
*
* @tparam LoadBalancerType The type of the load balancer
* @tparam ETLServiceType The type of the ETL service
* @tparam CountersType The type of the counters
*/
template <typename LoadBalancerType, typename ETLServiceType, typename CountersType>
template <typename CountersType>
class BaseServerInfoHandler {
static constexpr auto kBACKEND_COUNTERS_KEY = "backend_counters";
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
std::shared_ptr<LoadBalancerType> balancer_;
std::shared_ptr<ETLServiceType const> etl_;
std::shared_ptr<etlng::LoadBalancerInterface> balancer_;
std::shared_ptr<etlng::ETLServiceInterface const> etl_;
std::reference_wrapper<CountersType const> counters_;
public:
@@ -158,8 +154,8 @@ public:
BaseServerInfoHandler(
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<feed::SubscriptionManagerInterface> const& subscriptions,
std::shared_ptr<LoadBalancerType> const& balancer,
std::shared_ptr<ETLServiceType const> const& etl,
std::shared_ptr<etlng::LoadBalancerInterface> const& balancer,
std::shared_ptr<etlng::ETLServiceInterface const> const& etl,
CountersType const& counters
)
: backend_(backend)
@@ -352,6 +348,6 @@ private:
*
* For more details see: https://xrpl.org/server_info-clio.html
*/
using ServerInfoHandler = BaseServerInfoHandler<etl::LoadBalancer, etl::ETLService, Counters>;
using ServerInfoHandler = BaseServerInfoHandler<Counters>;
} // namespace rpc

26
src/rpc/handlers/Tx.hpp
View File

@@ -22,6 +22,7 @@
#include "data/BackendInterface.hpp"
#include "data/Types.hpp"
#include "etl/ETLService.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/JS.hpp"
#include "rpc/RPCHelpers.hpp"
@@ -52,14 +53,13 @@
namespace rpc {
/**
* @brief Contains common functionality for handling the `tx` command
* @brief The tx method retrieves information on a single transaction, by its identifying hash.
*
* @tparam ETLServiceType The type of the ETL service to use
* For more details see: https://xrpl.org/tx.html
*/
template <typename ETLServiceType>
class BaseTxHandler {
class TxHandler {
std::shared_ptr<BackendInterface> sharedPtrBackend_;
std::shared_ptr<ETLServiceType const> etl_;
std::shared_ptr<etlng::ETLServiceInterface const> etl_;
public:
/**
@@ -95,14 +95,14 @@ public:
using Result = HandlerReturnType<Output>;
/**
* @brief Construct a new BaseTxHandler object
* @brief Construct a new TxHandler object
*
* @param sharedPtrBackend The backend to use
* @param etl The ETL service to use
*/
BaseTxHandler(
TxHandler(
std::shared_ptr<BackendInterface> const& sharedPtrBackend,
std::shared_ptr<ETLServiceType const> const& etl
std::shared_ptr<etlng::ETLServiceInterface const> const& etl
)
: sharedPtrBackend_(sharedPtrBackend), etl_(etl)
{
@@ -183,7 +183,7 @@ public:
dbResponse = sharedPtrBackend_->fetchTransaction(ripple::uint256{input.transaction->c_str()}, ctx.yield);
}
auto output = BaseTxHandler::Output{.apiVersion = ctx.apiVersion};
auto output = TxHandler::Output{.apiVersion = ctx.apiVersion};
if (!dbResponse) {
if (rangeSupplied && input.transaction) // ranges not for ctid
@@ -320,7 +320,7 @@ private:
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv)
{
auto input = BaseTxHandler::Input{};
auto input = TxHandler::Input{};
auto const& jsonObject = jv.as_object();
if (jsonObject.contains(JS(transaction)))
@@ -344,10 +344,4 @@ private:
}
};
/**
* @brief The tx method retrieves information on a single transaction, by its identifying hash.
*
* For more details see: https://xrpl.org/tx.html
*/
using TxHandler = BaseTxHandler<etl::ETLService>;
} // namespace rpc

2
src/util/newconfig/ConfigDefinition.hpp
View File

@@ -293,7 +293,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{
{"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
{"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"__ng_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}},
{"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},
{"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},

7
src/web/RPCServerHandler.hpp
View File

@@ -20,6 +20,7 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/Factories.hpp"
#include "rpc/JS.hpp"
@@ -58,11 +59,11 @@ namespace web {
*
* Note: see @ref web::SomeServerHandler concept
*/
template <typename RPCEngineType, typename ETLType>
template <typename RPCEngineType>
class RPCServerHandler {
std::shared_ptr<BackendInterface const> const backend_;
std::shared_ptr<RPCEngineType> const rpcEngine_;
std::shared_ptr<ETLType const> const etl_;
std::shared_ptr<etlng::ETLServiceInterface const> const etl_;
util::TagDecoratorFactory const tagFactory_;
rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed
@@ -82,7 +83,7 @@ public:
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface const> const& backend,
std::shared_ptr<RPCEngineType> const& rpcEngine,
std::shared_ptr<ETLType const> const& etl
std::shared_ptr<etlng::ETLServiceInterface const> const& etl
)
: backend_(backend)
, rpcEngine_(rpcEngine)

7
src/web/ng/RPCServerHandler.hpp
View File

@@ -20,6 +20,7 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include "rpc/Errors.hpp"
#include "rpc/Factories.hpp"
#include "rpc/JS.hpp"
@@ -64,11 +65,11 @@ namespace web::ng {
*
* Note: see @ref web::SomeServerHandler concept
*/
template <typename RPCEngineType, typename ETLType>
template <typename RPCEngineType>
class RPCServerHandler {
std::shared_ptr<BackendInterface const> const backend_;
std::shared_ptr<RPCEngineType> const rpcEngine_;
std::shared_ptr<ETLType const> const etl_;
std::shared_ptr<etlng::ETLServiceInterface const> const etl_;
util::TagDecoratorFactory const tagFactory_;
rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed
@@ -88,7 +89,7 @@ public:
util::config::ClioConfigDefinition const& config,
std::shared_ptr<BackendInterface const> const& backend,
std::shared_ptr<RPCEngineType> const& rpcEngine,
std::shared_ptr<ETLType const> const& etl
std::shared_ptr<etlng::ETLServiceInterface const> const& etl
)
: backend_(backend)
, rpcEngine_(rpcEngine)

18
tests/common/util/MockETLService.hpp
View File

@@ -20,21 +20,21 @@
#pragma once
#include "etl/ETLState.hpp"
#include "etlng/ETLServiceInterface.hpp"
#include <boost/json.hpp>
#include <boost/json/object.hpp>
#include <gmock/gmock.h>
#include <chrono>
#include <cstdint>
#include <optional>
struct MockETLService {
MOCK_METHOD(boost::json::object, getInfo, (), (const));
MOCK_METHOD(std::chrono::time_point<std::chrono::system_clock>, getLastPublish, (), (const));
MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const));
MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const));
MOCK_METHOD(bool, isAmendmentBlocked, (), (const));
MOCK_METHOD(bool, isCorruptionDetected, (), (const));
MOCK_METHOD(std::optional<etl::ETLState>, getETLState, (), (const));
struct MockETLService : etlng::ETLServiceInterface {
MOCK_METHOD(void, run, (), (override));
MOCK_METHOD(void, stop, (), (override));
MOCK_METHOD(boost::json::object, getInfo, (), (const, override));
MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const, override));
MOCK_METHOD(bool, isAmendmentBlocked, (), (const, override));
MOCK_METHOD(bool, isCorruptionDetected, (), (const, override));
MOCK_METHOD(std::optional<etl::ETLState>, getETLState, (), (const, override));
};

19
tests/common/util/MockLoadBalancer.hpp
View File

@@ -38,22 +38,6 @@
#include <string>
#include <vector>
struct MockLoadBalancer {
using RawLedgerObjectType = FakeLedgerObject;
MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ());
MOCK_METHOD(std::optional<FakeFetchResponse>, fetchLedger, (uint32_t, bool, bool), ());
MOCK_METHOD(boost::json::value, toJson, (), (const));
using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
MOCK_METHOD(
ForwardToRippledReturnType,
forwardToRippled,
(boost::json::object const&, std::optional<std::string> const&, bool, boost::asio::yield_context),
(const)
);
};
struct MockNgLoadBalancer : etlng::LoadBalancerInterface {
using RawLedgerObjectType = FakeLedgerObject;
@@ -85,4 +69,7 @@ struct MockNgLoadBalancer : etlng::LoadBalancerInterface {
(boost::json::object const&, std::optional<std::string> const&, bool, boost::asio::yield_context),
(override)
);
MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
};
using MockLoadBalancer = MockNgLoadBalancer;

6
tests/common/util/MockSource.hpp
View File

@@ -60,7 +60,7 @@ struct MockSource : etl::SourceBase {
(uint32_t, bool, bool),
(override)
);
MOCK_METHOD((std::pair<std::vector<std::string>, bool>), loadInitialLedger, (uint32_t, uint32_t, bool), (override));
MOCK_METHOD((std::pair<std::vector<std::string>, bool>), loadInitialLedger, (uint32_t, uint32_t), (override));
using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
MOCK_METHOD(
@@ -132,9 +132,9 @@ public:
}
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, uint32_t maxLedger, bool getObjects) override
loadInitialLedger(uint32_t sequence, uint32_t maxLedger) override
{
return mock_->loadInitialLedger(sequence, maxLedger, getObjects);
return mock_->loadInitialLedger(sequence, maxLedger);
}
std::expected<boost::json::object, rpc::ClioError>

245
tests/common/util/MockSourceNg.hpp Normal file
View File

@@ -0,0 +1,245 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/Source.hpp"
#include "feed/SubscriptionManagerInterface.hpp"
#include "rpc/Errors.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/uuid/uuid.hpp>
#include <gmock/gmock.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <algorithm>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
struct MockSourceNg : etlng::SourceBase {
MOCK_METHOD(void, run, (), (override));
MOCK_METHOD(void, stop, (boost::asio::yield_context), (override));
MOCK_METHOD(bool, isConnected, (), (const, override));
MOCK_METHOD(void, setForwarding, (bool), (override));
MOCK_METHOD(boost::json::object, toJson, (), (const, override));
MOCK_METHOD(std::string, toString, (), (const, override));
MOCK_METHOD(bool, hasLedger, (uint32_t), (const, override));
MOCK_METHOD(
(std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>),
fetchLedger,
(uint32_t, bool, bool),
(override)
);
MOCK_METHOD(
(std::pair<std::vector<std::string>, bool>),
loadInitialLedger,
(uint32_t, uint32_t, etlng::InitialLoadObserverInterface&),
(override)
);
using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
MOCK_METHOD(
ForwardToRippledReturnType,
forwardToRippled,
(boost::json::object const&, std::optional<std::string> const&, std::string_view, boost::asio::yield_context),
(const, override)
);
};
template <template <typename> typename MockType>
using MockSourceNgPtr = std::shared_ptr<MockType<MockSourceNg>>;
template <template <typename> typename MockType>
class MockSourceNgWrapper : public etlng::SourceBase {
MockSourceNgPtr<MockType> mock_;
public:
MockSourceNgWrapper(MockSourceNgPtr<MockType> mockData) : mock_(std::move(mockData))
{
}
void
run() override
{
mock_->run();
}
void
stop(boost::asio::yield_context yield) override
{
mock_->stop(yield);
}
bool
isConnected() const override
{
return mock_->isConnected();
}
void
setForwarding(bool isForwarding) override
{
mock_->setForwarding(isForwarding);
}
boost::json::object
toJson() const override
{
return mock_->toJson();
}
std::string
toString() const override
{
return mock_->toString();
}
bool
hasLedger(uint32_t sequence) const override
{
return mock_->hasLedger(sequence);
}
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors) override
{
return mock_->fetchLedger(sequence, getObjects, getObjectNeighbors);
}
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, uint32_t maxLedger, etlng::InitialLoadObserverInterface& observer) override
{
return mock_->loadInitialLedger(sequence, maxLedger, observer);
}
std::expected<boost::json::object, rpc::ClioError>
forwardToRippled(
boost::json::object const& request,
std::optional<std::string> const& forwardToRippledClientIp,
std::string_view xUserValue,
boost::asio::yield_context yield
) const override
{
return mock_->forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield);
}
};
struct MockSourceNgCallbacks {
etlng::SourceBase::OnDisconnectHook onDisconnect;
etlng::SourceBase::OnConnectHook onConnect;
etlng::SourceBase::OnLedgerClosedHook onLedgerClosed;
};
template <template <typename> typename MockType>
struct MockSourceNgData {
MockSourceNgPtr<MockType> source = std::make_shared<MockType<MockSourceNg>>();
std::optional<MockSourceNgCallbacks> callbacks;
};
template <template <typename> typename MockType = testing::NiceMock>
class MockSourceNgFactoryImpl {
std::vector<MockSourceNgData<MockType>> mockData_;
public:
MockSourceNgFactoryImpl(size_t numSources)
{
setSourcesNumber(numSources);
ON_CALL(*this, makeSource)
.WillByDefault([this](
util::config::ObjectView const&,
boost::asio::io_context&,
std::shared_ptr<feed::SubscriptionManagerInterface>,
std::shared_ptr<etl::NetworkValidatedLedgersInterface>,
std::chrono::steady_clock::duration,
etlng::SourceBase::OnConnectHook onConnect,
etlng::SourceBase::OnDisconnectHook onDisconnect,
etlng::SourceBase::OnLedgerClosedHook onLedgerClosed
) {
auto it = std::ranges::find_if(mockData_, [](auto const& d) { return not d.callbacks.has_value(); });
[&]() { ASSERT_NE(it, mockData_.end()) << "Make source called more than expected"; }();
it->callbacks = MockSourceNgCallbacks{
.onDisconnect = std::move(onDisconnect),
.onConnect = std::move(onConnect),
.onLedgerClosed = std::move(onLedgerClosed)
};
return std::make_unique<MockSourceNgWrapper<MockType>>(it->source);
});
}
void
setSourcesNumber(size_t numSources)
{
mockData_.clear();
mockData_.reserve(numSources);
std::ranges::generate_n(std::back_inserter(mockData_), numSources, [] { return MockSourceNgData<MockType>{}; });
}
template <typename... Args>
etlng::SourcePtr
operator()(Args&&... args)
{
return makeSource(std::forward<Args>(args)...);
}
MOCK_METHOD(
etlng::SourcePtr,
makeSource,
(util::config::ObjectView const&,
boost::asio::io_context&,
std::shared_ptr<feed::SubscriptionManagerInterface>,
std::shared_ptr<etl::NetworkValidatedLedgersInterface>,
std::chrono::steady_clock::duration,
etlng::SourceBase::OnConnectHook,
etlng::SourceBase::OnDisconnectHook,
etlng::SourceBase::OnLedgerClosedHook)
);
MockType<MockSourceNg>&
sourceAt(size_t index)
{
return *mockData_.at(index).source;
}
MockSourceNgCallbacks&
callbacksAt(size_t index)
{
auto& callbacks = mockData_.at(index).callbacks;
[&]() { ASSERT_TRUE(callbacks.has_value()) << "Callbacks not set"; }();
return *callbacks;
}
};
using MockSourceNgFactory = testing::NiceMock<MockSourceNgFactoryImpl<>>;
using StrictMockSourceNgFactory = testing::StrictMock<MockSourceNgFactoryImpl<testing::StrictMock>>;

3
tests/unit/CMakeLists.txt
View File

@@ -38,13 +38,16 @@ target_sources(
# ETLng
etlng/AmendmentBlockHandlerTests.cpp
etlng/ExtractionTests.cpp
etlng/ForwardingSourceTests.cpp
etlng/GrpcSourceTests.cpp
etlng/RegistryTests.cpp
etlng/SchedulingTests.cpp
etlng/TaskManagerTests.cpp
etlng/LoadingTests.cpp
etlng/LoadBalancerTests.cpp
etlng/NetworkValidatedLedgersTests.cpp
etlng/MonitorTests.cpp
etlng/SourceImplTests.cpp
etlng/ext/CoreTests.cpp
etlng/ext/CacheTests.cpp
etlng/ext/NFTTests.cpp

14
tests/unit/app/StopperTests.cpp
View File

@@ -17,11 +17,11 @@
*/
//==============================================================================
#include "app/Stopper.hpp"
#include "etl/ETLService.hpp"
#include "etl/LoadBalancer.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/MockBackend.hpp"
#include "util/MockETLService.hpp"
#include "util/MockLoadBalancer.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
@@ -65,17 +65,11 @@ struct StopperMakeCallbackTest : util::prometheus::WithPrometheus, SyncAsioConte
struct ServerMock : web::ng::ServerTag {
MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
};
struct LoadBalancerMock : etl::LoadBalancerTag {
MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
};
struct ETLServiceMock : etl::ETLServiceTag {
MOCK_METHOD(void, stop, (), ());
};
protected:
testing::StrictMock<ServerMock> serverMock_;
testing::StrictMock<LoadBalancerMock> loadBalancerMock_;
testing::StrictMock<ETLServiceMock> etlServiceMock_;
testing::StrictMock<MockNgLoadBalancer> loadBalancerMock_;
testing::StrictMock<MockETLService> etlServiceMock_;
testing::StrictMock<MockSubscriptionManager> subscriptionManagerMock_;
testing::StrictMock<MockBackend> backendMock_{util::config::ClioConfigDefinition{}};
boost::asio::io_context ioContextToStop_;

1
tests/unit/etl/ETLStateTests.cpp
View File

@@ -18,6 +18,7 @@
//==============================================================================
#include "etl/ETLState.hpp"
#include "etl/Source.hpp"
#include "rpc/Errors.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/MockSource.hpp"

7
tests/unit/etl/GrpcSourceTests.cpp
View File

@@ -92,7 +92,7 @@ TEST_F(GrpcSourceTests, fetchLedgerNoStub)
TEST_F(GrpcSourceTests, loadInitialLedgerNoStub)
{
GrpcSource wrongGrpcSource{"wrong", "wrong", mockBackend_};
auto const [data, success] = wrongGrpcSource.loadInitialLedger(0, 0, false);
auto const [data, success] = wrongGrpcSource.loadInitialLedger(0, 0);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
}
@@ -101,7 +101,6 @@ struct GrpcSourceLoadInitialLedgerTests : GrpcSourceTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 4;
bool const cacheOnly_ = false;
};
TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
@@ -116,7 +115,7 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
return grpc::Status{grpc::StatusCode::NOT_FOUND, "Not found"};
});
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_, cacheOnly_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
}
@@ -147,7 +146,7 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, worksFine)
EXPECT_CALL(*mockBackend_, writeNFTs).Times(numMarkers_);
EXPECT_CALL(*mockBackend_, writeLedgerObject).Times(numMarkers_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_, cacheOnly_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
EXPECT_TRUE(success);
EXPECT_EQ(data, std::vector<std::string>(4, keyStr));

29
tests/unit/etl/LoadBalancerTests.cpp
View File

@@ -436,56 +436,50 @@ struct LoadBalancerLoadInitialLedgerTests : LoadBalancerOnConnectHookTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 16;
bool const cacheOnly_ = true;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
};
TEST_F(LoadBalancerLoadInitialLedgerTests, load)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0DoesntHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_bothSourcesDontHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_, std::chrono::milliseconds{1}), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, std::chrono::milliseconds{1}), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0ReturnsStatusFalse)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, false)));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
struct LoadBalancerLoadInitialLedgerCustomNumMarkersTests : LoadBalancerConstructorTests {
protected:
uint32_t const numMarkers_ = 16;
uint32_t const sequence_ = 123;
bool const cacheOnly_ = true;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
};
@@ -502,10 +496,9 @@ TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersTests, loadInitialLedger)
util::Random::setSeed(0);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_), response_.first);
}
struct LoadBalancerFetchLegerTests : LoadBalancerOnConnectHookTests {

4
tests/unit/etl/SourceImplTests.cpp
View File

@@ -48,7 +48,7 @@ struct GrpcSourceMock {
MOCK_METHOD(FetchLedgerReturnType, fetchLedger, (uint32_t, bool, bool));
using LoadLedgerReturnType = std::pair<std::vector<std::string>, bool>;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t, bool));
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t));
};
struct SubscriptionSourceMock {
@@ -174,7 +174,7 @@ TEST_F(SourceImplTest, loadInitialLedger)
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, false))
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, true)));
auto const [actualLedgers, actualSuccess] = source_.loadInitialLedger(ledgerSeq, numMarkers);

196
tests/unit/etlng/ForwardingSourceTests.cpp Normal file
View File

@@ -0,0 +1,196 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ForwardingSource.hpp"
#include "rpc/Errors.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/TestWsServer.hpp"
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/serialize.hpp>
#include <gtest/gtest.h>
#include <algorithm>
#include <chrono>
#include <memory>
#include <optional>
#include <string>
#include <utility>
using namespace etlng::impl;
struct ForwardingSourceNgTests : SyncAsioContextTest {
protected:
TestWsServer server_{ctx_, "0.0.0.0"};
ForwardingSource forwardingSource_{
"127.0.0.1",
server_.port(),
std::chrono::milliseconds{20},
std::chrono::milliseconds{20}
};
};
TEST_F(ForwardingSourceNgTests, ConnectionFailed)
{
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled({}, {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlConnectionError);
});
}
struct ForwardingSourceOperationsNgTests : ForwardingSourceNgTests {
TestWsConnection
serverConnection(boost::asio::yield_context yield)
{
// First connection attempt is SSL handshake so it will fail
auto failedConnection = server_.acceptConnection(yield);
[&]() { ASSERT_FALSE(failedConnection); }();
auto connection = server_.acceptConnection(yield);
[&]() { ASSERT_TRUE(connection) << connection.error().message(); }();
return std::move(connection).value();
}
protected:
std::string const message_ = R"({"data": "some_data"})";
boost::json::object const reply_ = {{"reply", "some_reply"}};
};
TEST_F(ForwardingSourceOperationsNgTests, XUserHeader)
{
std::string const xUserValue = "some_user";
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto headers = connection.headers();
ASSERT_FALSE(headers.empty());
auto it = std::ranges::find_if(headers, [](auto const& header) {
return std::holds_alternative<std::string>(header.name) && std::get<std::string>(header.name) == "X-User";
});
ASSERT_FALSE(it == headers.end());
EXPECT_EQ(std::get<std::string>(it->name), "X-User");
EXPECT_EQ(it->value, xUserValue);
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result =
forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, xUserValue, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestError);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ReadFailed)
{
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestError);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ReadTimeout)
{
TestWsConnectionPtr connection;
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
connection = std::make_unique<TestWsConnection>(serverConnection(yield));
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestTimeout);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ParseFailed)
{
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send("invalid_json", yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlInvalidResponse);
});
}
TEST_F(ForwardingSourceOperationsNgTests, GotNotAnObject)
{
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send(R"(["some_value"])", yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlInvalidResponse);
});
}
TEST_F(ForwardingSourceOperationsNgTests, Success)
{
boost::asio::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send(boost::json::serialize(reply_), yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
});
runSpawn([&](boost::asio::yield_context yield) {
auto result =
forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), "some_ip", {}, yield);
[&]() { ASSERT_TRUE(result); }();
auto expectedReply = reply_;
expectedReply["forwarded"] = true;
EXPECT_EQ(*result, expectedReply) << *result;
});
}

821
tests/unit/etlng/LoadBalancerTests.cpp Normal file
View File

@@ -0,0 +1,821 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/Models.hpp"
#include "etlng/Source.hpp"
#include "rpc/Errors.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockSourceNg.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/NameGenerator.hpp"
#include "util/Random.hpp"
#include "util/newconfig/Array.hpp"
#include "util/newconfig/ConfigConstraints.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include "util/newconfig/ConfigFileJson.hpp"
#include "util/newconfig/ConfigValue.hpp"
#include "util/newconfig/Types.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/array.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp>
#include <gmock/gmock.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
using namespace etlng;
using namespace util::config;
using testing::Return;
namespace {
constinit auto const kTWO_SOURCES_LEDGER_RESPONSE = R"({
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source1"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source2"
}
]
})";
constinit auto const kTHREE_SOURCES_LEDGER_RESPONSE = R"({
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source1"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source2"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source3"
}
]
})";
inline ClioConfigDefinition
getParseLoadBalancerConfig(boost::json::value val)
{
ClioConfigDefinition config{
{{"forwarding.cache_timeout",
ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(gValidatePositiveDouble)},
{"forwarding.request_timeout",
ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(gValidatePositiveDouble)},
{"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}},
{"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},
{"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional()}},
{"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNumMarkers)}}
};
auto const errors = config.parse(ConfigFileJson{val.as_object()});
[&]() { ASSERT_FALSE(errors.has_value()); }();
return config;
}
struct InitialLoadObserverMock : etlng::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etlng::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etlng::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct LoadBalancerConstructorNgTests : util::prometheus::WithPrometheus, MockBackendTestStrict {
std::unique_ptr<LoadBalancer>
makeLoadBalancer()
{
auto const cfg = getParseLoadBalancerConfig(configJson_);
return std::make_unique<LoadBalancer>(
cfg,
ioContext_,
backend_,
subscriptionManager_,
networkManager_,
[this](auto&&... args) -> SourcePtr { return sourceFactory_(std::forward<decltype(args)>(args)...); }
);
}
protected:
StrictMockSubscriptionManagerSharedPtr subscriptionManager_;
StrictMockNetworkValidatedLedgersPtr networkManager_;
StrictMockSourceNgFactory sourceFactory_{2};
boost::asio::io_context ioContext_;
boost::json::value configJson_ = boost::json::parse(kTWO_SOURCES_LEDGER_RESPONSE);
};
TEST_F(LoadBalancerConstructorNgTests, construct)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, forwardingTimeoutPassedToSourceFactory)
{
auto const forwardingTimeout = 10;
configJson_.as_object()["forwarding"] = boost::json::object{{"timeout", float{forwardingTimeout}}};
EXPECT_CALL(
sourceFactory_,
makeSource(
testing::_,
testing::_,
testing::_,
testing::_,
std::chrono::steady_clock::duration{std::chrono::seconds{forwardingTimeout}},
testing::_,
testing::_,
testing::_
)
)
.Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesFail)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesReturnError)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(boost::json::object{{"error", "some error"}}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(boost::json::object{{"error", "some error"}}));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_Source1Fails0OK)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_Source0Fails1OK)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_DifferentNetworkID)
{
auto const source1Json = boost::json::parse(R"({"result": {"info": {"network_id": 0}}})");
auto const source2Json = boost::json::parse(R"({"result": {"info": {"network_id": 1}}})");
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(source1Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(source2Json.as_object()));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesFailButAllowNoEtlIsTrue)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
configJson_.as_object()["allow_no_etl"] = true;
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_DifferentNetworkIDButAllowNoEtlIsTrue)
{
auto const source1Json = boost::json::parse(R"({"result": {"info": {"network_id": 0}}})");
auto const source2Json = boost::json::parse(R"({"result": {"info": {"network_id": 1}}})");
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(source1Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(source2Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
configJson_.as_object()["allow_no_etl"] = true;
makeLoadBalancer();
}
struct LoadBalancerOnConnectHookNgTests : LoadBalancerConstructorNgTests {
LoadBalancerOnConnectHookNgTests()
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
loadBalancer_ = makeLoadBalancer();
}
protected:
std::unique_ptr<LoadBalancer> loadBalancer_;
};
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect_Source0IsNotConnected)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect(); // assuming it connects and disconnects immediately
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(1).onConnect();
// Nothing is called on another connect
sourceFactory_.callbacksAt(0).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect_BothSourcesAreNotConnected)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(1).onConnect();
// Then source 0 got connected
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
}
struct LoadBalancerStopNgTests : LoadBalancerOnConnectHookNgTests, SyncAsioContextTest {};
TEST_F(LoadBalancerStopNgTests, stopCallsSourcesStop)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), stop);
EXPECT_CALL(sourceFactory_.sourceAt(1), stop);
runSyncOperation([this](boost::asio::yield_context yield) { loadBalancer_->stop(yield); });
}
struct LoadBalancerOnDisconnectHookNgTests : LoadBalancerOnConnectHookNgTests {
LoadBalancerOnDisconnectHookNgTests()
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
// nothing happens on source 1 connect
sourceFactory_.callbacksAt(1).onConnect();
}
};
TEST_F(LoadBalancerOnDisconnectHookNgTests, source0Disconnects)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(0).onDisconnect(true);
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source1Disconnects)
{
sourceFactory_.callbacksAt(1).onDisconnect(false);
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source0DisconnectsAndConnectsBack)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(0).onDisconnect(true);
sourceFactory_.callbacksAt(0).onConnect();
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source1DisconnectsAndConnectsBack)
{
sourceFactory_.callbacksAt(1).onDisconnect(false);
sourceFactory_.callbacksAt(1).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, bothSourcesDisconnectAndConnectBack)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onDisconnect(true);
sourceFactory_.callbacksAt(1).onDisconnect(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
}
struct LoadBalancer3SourcesNgTests : LoadBalancerConstructorNgTests {
LoadBalancer3SourcesNgTests()
{
sourceFactory_.setSourcesNumber(3);
configJson_ = boost::json::parse(kTHREE_SOURCES_LEDGER_RESPONSE);
EXPECT_CALL(sourceFactory_, makeSource).Times(3);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
EXPECT_CALL(sourceFactory_.sourceAt(2), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(2), run);
loadBalancer_ = makeLoadBalancer();
}
protected:
std::unique_ptr<LoadBalancer> loadBalancer_;
};
TEST_F(LoadBalancer3SourcesNgTests, forwardingUpdate)
{
// Source 2 is connected first
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(2), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(2), setForwarding(true));
sourceFactory_.callbacksAt(2).onConnect();
// Then source 0 and 1 are getting connected, but nothing should happen
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
// Source 0 got disconnected
sourceFactory_.callbacksAt(0).onDisconnect(false);
}
struct LoadBalancerLoadInitialLedgerNgTests : LoadBalancerOnConnectHookNgTests {
LoadBalancerLoadInitialLedgerNgTests()
{
util::Random::setSeed(0);
}
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 16;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_source0DoesntHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_bothSourcesDontHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_source0ReturnsStatusFalse)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, false)));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.first);
}
struct LoadBalancerLoadInitialLedgerCustomNumMarkersNgTests : LoadBalancerConstructorNgTests {
protected:
uint32_t const numMarkers_ = 16;
uint32_t const sequence_ = 123;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersNgTests, loadInitialLedger)
{
configJson_.as_object()["num_markers"] = numMarkers_;
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
auto loadBalancer = makeLoadBalancer();
util::Random::setSeed(0);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.first);
}
struct LoadBalancerFetchLegerNgTests : LoadBalancerOnConnectHookNgTests {
LoadBalancerFetchLegerNgTests()
{
util::Random::setSeed(0);
response_.second.set_validated(true);
}
protected:
uint32_t const sequence_ = 123;
bool const getObjects_ = true;
bool const getObjectNeighbors_ = false;
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse> response_ =
std::make_pair(grpc::Status::OK, org::xrpl::rpc::v1::GetLedgerResponse{});
};
TEST_F(LoadBalancerFetchLegerNgTests, fetch)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_Source0ReturnsBadStatus)
{
auto source0Response = response_;
source0Response.first = grpc::Status::CANCELLED;
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(source0Response));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_Source0ReturnsNotValidated)
{
auto source0Response = response_;
source0Response.second.set_validated(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(source0Response));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_bothSourcesFail)
{
auto badResponse = response_;
badResponse.second.set_validated(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(badResponse))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(badResponse));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_, std::chrono::milliseconds{1})
.has_value());
}
struct LoadBalancerForwardToRippledNgTests : LoadBalancerConstructorNgTests, SyncAsioContextTest {
LoadBalancerForwardToRippledNgTests()
{
util::Random::setSeed(0);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
}
protected:
boost::json::object const request_{{"command", "value"}};
std::optional<std::string> const clientIP_ = "some_ip";
boost::json::object const response_{{"response", "other_value"}};
};
TEST_F(LoadBalancerForwardToRippledNgTests, forward)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kADMIN_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, true, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardWithXUserHeader)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, false, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, source0Fails)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, false, yield), response_);
});
}
struct LoadBalancerForwardToRippledErrorNgTestBundle {
std::string testName;
rpc::ClioError firstSourceError;
rpc::ClioError secondSourceError;
rpc::ClioError responseExpectedError;
};
struct LoadBalancerForwardToRippledErrorNgTests
: LoadBalancerForwardToRippledNgTests,
testing::WithParamInterface<LoadBalancerForwardToRippledErrorNgTestBundle> {};
INSTANTIATE_TEST_SUITE_P(
LoadBalancerForwardToRippledErrorNgTests,
LoadBalancerForwardToRippledErrorNgTests,
testing::Values(
LoadBalancerForwardToRippledErrorNgTestBundle{
"ConnectionError_RequestError",
rpc::ClioError::EtlConnectionError,
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlRequestError
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"RequestError_RequestTimeout",
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"RequestTimeout_InvalidResponse",
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlInvalidResponse,
rpc::ClioError::EtlInvalidResponse
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"BothRequestTimeout",
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"InvalidResponse_RequestError",
rpc::ClioError::EtlInvalidResponse,
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlInvalidResponse
}
),
tests::util::kNAME_GENERATOR
);
TEST_P(LoadBalancerForwardToRippledErrorNgTests, bothSourcesFail)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{GetParam().firstSourceError}));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{GetParam().secondSourceError}));
runSpawn([&](boost::asio::yield_context yield) {
auto const response = loadBalancer->forwardToRippled(request_, clientIP_, false, yield);
ASSERT_FALSE(response);
EXPECT_EQ(response.error(), GetParam().responseExpectedError);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardingCacheEnabled)
{
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", 10.}};
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command", "server_info"}};
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardingCacheDisabledOnLedgerClosedHookCalled)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_NO_THROW(sourceFactory_.callbacksAt(0).onLedgerClosed());
}
TEST_F(LoadBalancerForwardToRippledNgTests, onLedgerClosedHookInvalidatesCache)
{
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", 10.}};
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command", "server_info"}};
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(boost::json::object{}));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
sourceFactory_.callbacksAt(0).onLedgerClosed();
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), boost::json::object{});
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, commandLineMissing)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command2", "server_info"}};
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(
loadBalancer->forwardToRippled(request, clientIP_, false, yield).error(),
rpc::ClioError::RpcCommandIsMissing
);
});
}
struct LoadBalancerToJsonNgTests : LoadBalancerOnConnectHookNgTests {};
TEST_F(LoadBalancerToJsonNgTests, toJson)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), toJson).WillOnce(Return(boost::json::object{{"source1", "value1"}}));
EXPECT_CALL(sourceFactory_.sourceAt(1), toJson).WillOnce(Return(boost::json::object{{"source2", "value2"}}));
auto const expectedJson =
boost::json::array({boost::json::object{{"source1", "value1"}}, boost::json::object{{"source2", "value2"}}});
EXPECT_EQ(loadBalancer_->toJson(), expectedJson);
}

223
tests/unit/etlng/SourceImplTests.cpp Normal file
View File

@@ -0,0 +1,223 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/SourceImpl.hpp"
#include "rpc/Errors.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/json/value_to.hpp>
#include <gmock/gmock.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
using namespace etlng::impl;
using testing::Return;
using testing::StrictMock;
namespace {
struct GrpcSourceMock {
using FetchLedgerReturnType = std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>;
MOCK_METHOD(FetchLedgerReturnType, fetchLedger, (uint32_t, bool, bool));
using LoadLedgerReturnType = std::pair<std::vector<std::string>, bool>;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&));
};
struct SubscriptionSourceMock {
MOCK_METHOD(void, run, ());
MOCK_METHOD(bool, hasLedger, (uint32_t), (const));
MOCK_METHOD(bool, isConnected, (), (const));
MOCK_METHOD(void, setForwarding, (bool));
MOCK_METHOD(std::chrono::steady_clock::time_point, lastMessageTime, (), (const));
MOCK_METHOD(std::string, validatedRange, (), (const));
MOCK_METHOD(void, stop, (boost::asio::yield_context));
};
struct ForwardingSourceMock {
MOCK_METHOD(void, constructor, (std::string const&, std::string const&, std::chrono::steady_clock::duration));
using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
using ClientIpOpt = std::optional<std::string>;
MOCK_METHOD(
ForwardToRippledReturnType,
forwardToRippled,
(boost::json::object const&, ClientIpOpt const&, std::string_view, boost::asio::yield_context),
(const)
);
};
struct InitialLoadObserverMock : etlng::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etlng::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etlng::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct SourceImplNgTest : public ::testing::Test {
protected:
boost::asio::io_context ioc_;
StrictMock<GrpcSourceMock> grpcSourceMock_;
std::shared_ptr<StrictMock<SubscriptionSourceMock>> subscriptionSourceMock_ =
std::make_shared<StrictMock<SubscriptionSourceMock>>();
StrictMock<ForwardingSourceMock> forwardingSourceMock_;
SourceImpl<
StrictMock<GrpcSourceMock>&,
std::shared_ptr<StrictMock<SubscriptionSourceMock>>,
StrictMock<ForwardingSourceMock>&>
source_{
"some_ip",
"some_ws_port",
"some_grpc_port",
grpcSourceMock_,
subscriptionSourceMock_,
forwardingSourceMock_
};
};
TEST_F(SourceImplNgTest, run)
{
EXPECT_CALL(*subscriptionSourceMock_, run());
source_.run();
}
TEST_F(SourceImplNgTest, stop)
{
EXPECT_CALL(*subscriptionSourceMock_, stop);
boost::asio::io_context ctx;
boost::asio::spawn(ctx, [&](boost::asio::yield_context yield) { source_.stop(yield); });
ctx.run();
}
TEST_F(SourceImplNgTest, isConnected)
{
EXPECT_CALL(*subscriptionSourceMock_, isConnected()).WillOnce(testing::Return(true));
EXPECT_TRUE(source_.isConnected());
}
TEST_F(SourceImplNgTest, setForwarding)
{
EXPECT_CALL(*subscriptionSourceMock_, setForwarding(true));
source_.setForwarding(true);
}
TEST_F(SourceImplNgTest, toJson)
{
EXPECT_CALL(*subscriptionSourceMock_, validatedRange()).WillOnce(Return(std::string("some_validated_range")));
EXPECT_CALL(*subscriptionSourceMock_, isConnected()).WillOnce(Return(true));
auto const lastMessageTime = std::chrono::steady_clock::now();
EXPECT_CALL(*subscriptionSourceMock_, lastMessageTime()).WillOnce(Return(lastMessageTime));
auto const json = source_.toJson();
EXPECT_EQ(boost::json::value_to<std::string>(json.at("validated_range")), "some_validated_range");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("is_connected")), "1");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("ip")), "some_ip");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("ws_port")), "some_ws_port");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("grpc_port")), "some_grpc_port");
auto lastMessageAgeStr = boost::json::value_to<std::string>(json.at("last_msg_age_seconds"));
EXPECT_GE(std::stoi(lastMessageAgeStr), 0);
}
TEST_F(SourceImplNgTest, toString)
{
EXPECT_CALL(*subscriptionSourceMock_, validatedRange()).WillOnce(Return(std::string("some_validated_range")));
auto const str = source_.toString();
EXPECT_EQ(
str,
"{validated range: some_validated_range, ip: some_ip, web socket port: some_ws_port, grpc port: some_grpc_port}"
);
}
TEST_F(SourceImplNgTest, hasLedger)
{
uint32_t const ledgerSeq = 123;
EXPECT_CALL(*subscriptionSourceMock_, hasLedger(ledgerSeq)).WillOnce(Return(true));
EXPECT_TRUE(source_.hasLedger(ledgerSeq));
}
TEST_F(SourceImplNgTest, fetchLedger)
{
uint32_t const ledgerSeq = 123;
EXPECT_CALL(grpcSourceMock_, fetchLedger(ledgerSeq, true, false));
auto const [actualStatus, actualResponse] = source_.fetchLedger(ledgerSeq);
EXPECT_EQ(actualStatus.error_code(), grpc::StatusCode::OK);
}
TEST_F(SourceImplNgTest, loadInitialLedger)
{
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
auto observerMock = testing::StrictMock<InitialLoadObserverMock>();
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, testing::_))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, true)));
auto const [actualLedgers, actualSuccess] = source_.loadInitialLedger(ledgerSeq, numMarkers, observerMock);
EXPECT_TRUE(actualLedgers.empty());
EXPECT_TRUE(actualSuccess);
}
TEST_F(SourceImplNgTest, forwardToRippled)
{
boost::json::object const request = {{"some_key", "some_value"}};
std::optional<std::string> const clientIp = "some_client_ip";
std::string_view xUserValue = "some_user";
EXPECT_CALL(forwardingSourceMock_, forwardToRippled(request, clientIp, xUserValue, testing::_))
.WillOnce(Return(request));
boost::asio::io_context ioContext;
boost::asio::spawn(ioContext, [&](boost::asio::yield_context yield) {
auto const response = source_.forwardToRippled(request, clientIp, xUserValue, yield);
EXPECT_EQ(response, request);
});
ioContext.run();
}

6
tests/unit/rpc/ForwardingProxyTests.cpp
View File

@@ -60,11 +60,7 @@ protected:
ClioConfigDefinition const config_{{"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none")}};
util::TagDecoratorFactory tagFactory_{config_};
rpc::impl::ForwardingProxy<MockLoadBalancer, MockCounters, MockHandlerProvider> proxy_{
loadBalancer_,
counters_,
handlerProvider_
};
rpc::impl::ForwardingProxy<MockCounters, MockHandlerProvider> proxy_{loadBalancer_, counters_, handlerProvider_};
};
struct ShouldForwardParamTestCaseBundle {

61
tests/unit/rpc/RPCEngineTests.cpp
View File

@@ -65,7 +65,7 @@ using namespace util::config;
namespace {
constexpr auto kFORWARD_REPLY = R"JSON({
"result":
"result":
{
"status": "success",
"forwarded": true
@@ -200,16 +200,15 @@ TEST_P(RPCEngineFlowParameterTest, Test)
{
auto const& testBundle = GetParam();
std::shared_ptr<RPCEngine<MockLoadBalancer, MockCounters>> engine =
RPCEngine<MockLoadBalancer, MockCounters>::makeRPCEngine(
generateDefaultRPCEngineConfig(),
backend_,
mockLoadBalancerPtr_,
dosGuard,
queue,
*mockCountersPtr_,
handlerProvider
);
std::shared_ptr<RPCEngine<MockCounters>> engine = RPCEngine<MockCounters>::makeRPCEngine(
generateDefaultRPCEngineConfig(),
backend_,
mockLoadBalancerPtr_,
dosGuard,
queue,
*mockCountersPtr_,
handlerProvider
);
if (testBundle.forwarded) {
EXPECT_CALL(*mockLoadBalancerPtr_, forwardToRippled)
@@ -272,10 +271,9 @@ TEST_P(RPCEngineFlowParameterTest, Test)
TEST_F(RPCEngineTest, ThrowDatabaseError)
{
auto const method = "subscribe";
std::shared_ptr<RPCEngine<MockLoadBalancer, MockCounters>> engine =
RPCEngine<MockLoadBalancer, MockCounters>::makeRPCEngine(
cfg, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
std::shared_ptr<RPCEngine<MockCounters>> engine = RPCEngine<MockCounters>::makeRPCEngine(
cfg, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
EXPECT_CALL(*backend_, isTooBusy).WillOnce(Return(false));
EXPECT_CALL(*handlerProvider, getHandler(method)).WillOnce(Return(AnyHandler{tests::common::FailingHandlerFake{}}));
EXPECT_CALL(*mockCountersPtr_, rpcErrored(method)).WillOnce(Throw(data::DatabaseTimeout{}));
@@ -305,10 +303,9 @@ TEST_F(RPCEngineTest, ThrowDatabaseError)
TEST_F(RPCEngineTest, ThrowException)
{
auto const method = "subscribe";
std::shared_ptr<RPCEngine<MockLoadBalancer, MockCounters>> engine =
RPCEngine<MockLoadBalancer, MockCounters>::makeRPCEngine(
cfg, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
std::shared_ptr<RPCEngine<MockCounters>> engine = RPCEngine<MockCounters>::makeRPCEngine(
cfg, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
EXPECT_CALL(*backend_, isTooBusy).WillOnce(Return(false));
EXPECT_CALL(*handlerProvider, getHandler(method)).WillOnce(Return(AnyHandler{tests::common::FailingHandlerFake{}}));
EXPECT_CALL(*mockCountersPtr_, rpcErrored(method)).WillOnce(Throw(std::exception{}));
@@ -353,14 +350,14 @@ generateCacheTestValuesForParametersTest()
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc":
"rpc":
{"cache_timeout": 10}
})JSON",
.method = "server_info",
.isAdmin = false,
.expectedCacheEnabled = true},
{.testName = "CacheDisabledWhenNoConfig",
.config = R"JSON({
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc": {"cache_timeout": 0}
@@ -369,7 +366,7 @@ generateCacheTestValuesForParametersTest()
.isAdmin = false,
.expectedCacheEnabled = false},
{.testName = "CacheDisabledWhenNoTimeout",
.config = R"JSON({
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc": {"cache_timeout": 0}
@@ -378,7 +375,7 @@ generateCacheTestValuesForParametersTest()
.isAdmin = false,
.expectedCacheEnabled = false},
{.testName = "CacheDisabledWhenTimeoutIsZero",
.config = R"JSON({
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc": {"cache_timeout": 0}
@@ -387,7 +384,7 @@ generateCacheTestValuesForParametersTest()
.isAdmin = false,
.expectedCacheEnabled = false},
{.testName = "CacheNotWorkForAdmin",
.config = R"JSON({
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc": { "cache_timeout": 10}
@@ -396,7 +393,7 @@ generateCacheTestValuesForParametersTest()
.isAdmin = true,
.expectedCacheEnabled = false},
{.testName = "CacheDisabledWhenCmdNotMatch",
.config = R"JSON({
.config = R"JSON({
"server": {"max_queue_size": 2},
"workers": 4,
"rpc": {"cache_timeout": 10}
@@ -425,10 +422,9 @@ TEST_P(RPCEngineCacheParameterTest, Test)
auto const admin = testParam.isAdmin;
auto const method = testParam.method;
std::shared_ptr<RPCEngine<MockLoadBalancer, MockCounters>> engine =
RPCEngine<MockLoadBalancer, MockCounters>::makeRPCEngine(
cfgCache, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
std::shared_ptr<RPCEngine<MockCounters>> engine = RPCEngine<MockCounters>::makeRPCEngine(
cfgCache, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
int callTime = 2;
EXPECT_CALL(*handlerProvider, isClioOnly).Times(callTime).WillRepeatedly(Return(false));
if (testParam.expectedCacheEnabled) {
@@ -474,10 +470,9 @@ TEST_F(RPCEngineTest, NotCacheIfErrorHappen)
auto const notAdmin = false;
auto const method = "server_info";
std::shared_ptr<RPCEngine<MockLoadBalancer, MockCounters>> engine =
RPCEngine<MockLoadBalancer, MockCounters>::makeRPCEngine(
cfgCache, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
std::shared_ptr<RPCEngine<MockCounters>> engine = RPCEngine<MockCounters>::makeRPCEngine(
cfgCache, backend_, mockLoadBalancerPtr_, dosGuard, queue, *mockCountersPtr_, handlerProvider
);
int callTime = 2;
EXPECT_CALL(*backend_, isTooBusy).Times(callTime).WillRepeatedly(Return(false));

2
tests/unit/rpc/handlers/AllHandlerTests.cpp
View File

@@ -74,7 +74,7 @@
using ::testing::Types;
using namespace rpc;
using TestServerInfoHandler = BaseServerInfoHandler<MockLoadBalancer, MockETLService, MockCounters>;
using TestServerInfoHandler = BaseServerInfoHandler<MockCounters>;
constexpr static auto kINDEX1 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DD";
constexpr static auto kAMM_ACCOUNT = "rLcS7XL6nxRAi7JcbJcn1Na179oF3vdfbh";

2
tests/unit/rpc/handlers/ServerInfoTests.cpp
View File

@@ -46,7 +46,7 @@ using namespace data;
namespace json = boost::json;
using namespace testing;
using TestServerInfoHandler = BaseServerInfoHandler<MockLoadBalancer, MockETLService, MockCounters>;
using TestServerInfoHandler = BaseServerInfoHandler<MockCounters>;
namespace {

56
tests/unit/rpc/handlers/TxTests.cpp
View File

@@ -44,7 +44,7 @@ using namespace data;
namespace json = boost::json;
using namespace testing;
using TestTxHandler = BaseTxHandler<MockETLService>;
using TestTxHandler = TxHandler;
namespace {
@@ -148,7 +148,7 @@ TEST_F(RPCTxTest, ExcessiveLgrRange)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"min_ledger": 1,
@@ -182,7 +182,7 @@ TEST_F(RPCTxTest, InvalidBinaryV1)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"binary": 12
@@ -199,7 +199,7 @@ TEST_F(RPCTxTest, InvalidBinaryV2)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"binary": 12
@@ -220,7 +220,7 @@ TEST_F(RPCTxTest, InvalidLgrRange)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"max_ledger": 1,
@@ -249,7 +249,7 @@ TEST_F(RPCTxTest, TxnNotFound)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -277,7 +277,7 @@ TEST_F(RPCTxTest, TxnNotFoundInGivenRangeSearchAllFalse)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"min_ledger": 1,
@@ -308,7 +308,7 @@ TEST_F(RPCTxTest, TxnNotFoundInGivenRangeSearchAllTrue)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"min_ledger": 1,
@@ -340,7 +340,7 @@ TEST_F(RPCTxTest, CtidNotFoundSearchAllFalse)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"ctid": "{}",
"min_ledger": 1,
"max_ledger": 1000
@@ -375,7 +375,7 @@ TEST_F(RPCTxTest, DefaultParameter_API_v1)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -405,7 +405,7 @@ TEST_F(RPCTxTest, PaymentTx_API_v1)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -436,7 +436,7 @@ TEST_F(RPCTxTest, PaymentTx_API_v2)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -470,7 +470,7 @@ TEST_F(RPCTxTest, DefaultParameter_API_v2)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -510,7 +510,7 @@ TEST_F(RPCTxTest, ReturnBinary)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"binary": true
@@ -553,7 +553,7 @@ TEST_F(RPCTxTest, ReturnBinaryWithCTID)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"binary": true
@@ -585,14 +585,14 @@ TEST_F(RPCTxTest, MintNFT)
"FinalFields": {{
"NFTokens": [
{{
"NFToken":
"NFToken":
{{
"NFTokenID": "{}",
"URI": "7465737475726C"
}}
}},
{{
"NFToken":
"NFToken":
{{
"NFTokenID": "1B8590C01B0006EDFA9ED60296DD052DC5E90F99659B25014D08E1BC983515BC",
"URI": "7465737475726C"
@@ -604,7 +604,7 @@ TEST_F(RPCTxTest, MintNFT)
"PreviousFields": {{
"NFTokens": [
{{
"NFToken":
"NFToken":
{{
"NFTokenID": "1B8590C01B0006EDFA9ED60296DD052DC5E90F99659B25014D08E1BC983515BC",
"URI": "7465737475726C"
@@ -640,7 +640,7 @@ TEST_F(RPCTxTest, MintNFT)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -667,7 +667,7 @@ TEST_F(RPCTxTest, NFTAcceptOffer)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -695,7 +695,7 @@ TEST_F(RPCTxTest, NFTCancelOffer)
runSpawn([this, &ids](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -730,7 +730,7 @@ TEST_F(RPCTxTest, NFTCreateOffer)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -747,7 +747,7 @@ TEST_F(RPCTxTest, CTIDAndTransactionBothProvided)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}",
"ctid": "{}"
@@ -819,7 +819,7 @@ TEST_F(RPCTxTest, CTIDNotMatch)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"ctid": "{}"
}})",
@@ -899,7 +899,7 @@ TEST_F(RPCTxTest, ReturnCTIDForTxInput)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -972,7 +972,7 @@ TEST_F(RPCTxTest, NotReturnCTIDIfETLNotAvaiable)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"transaction": "{}"
}})",
@@ -1057,7 +1057,7 @@ TEST_F(RPCTxTest, ViaCTID)
runSpawn([this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"ctid": "{}"
}})",
@@ -1095,7 +1095,7 @@ TEST_F(RPCTxTest, ViaLowercaseCTID)
runSpawn([&, this](auto yield) {
auto const handler = AnyHandler{TestTxHandler{backend_, mockETLServicePtr_}};
auto const req = json::parse(fmt::format(
R"({{
R"({{
"command": "tx",
"ctid": "{}"
}})",

10
tests/unit/web/RPCServerHandlerTests.cpp
View File

@@ -95,8 +95,8 @@ struct WebRPCServerHandlerTest : util::prometheus::WithPrometheus, MockBackendTe
std::shared_ptr<MockAsyncRPCEngine> rpcEngine = std::make_shared<MockAsyncRPCEngine>();
std::shared_ptr<MockETLService> etl = std::make_shared<MockETLService>();
std::shared_ptr<util::TagDecoratorFactory> tagFactory = std::make_shared<util::TagDecoratorFactory>(cfg);
std::shared_ptr<RPCServerHandler<MockAsyncRPCEngine, MockETLService>> handler =
std::make_shared<RPCServerHandler<MockAsyncRPCEngine, MockETLService>>(cfg, backend_, rpcEngine, etl);
std::shared_ptr<RPCServerHandler<MockAsyncRPCEngine>> handler =
std::make_shared<RPCServerHandler<MockAsyncRPCEngine>>(cfg, backend_, rpcEngine, etl);
std::shared_ptr<MockWsBase> session = std::make_shared<MockWsBase>(*tagFactory);
};
@@ -756,8 +756,7 @@ TEST_F(WebRPCServerHandlerTest, WsTooBusy)
session->upgraded = true;
auto localRpcEngine = std::make_shared<MockRPCEngine>();
auto localHandler =
std::make_shared<RPCServerHandler<MockRPCEngine, MockETLService>>(cfg, backend_, localRpcEngine, etl);
auto localHandler = std::make_shared<RPCServerHandler<MockRPCEngine>>(cfg, backend_, localRpcEngine, etl);
static constexpr auto kREQUEST = R"({
"command": "server_info",
"id": 99
@@ -784,8 +783,7 @@ TEST_F(WebRPCServerHandlerTest, WsTooBusy)
TEST_F(WebRPCServerHandlerTest, HTTPTooBusy)
{
auto localRpcEngine = std::make_shared<MockRPCEngine>();
auto localHandler =
std::make_shared<RPCServerHandler<MockRPCEngine, MockETLService>>(cfg, backend_, localRpcEngine, etl);
auto localHandler = std::make_shared<RPCServerHandler<MockRPCEngine>>(cfg, backend_, localRpcEngine, etl);
static constexpr auto kREQUEST = R"({
"method": "server_info",
"params": [{}]

2
tests/unit/web/ng/RPCServerHandlerTests.cpp
View File

@@ -72,7 +72,7 @@ protected:
std::shared_ptr<testing::StrictMock<MockRPCEngine>> rpcEngine_ =
std::make_shared<testing::StrictMock<MockRPCEngine>>();
std::shared_ptr<StrictMock<MockETLService>> etl_ = std::make_shared<StrictMock<MockETLService>>();
RPCServerHandler<MockRPCEngine, MockETLService> rpcServerHandler_{config, backend_, rpcEngine_, etl_};
RPCServerHandler<MockRPCEngine> rpcServerHandler_{config, backend_, rpcEngine_, etl_};
util::TagDecoratorFactory tagFactory_{config};
StrictMockConnectionMetadata connectionMetadata_{"some ip", tagFactory_};