mirror of
https://github.com/XRPLF/clio.git
synced 2026-06-05 09:46:48 +00:00
@@ -99,7 +99,8 @@ public:
|
||||
bool cacheOnly = false
|
||||
)
|
||||
{
|
||||
LOG(log_.trace()) << "Processing response. " << "Marker prefix = " << getMarkerPrefix();
|
||||
LOG(log_.trace()) << "Processing response. "
|
||||
<< "Marker prefix = " << getMarkerPrefix();
|
||||
if (abort) {
|
||||
LOG(log_.error()) << "AsyncCallData aborted";
|
||||
return CallStatus::ERRORED;
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "etl/NetworkValidatedLedgersInterface.hpp"
|
||||
#include "etl/SystemState.hpp"
|
||||
#include "util/Assert.hpp"
|
||||
#include "util/Profiler.hpp"
|
||||
@@ -39,12 +40,12 @@ namespace etl::impl {
|
||||
/**
|
||||
* @brief Extractor thread that is fetching GRPC data and enqueue it on the DataPipeType
|
||||
*/
|
||||
template <typename DataPipeType, typename NetworkValidatedLedgersType, typename LedgerFetcherType>
|
||||
template <typename DataPipeType, typename LedgerFetcherType>
|
||||
class Extractor {
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
std::reference_wrapper<DataPipeType> pipe_;
|
||||
std::shared_ptr<NetworkValidatedLedgersType> networkValidatedLedgers_;
|
||||
std::shared_ptr<NetworkValidatedLedgersInterface> networkValidatedLedgers_;
|
||||
std::reference_wrapper<LedgerFetcherType> ledgerFetcher_;
|
||||
uint32_t startSequence_;
|
||||
std::optional<uint32_t> finishSequence_;
|
||||
@@ -55,7 +56,7 @@ class Extractor {
|
||||
public:
|
||||
Extractor(
|
||||
DataPipeType& pipe,
|
||||
std::shared_ptr<NetworkValidatedLedgersType> networkValidatedLedgers,
|
||||
std::shared_ptr<NetworkValidatedLedgersInterface> networkValidatedLedgers,
|
||||
LedgerFetcherType& ledgerFetcher,
|
||||
uint32_t startSequence,
|
||||
std::optional<uint32_t> finishSequence,
|
||||
|
||||
@@ -197,58 +197,55 @@ public:
|
||||
// asyncWriter consumes from the queue and inserts the data into the
|
||||
// Ledger object. Once the below call returns, all data has been pushed
|
||||
// into the queue
|
||||
auto [edgeKeys, success] = loadBalancer_->loadInitialLedger(sequence);
|
||||
auto edgeKeys = loadBalancer_->loadInitialLedger(sequence);
|
||||
|
||||
if (success) {
|
||||
size_t numWrites = 0;
|
||||
backend_->cache().setFull();
|
||||
size_t numWrites = 0;
|
||||
backend_->cache().setFull();
|
||||
|
||||
auto seconds = ::util::timed<std::chrono::seconds>([this, keys = &edgeKeys, sequence, &numWrites] {
|
||||
for (auto& key : *keys) {
|
||||
LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
|
||||
auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
|
||||
if (succ)
|
||||
backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
|
||||
}
|
||||
auto seconds = ::util::timed<std::chrono::seconds>([this, keys = std::move(edgeKeys), sequence, &numWrites](
|
||||
) mutable {
|
||||
for (auto& key : keys) {
|
||||
LOG(log_.debug()) << "Writing edge key = " << ripple::strHex(key);
|
||||
auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
|
||||
if (succ)
|
||||
backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
|
||||
}
|
||||
|
||||
ripple::uint256 prev = data::firstKey;
|
||||
while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
|
||||
ASSERT(cur.has_value(), "Succesor for key {} must exist", ripple::strHex(prev));
|
||||
if (prev == data::firstKey)
|
||||
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
|
||||
ripple::uint256 prev = data::firstKey;
|
||||
while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
|
||||
ASSERT(cur.has_value(), "Succesor for key {} must exist", ripple::strHex(prev));
|
||||
if (prev == data::firstKey)
|
||||
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
|
||||
|
||||
if (isBookDir(cur->key, cur->blob)) {
|
||||
auto base = getBookBase(cur->key);
|
||||
// make sure the base is not an actual object
|
||||
if (!backend_->cache().get(base, sequence)) {
|
||||
auto succ = backend_->cache().getSuccessor(base, sequence);
|
||||
ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
|
||||
if (succ->key == cur->key) {
|
||||
LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base) << " - "
|
||||
<< ripple::strHex(cur->key);
|
||||
if (isBookDir(cur->key, cur->blob)) {
|
||||
auto base = getBookBase(cur->key);
|
||||
// make sure the base is not an actual object
|
||||
if (!backend_->cache().get(base, sequence)) {
|
||||
auto succ = backend_->cache().getSuccessor(base, sequence);
|
||||
ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
|
||||
if (succ->key == cur->key) {
|
||||
LOG(log_.debug()) << "Writing book successor = " << ripple::strHex(base) << " - "
|
||||
<< ripple::strHex(cur->key);
|
||||
|
||||
backend_->writeSuccessor(
|
||||
uint256ToString(base), sequence, uint256ToString(cur->key)
|
||||
);
|
||||
}
|
||||
backend_->writeSuccessor(uint256ToString(base), sequence, uint256ToString(cur->key));
|
||||
}
|
||||
|
||||
++numWrites;
|
||||
}
|
||||
|
||||
prev = cur->key;
|
||||
static constexpr std::size_t LOG_INTERVAL = 100000;
|
||||
if (numWrites % LOG_INTERVAL == 0 && numWrites != 0)
|
||||
LOG(log_.info()) << "Wrote " << numWrites << " book successors";
|
||||
++numWrites;
|
||||
}
|
||||
|
||||
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
|
||||
++numWrites;
|
||||
});
|
||||
prev = cur->key;
|
||||
static constexpr std::size_t LOG_INTERVAL = 100000;
|
||||
if (numWrites % LOG_INTERVAL == 0 && numWrites != 0)
|
||||
LOG(log_.info()) << "Wrote " << numWrites << " book successors";
|
||||
}
|
||||
|
||||
LOG(log_.info()) << "Looping through cache and submitting all writes took " << seconds
|
||||
<< " seconds. numWrites = " << std::to_string(numWrites);
|
||||
}
|
||||
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
|
||||
++numWrites;
|
||||
});
|
||||
|
||||
LOG(log_.info()) << "Looping through cache and submitting all writes took " << seconds
|
||||
<< " seconds. numWrites = " << std::to_string(numWrites);
|
||||
|
||||
LOG(log_.debug()) << "Loaded initial ledger";
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etl/SystemState.hpp"
|
||||
#include "feed/SubscriptionManagerInterface.hpp"
|
||||
#include "util/Assert.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
#include "util/prometheus/Counter.hpp"
|
||||
@@ -63,7 +64,7 @@ namespace etl::impl {
|
||||
* includes reading all of the transactions from the database) is done from the application wide asio io_service, and a
|
||||
* strand is used to ensure ledgers are published in order.
|
||||
*/
|
||||
template <typename SubscriptionManagerType, typename CacheType>
|
||||
template <typename CacheType>
|
||||
class LedgerPublisher {
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
@@ -71,7 +72,7 @@ class LedgerPublisher {
|
||||
|
||||
std::shared_ptr<BackendInterface> backend_;
|
||||
std::reference_wrapper<CacheType> cache_;
|
||||
std::shared_ptr<SubscriptionManagerType> subscriptions_;
|
||||
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
|
||||
std::reference_wrapper<SystemState const> state_; // shared state for ETL
|
||||
|
||||
std::chrono::time_point<ripple::NetClock> lastCloseTime_;
|
||||
@@ -94,7 +95,7 @@ public:
|
||||
boost::asio::io_context& ioc,
|
||||
std::shared_ptr<BackendInterface> backend,
|
||||
CacheType& cache,
|
||||
std::shared_ptr<SubscriptionManagerType> subscriptions,
|
||||
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
|
||||
SystemState const& state
|
||||
)
|
||||
: publishStrand_{boost::asio::make_strand(ioc)}
|
||||
@@ -110,11 +111,16 @@ public:
|
||||
* stream.
|
||||
*
|
||||
* @param ledgerSequence the sequence of the ledger to publish
|
||||
* @param maxAttempts the number of times to attempt to read the ledger from the database. 1 attempt per second
|
||||
* @param maxAttempts the number of times to attempt to read the ledger from the database
|
||||
* @param attemptsDelay the delay between attempts to read the ledger from the database
|
||||
* @return Whether the ledger was found in the database and published
|
||||
*/
|
||||
bool
|
||||
publish(uint32_t ledgerSequence, std::optional<uint32_t> maxAttempts)
|
||||
publish(
|
||||
uint32_t ledgerSequence,
|
||||
std::optional<uint32_t> maxAttempts,
|
||||
std::chrono::steady_clock::duration attemptsDelay = std::chrono::seconds{1}
|
||||
)
|
||||
{
|
||||
LOG(log_.info()) << "Attempting to publish ledger = " << ledgerSequence;
|
||||
size_t numAttempts = 0;
|
||||
@@ -130,7 +136,7 @@ public:
|
||||
LOG(log_.debug()) << "Failed to publish ledger after " << numAttempts << " attempts.";
|
||||
return false;
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::seconds(1));
|
||||
std::this_thread::sleep_for(attemptsDelay);
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
219
src/etl/impl/SourceImpl.hpp
Normal file
219
src/etl/impl/SourceImpl.hpp
Normal file
@@ -0,0 +1,219 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2024, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "etl/Source.hpp"
|
||||
#include "etl/impl/ForwardingSource.hpp"
|
||||
#include "etl/impl/GrpcSource.hpp"
|
||||
#include "etl/impl/SubscriptionSource.hpp"
|
||||
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/json/object.hpp>
|
||||
#include <grpcpp/support/status.h>
|
||||
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
namespace etl::impl {
|
||||
|
||||
/**
|
||||
* @brief Provides an implementation of a ETL source
|
||||
*
|
||||
* @tparam GrpcSourceType The type of the gRPC source
|
||||
* @tparam SubscriptionSourceTypePtr The type of the subscription source
|
||||
* @tparam ForwardingSourceType The type of the forwarding source
|
||||
*/
|
||||
template <
|
||||
typename GrpcSourceType = GrpcSource,
|
||||
typename SubscriptionSourceTypePtr = std::unique_ptr<SubscriptionSource>,
|
||||
typename ForwardingSourceType = ForwardingSource>
|
||||
class SourceImpl : public SourceBase {
|
||||
std::string ip_;
|
||||
std::string wsPort_;
|
||||
std::string grpcPort_;
|
||||
|
||||
GrpcSourceType grpcSource_;
|
||||
SubscriptionSourceTypePtr subscriptionSource_;
|
||||
ForwardingSourceType forwardingSource_;
|
||||
|
||||
public:
|
||||
/**
|
||||
* @brief Construct a new SourceImpl object
|
||||
*
|
||||
* @param ip The IP of the source
|
||||
* @param wsPort The web socket port of the source
|
||||
* @param grpcPort The gRPC port of the source
|
||||
* @param grpcSource The gRPC source
|
||||
* @param subscriptionSource The subscription source
|
||||
* @param forwardingSource The forwarding source
|
||||
*/
|
||||
template <typename SomeGrpcSourceType, typename SomeForwardingSourceType>
|
||||
requires std::is_same_v<GrpcSourceType, SomeGrpcSourceType> and
|
||||
std::is_same_v<ForwardingSourceType, SomeForwardingSourceType>
|
||||
SourceImpl(
|
||||
std::string ip,
|
||||
std::string wsPort,
|
||||
std::string grpcPort,
|
||||
SomeGrpcSourceType&& grpcSource,
|
||||
SubscriptionSourceTypePtr subscriptionSource,
|
||||
SomeForwardingSourceType&& forwardingSource
|
||||
)
|
||||
: ip_(std::move(ip))
|
||||
, wsPort_(std::move(wsPort))
|
||||
, grpcPort_(std::move(grpcPort))
|
||||
, grpcSource_(std::forward<SomeGrpcSourceType>(grpcSource))
|
||||
, subscriptionSource_(std::move(subscriptionSource))
|
||||
, forwardingSource_(std::forward<SomeForwardingSourceType>(forwardingSource))
|
||||
{
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Run subscriptions loop of the source
|
||||
*/
|
||||
void
|
||||
run() final
|
||||
{
|
||||
subscriptionSource_->run();
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Check if source is connected
|
||||
*
|
||||
* @return true if source is connected; false otherwise
|
||||
*/
|
||||
bool
|
||||
isConnected() const final
|
||||
{
|
||||
return subscriptionSource_->isConnected();
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Set the forwarding state of the source.
|
||||
*
|
||||
* @param isForwarding Whether to forward or not
|
||||
*/
|
||||
void
|
||||
setForwarding(bool isForwarding) final
|
||||
{
|
||||
subscriptionSource_->setForwarding(isForwarding);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Represent the source as a JSON object
|
||||
*
|
||||
* @return JSON representation of the source
|
||||
*/
|
||||
boost::json::object
|
||||
toJson() const final
|
||||
{
|
||||
boost::json::object res;
|
||||
|
||||
res["validated_range"] = subscriptionSource_->validatedRange();
|
||||
res["is_connected"] = std::to_string(static_cast<int>(subscriptionSource_->isConnected()));
|
||||
res["ip"] = ip_;
|
||||
res["ws_port"] = wsPort_;
|
||||
res["grpc_port"] = grpcPort_;
|
||||
|
||||
auto last = subscriptionSource_->lastMessageTime();
|
||||
if (last.time_since_epoch().count() != 0) {
|
||||
res["last_msg_age_seconds"] = std::to_string(
|
||||
std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - last).count()
|
||||
);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
/** @return String representation of the source (for debug) */
|
||||
std::string
|
||||
toString() const final
|
||||
{
|
||||
return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ +
|
||||
", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}";
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Check if ledger is known by this source.
|
||||
*
|
||||
* @param sequence The ledger sequence to check
|
||||
* @return true if ledger is in the range of this source; false otherwise
|
||||
*/
|
||||
bool
|
||||
hasLedger(uint32_t sequence) const final
|
||||
{
|
||||
return subscriptionSource_->hasLedger(sequence);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Fetch data for a specific ledger.
|
||||
*
|
||||
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
|
||||
* is found in the database, or the server is shutting down.
|
||||
*
|
||||
* @param sequence Sequence of the ledger to fetch
|
||||
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
|
||||
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
|
||||
* @return A std::pair of the response status and the response itself
|
||||
*/
|
||||
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
|
||||
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final
|
||||
{
|
||||
return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Download a ledger in full.
|
||||
*
|
||||
* @param sequence Sequence of the ledger to download
|
||||
* @param numMarkers Number of markers to generate for async calls
|
||||
* @param cacheOnly Only insert into cache, not the DB; defaults to false
|
||||
* @return A std::pair of the data and a bool indicating whether the download was successful
|
||||
*/
|
||||
std::pair<std::vector<std::string>, bool>
|
||||
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final
|
||||
{
|
||||
return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Forward a request to rippled.
|
||||
*
|
||||
* @param request The request to forward
|
||||
* @param forwardToRippledClientIp IP of the client forwarding this request if known
|
||||
* @param yield The coroutine context
|
||||
* @return Response wrapped in an optional on success; nullopt otherwise
|
||||
*/
|
||||
std::optional<boost::json::object>
|
||||
forwardToRippled(
|
||||
boost::json::object const& request,
|
||||
std::optional<std::string> const& forwardToRippledClientIp,
|
||||
boost::asio::yield_context yield
|
||||
) const final
|
||||
{
|
||||
return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, yield);
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace etl::impl
|
||||
@@ -19,6 +19,8 @@
|
||||
|
||||
#include "etl/impl/SubscriptionSource.hpp"
|
||||
|
||||
#include "etl/ETLHelpers.hpp"
|
||||
#include "feed/SubscriptionManagerInterface.hpp"
|
||||
#include "rpc/JS.hpp"
|
||||
#include "util/Retry.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
@@ -28,8 +30,11 @@
|
||||
#include <boost/algorithm/string/split.hpp>
|
||||
#include <boost/asio/error.hpp>
|
||||
#include <boost/asio/executor_work_guard.hpp>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/asio/strand.hpp>
|
||||
#include <boost/asio/use_future.hpp>
|
||||
#include <boost/beast/http/field.hpp>
|
||||
#include <boost/json/object.hpp>
|
||||
#include <boost/json/parse.hpp>
|
||||
#include <boost/json/serialize.hpp>
|
||||
@@ -52,6 +57,33 @@
|
||||
|
||||
namespace etl::impl {
|
||||
|
||||
SubscriptionSource::SubscriptionSource(
|
||||
boost::asio::io_context& ioContext,
|
||||
std::string const& ip,
|
||||
std::string const& wsPort,
|
||||
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
|
||||
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
|
||||
OnConnectHook onConnect,
|
||||
OnDisconnectHook onDisconnect,
|
||||
OnLedgerClosedHook onLedgerClosed,
|
||||
std::chrono::steady_clock::duration const connectionTimeout,
|
||||
std::chrono::steady_clock::duration const retryDelay
|
||||
)
|
||||
: log_(fmt::format("GrpcSource[{}:{}]", ip, wsPort))
|
||||
, wsConnectionBuilder_(ip, wsPort)
|
||||
, validatedLedgers_(std::move(validatedLedgers))
|
||||
, subscriptions_(std::move(subscriptions))
|
||||
, strand_(boost::asio::make_strand(ioContext))
|
||||
, retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
|
||||
, onConnect_(std::move(onConnect))
|
||||
, onDisconnect_(std::move(onDisconnect))
|
||||
, onLedgerClosed_(std::move(onLedgerClosed))
|
||||
{
|
||||
wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
|
||||
.addHeader({"X-User", "clio-client"})
|
||||
.setConnectionTimeout(connectionTimeout);
|
||||
}
|
||||
|
||||
SubscriptionSource::~SubscriptionSource()
|
||||
{
|
||||
stop();
|
||||
@@ -91,6 +123,12 @@ SubscriptionSource::isConnected() const
|
||||
return isConnected_;
|
||||
}
|
||||
|
||||
bool
|
||||
SubscriptionSource::isForwarding() const
|
||||
{
|
||||
return isForwarding_;
|
||||
}
|
||||
|
||||
void
|
||||
SubscriptionSource::setForwarding(bool isForwarding)
|
||||
{
|
||||
@@ -203,18 +241,18 @@ SubscriptionSource::handleMessage(std::string const& message)
|
||||
} else {
|
||||
if (isForwarding_) {
|
||||
if (object.contains(JS(transaction))) {
|
||||
dependencies_.forwardProposedTransaction(object);
|
||||
subscriptions_->forwardProposedTransaction(object);
|
||||
} else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ValidationReceived) {
|
||||
dependencies_.forwardValidation(object);
|
||||
subscriptions_->forwardValidation(object);
|
||||
} else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ManifestReceived) {
|
||||
dependencies_.forwardManifest(object);
|
||||
subscriptions_->forwardManifest(object);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ledgerIndex != 0) {
|
||||
LOG(log_.trace()) << "Pushing ledger sequence = " << ledgerIndex;
|
||||
dependencies_.pushValidatedLedger(ledgerIndex);
|
||||
validatedLedgers_->push(ledgerIndex);
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
@@ -228,9 +266,9 @@ void
|
||||
SubscriptionSource::handleError(util::requests::RequestError const& error, boost::asio::yield_context yield)
|
||||
{
|
||||
isConnected_ = false;
|
||||
isForwarding_ = false;
|
||||
if (not stop_) {
|
||||
onDisconnect_();
|
||||
isForwarding_ = false;
|
||||
}
|
||||
|
||||
if (wsConnection_ != nullptr) {
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "etl/impl/SubscriptionSourceDependencies.hpp"
|
||||
#include "etl/ETLHelpers.hpp"
|
||||
#include "etl/Source.hpp"
|
||||
#include "feed/SubscriptionManagerInterface.hpp"
|
||||
#include "util/Mutex.hpp"
|
||||
#include "util/Retry.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
@@ -35,7 +37,6 @@
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <future>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
@@ -50,9 +51,9 @@ namespace etl::impl {
|
||||
*/
|
||||
class SubscriptionSource {
|
||||
public:
|
||||
using OnConnectHook = std::function<void()>;
|
||||
using OnDisconnectHook = std::function<void()>;
|
||||
using OnLedgerClosedHook = std::function<void()>;
|
||||
using OnConnectHook = SourceBase::OnConnectHook;
|
||||
using OnDisconnectHook = SourceBase::OnDisconnectHook;
|
||||
using OnLedgerClosedHook = SourceBase::OnLedgerClosedHook;
|
||||
|
||||
private:
|
||||
util::Logger log_;
|
||||
@@ -65,7 +66,8 @@ private:
|
||||
};
|
||||
util::Mutex<ValidatedLedgersData> validatedLedgersData_;
|
||||
|
||||
SubscriptionSourceDependencies dependencies_;
|
||||
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers_;
|
||||
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions_;
|
||||
|
||||
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
|
||||
|
||||
@@ -92,7 +94,6 @@ public:
|
||||
* @brief Construct a new Subscription Source object
|
||||
*
|
||||
* @tparam NetworkValidatedLedgersType The type of the network validated ledgers object
|
||||
* @tparam SubscriptionManagerType The type of the subscription manager object
|
||||
* @param ioContext The io_context to use
|
||||
* @param ip The ip address of the source
|
||||
* @param wsPort The port of the source
|
||||
@@ -105,32 +106,18 @@ public:
|
||||
* @param connectionTimeout The connection timeout. Defaults to 30 seconds
|
||||
* @param retryDelay The retry delay. Defaults to 1 second
|
||||
*/
|
||||
template <typename NetworkValidatedLedgersType, typename SubscriptionManagerType>
|
||||
SubscriptionSource(
|
||||
boost::asio::io_context& ioContext,
|
||||
std::string const& ip,
|
||||
std::string const& wsPort,
|
||||
std::shared_ptr<NetworkValidatedLedgersType> validatedLedgers,
|
||||
std::shared_ptr<SubscriptionManagerType> subscriptions,
|
||||
std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
|
||||
std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
|
||||
OnConnectHook onConnect,
|
||||
OnDisconnectHook onDisconnect,
|
||||
OnLedgerClosedHook onLedgerClosed,
|
||||
std::chrono::steady_clock::duration const connectionTimeout = CONNECTION_TIMEOUT,
|
||||
std::chrono::steady_clock::duration const retryDelay = RETRY_DELAY
|
||||
)
|
||||
: log_(fmt::format("GrpcSource[{}:{}]", ip, wsPort))
|
||||
, wsConnectionBuilder_(ip, wsPort)
|
||||
, dependencies_(std::move(validatedLedgers), std::move(subscriptions))
|
||||
, strand_(boost::asio::make_strand(ioContext))
|
||||
, retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
|
||||
, onConnect_(std::move(onConnect))
|
||||
, onDisconnect_(std::move(onDisconnect))
|
||||
, onLedgerClosed_(std::move(onLedgerClosed))
|
||||
{
|
||||
wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
|
||||
.addHeader({"X-User", "clio-client"})
|
||||
.setConnectionTimeout(connectionTimeout);
|
||||
}
|
||||
);
|
||||
|
||||
/**
|
||||
* @brief Destroy the Subscription Source object
|
||||
@@ -162,6 +149,14 @@ public:
|
||||
bool
|
||||
isConnected() const;
|
||||
|
||||
/**
|
||||
* @brief Get whether the source is forwarding
|
||||
*
|
||||
* @return true if the source is forwarding, false otherwise
|
||||
*/
|
||||
bool
|
||||
isForwarding() const;
|
||||
|
||||
/**
|
||||
* @brief Set source forwarding
|
||||
*
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2024, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <boost/json/object.hpp>
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
namespace etl::impl {
|
||||
|
||||
class SubscriptionSourceDependencies {
|
||||
struct Concept;
|
||||
std::unique_ptr<Concept> pImpl_;
|
||||
|
||||
public:
|
||||
template <typename NetworkValidatedLedgersType, typename SubscriptionManagerType>
|
||||
SubscriptionSourceDependencies(
|
||||
std::shared_ptr<NetworkValidatedLedgersType> networkValidatedLedgers,
|
||||
std::shared_ptr<SubscriptionManagerType> subscriptions
|
||||
)
|
||||
: pImpl_{std::make_unique<Model<NetworkValidatedLedgersType, SubscriptionManagerType>>(
|
||||
std::move(networkValidatedLedgers),
|
||||
std::move(subscriptions)
|
||||
)}
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
forwardProposedTransaction(boost::json::object const& receivedTxJson)
|
||||
{
|
||||
pImpl_->forwardProposedTransaction(receivedTxJson);
|
||||
}
|
||||
|
||||
void
|
||||
forwardValidation(boost::json::object const& validationJson) const
|
||||
{
|
||||
pImpl_->forwardValidation(validationJson);
|
||||
}
|
||||
void
|
||||
forwardManifest(boost::json::object const& manifestJson) const
|
||||
{
|
||||
pImpl_->forwardManifest(manifestJson);
|
||||
}
|
||||
void
|
||||
pushValidatedLedger(uint32_t const idx)
|
||||
{
|
||||
pImpl_->pushValidatedLedger(idx);
|
||||
}
|
||||
|
||||
private:
|
||||
struct Concept {
|
||||
virtual ~Concept() = default;
|
||||
|
||||
virtual void
|
||||
forwardProposedTransaction(boost::json::object const& receivedTxJson) = 0;
|
||||
virtual void
|
||||
forwardValidation(boost::json::object const& validationJson) const = 0;
|
||||
virtual void
|
||||
forwardManifest(boost::json::object const& manifestJson) const = 0;
|
||||
virtual void
|
||||
pushValidatedLedger(uint32_t idx) = 0;
|
||||
};
|
||||
|
||||
template <typename SomeNetworkValidatedLedgersType, typename SomeSubscriptionManagerType>
|
||||
class Model : public Concept {
|
||||
std::shared_ptr<SomeNetworkValidatedLedgersType> networkValidatedLedgers_;
|
||||
std::shared_ptr<SomeSubscriptionManagerType> subscriptions_;
|
||||
|
||||
public:
|
||||
Model(
|
||||
std::shared_ptr<SomeNetworkValidatedLedgersType> networkValidatedLedgers,
|
||||
std::shared_ptr<SomeSubscriptionManagerType> subscriptions
|
||||
)
|
||||
: networkValidatedLedgers_{std::move(networkValidatedLedgers)}, subscriptions_{std::move(subscriptions)}
|
||||
{
|
||||
}
|
||||
void
|
||||
forwardProposedTransaction(boost::json::object const& receivedTxJson) override
|
||||
{
|
||||
subscriptions_->forwardProposedTransaction(receivedTxJson);
|
||||
}
|
||||
void
|
||||
forwardValidation(boost::json::object const& validationJson) const override
|
||||
{
|
||||
subscriptions_->forwardValidation(validationJson);
|
||||
}
|
||||
void
|
||||
forwardManifest(boost::json::object const& manifestJson) const override
|
||||
{
|
||||
subscriptions_->forwardManifest(manifestJson);
|
||||
}
|
||||
void
|
||||
pushValidatedLedger(uint32_t idx) override
|
||||
{
|
||||
networkValidatedLedgers_->push(idx);
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace etl::impl
|
||||
Reference in New Issue
Block a user