diff --git a/.github/actions/prepare_runner/action.yml b/.github/actions/prepare_runner/action.yml index b74ed1def..73fcd22ef 100644 --- a/.github/actions/prepare_runner/action.yml +++ b/.github/actions/prepare_runner/action.yml @@ -11,9 +11,35 @@ runs: if: ${{ runner.os == 'macOS' }} shell: bash run: | - brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates + brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH + - name: Install CMake 3.31.6 on mac + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + # Uninstall any existing cmake + brew uninstall cmake --ignore-dependencies || true + + # Download specific cmake formula + FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb" + FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4" + + mkdir -p /tmp/homebrew-formula + curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb + + # Verify the downloaded formula + ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1) + if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then + echo "Error: Formula checksum mismatch" + echo "Expected: $FORMULA_EXPECTED_SHA256" + echo "Actual: $ACTUAL_SHA256" + exit 1 + fi + + # Install cmake from the specific formula with force flag + brew install --force /tmp/homebrew-formula/cmake.rb + - name: Fix git permissions on Linux if: ${{ runner.os == 'Linux' }} shell: bash diff --git a/src/app/ClioApplication.cpp b/src/app/ClioApplication.cpp index e68a2797a..72c0eb9d1 100644 --- a/src/app/ClioApplication.cpp +++ b/src/app/ClioApplication.cpp @@ -27,6 +27,8 @@ #include "etl/ETLService.hpp" #include "etl/LoadBalancer.hpp" #include "etl/NetworkValidatedLedgers.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManager.hpp" #include "migration/MigrationInspectorFactory.hpp" #include "rpc/Counters.hpp" @@ -42,6 +44,7 @@ #include "web/Server.hpp" #include "web/dosguard/DOSGuard.hpp" #include "web/dosguard/IntervalSweepHandler.hpp" +#include "web/dosguard/Weights.hpp" #include "web/dosguard/WhitelistHandler.hpp" #include "web/ng/RPCServerHandler.hpp" #include "web/ng/Server.hpp" @@ -101,7 +104,8 @@ ClioApplication::run(bool const useNgWebServer) // Rate limiter, to prevent abuse auto whitelistHandler = web::dosguard::WhitelistHandler{config_}; - auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler}; + auto const dosguardWeights = web::dosguard::Weights::make(config_); + auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler, dosguardWeights}; auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard}; auto cache = data::LedgerCache{}; @@ -130,7 +134,12 @@ ClioApplication::run(bool const useNgWebServer) // ETL uses the balancer to extract data. // The server uses the balancer to forward RPCs to a rippled node. // The balancer itself publishes to streams (transactions_proposed and accounts_proposed) - auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + auto balancer = [&] -> std::shared_ptr { + if (config_.get("__ng_etl")) + return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + + return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + }(); // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers); @@ -142,12 +151,12 @@ ClioApplication::run(bool const useNgWebServer) config_, backend, subscriptions, balancer, etl, amendmentCenter, counters ); - using RPCEngineType = rpc::RPCEngine; + using RPCEngineType = rpc::RPCEngine; auto const rpcEngine = RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider); if (useNgWebServer or config_.get("server.__ng_web_server")) { - web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl}; + web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl, dosGuard}; auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_); if (not expectedAdminVerifier.has_value()) { @@ -165,7 +174,7 @@ ClioApplication::run(bool const useNgWebServer) httpServer->onGet("/metrics", MetricsHandler{adminVerifier}); httpServer->onGet("/health", HealthCheckHandler{}); - auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard}; + auto requestHandler = RequestHandler{adminVerifier, handler}; httpServer->onPost("/", requestHandler); httpServer->onWs(std::move(requestHandler)); @@ -188,8 +197,7 @@ ClioApplication::run(bool const useNgWebServer) } // Init the web server - auto handler = - std::make_shared>(config_, backend, rpcEngine, etl); + auto handler = std::make_shared>(config_, backend, rpcEngine, etl, dosGuard); auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler); diff --git a/src/app/Stopper.hpp b/src/app/Stopper.hpp index b4faa1377..daba1164e 100644 --- a/src/app/Stopper.hpp +++ b/src/app/Stopper.hpp @@ -20,8 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" -#include "etl/LoadBalancer.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/CoroutineGroup.hpp" #include "util/log/Logger.hpp" @@ -74,15 +74,12 @@ public: * @param ioc The io_context to stop. * @return The callback to be called on application stop. */ - template < - web::ng::SomeServer ServerType, - etl::SomeLoadBalancer LoadBalancerType, - etl::SomeETLService ETLServiceType> + template static std::function makeOnStopCallback( ServerType& server, - LoadBalancerType& balancer, - ETLServiceType& etl, + etlng::LoadBalancerInterface& balancer, + etlng::ETLServiceInterface& etl, feed::SubscriptionManagerInterface& subscriptions, data::BackendInterface& backend, boost::asio::io_context& ioc diff --git a/src/app/WebHandlers.hpp b/src/app/WebHandlers.hpp index ac7958ac0..5dacc222a 100644 --- a/src/app/WebHandlers.hpp +++ b/src/app/WebHandlers.hpp @@ -147,7 +147,6 @@ class RequestHandler { util::Logger webServerLog_{"WebServer"}; std::shared_ptr adminVerifier_; std::reference_wrapper rpcHandler_; - std::reference_wrapper dosguard_; public: /** @@ -155,14 +154,9 @@ public: * * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access. * @param rpcHandler The RPC handler to use for handling the request. - * @param dosguard The DOSGuardInterface to use for checking the connection. */ - RequestHandler( - std::shared_ptr adminVerifier, - RpcHandlerType& rpcHandler, - web::dosguard::DOSGuardInterface& dosguard - ) - : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard) + RequestHandler(std::shared_ptr adminVerifier, RpcHandlerType& rpcHandler) + : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler) { } @@ -183,21 +177,6 @@ public: boost::asio::yield_context yield ) { - if (not dosguard_.get().request(connectionMetadata.ip())) { - auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN); - - if (not request.isHttp()) { - try { - auto requestJson = boost::json::parse(request.message()); - if (requestJson.is_object() && requestJson.as_object().contains("id")) - error["id"] = requestJson.as_object().at("id"); - error["request"] = request.message(); - } catch (std::exception const&) { - error["request"] = request.message(); - } - } - return web::ng::Response{boost::beast::http::status::service_unavailable, error, request}; - } LOG(webServerLog_.info()) << connectionMetadata.tag() << "Received request from ip = " << connectionMetadata.ip() << " - posting to WorkQueue"; @@ -207,20 +186,7 @@ public: }); try { - auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield); - - if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) { - auto jsonResponse = boost::json::parse(response.message()).as_object(); - jsonResponse["warning"] = "load"; - if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) { - jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit)); - } else { - jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)}; - } - response.setMessage(jsonResponse); - } - - return response; + return rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield); } catch (std::exception const&) { return web::ng::Response{ boost::beast::http::status::internal_server_error, diff --git a/src/data/BackendInterface.hpp b/src/data/BackendInterface.hpp index 387cd41dd..5c376d798 100644 --- a/src/data/BackendInterface.hpp +++ b/src/data/BackendInterface.hpp @@ -154,7 +154,7 @@ public: } virtual ~BackendInterface() = default; - // TODO: Remove this hack once old ETL is removed. + // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed. // Cache should not be exposed thru BackendInterface /** @@ -648,6 +648,14 @@ public: virtual void writeAccountTransactions(std::vector data) = 0; + /** + * @brief Write a new account transaction. + * + * @param record An object representing the account transaction + */ + virtual void + writeAccountTransaction(AccountTransactionsData record) = 0; + /** * @brief Write NFTs transactions. * diff --git a/src/data/CassandraBackend.hpp b/src/data/CassandraBackend.hpp index 4bba53301..7ea02475c 100644 --- a/src/data/CassandraBackend.hpp +++ b/src/data/CassandraBackend.hpp @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -906,30 +907,42 @@ public: statements.reserve(data.size() * 10); // assume 10 transactions avg for (auto& record : data) { - std::transform( - std::begin(record.accounts), - std::end(record.accounts), - std::back_inserter(statements), - [this, &record](auto&& account) { - return schema_->insertAccountTx.bind( - std::forward(account), - std::make_tuple(record.ledgerSequence, record.transactionIndex), - record.txHash - ); - } - ); + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); } executor_.write(std::move(statements)); } + void + writeAccountTransaction(AccountTransactionsData record) override + { + std::vector statements; + statements.reserve(record.accounts.size()); + + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); + + executor_.write(std::move(statements)); + } + void writeNFTTransactions(std::vector const& data) override { std::vector statements; statements.reserve(data.size()); - std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) { + std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) { return schema_->insertNFTTx.bind( record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash ); @@ -999,7 +1012,7 @@ public: std::vector statements; statements.reserve(data.size()); for (auto [mptId, holder] : data) - statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder))); + statements.push_back(schema_->insertMPTHolder.bind(mptId, holder)); executor_.write(std::move(statements)); } diff --git a/src/data/DBHelpers.hpp b/src/data/DBHelpers.hpp index cba1ec1ac..76b5acedf 100644 --- a/src/data/DBHelpers.hpp +++ b/src/data/DBHelpers.hpp @@ -54,7 +54,7 @@ struct AccountTransactionsData { * @param meta The transaction metadata * @param txHash The transaction hash */ - AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash) + AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash) : accounts(meta.getAffectedAccounts()) , ledgerSequence(meta.getLgrSeq()) , transactionIndex(meta.getIndex()) diff --git a/src/data/LedgerCache.cpp b/src/data/LedgerCache.cpp index 11f0b8162..7180c4e9d 100644 --- a/src/data/LedgerCache.cpp +++ b/src/data/LedgerCache.cpp @@ -20,6 +20,7 @@ #include "data/LedgerCache.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/Assert.hpp" #include @@ -87,6 +88,42 @@ LedgerCache::update(std::vector const& objs, uint32_t seq, bool is } } +void +LedgerCache::update(std::vector const& objs, uint32_t seq) +{ + if (disabled_) + return; + + std::scoped_lock const lck{mtx_}; + if (seq > latestSeq_) { + ASSERT( + seq == latestSeq_ + 1 || latestSeq_ == 0, + "New sequence must be either next or first. seq = {}, latestSeq_ = {}", + seq, + latestSeq_ + ); + latestSeq_ = seq; + } + + deleted_.clear(); // previous update's deletes no longer needed + + for (auto const& obj : objs) { + if (!obj.data.empty()) { + auto& e = map_[obj.key]; + if (seq > e.seq) + e = {.seq = seq, .blob = obj.data}; + } else { + if (map_.contains(obj.key)) + deleted_[obj.key] = map_[obj.key]; + + map_.erase(obj.key); + if (!full_) + deletes_.insert(obj.key); + } + } + cv_.notify_all(); +} + std::optional LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const { @@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const return {e->second.blob}; } +std::optional +LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const +{ + if (disabled_) + return std::nullopt; + + std::shared_lock const lck{mtx_}; + if (seq > latestSeq_) + return std::nullopt; + + ++objectReqCounter_.get(); + + auto e = deleted_.find(key); + if (e == deleted_.end()) + return std::nullopt; + + if (seq < e->second.seq) + return std::nullopt; + + ++objectHitCounter_.get(); + return {e->second.blob}; +} + void LedgerCache::setDisabled() { diff --git a/src/data/LedgerCache.hpp b/src/data/LedgerCache.hpp index 8a3c40688..bf957af28 100644 --- a/src/data/LedgerCache.hpp +++ b/src/data/LedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Label.hpp" @@ -29,7 +30,6 @@ #include #include -#include #include #include #include @@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface { )}; std::map map_; + std::map deleted_; mutable std::shared_mutex mtx_; std::condition_variable_any cv_; @@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface { public: void - update(std::vector const& objs, uint32_t seq, bool isBackground = false) override; + update(std::vector const& objs, uint32_t seq, bool isBackground) override; + + void + update(std::vector const& objs, uint32_t seq) override; std::optional get(ripple::uint256 const& key, uint32_t seq) const override; + std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const override; + std::optional getSuccessor(ripple::uint256 const& key, uint32_t seq) const override; diff --git a/src/data/LedgerCacheInterface.hpp b/src/data/LedgerCacheInterface.hpp index e5ecee49d..850403bcc 100644 --- a/src/data/LedgerCacheInterface.hpp +++ b/src/data/LedgerCacheInterface.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -55,6 +56,15 @@ public: virtual void update(std::vector const& objs, uint32_t seq, bool isBackground = false) = 0; + /** + * @brief Update the cache with new ledger objects. + * + * @param objs The ledger objects to update cache with + * @param seq The sequence to update cache for + */ + virtual void + update(std::vector const& objs, uint32_t seq) = 0; + /** * @brief Fetch a cached object by its key and sequence number. * @@ -65,6 +75,16 @@ public: virtual std::optional get(ripple::uint256 const& key, uint32_t seq) const = 0; + /** + * @brief Fetch a recently deleted object by its key and sequence number. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned + */ + virtual std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0; + /** * @brief Gets a cached successor. * diff --git a/src/etl/ETLService.cpp b/src/etl/ETLService.cpp index 6eca76339..a7207c99b 100644 --- a/src/etl/ETLService.cpp +++ b/src/etl/ETLService.cpp @@ -22,6 +22,7 @@ #include "data/BackendInterface.hpp" #include "etl/CorruptionDetector.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/Assert.hpp" #include "util/Constants.hpp" @@ -43,6 +44,7 @@ #include namespace etl { + // Database must be populated when this starts std::optional ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors) @@ -265,7 +267,7 @@ ETLService::ETLService( boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) : backend_(backend) diff --git a/src/etl/ETLService.hpp b/src/etl/ETLService.hpp index d1d865475..cc03770ff 100644 --- a/src/etl/ETLService.hpp +++ b/src/etl/ETLService.hpp @@ -32,7 +32,12 @@ #include "etl/impl/LedgerLoader.hpp" #include "etl/impl/LedgerPublisher.hpp" #include "etl/impl/Transformer.hpp" +#include "etlng/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" #include "util/log/Logger.hpp" #include @@ -41,7 +46,6 @@ #include #include -#include #include #include #include @@ -81,14 +85,13 @@ concept SomeETLService = std::derived_from; * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring * to writing and from writing to monitoring, based on the activity of other processes running on different machines. */ -class ETLService : public ETLServiceTag { +class ETLService : public etlng::ETLServiceInterface, ETLServiceTag { // TODO: make these template parameters in ETLService - using LoadBalancerType = LoadBalancer; using DataPipeType = etl::impl::ExtractionDataPipe; using CacheLoaderType = etl::CacheLoader<>; - using LedgerFetcherType = etl::impl::LedgerFetcher; + using LedgerFetcherType = etl::impl::LedgerFetcher; using ExtractorType = etl::impl::Extractor; - using LedgerLoaderType = etl::impl::LedgerLoader; + using LedgerLoaderType = etl::impl::LedgerLoader; using LedgerPublisherType = etl::impl::LedgerPublisher; using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler; using TransformerType = @@ -97,7 +100,7 @@ class ETLService : public ETLServiceTag { util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::shared_ptr networkValidatedLedgers_; std::uint32_t extractorThreads_ = 1; @@ -132,7 +135,7 @@ public: boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ); @@ -154,20 +157,37 @@ public: * @param ledgers The network validated ledgers datastructure * @return A shared pointer to a new instance of ETLService */ - static std::shared_ptr + static std::shared_ptr makeETLService( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) { - auto etl = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); - etl->run(); + std::shared_ptr ret; - return etl; + if (config.get("__ng_etl")) { + ASSERT( + std::dynamic_pointer_cast(balancer), + "LoadBalancer type must be etlng::LoadBalancer" + ); + ret = std::make_shared(config, backend, subscriptions, balancer, ledgers); + } else { + ASSERT( + std::dynamic_pointer_cast(balancer), "LoadBalancer type must be etl::LoadBalancer" + ); + ret = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); + } + + // inject networkID into subscriptions, as transaction feed require it to inject CTID in response + if (auto const state = ret->getETLState(); state) + subscriptions->setNetworkID(state->networkID); + + ret->run(); + return ret; } /** @@ -184,7 +204,7 @@ public: * @note This method blocks until the ETL service has stopped. */ void - stop() + stop() override { LOG(log_.info()) << "Stop called"; @@ -203,7 +223,7 @@ public: * @return Time passed since last ledger close */ std::uint32_t - lastCloseAgeSeconds() const + lastCloseAgeSeconds() const override { return ledgerPublisher_.lastCloseAgeSeconds(); } @@ -214,7 +234,7 @@ public: * @return true if currently amendment blocked; false otherwise */ bool - isAmendmentBlocked() const + isAmendmentBlocked() const override { return state_.isAmendmentBlocked; } @@ -225,7 +245,7 @@ public: * @return true if corruption of DB was detected and cache was stopped. */ bool - isCorruptionDetected() const + isCorruptionDetected() const override { return state_.isCorruptionDetected; } @@ -236,7 +256,7 @@ public: * @return The state of ETL as a JSON object */ boost::json::object - getInfo() const + getInfo() const override { boost::json::object result; @@ -254,11 +274,17 @@ public: * @return The etl nodes' state, nullopt if etl nodes are not connected */ std::optional - getETLState() const noexcept + getETLState() const noexcept override { return loadBalancer_->getETLState(); } + /** + * @brief Start all components to run ETL service. + */ + void + run() override; + private: /** * @brief Run the ETL pipeline. @@ -325,12 +351,6 @@ private: return numMarkers_; } - /** - * @brief Start all components to run ETL service. - */ - void - run(); - /** * @brief Spawn the worker thread and start monitoring. */ diff --git a/src/etl/ETLState.cpp b/src/etl/ETLState.cpp index cd0bbe0f8..f0ca5cd9f 100644 --- a/src/etl/ETLState.cpp +++ b/src/etl/ETLState.cpp @@ -40,7 +40,7 @@ tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) { auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object(); if (rippledInfo.contains(JS(network_id))) - state.networkID.emplace(boost::json::value_to(rippledInfo.at(JS(network_id)))); + state.networkID = boost::json::value_to(rippledInfo.at(JS(network_id))); } return state; diff --git a/src/etl/ETLState.hpp b/src/etl/ETLState.hpp index 0302aab5f..3b447d985 100644 --- a/src/etl/ETLState.hpp +++ b/src/etl/ETLState.hpp @@ -38,7 +38,12 @@ namespace etl { * @brief This class is responsible for fetching and storing the state of the ETL information, such as the network id */ struct ETLState { - std::optional networkID; + /* + * NOTE: Rippled NetworkID: Mainnet = 0; Testnet = 1; Devnet = 2 + * However, if rippled is running on neither of these (ie. standalone mode) rippled will default to 0, but + * is not included in the stateOpt response. Must manually add it here. + */ + uint32_t networkID{0}; /** * @brief Fetch the ETL state from the rippled server diff --git a/src/etl/LoadBalancer.cpp b/src/etl/LoadBalancer.cpp index 7dc0d1925..fa0010a0b 100644 --- a/src/etl/LoadBalancer.cpp +++ b/src/etl/LoadBalancer.cpp @@ -23,6 +23,7 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "util/Assert.hpp" @@ -59,7 +60,7 @@ using namespace util::config; namespace etl { -std::shared_ptr +std::shared_ptr LoadBalancer::makeLoadBalancer( ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -141,12 +142,11 @@ LoadBalancer::LoadBalancer( if (!stateOpt) { LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString() << " Please check the configuration and network"; - } else if (etlState_ && etlState_->networkID && stateOpt->networkID && - etlState_->networkID != stateOpt->networkID) { + } else if (etlState_ && etlState_->networkID != stateOpt->networkID) { checkOnETLFailure(fmt::format( "ETL sources must be on the same network. Source network id = {} does not match others network id = {}", - *(stateOpt->networkID), - *(etlState_->networkID) + stateOpt->networkID, + etlState_->networkID )); } else { etlState_ = stateOpt; @@ -175,12 +175,12 @@ LoadBalancer::~LoadBalancer() } std::vector -LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly, std::chrono::steady_clock::duration retryAfter) +LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter) { std::vector response; execute( - [this, &response, &sequence, cacheOnly](auto& source) { - auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly); + [this, &response, &sequence](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_); if (!res) { LOG(log_.error()) << "Failed to download initial ledger." diff --git a/src/etl/LoadBalancer.hpp b/src/etl/LoadBalancer.hpp index cfe01d019..169636685 100644 --- a/src/etl/LoadBalancer.hpp +++ b/src/etl/LoadBalancer.hpp @@ -23,8 +23,11 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" +#include "util/Assert.hpp" #include "util/Mutex.hpp" #include "util/ResponseExpirationCache.hpp" #include "util/log/Logger.hpp" @@ -41,13 +44,13 @@ #include #include -#include #include #include #include #include #include #include +#include #include namespace etl { @@ -69,7 +72,7 @@ concept SomeLoadBalancer = std::derived_from; * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also * allows requests for ledger data to be load balanced across all possible ETL sources. */ -class LoadBalancer : public LoadBalancerTag { +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { public: using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; @@ -133,7 +136,7 @@ public: * @param sourceFactory A factory function to create a source * @return A shared pointer to a new instance of LoadBalancer */ - static std::shared_ptr + static std::shared_ptr makeLoadBalancer( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -150,16 +153,32 @@ public: * @note This function will retry indefinitely until the ledger is downloaded. * * @param sequence Sequence of ledger to download - * @param cacheOnly Whether to only write to cache and not to the DB; defaults to false + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}) + override; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param observer The observer to notify of progress * @param retryAfter Time to wait between retries (2 seconds by default) * @return A std::vector The ledger data */ std::vector loadInitialLedger( - uint32_t sequence, - bool cacheOnly = false, - std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] etlng::InitialLoadObserverInterface& observer, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for old ETL"); + std::unreachable(); + } /** * @brief Fetch data for a specific ledger. @@ -180,7 +199,7 @@ public: bool getObjects, bool getObjectNeighbors, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + ) override; /** * @brief Represent the state of this load balancer as a JSON object @@ -188,7 +207,7 @@ public: * @return JSON representation of the state of this load balancer. */ boost::json::value - toJson() const; + toJson() const override; /** * @brief Forward a JSON RPC request to a randomly selected rippled node. @@ -205,14 +224,14 @@ public: std::optional const& clientIp, bool isAdmin, boost::asio::yield_context yield - ); + ) override; /** * @brief Return state of ETL nodes. * @return ETL state, nullopt if etl nodes not available */ std::optional - getETLState() noexcept; + getETLState() noexcept override; /** * @brief Stop the load balancer. This will stop all subscription sources. @@ -221,7 +240,7 @@ public: * @param yield The coroutine context */ void - stop(boost::asio::yield_context yield); + stop(boost::asio::yield_context yield) override; private: /** diff --git a/src/etl/NetworkValidatedLedgersInterface.hpp b/src/etl/NetworkValidatedLedgersInterface.hpp index d2ab9d12b..067407c8c 100644 --- a/src/etl/NetworkValidatedLedgersInterface.hpp +++ b/src/etl/NetworkValidatedLedgersInterface.hpp @@ -26,6 +26,7 @@ #include #include + namespace etl { /** diff --git a/src/etl/Source.hpp b/src/etl/Source.hpp index 91d9bf817..6ff7e7285 100644 --- a/src/etl/Source.hpp +++ b/src/etl/Source.hpp @@ -23,8 +23,6 @@ #include "etl/NetworkValidatedLedgersInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" -#include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "util/newconfig/ObjectView.hpp" #include @@ -130,11 +128,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ virtual std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0; + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) = 0; /** * @brief Forward a request to rippled. diff --git a/src/etl/impl/GrpcSource.cpp b/src/etl/impl/GrpcSource.cpp index 29be5b224..539cdf481 100644 --- a/src/etl/impl/GrpcSource.cpp +++ b/src/etl/impl/GrpcSource.cpp @@ -98,7 +98,7 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb } std::pair, bool> -GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers, bool const cacheOnly) +GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers) { if (!stub_) return {{}, false}; @@ -130,7 +130,7 @@ GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix(); - auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly); + auto result = ptr->process(stub_, cq, *backend_, abort); if (result != etl::impl::AsyncCallData::CallStatus::MORE) { ++numFinished; LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished; diff --git a/src/etl/impl/GrpcSource.hpp b/src/etl/impl/GrpcSource.hpp index 5d41f193d..248f4191d 100644 --- a/src/etl/impl/GrpcSource.hpp +++ b/src/etl/impl/GrpcSource.hpp @@ -60,11 +60,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly = false); + loadInitialLedger(uint32_t sequence, uint32_t numMarkers); }; } // namespace etl::impl diff --git a/src/etl/impl/LedgerFetcher.hpp b/src/etl/impl/LedgerFetcher.hpp index 9d8a0df88..acb792ac2 100644 --- a/src/etl/impl/LedgerFetcher.hpp +++ b/src/etl/impl/LedgerFetcher.hpp @@ -20,6 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/log/Logger.hpp" #include @@ -34,22 +36,18 @@ namespace etl::impl { /** * @brief GRPC Ledger data fetcher */ -template -class LedgerFetcher { -public: - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - +class LedgerFetcher : public LedgerFetcherInterface { private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; public: /** * @brief Create an instance of the fetcher */ - LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) + LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) : backend_(std::move(backend)), loadBalancer_(std::move(balancer)) { } @@ -64,7 +62,7 @@ public: * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchData(uint32_t sequence) + fetchData(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; @@ -84,7 +82,7 @@ public: * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchDataAndDiff(uint32_t sequence) + fetchDataAndDiff(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; diff --git a/src/etl/impl/LedgerLoader.hpp b/src/etl/impl/LedgerLoader.hpp index e6d88ee8e..6be0f30e3 100644 --- a/src/etl/impl/LedgerLoader.hpp +++ b/src/etl/impl/LedgerLoader.hpp @@ -26,6 +26,7 @@ #include "etl/NFTHelpers.hpp" #include "etl/SystemState.hpp" #include "etl/impl/LedgerFetcher.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/Assert.hpp" #include "util/LedgerUtils.hpp" #include "util/Profiler.hpp" @@ -65,18 +66,18 @@ namespace etl::impl { /** * @brief Loads ledger data into the DB */ -template +template class LedgerLoader { public: - using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType; - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType; + using GetLedgerResponseType = etlng::LoadBalancerInterface::GetLedgerResponseType; + using OptionalGetLedgerResponseType = etlng::LoadBalancerInterface::OptionalGetLedgerResponseType; + using RawLedgerObjectType = etlng::LoadBalancerInterface::RawLedgerObjectType; private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::reference_wrapper fetcher_; std::reference_wrapper state_; // shared state for ETL @@ -86,7 +87,7 @@ public: */ LedgerLoader( std::shared_ptr backend, - std::shared_ptr balancer, + std::shared_ptr balancer, LedgerFetcherType& fetcher, SystemState const& state ) @@ -121,7 +122,7 @@ public: LOG(log_.trace()) << "Inserting transaction = " << sttx.getTransactionID(); - ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; + ripple::TxMeta const txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx); result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end()); diff --git a/src/etl/impl/SourceImpl.hpp b/src/etl/impl/SourceImpl.hpp index 3f2f25a33..a2b4175e0 100644 --- a/src/etl/impl/SourceImpl.hpp +++ b/src/etl/impl/SourceImpl.hpp @@ -202,9 +202,9 @@ public: * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) final { - return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly); + return grpcSource_.loadInitialLedger(sequence, numMarkers); } /** diff --git a/src/etlng/CMakeLists.txt b/src/etlng/CMakeLists.txt index 4b97c43ab..49e443a34 100644 --- a/src/etlng/CMakeLists.txt +++ b/src/etlng/CMakeLists.txt @@ -2,13 +2,20 @@ add_library(clio_etlng) target_sources( clio_etlng - PRIVATE impl/AmendmentBlockHandler.cpp + PRIVATE LoadBalancer.cpp + Source.cpp + impl/AmendmentBlockHandler.cpp impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp + impl/ForwardingSource.cpp impl/Loading.cpp impl/Monitor.cpp impl/TaskManager.cpp + impl/ext/Cache.cpp + impl/ext/Core.cpp + impl/ext/NFT.cpp + impl/ext/Successor.cpp ) target_link_libraries(clio_etlng PUBLIC clio_data) diff --git a/src/etlng/ETLService.hpp b/src/etlng/ETLService.hpp new file mode 100644 index 000000000..960d6e456 --- /dev/null +++ b/src/etlng/ETLService.hpp @@ -0,0 +1,283 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCache.hpp" +#include "data/Types.hpp" +#include "etl/CacheLoader.hpp" +#include "etl/ETLState.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/SystemState.hpp" +#include "etl/impl/AmendmentBlockHandler.hpp" +#include "etl/impl/LedgerFetcher.hpp" +#include "etl/impl/LedgerPublisher.hpp" +#include "etlng/AmendmentBlockHandlerInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/ExtractorInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/impl/AmendmentBlockHandler.hpp" +#include "etlng/impl/Extraction.hpp" +#include "etlng/impl/Loading.hpp" +#include "etlng/impl/Monitor.hpp" +#include "etlng/impl/Registry.hpp" +#include "etlng/impl/Scheduling.hpp" +#include "etlng/impl/TaskManager.hpp" +#include "etlng/impl/ext/Cache.hpp" +#include "etlng/impl/ext/Core.hpp" +#include "etlng/impl/ext/NFT.hpp" +#include "etlng/impl/ext/Successor.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" +#include "util/Profiler.hpp" +#include "util/async/context/BasicExecutionContext.hpp" +#include "util/config/Config.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + * + * Usually, multiple different processes share access to the same network accessible databases, in which case only one + * such process is performing ETL and writing to the database. The other processes simply monitor the database for new + * ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the + * ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer. + * + * If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and + * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring + * to writing and from writing to monitoring, based on the activity of other processes running on different machines. + */ +class ETLService : public ETLServiceInterface { + util::Logger log_{"ETL"}; + + std::shared_ptr backend_; + std::shared_ptr subscriptions_; + std::shared_ptr balancer_; + std::shared_ptr ledgers_; + std::shared_ptr> cacheLoader_; + + std::shared_ptr fetcher_; + std::shared_ptr extractor_; + + etl::SystemState state_; + util::async::CoroExecutionContext ctx_{8}; + + std::shared_ptr amendmentBlockHandler_; + std::shared_ptr loader_; + + std::optional> mainLoop_; + +public: + /** + * @brief Create an instance of ETLService. + * + * @param config The configuration to use + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param balancer Load balancer to use + * @param ledgers The network validated ledgers datastructure + */ + ETLService( + util::config::ClioConfigDefinition const& config, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr balancer, + std::shared_ptr ledgers + ) + : backend_(std::move(backend)) + , subscriptions_(std::move(subscriptions)) + , balancer_(std::move(balancer)) + , ledgers_(std::move(ledgers)) + , cacheLoader_(std::make_shared>(config, backend_, backend_->cache())) + , fetcher_(std::make_shared(backend_, balancer_)) + , extractor_(std::make_shared(fetcher_)) + , amendmentBlockHandler_(std::make_shared(ctx_, state_)) + , loader_(std::make_shared( + backend_, + fetcher_, + impl::makeRegistry( + impl::CacheExt{backend_->cache()}, + impl::CoreExt{backend_}, + impl::SuccessorExt{backend_, backend_->cache()}, + impl::NFTExt{backend_} + ), + amendmentBlockHandler_ + )) + { + LOG(log_.info()) << "Creating ETLng..."; + } + + ~ETLService() override + { + LOG(log_.debug()) << "Stopping ETLng"; + } + + void + run() override + { + LOG(log_.info()) << "run() in ETLng..."; + + mainLoop_.emplace(ctx_.execute([this] { + auto const rng = loadInitialLedgerIfNeeded(); + + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + std::optional mostRecentValidated = ledgers_->getMostRecent(); + + if (not mostRecentValidated) { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + return; + } + + ASSERT(rng.has_value(), "Ledger range can't be null"); + auto const nextSequence = rng->maxSequence + 1; + + LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence; + + auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence} + // impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000}, + // TODO lift limit and start with rng.minSeq + ); + + auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_); + + // TODO: figure out this: std::make_shared(backend_, ledgers_, nextSequence) + man.run({}); // TODO: needs to be interruptable and fill out settings + })); + } + + void + stop() override + { + LOG(log_.info()) << "Stop called"; + // TODO: stop the service correctly + } + + boost::json::object + getInfo() const override + { + // TODO + return {{"ok", true}}; + } + + bool + isAmendmentBlocked() const override + { + // TODO + return false; + } + + bool + isCorruptionDetected() const override + { + // TODO + return false; + } + + std::optional + getETLState() const override + { + // TODO + return std::nullopt; + } + + std::uint32_t + lastCloseAgeSeconds() const override + { + // TODO + return 0; + } + +private: + // TODO: this better be std::expected + std::optional + loadInitialLedgerIfNeeded() + { + if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) { + LOG(log_.info()) << "Database is empty. Will download a ledger from the network."; + + try { + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) { + auto const seq = *mostRecentValidated; + LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... "; + + auto [ledger, timeDiff] = ::util::timed>([this, seq]() { + return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) { + // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar + data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_); + + // TODO: this should be interruptable for graceful shutdown + return loader_->loadInitialLedger(data); + }); + }); + + LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff; + LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size(); + + if (ledger.has_value()) + return backend_->hardFetchLedgerRangeNoThrow(); + + LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop"; + } else { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + } + } catch (std::runtime_error const& e) { + LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what(); + amendmentBlockHandler_->notifyAmendmentBlocked(); + } + } else { + LOG(log_.info()) << "Database already populated. Picking up from the tip of history"; + cacheLoader_->load(rng->maxSequence); + + return rng; + } + + return std::nullopt; + } +}; +} // namespace etlng diff --git a/src/etlng/ETLServiceInterface.hpp b/src/etlng/ETLServiceInterface.hpp new file mode 100644 index 000000000..7f0b2fc2b --- /dev/null +++ b/src/etlng/ETLServiceInterface.hpp @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/ETLState.hpp" + +#include + +#include +#include + +namespace etlng { + +/** + * @brief This is a base class for any ETL service implementations. + * @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + */ +struct ETLServiceInterface { + virtual ~ETLServiceInterface() = default; + + /** + * @brief Start all components to run ETL service. + */ + virtual void + run() = 0; + + /** + * @brief Stop the ETL service. + * @note This method blocks until the ETL service has stopped. + */ + virtual void + stop() = 0; + + /** + * @brief Get state of ETL as a JSON object + * + * @return The state of ETL as a JSON object + */ + [[nodiscard]] virtual boost::json::object + getInfo() const = 0; + + /** + * @brief Check for the amendment blocked state. + * + * @return true if currently amendment blocked; false otherwise + */ + [[nodiscard]] virtual bool + isAmendmentBlocked() const = 0; + + /** + * @brief Check whether Clio detected DB corruptions. + * + * @return true if corruption of DB was detected and cache was stopped. + */ + [[nodiscard]] virtual bool + isCorruptionDetected() const = 0; + + /** + * @brief Get the etl nodes' state + * @return The etl nodes' state, nullopt if etl nodes are not connected + */ + [[nodiscard]] virtual std::optional + getETLState() const = 0; + + /** + * @brief Get time passed since last ledger close, in seconds. + * + * @return Time passed since last ledger close + */ + [[nodiscard]] virtual std::uint32_t + lastCloseAgeSeconds() const = 0; +}; + +} // namespace etlng diff --git a/src/etlng/LoadBalancer.cpp b/src/etlng/LoadBalancer.cpp new file mode 100644 index 000000000..d6b96c7f9 --- /dev/null +++ b/src/etlng/LoadBalancer.cpp @@ -0,0 +1,370 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/LoadBalancer.hpp" + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/CoroutineGroup.hpp" +#include "util/Random.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace util::config; + +namespace etlng { + +std::shared_ptr +LoadBalancer::makeLoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) +{ + return std::make_shared( + config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory) + ); +} + +LoadBalancer::LoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) +{ + auto const forwardingCacheTimeout = config.get("forwarding.cache_timeout"); + if (forwardingCacheTimeout > 0.f) { + forwardingCache_ = util::ResponseExpirationCache{ + util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout), + {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"} + }; + } + + auto const numMarkers = config.getValueView("num_markers"); + if (numMarkers.hasValue()) { + auto const value = numMarkers.asIntType(); + downloadRanges_ = value; + } else if (backend->fetchLedgerRange()) { + downloadRanges_ = 4; + } + + auto const allowNoEtl = config.get("allow_no_etl"); + + auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) { + LOG(log_.warn()) << log; + + if (!allowNoEtl) { + LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources."; + throw std::logic_error("ETL configuration error."); + } + }; + + auto const forwardingTimeout = + ClioConfigDefinition::toMilliseconds(config.get("forwarding.request_timeout")); + auto const etlArray = config.getArray("etl_sources"); + for (auto it = etlArray.begin(); it != etlArray.end(); ++it) { + auto source = sourceFactory( + *it, + ioc, + subscriptions, + validatedLedgers, + forwardingTimeout, + [this]() { + if (not hasForwardingSource_.lock().get()) + chooseForwardingSource(); + }, + [this](bool wasForwarding) { + if (wasForwarding) + chooseForwardingSource(); + }, + [this]() { + if (forwardingCache_.has_value()) + forwardingCache_->invalidate(); + } + ); + + // checking etl node validity + auto const stateOpt = etl::ETLState::fetchETLStateFromSource(*source); + + if (!stateOpt) { + LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString() + << " Please check the configuration and network"; + } else if (etlState_ && etlState_->networkID != stateOpt->networkID) { + checkOnETLFailure(fmt::format( + "ETL sources must be on the same network. Source network id = {} does not match others network id = {}", + stateOpt->networkID, + etlState_->networkID + )); + } else { + etlState_ = stateOpt; + } + + sources_.push_back(std::move(source)); + LOG(log_.info()) << "Added etl source - " << sources_.back()->toString(); + } + + if (!etlState_) + checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network"); + + if (sources_.empty()) + checkOnETLFailure("No ETL sources configured. Please check the configuration"); + + // This is made separate from source creation to prevent UB in case one of the sources will call + // chooseForwardingSource while we are still filling the sources_ vector + for (auto const& source : sources_) { + source->run(); + } +} + +LoadBalancer::~LoadBalancer() +{ + sources_.clear(); +} + +std::vector +LoadBalancer::loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& loadObserver, + std::chrono::steady_clock::duration retryAfter +) +{ + std::vector response; + execute( + [this, &response, &sequence, &loadObserver](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, loadObserver); + + if (!res) { + LOG(log_.error()) << "Failed to download initial ledger." + << " Sequence = " << sequence << " source = " << source->toString(); + } else { + response = std::move(data); + } + + return res; + }, + sequence, + retryAfter + ); + return response; +} + +LoadBalancer::OptionalGetLedgerResponseType +LoadBalancer::fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter +) +{ + GetLedgerResponseType response; + execute( + [&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) { + auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors); + response = std::move(data); + if (status.ok() && response.validated()) { + LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence + << " from source = " << source->toString(); + return true; + } + + LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString() + << ", error_code: " << status.error_code() << ", error_msg: " << status.error_message() + << ", source = " << source->toString(); + return false; + }, + ledgerSequence, + retryAfter + ); + return response; +} + +std::expected +LoadBalancer::forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield +) +{ + if (not request.contains("command")) + return std::unexpected{rpc::ClioError::RpcCommandIsMissing}; + + auto const cmd = boost::json::value_to(request.at("command")); + if (forwardingCache_) { + if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) { + return std::move(cachedResponse).value(); + } + } + + ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests."); + std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + auto numAttempts = 0u; + + auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE; + + std::optional response; + rpc::ClioError error = rpc::ClioError::EtlConnectionError; + while (numAttempts < sources_.size()) { + auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); + if (res) { + response = std::move(res).value(); + break; + } + error = std::max(error, res.error()); // Choose the best result between all sources + + sourceIdx = (sourceIdx + 1) % sources_.size(); + ++numAttempts; + } + + if (response) { + if (forwardingCache_ and not response->contains("error")) + forwardingCache_->put(cmd, *response); + return std::move(response).value(); + } + + return std::unexpected{error}; +} + +boost::json::value +LoadBalancer::toJson() const +{ + boost::json::array ret; + for (auto& src : sources_) + ret.push_back(src->toJson()); + + return ret; +} + +template +void +LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter) +{ + ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions."); + size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + size_t numAttempts = 0; + + while (true) { + auto& source = sources_[sourceIdx]; + + LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence + << " - source = " << source->toString(); + // Originally, it was (source->hasLedger(ledgerSequence) || true) + /* Sometimes rippled has ledger but doesn't actually know. However, + but this does NOT happen in the normal case and is safe to remove + This || true is only needed when loading full history standalone */ + if (source->hasLedger(ledgerSequence)) { + bool const res = f(source); + if (res) { + LOG(log_.debug()) << "Successfully executed func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + break; + } + + LOG(log_.warn()) << "Failed to execute func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } else { + LOG(log_.warn()) << "Ledger not present at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } + sourceIdx = (sourceIdx + 1) % sources_.size(); + numAttempts++; + if (numAttempts % sources_.size() == 0) { + LOG(log_.info()) << "Ledger sequence " << ledgerSequence + << " is not yet available from any configured sources. Sleeping and trying again"; + std::this_thread::sleep_for(retryAfter); + } + } +} + +std::optional +LoadBalancer::getETLState() noexcept +{ + if (!etlState_) { + // retry ETLState fetch + etlState_ = etl::ETLState::fetchETLStateFromSource(*this); + } + return etlState_; +} + +void +LoadBalancer::stop(boost::asio::yield_context yield) +{ + util::CoroutineGroup group{yield}; + std::ranges::for_each(sources_, [&group, yield](auto& source) { + group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); }); + }); + group.asyncWait(yield); +} + +void +LoadBalancer::chooseForwardingSource() +{ + LOG(log_.info()) << "Choosing a new source to forward subscriptions"; + auto hasForwardingSourceLock = hasForwardingSource_.lock(); + hasForwardingSourceLock.get() = false; + for (auto& source : sources_) { + if (not hasForwardingSourceLock.get() and source->isConnected()) { + source->setForwarding(true); + hasForwardingSourceLock.get() = true; + } else { + source->setForwarding(false); + } + } +} + +} // namespace etlng diff --git a/src/etlng/LoadBalancer.hpp b/src/etlng/LoadBalancer.hpp new file mode 100644 index 000000000..ac01100da --- /dev/null +++ b/src/etlng/LoadBalancer.hpp @@ -0,0 +1,271 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2022, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/Mutex.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief A tag class to help identify LoadBalancer in templated code. + */ +struct LoadBalancerTag { + virtual ~LoadBalancerTag() = default; +}; + +template +concept SomeLoadBalancer = std::derived_from; + +/** + * @brief This class is used to manage connections to transaction processing processes. + * + * This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of + * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also + * allows requests for ledger data to be load balanced across all possible ETL sources. + */ +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { +public: + using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; + using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; + using OptionalGetLedgerResponseType = std::optional; + +private: + static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16; + + util::Logger log_{"ETL"}; + // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache + std::optional forwardingCache_; + std::optional forwardingXUserValue_; + + std::vector sources_; + std::optional etlState_; + std::uint32_t downloadRanges_ = + kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */ + + // Using mutext instead of atomic_bool because choosing a new source to + // forward messages should be done with a mutual exclusion otherwise there will be a race condition + util::Mutex hasForwardingSource_{false}; + +public: + /** + * @brief Value for the X-User header when forwarding admin requests + */ + static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin"; + + /** + * @brief Value for the X-User header when forwarding user requests + */ + static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user"; + + /** + * @brief Create an instance of the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + */ + LoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + /** + * @brief A factory function for the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + * @return A shared pointer to a new instance of LoadBalancer + */ + static std::shared_ptr + makeLoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + ~LoadBalancer() override; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for new ETL"); + std::unreachable(); + }; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param observer The observer to notify of progress + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& observer, + std::chrono::steady_clock::duration retryAfter + ) override; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param ledgerSequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one + * @param getObjectNeighbors Whether to request object neighbors + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return The extracted data, if extraction was successful. If the ledger was found + * in the database or the server is shutting down, the optional will be empty + */ + OptionalGetLedgerResponseType + fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} + ) override; + + /** + * @brief Represent the state of this load balancer as a JSON object + * + * @return JSON representation of the state of this load balancer. + */ + boost::json::value + toJson() const override; + + /** + * @brief Forward a JSON RPC request to a randomly selected rippled node. + * + * @param request JSON-RPC request to forward + * @param clientIp The IP address of the peer, if known + * @param isAdmin Whether the request is from an admin + * @param yield The coroutine context + * @return Response received from rippled node as JSON object on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield + ) override; + + /** + * @brief Return state of ETL nodes. + * @return ETL state, nullopt if etl nodes not available + */ + std::optional + getETLState() noexcept override; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + void + stop(boost::asio::yield_context yield) override; + +private: + /** + * @brief Execute a function on a randomly selected source. + * + * @note f is a function that takes an Source as an argument and returns a bool. + * Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another + * randomly chosen Source is used. The process repeats until f returns true. + * + * @param f Function to execute. This function takes the ETL source as an argument, and returns a bool + * @param ledgerSequence f is executed for each Source that has this ledger + * @param retryAfter Time to wait between retries (2 seconds by default) + * server is shutting down + */ + template + void + execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}); + + /** + * @brief Choose a new source to forward requests + */ + void + chooseForwardingSource(); +}; + +} // namespace etlng diff --git a/src/etlng/LoadBalancerInterface.hpp b/src/etlng/LoadBalancerInterface.hpp index 3466a6e79..20623d456 100644 --- a/src/etlng/LoadBalancerInterface.hpp +++ b/src/etlng/LoadBalancerInterface.hpp @@ -129,6 +129,15 @@ public: */ virtual std::optional getETLState() noexcept = 0; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + virtual void + stop(boost::asio::yield_context yield) = 0; }; } // namespace etlng diff --git a/src/etlng/LoaderInterface.hpp b/src/etlng/LoaderInterface.hpp index 929d71679..72cad3192 100644 --- a/src/etlng/LoaderInterface.hpp +++ b/src/etlng/LoaderInterface.hpp @@ -45,7 +45,7 @@ struct LoaderInterface { * @param data The data to load * @return Optional ledger header */ - virtual std::optional + [[nodiscard]] virtual std::optional loadInitialLedger(model::LedgerData const& data) = 0; }; diff --git a/src/etlng/Source.cpp b/src/etlng/Source.cpp new file mode 100644 index 000000000..3f9395446 --- /dev/null +++ b/src/etlng/Source.cpp @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Source.hpp" + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "etlng/impl/SourceImpl.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include + +#include +#include +#include +#include + +namespace etlng { + +SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +) +{ + auto const ip = config.get("ip"); + auto const wsPort = config.get("ws_port"); + auto const grpcPort = config.get("grpc_port"); + + etl::impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout}; + impl::GrpcSource grpcSource{ip, grpcPort}; + auto subscriptionSource = std::make_unique( + ioc, + ip, + wsPort, + std::move(validatedLedgers), + std::move(subscriptions), + std::move(onConnect), + std::move(onDisconnect), + std::move(onLedgerClosed) + ); + + return std::make_unique>( + ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource) + ); +} + +} // namespace etlng diff --git a/src/etlng/Source.hpp b/src/etlng/Source.hpp new file mode 100644 index 000000000..52c07ef9b --- /dev/null +++ b/src/etlng/Source.hpp @@ -0,0 +1,194 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief Provides an implementation of a ETL source + */ +class SourceBase { +public: + using OnConnectHook = std::function; + using OnDisconnectHook = std::function; + using OnLedgerClosedHook = std::function; + + virtual ~SourceBase() = default; + + /** + * @brief Run subscriptions loop of the source + */ + virtual void + run() = 0; + + /** + * @brief Stop Source. + * @note This method will asynchronously wait for source to be stopped. + * + * @param yield The coroutine context. + */ + virtual void + stop(boost::asio::yield_context yield) = 0; + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + [[nodiscard]] virtual bool + isConnected() const = 0; + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + virtual void + setForwarding(bool isForwarding) = 0; + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + [[nodiscard]] virtual boost::json::object + toJson() const = 0; + + /** @return String representation of the source (for debug) */ + [[nodiscard]] virtual std::string + toString() const = 0; + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + [[nodiscard]] virtual bool + hasLedger(uint32_t sequence) const = 0; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + [[nodiscard]] virtual std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0; + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + virtual std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) = 0; + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Value of the X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + [[nodiscard]] virtual std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const = 0; +}; + +using SourcePtr = std::unique_ptr; + +using SourceFactory = std::function subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +)>; + +/** + * @brief Create a source + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers data structure + * @param forwardingTimeout The timeout for forwarding to rippled + * @param onConnect The hook to call on connect + * @param onDisconnect The hook to call on disconnect + * @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set + * as forwarding. + * @return The created source + */ +[[nodiscard]] SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +); + +} // namespace etlng diff --git a/src/etlng/impl/ForwardingSource.cpp b/src/etlng/impl/ForwardingSource.cpp new file mode 100644 index 000000000..b4fb024bf --- /dev/null +++ b/src/etlng/impl/ForwardingSource.cpp @@ -0,0 +1,116 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ForwardingSource.hpp" + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +ForwardingSource::ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout +) + : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort)) + , connectionBuilder_(std::move(ip), std::move(wsPort)) + , forwardingTimeout_{forwardingTimeout} +{ + connectionBuilder_.setConnectionTimeout(connTimeout) + .addHeader( + {boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)} + ); +} + +std::expected +ForwardingSource::forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield +) const +{ + auto connectionBuilder = connectionBuilder_; + if (forwardToRippledClientIp) { + connectionBuilder.addHeader( + {boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)} + ); + } + + connectionBuilder.addHeader({"X-User", std::string{xUserValue}}); + + auto expectedConnection = connectionBuilder.connect(yield); + if (not expectedConnection) { + LOG(log_.debug()) << "Couldn't connect to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlConnectionError}; + } + auto& connection = expectedConnection.value(); + + auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_); + if (writeError) { + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + auto response = connection->read(yield, forwardingTimeout_); + if (not response) { + if (auto errorCode = response.error().errorCode(); + errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) { + LOG(log_.debug()) << "Request to rippled timed out"; + return std::unexpected{rpc::ClioError::EtlRequestTimeout}; + } + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + boost::json::value parsedResponse; + try { + parsedResponse = boost::json::parse(*response); + if (not parsedResponse.is_object()) + throw std::runtime_error("response is not an object"); + } catch (std::exception const& e) { + LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response; + return std::unexpected{rpc::ClioError::EtlInvalidResponse}; + } + + auto responseObject = parsedResponse.as_object(); + responseObject["forwarded"] = true; + + return responseObject; +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ForwardingSource.hpp b/src/etlng/impl/ForwardingSource.hpp new file mode 100644 index 000000000..8624e4e50 --- /dev/null +++ b/src/etlng/impl/ForwardingSource.hpp @@ -0,0 +1,70 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" +#include "util/requests/WsConnection.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class ForwardingSource { + util::Logger log_; + util::requests::WsConnectionBuilder connectionBuilder_; + std::chrono::steady_clock::duration forwardingTimeout_; + + static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3}; + +public: + ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT + ); + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value for X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/Loading.hpp b/src/etlng/impl/Loading.hpp index 435dcb4e4..caa677bce 100644 --- a/src/etlng/impl/Loading.hpp +++ b/src/etlng/impl/Loading.hpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/src/etlng/impl/Registry.hpp b/src/etlng/impl/Registry.hpp index d5d262388..921b70811 100644 --- a/src/etlng/impl/Registry.hpp +++ b/src/etlng/impl/Registry.hpp @@ -24,7 +24,6 @@ #include -#include #include #include #include @@ -81,7 +80,7 @@ concept ContainsValidHook = HasLedgerDataHook or HasInitialDataHook or template concept NoTwoOfKind = not(HasLedgerDataHook and HasTransactionHook) and - not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialDataHook and HasObjectHook) and + not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialObjectsHook and HasInitialObjectHook); template @@ -216,4 +215,10 @@ public: } }; +static auto +makeRegistry(auto&&... exts) +{ + return std::make_unique...>>(std::forward(exts)...); +} + } // namespace etlng::impl diff --git a/src/etlng/impl/SourceImpl.hpp b/src/etlng/impl/SourceImpl.hpp new file mode 100644 index 000000000..1c99d973b --- /dev/null +++ b/src/etlng/impl/SourceImpl.hpp @@ -0,0 +1,232 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "rpc/Errors.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +/** + * @brief Provides an implementation of a ETL source + * + * @tparam GrpcSourceType The type of the gRPC source + * @tparam SubscriptionSourceTypePtr The type of the subscription source + * @tparam ForwardingSourceType The type of the forwarding source + */ +template < + typename GrpcSourceType = GrpcSource, + typename SubscriptionSourceTypePtr = std::unique_ptr, + typename ForwardingSourceType = etl::impl::ForwardingSource> +class SourceImpl : public SourceBase { + std::string ip_; + std::string wsPort_; + std::string grpcPort_; + + GrpcSourceType grpcSource_; + SubscriptionSourceTypePtr subscriptionSource_; + ForwardingSourceType forwardingSource_; + +public: + /** + * @brief Construct a new SourceImpl object + * + * @param ip The IP of the source + * @param wsPort The web socket port of the source + * @param grpcPort The gRPC port of the source + * @param grpcSource The gRPC source + * @param subscriptionSource The subscription source + * @param forwardingSource The forwarding source + */ + template + requires std::is_same_v and + std::is_same_v + SourceImpl( + std::string ip, + std::string wsPort, + std::string grpcPort, + SomeGrpcSourceType&& grpcSource, + SubscriptionSourceTypePtr subscriptionSource, + SomeForwardingSourceType&& forwardingSource + ) + : ip_(std::move(ip)) + , wsPort_(std::move(wsPort)) + , grpcPort_(std::move(grpcPort)) + , grpcSource_(std::forward(grpcSource)) + , subscriptionSource_(std::move(subscriptionSource)) + , forwardingSource_(std::forward(forwardingSource)) + { + } + + /** + * @brief Run subscriptions loop of the source + */ + void + run() final + { + subscriptionSource_->run(); + } + + void + stop(boost::asio::yield_context yield) final + { + subscriptionSource_->stop(yield); + } + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + bool + isConnected() const final + { + return subscriptionSource_->isConnected(); + } + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + void + setForwarding(bool isForwarding) final + { + subscriptionSource_->setForwarding(isForwarding); + } + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + boost::json::object + toJson() const final + { + boost::json::object res; + + res["validated_range"] = subscriptionSource_->validatedRange(); + res["is_connected"] = std::to_string(static_cast(subscriptionSource_->isConnected())); + res["ip"] = ip_; + res["ws_port"] = wsPort_; + res["grpc_port"] = grpcPort_; + + auto last = subscriptionSource_->lastMessageTime(); + if (last.time_since_epoch().count() != 0) { + res["last_msg_age_seconds"] = std::to_string( + std::chrono::duration_cast(std::chrono::steady_clock::now() - last).count() + ); + } + + return res; + } + + /** @return String representation of the source (for debug) */ + std::string + toString() const final + { + return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ + + ", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}"; + } + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + bool + hasLedger(uint32_t sequence) const final + { + return subscriptionSource_->hasLedger(sequence); + } + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final + { + return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors); + } + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) final + { + return grpcSource_.loadInitialLedger(sequence, numMarkers, loader); + } + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value of the X-User header + * @param yield The coroutine context + * @return Response or ClioError + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const final + { + return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield); + } +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Cache.cpp b/src/etlng/impl/ext/Cache.cpp new file mode 100644 index 000000000..d7c2e5f51 --- /dev/null +++ b/src/etlng/impl/ext/Cache.cpp @@ -0,0 +1,59 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Cache.hpp" + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include + +namespace etlng::impl { + +CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache) +{ +} + +void +CacheExt::onLedgerData(model::LedgerData const& data) const +{ + cache_.get().update(data.objects, data.seq); + LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size(); +} + +void +CacheExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size(); + cache_.get().update(data.objects, data.seq); + cache_.get().setFull(); +} + +void +CacheExt::onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) + const +{ + LOG(log_.trace()) << "got initial objects cnt = " << objs.size(); + cache_.get().update(objs, seq); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Cache.hpp b/src/etlng/impl/ext/Cache.hpp new file mode 100644 index 000000000..fc9fbbeb4 --- /dev/null +++ b/src/etlng/impl/ext/Cache.hpp @@ -0,0 +1,51 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +class CacheExt { + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + CacheExt(data::LedgerCacheInterface& cache); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.cpp b/src/etlng/impl/ext/Core.cpp new file mode 100644 index 000000000..3049961fd --- /dev/null +++ b/src/etlng/impl/ext/Core.cpp @@ -0,0 +1,83 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Core.hpp" + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include + +namespace etlng::impl { + +CoreExt::CoreExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +CoreExt::onLedgerData(model::LedgerData const& data) const +{ + LOG(log_.debug()) << "Loading ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.info()) << "Loading initial ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::onObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::insertTransactions(model::LedgerData const& data) const +{ + for (auto const& txn : data.transactions) { + LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID(); + + backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()}); + backend_->writeTransaction( + auto{txn.key}, + data.seq, + data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction' + auto{txn.raw}, + auto{txn.metaRaw} + ); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.hpp b/src/etlng/impl/ext/Core.hpp new file mode 100644 index 000000000..7913c15ef --- /dev/null +++ b/src/etlng/impl/ext/Core.hpp @@ -0,0 +1,58 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include + +#include +#include + +namespace etlng::impl { + +class CoreExt { + std::shared_ptr backend_; + + util::Logger log_{"ETL"}; + +public: + CoreExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onObject(uint32_t seq, model::Object const& obj) const; + +private: + void + insertTransactions(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.cpp b/src/etlng/impl/ext/NFT.cpp new file mode 100644 index 000000000..30566acf9 --- /dev/null +++ b/src/etlng/impl/ext/NFT.cpp @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/NFT.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "etl/NFTHelpers.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +NFTExt::NFTExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +NFTExt::onLedgerData(model::LedgerData const& data) const +{ + writeNFTs(data); +} + +void +NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial object with key = " << obj.key; + backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw)); +} + +void +NFTExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size(); + writeNFTs(data); +} + +void +NFTExt::writeNFTs(model::LedgerData const& data) const +{ + std::vector nfts; + std::vector nftTxs; + + for (auto const& tx : data.transactions) { + auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx); + nftTxs.insert(nftTxs.end(), txs.begin(), txs.end()); + if (maybeNFT) + nfts.push_back(*maybeNFT); + } + + // This is uniqued so that we only write latest modification (as in previous implementation) + backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts)); + backend_->writeNFTTransactions(nftTxs); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.hpp b/src/etlng/impl/ext/NFT.hpp new file mode 100644 index 000000000..6d6a146cf --- /dev/null +++ b/src/etlng/impl/ext/NFT.hpp @@ -0,0 +1,52 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +namespace etlng::impl { + +class NFTExt { + std::shared_ptr backend_; + util::Logger log_{"ETL"}; + +public: + NFTExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onInitialData(model::LedgerData const& data) const; + +private: + void + writeNFTs(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.cpp b/src/etlng/impl/ext/Successor.cpp new file mode 100644 index 000000000..a46d3158b --- /dev/null +++ b/src/etlng/impl/ext/Successor.cpp @@ -0,0 +1,222 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Successor.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/Assert.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +SuccessorExt::SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache) + : backend_(std::move(backend)), cache_(cache) +{ +} + +void +SuccessorExt::onInitialData(model::LedgerData const& data) const +{ + ASSERT(cache_.get().isFull(), "Cache must be full at this point"); + ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load"); + ASSERT(data.objects.empty(), "Should not have objects from initial data"); + writeSuccessors(data.seq); + writeEdgeKeys(data.seq, data.edgeKeys.value()); +} + +void +SuccessorExt::onInitialObjects( + uint32_t seq, + [[maybe_unused]] std::vector const& objs, + std::string lastKey +) const +{ + for (auto const& obj : objs) { + if (!lastKey.empty()) + backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw}); + lastKey = obj.keyRaw; + } +} + +void +SuccessorExt::onLedgerData(model::LedgerData const& data) const +{ + namespace vs = std::views; + + LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size() + << "; got successors = " << data.successors.has_value() << "; cache is " + << (cache_.get().isFull() ? "FULL" : "Not full"); + + auto filteredObjects = data.objects // + | vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; }); + + if (data.successors.has_value()) { + for (auto const& successor : data.successors.value()) + writeIncludedSuccessor(data.seq, successor); + + for (auto const& obj : filteredObjects) + writeIncludedSuccessor(data.seq, obj); + } else { + if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq) + throw std::logic_error("Cache is not full, but object neighbors were not included"); + + for (auto const& obj : filteredObjects) + updateSuccessorFromCache(data.seq, obj); + } +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const +{ + auto firstBook = succ.firstBook; + if (firstBook.empty()) + firstBook = uint256ToString(data::kLAST_KEY); + + backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook)); +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const +{ + ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object"); + + // TODO: perhaps make these optionals inside of obj and move value_or here + auto pred = obj.predecessor; + auto succ = obj.successor; + + if (obj.type == model::Object::ModType::Deleted) { + backend_->writeSuccessor(std::move(pred), seq, std::move(succ)); + } else if (obj.type == model::Object::ModType::Created) { + backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw}); + backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ)); + } +} + +void +SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const +{ + auto const lb = + cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}}); + auto const ub = + cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}}); + + auto checkBookBase = false; + auto const isDeleted = obj.data.empty(); + + if (isDeleted) { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key)); + } else { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key)); + backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key)); + } + + if (isDeleted) { + auto const old = cache_.get().getDeleted(obj.key, seq - 1); + ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key)); + + checkBookBase = isBookDir(obj.key, *old); + } else { + checkBookBase = isBookDir(obj.key, obj.data); + } + + if (checkBookBase) { + auto const current = cache_.get().get(obj.key, seq); + auto const bookBase = getBookBase(obj.key); + + if (isDeleted and not current.has_value()) { + updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase); + } else if (current.has_value()) { + auto const successor = cache_.get().getSuccessor(bookBase, seq); + ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq); + + if (successor->key == obj.key) { + updateBookSuccessor(successor, seq, bookBase); + } + } + } +} + +void +SuccessorExt::updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase +) const +{ + if (maybeSuccessor.has_value()) { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key)); + } else { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY)); + } +} + +void +SuccessorExt::writeSuccessors(uint32_t seq) const +{ + ripple::uint256 prev = data::kFIRST_KEY; + while (auto cur = cache_.get().getSuccessor(prev, seq)) { + if (prev == data::kFIRST_KEY) + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key)); + + if (isBookDir(cur->key, cur->blob)) { + auto base = getBookBase(cur->key); + + // make sure the base is not an actual object + if (not cache_.get().get(base, seq)) { + auto succ = cache_.get().getSuccessor(base, seq); + ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base)); + + if (succ->key == cur->key) + backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key)); + } + } + + prev = cur->key; + } + + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY)); +} + +void +SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const +{ + for (auto const& key : edgeKeys) { + auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq); + if (succ) + backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key)); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.hpp b/src/etlng/impl/ext/Successor.hpp new file mode 100644 index 000000000..5ba8d7022 --- /dev/null +++ b/src/etlng/impl/ext/Successor.hpp @@ -0,0 +1,82 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class SuccessorExt { + std::shared_ptr backend_; + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache); + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector const& objs, std::string lastKey) const; + + void + onLedgerData(model::LedgerData const& data) const; + +private: + void + writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const; + + void + writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const; + + void + updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const; + + void + updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase + ) const; + + void + writeSuccessors(uint32_t seq) const; + + void + writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const; +}; + +} // namespace etlng::impl diff --git a/src/feed/SubscriptionManager.cpp b/src/feed/SubscriptionManager.cpp index 7db63718a..af7bd574f 100644 --- a/src/feed/SubscriptionManager.cpp +++ b/src/feed/SubscriptionManager.cpp @@ -191,7 +191,7 @@ SubscriptionManager::unsubBook(ripple::Book const& book, SubscriberSharedPtr con void SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo) { - transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_); + transactionFeed_.pub(txMeta, lgrInfo, backend_, amendmentCenter_, networkID_); } boost::json::object @@ -210,4 +210,16 @@ SubscriptionManager::report() const }; } +void +SubscriptionManager::setNetworkID(uint32_t const networkID) +{ + networkID_ = networkID; +} + +uint32_t +SubscriptionManager::getNetworkID() const +{ + return networkID_; +} + } // namespace feed diff --git a/src/feed/SubscriptionManager.hpp b/src/feed/SubscriptionManager.hpp index 09c004a5b..0108ef0cf 100644 --- a/src/feed/SubscriptionManager.hpp +++ b/src/feed/SubscriptionManager.hpp @@ -69,6 +69,7 @@ class SubscriptionManager : public SubscriptionManagerInterface { impl::BookChangesFeed bookChangesFeed_; impl::TransactionFeed transactionFeed_; impl::ProposedTransactionFeed proposedTransactionFeed_; + uint32_t networkID_{0}; public: /** @@ -332,6 +333,21 @@ public: */ boost::json::object report() const final; + + /** + * @brief Set the networkID. + * @param networkID The network id to set. + */ + void + setNetworkID(uint32_t networkID) final; + + /** + * @brief Get the networkID. + * + * @return The network id. + */ + uint32_t + getNetworkID() const final; }; } // namespace feed diff --git a/src/feed/SubscriptionManagerInterface.hpp b/src/feed/SubscriptionManagerInterface.hpp index 4339e5951..8303fb693 100644 --- a/src/feed/SubscriptionManagerInterface.hpp +++ b/src/feed/SubscriptionManagerInterface.hpp @@ -244,10 +244,25 @@ public: /** * @brief Get the number of subscribers. * - * @return The report of the number of subscribers + * @return The report of the number of subscribers. */ virtual boost::json::object report() const = 0; + + /** + * @brief Set the networkID. + * @param networkID The network id to set. + */ + virtual void + setNetworkID(uint32_t networkID) = 0; + + /** + * @brief Get the networkID. + * + * @return The network id. + */ + virtual uint32_t + getNetworkID() const = 0; }; } // namespace feed diff --git a/src/feed/impl/TransactionFeed.cpp b/src/feed/impl/TransactionFeed.cpp index ac65cca6f..b2b85db4b 100644 --- a/src/feed/impl/TransactionFeed.cpp +++ b/src/feed/impl/TransactionFeed.cpp @@ -25,6 +25,7 @@ #include "feed/Types.hpp" #include "rpc/JS.hpp" #include "rpc/RPCHelpers.hpp" +#include "util/Assert.hpp" #include "util/log/Logger.hpp" #include @@ -176,7 +177,8 @@ TransactionFeed::pub( data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo, std::shared_ptr const& backend, - std::shared_ptr const& amendmentCenter + std::shared_ptr const& amendmentCenter, + uint32_t const networkID ) { auto [tx, meta] = rpc::deserializeTxPlusMeta(txMeta, lgrInfo.seq); @@ -205,6 +207,15 @@ TransactionFeed::pub( rpc::insertDeliverMaxAlias(pubObj[txKey].as_object(), version); rpc::insertMPTIssuanceID(pubObj[JS(meta)].as_object(), tx, meta); + auto const& metaObj = pubObj[JS(meta)]; + ASSERT(metaObj.is_object(), "meta must be an obj in rippled and clio"); + if (metaObj.as_object().contains("TransactionIndex") && metaObj.as_object().at("TransactionIndex").is_int64()) { + if (auto const& ctid = + rpc::encodeCTID(lgrInfo.seq, metaObj.as_object().at("TransactionIndex").as_int64(), networkID); + ctid) + pubObj[JS(ctid)] = ctid.value(); + } + pubObj[JS(type)] = "transaction"; pubObj[JS(validated)] = true; pubObj[JS(status)] = "closed"; diff --git a/src/feed/impl/TransactionFeed.hpp b/src/feed/impl/TransactionFeed.hpp index 4500575d4..ac16ca79a 100644 --- a/src/feed/impl/TransactionFeed.hpp +++ b/src/feed/impl/TransactionFeed.hpp @@ -182,12 +182,14 @@ public: * @param txMeta The transaction and metadata. * @param lgrInfo The ledger header. * @param backend The backend. + * @param networkID The network ID. */ void pub(data::TransactionAndMetadata const& txMeta, ripple::LedgerHeader const& lgrInfo, std::shared_ptr const& backend, - std::shared_ptr const& amendmentCenter); + std::shared_ptr const& amendmentCenter, + uint32_t networkID); /** * @brief Get the number of subscribers of the transaction feed. diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 557e843d6..bce9f8475 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -1,3 +1,8 @@ +# Have to use RPCCenter as a separate library since it is used in util +add_library(clio_rpc_center) +target_sources(clio_rpc_center PRIVATE RPCCenter.cpp) +target_include_directories(clio_rpc_center PUBLIC "${CMAKE_SOURCE_DIR}/src") + add_library(clio_rpc) target_sources( diff --git a/src/rpc/RPCCenter.cpp b/src/rpc/RPCCenter.cpp new file mode 100644 index 000000000..74e079b13 --- /dev/null +++ b/src/rpc/RPCCenter.cpp @@ -0,0 +1,112 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "rpc/RPCCenter.hpp" + +#include +#include + +namespace rpc { + +namespace { + +std::unordered_set const& +handledRpcs() +{ + static std::unordered_set kHANDLED_RPCS = { + "account_channels", + "account_currencies", + "account_info", + "account_lines", + "account_nfts", + "account_objects", + "account_offers", + "account_tx", + "amm_info", + "book_changes", + "book_offers", + "deposit_authorized", + "feature", + "gateway_balances", + "get_aggregate_price", + "ledger", + "ledger_data", + "ledger_entry", + "ledger_index", + "ledger_range", + "mpt_holders", + "nfts_by_issuer", + "nft_history", + "nft_buy_offers", + "nft_info", + "nft_sell_offers", + "noripple_check", + "ping", + "random", + "server_info", + "transaction_entry", + "tx", + "subscribe", + "unsubscribe", + "version", + }; + return kHANDLED_RPCS; +} + +std::unordered_set const& +forwardedRpcs() +{ + static std::unordered_set const kFORWARDED_RPCS = { + "server_definitions", + "server_state", + "submit", + "submit_multisigned", + "fee", + "ledger_closed", + "ledger_current", + "ripple_path_find", + "manifest", + "channel_authorize", + "channel_verify", + "simulate", + }; + return kFORWARDED_RPCS; +} + +} // namespace + +bool +RPCCenter::isRpcName(std::string_view s) +{ + return isHandled(s) || isForwarded(s); +} + +bool +RPCCenter::isHandled(std::string_view s) +{ + return handledRpcs().contains(s); +} + +bool +RPCCenter::isForwarded(std::string_view s) +{ + return forwardedRpcs().contains(s); +} + +} // namespace rpc diff --git a/src/rpc/RPCCenter.hpp b/src/rpc/RPCCenter.hpp new file mode 100644 index 000000000..c772c5a25 --- /dev/null +++ b/src/rpc/RPCCenter.hpp @@ -0,0 +1,61 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include + +namespace rpc { + +/** + * @brief Registry of RPC commands supported by Clio + * + * The RPCCenter maintains lists of RPC commands that can be handled locally + * and those that need to be forwarded to rippled. + */ +struct RPCCenter { + /** + * @brief Checks if a string is a valid RPC command name + * + * @param s The string to check + * @return true if the string is a recognized RPC name, false otherwise + */ + static bool + isRpcName(std::string_view s); + + /** + * @brief Checks if a string is a RPC command handled by Clio without forwarding to rippled + * + * @param s The string to check + * @return true if the string is a handled RPC command, false otherwise + */ + static bool + isHandled(std::string_view s); + + /** + * @brief Checks if a string is a RPC command that will be forwarded to rippled + * + * @param s The string to check + * @return true if the string is a forwarded RPC command, false otherwise + */ + static bool + isForwarded(std::string_view s); +}; + +} // namespace rpc diff --git a/src/rpc/RPCEngine.hpp b/src/rpc/RPCEngine.hpp index 13216c20d..fe46db104 100644 --- a/src/rpc/RPCEngine.hpp +++ b/src/rpc/RPCEngine.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/WorkQueue.hpp" @@ -55,7 +56,7 @@ namespace rpc { /** * @brief The RPC engine that ties all RPC-related functionality together. */ -template +template class RPCEngine { util::Logger perfLog_{"Performance"}; util::Logger log_{"RPC"}; @@ -67,7 +68,7 @@ class RPCEngine { std::shared_ptr handlerProvider_; - impl::ForwardingProxy forwardingProxy_; + impl::ForwardingProxy forwardingProxy_; std::optional responseCache_; @@ -86,7 +87,7 @@ public: RPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, @@ -128,7 +129,7 @@ public: makeRPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, diff --git a/src/rpc/RPCHelpers.cpp b/src/rpc/RPCHelpers.cpp index 651b84120..6d1ec3503 100644 --- a/src/rpc/RPCHelpers.cpp +++ b/src/rpc/RPCHelpers.cpp @@ -290,7 +290,10 @@ std::optional encodeCTID(uint32_t ledgerSeq, uint16_t txnIndex, uint16_t networkId) noexcept { static constexpr uint32_t kMAX_LEDGER_SEQ = 0x0FFF'FFFF; - if (ledgerSeq > kMAX_LEDGER_SEQ) + static constexpr uint32_t kMAX_TXN_INDEX = 0xFFFF; + static constexpr uint32_t kMAX_NETWORK_ID = 0xFFFF; + + if (ledgerSeq > kMAX_LEDGER_SEQ || txnIndex > kMAX_TXN_INDEX || networkId > kMAX_NETWORK_ID) return {}; static constexpr uint64_t kCTID_PREFIX = 0xC000'0000; diff --git a/src/rpc/RPCHelpers.hpp b/src/rpc/RPCHelpers.hpp index 040a211a0..3b72d3f8d 100644 --- a/src/rpc/RPCHelpers.hpp +++ b/src/rpc/RPCHelpers.hpp @@ -30,6 +30,7 @@ #include "rpc/Errors.hpp" #include "rpc/common/Types.hpp" #include "util/JsonUtils.hpp" +#include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "web/Context.hpp" @@ -744,12 +745,13 @@ decodeCTID(T const ctid) noexcept * @brief Log the duration of the request processing * * @tparam T The type of the duration - * @param ctx The context of the request + * @param request The request to log + * @param tag The tag of the context of the request * @param dur The duration to log */ -template +template void -logDuration(web::Context const& ctx, T const& dur) +logDuration(boost::json::object const& request, util::BaseTagDecorator const& tag, DurationType const& dur) { using boost::json::serialize; @@ -759,15 +761,15 @@ logDuration(web::Context const& ctx, T const& dur) auto const millis = std::chrono::duration_cast(dur).count(); auto const seconds = std::chrono::duration_cast(dur).count(); auto const msg = fmt::format( - "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(ctx.params)) + "Request processing duration = {} milliseconds. request = {}", millis, serialize(util::removeSecret(request)) ); if (seconds > kDURATION_ERROR_THRESHOLD_SECONDS) { - LOG(log.error()) << ctx.tag() << msg; + LOG(log.error()) << tag << msg; } else if (seconds > 1) { - LOG(log.warn()) << ctx.tag() << msg; + LOG(log.warn()) << tag << msg; } else - LOG(log.info()) << ctx.tag() << msg; + LOG(log.info()) << tag << msg; } /** diff --git a/src/rpc/common/impl/ForwardingProxy.hpp b/src/rpc/common/impl/ForwardingProxy.hpp index c60773df1..8bf96ceb0 100644 --- a/src/rpc/common/impl/ForwardingProxy.hpp +++ b/src/rpc/common/impl/ForwardingProxy.hpp @@ -19,7 +19,9 @@ #pragma once +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" +#include "rpc/RPCCenter.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/common/Types.hpp" #include "util/log/Logger.hpp" @@ -31,20 +33,21 @@ #include #include #include +#include namespace rpc::impl { -template +template class ForwardingProxy { util::Logger log_{"RPC"}; - std::shared_ptr balancer_; + std::shared_ptr balancer_; std::reference_wrapper counters_; std::shared_ptr handlerProvider_; public: ForwardingProxy( - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, CountersType& counters, std::shared_ptr const& handlerProvider ) @@ -104,22 +107,7 @@ public: bool isProxied(std::string const& method) const { - static std::unordered_set const kPROXIED_COMMANDS{ - "server_definitions", - "server_state", - "submit", - "submit_multisigned", - "fee", - "ledger_closed", - "ledger_current", - "ripple_path_find", - "manifest", - "channel_authorize", - "channel_verify", - "simulate", - }; - - return kPROXIED_COMMANDS.contains(method); + return RPCCenter::isForwarded(method); } private: diff --git a/src/rpc/common/impl/HandlerProvider.cpp b/src/rpc/common/impl/HandlerProvider.cpp index cf4485cc2..2f360a9a7 100644 --- a/src/rpc/common/impl/HandlerProvider.cpp +++ b/src/rpc/common/impl/HandlerProvider.cpp @@ -21,7 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Counters.hpp" #include "rpc/common/AnyHandler.hpp" @@ -65,6 +66,7 @@ #include #include #include +#include namespace rpc::impl { @@ -72,8 +74,8 @@ ProductionHandlerProvider::ProductionHandlerProvider( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ) @@ -85,7 +87,7 @@ ProductionHandlerProvider::ProductionHandlerProvider( {"account_nfts", {.handler = AccountNFTsHandler{backend}}}, {"account_objects", {.handler = AccountObjectsHandler{backend}}}, {"account_offers", {.handler = AccountOffersHandler{backend}}}, - {"account_tx", {.handler = AccountTxHandler{backend}}}, + {"account_tx", {.handler = AccountTxHandler{backend, etl}}}, {"amm_info", {.handler = AMMInfoHandler{backend, amendmentCenter}}}, {"book_changes", {.handler = BookChangesHandler{backend}}}, {"book_offers", {.handler = BookOffersHandler{backend, amendmentCenter}}}, @@ -138,4 +140,13 @@ ProductionHandlerProvider::isClioOnly(std::string const& command) const return handlerMap_.contains(command) && handlerMap_.at(command).isClioOnly; } +std::unordered_set +ProductionHandlerProvider::handlerNames() const +{ + std::unordered_set result; + for (auto const& [name, handler] : handlerMap_) + result.insert(name); + return result; +} + } // namespace rpc::impl diff --git a/src/rpc/common/impl/HandlerProvider.hpp b/src/rpc/common/impl/HandlerProvider.hpp index 89ea5661f..0b54be3c1 100644 --- a/src/rpc/common/impl/HandlerProvider.hpp +++ b/src/rpc/common/impl/HandlerProvider.hpp @@ -21,6 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/common/AnyHandler.hpp" #include "rpc/common/HandlerProvider.hpp" @@ -31,11 +33,8 @@ #include #include #include +#include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -55,8 +54,8 @@ public: util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ); @@ -69,6 +68,9 @@ public: bool isClioOnly(std::string const& command) const override; + + std::unordered_set + handlerNames() const; }; } // namespace rpc::impl diff --git a/src/rpc/handlers/AccountLines.cpp b/src/rpc/handlers/AccountLines.cpp index a5ab872d4..6459e27d6 100644 --- a/src/rpc/handlers/AccountLines.cpp +++ b/src/rpc/handlers/AccountLines.cpp @@ -86,8 +86,9 @@ AccountLinesHandler::addLine( bool const lineNoRipplePeer = (flags & (not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple)) != 0u; bool const lineFreeze = (flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; bool const lineFreezePeer = (flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreezePeer = (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; + bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; + bool const lineDeepFreezePeer = + (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; ripple::STAmount const& saBalance = balance; ripple::STAmount const& saLimit = lineLimit; @@ -102,6 +103,12 @@ AccountLinesHandler::addLine( line.qualityIn = lineQualityIn; line.qualityOut = lineQualityOut; + if (lineNoRipple) + line.noRipple = true; + + if (lineNoRipplePeer) + line.noRipplePeer = true; + if (lineAuth) line.authorized = true; @@ -120,8 +127,6 @@ AccountLinesHandler::addLine( if (lineDeepFreezePeer) line.deepFreezePeer = true; - line.noRipple = lineNoRipple; - line.noRipplePeer = lineNoRipplePeer; lines.push_back(line); } @@ -257,8 +262,11 @@ tag_invoke( {JS(quality_out), line.qualityOut}, }; - obj[JS(no_ripple)] = line.noRipple; - obj[JS(no_ripple_peer)] = line.noRipplePeer; + if (line.noRipple) + obj[JS(no_ripple)] = *(line.noRipple); + + if (line.noRipplePeer) + obj[JS(no_ripple_peer)] = *(line.noRipplePeer); if (line.authorized) obj[JS(authorized)] = *(line.authorized); diff --git a/src/rpc/handlers/AccountLines.hpp b/src/rpc/handlers/AccountLines.hpp index 3388f3166..38d6f5231 100644 --- a/src/rpc/handlers/AccountLines.hpp +++ b/src/rpc/handlers/AccountLines.hpp @@ -70,8 +70,8 @@ public: std::string limitPeer; uint32_t qualityIn{}; uint32_t qualityOut{}; - bool noRipple{}; - bool noRipplePeer{}; + std::optional noRipple; + std::optional noRipplePeer; std::optional authorized; std::optional peerAuthorized; std::optional freeze; diff --git a/src/rpc/handlers/AccountTx.cpp b/src/rpc/handlers/AccountTx.cpp index 9b9c68adf..882d5a983 100644 --- a/src/rpc/handlers/AccountTx.cpp +++ b/src/rpc/handlers/AccountTx.cpp @@ -161,6 +161,18 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) con auto const txKey = ctx.apiVersion < 2u ? JS(tx) : JS(tx_json); obj[JS(meta)] = std::move(meta); obj[txKey] = std::move(txn); + + // Put CTID into tx or tx_json + if (obj[JS(meta)].as_object().contains("TransactionIndex")) { + auto networkID = 0u; + if (auto const& etlState = etl_->getETLState(); etlState.has_value()) + networkID = etlState->networkID; + + auto const txnIdx = obj[JS(meta)].as_object().at("TransactionIndex").as_int64(); + if (auto const& ctid = rpc::encodeCTID(txnPlusMeta.ledgerSequence, txnIdx, networkID); ctid) + obj[txKey].as_object()[JS(ctid)] = ctid.value(); + } + obj[txKey].as_object()[JS(date)] = txnPlusMeta.date; obj[txKey].as_object()[JS(ledger_index)] = txnPlusMeta.ledgerSequence; diff --git a/src/rpc/handlers/AccountTx.hpp b/src/rpc/handlers/AccountTx.hpp index 0e620a8ca..14551c424 100644 --- a/src/rpc/handlers/AccountTx.hpp +++ b/src/rpc/handlers/AccountTx.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" #include "rpc/common/JsonBool.hpp" @@ -55,6 +56,7 @@ namespace rpc { class AccountTxHandler { util::Logger log_{"RPC"}; std::shared_ptr sharedPtrBackend_; + std::shared_ptr etl_; public: static constexpr auto kLIMIT_MIN = 1; @@ -109,8 +111,13 @@ public: * @brief Construct a new AccountTxHandler object * * @param sharedPtrBackend The backend to use + * @param etl The ETL service to use */ - AccountTxHandler(std::shared_ptr const& sharedPtrBackend) : sharedPtrBackend_(sharedPtrBackend) + AccountTxHandler( + std::shared_ptr const& sharedPtrBackend, + std::shared_ptr const& etl + ) + : sharedPtrBackend_(sharedPtrBackend), etl_{etl} { } @@ -130,6 +137,7 @@ public: {JS(ledger_index), validation::CustomValidators::ledgerIndexValidator}, {JS(ledger_index_min), validation::Type{}}, {JS(ledger_index_max), validation::Type{}}, + {JS(ctid), validation::Type{}}, {JS(limit), validation::Type{}, validation::Min(1u), diff --git a/src/rpc/handlers/ServerInfo.hpp b/src/rpc/handlers/ServerInfo.hpp index 008450721..085066f7c 100644 --- a/src/rpc/handlers/ServerInfo.hpp +++ b/src/rpc/handlers/ServerInfo.hpp @@ -21,6 +21,8 @@ #include "data/BackendInterface.hpp" #include "data/DBHelpers.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" @@ -49,10 +51,6 @@ #include #include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -62,18 +60,16 @@ namespace rpc { /** * @brief Contains common functionality for handling the `server_info` command * - * @tparam LoadBalancerType The type of the load balancer - * @tparam ETLServiceType The type of the ETL service * @tparam CountersType The type of the counters */ -template +template class BaseServerInfoHandler { static constexpr auto kBACKEND_COUNTERS_KEY = "backend_counters"; std::shared_ptr backend_; std::shared_ptr subscriptions_; - std::shared_ptr balancer_; - std::shared_ptr etl_; + std::shared_ptr balancer_; + std::shared_ptr etl_; std::reference_wrapper counters_; public: @@ -158,8 +154,8 @@ public: BaseServerInfoHandler( std::shared_ptr const& backend, std::shared_ptr const& subscriptions, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, CountersType const& counters ) : backend_(backend) @@ -352,6 +348,6 @@ private: * * For more details see: https://xrpl.org/server_info-clio.html */ -using ServerInfoHandler = BaseServerInfoHandler; +using ServerInfoHandler = BaseServerInfoHandler; } // namespace rpc diff --git a/src/rpc/handlers/Tx.hpp b/src/rpc/handlers/Tx.hpp index b05a64c1e..26da69a14 100644 --- a/src/rpc/handlers/Tx.hpp +++ b/src/rpc/handlers/Tx.hpp @@ -21,7 +21,7 @@ #include "data/BackendInterface.hpp" #include "data/Types.hpp" -#include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" #include "rpc/RPCHelpers.hpp" @@ -37,7 +37,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -52,14 +54,13 @@ namespace rpc { /** - * @brief Contains common functionality for handling the `tx` command + * @brief The tx method retrieves information on a single transaction, by its identifying hash. * - * @tparam ETLServiceType The type of the ETL service to use + * For more details see: https://xrpl.org/tx.html */ -template -class BaseTxHandler { +class TxHandler { std::shared_ptr sharedPtrBackend_; - std::shared_ptr etl_; + std::shared_ptr etl_; public: /** @@ -95,14 +96,14 @@ public: using Result = HandlerReturnType; /** - * @brief Construct a new BaseTxHandler object + * @brief Construct a new TxHandler object * * @param sharedPtrBackend The backend to use * @param etl The ETL service to use */ - BaseTxHandler( + TxHandler( std::shared_ptr const& sharedPtrBackend, - std::shared_ptr const& etl + std::shared_ptr const& etl ) : sharedPtrBackend_(sharedPtrBackend), etl_(etl) { @@ -183,7 +184,7 @@ public: dbResponse = sharedPtrBackend_->fetchTransaction(ripple::uint256{input.transaction->c_str()}, ctx.yield); } - auto output = BaseTxHandler::Output{.apiVersion = ctx.apiVersion}; + auto output = TxHandler::Output{.apiVersion = ctx.apiVersion}; if (!dbResponse) { if (rangeSupplied && input.transaction) // ranges not for ctid @@ -214,17 +215,15 @@ public: // input.transaction might be not available, get hash via tx object if (txn.contains(JS(hash))) output.hash = txn.at(JS(hash)).as_string(); + } - // append ctid here to mimic rippled 1.12 behavior: return ctid even binary=true - // rippled will change it in the future, ctid should be part of tx json which not available in binary - // mode - auto const txnIdx = boost::json::value_to(meta.at("TransactionIndex")); - if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId && - *currentNetId <= 0xFFFFU) { - output.ctid = rpc::encodeCTID( - dbResponse->ledgerSequence, static_cast(txnIdx), static_cast(*currentNetId) - ); - } + // append ctid here to mimic rippled behavior + auto const txnIdx = boost::json::value_to(meta.at("TransactionIndex")); + if (txnIdx <= 0xFFFFU && dbResponse->ledgerSequence < 0x0FFF'FFFFUL && currentNetId && + *currentNetId <= 0xFFFFU) { + output.ctid = rpc::encodeCTID( + dbResponse->ledgerSequence, static_cast(txnIdx), static_cast(*currentNetId) + ); } output.date = dbResponse->date; @@ -281,12 +280,10 @@ private: if (output.tx) { obj[JS(tx_json)] = *output.tx; obj[JS(tx_json)].as_object()[JS(date)] = output.date; + if (output.ctid) + obj[JS(tx_json)].as_object()[JS(ctid)] = *output.ctid; + obj[JS(tx_json)].as_object()[JS(ledger_index)] = output.ledgerIndex; - // move ctid from tx_json to root - if (obj[JS(tx_json)].as_object().contains(JS(ctid))) { - obj[JS(ctid)] = obj[JS(tx_json)].as_object()[JS(ctid)]; - obj[JS(tx_json)].as_object().erase(JS(ctid)); - } // move hash from tx_json to root if (obj[JS(tx_json)].as_object().contains(JS(hash))) { obj[JS(hash)] = obj[JS(tx_json)].as_object()[JS(hash)]; @@ -320,7 +317,7 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) { - auto input = BaseTxHandler::Input{}; + auto input = TxHandler::Input{}; auto const& jsonObject = jv.as_object(); if (jsonObject.contains(JS(transaction))) @@ -344,10 +341,4 @@ private: } }; -/** - * @brief The tx method retrieves information on a single transaction, by its identifying hash. - * - * For more details see: https://xrpl.org/tx.html - */ -using TxHandler = BaseTxHandler; } // namespace rpc diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 51d44765f..838b5b11f 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -24,6 +24,7 @@ target_sources( ResponseExpirationCache.cpp SignalsHandler.cpp StopHelper.cpp + StringHash.cpp Taggable.cpp TerminationHandler.cpp TimeUtils.cpp @@ -41,7 +42,7 @@ target_sources( # This must be above the target_link_libraries call otherwise backtrace doesn't work if ("${san}" STREQUAL "") - target_link_libraries(clio_util PUBLIC Boost::stacktrace_backtrace dl libbacktrace::libbacktrace) + target_link_libraries(clio_util PUBLIC Boost::stacktrace_backtrace dl libbacktrace::libbacktrace clio_rpc_center) endif () target_link_libraries( diff --git a/src/util/StringHash.cpp b/src/util/StringHash.cpp new file mode 100644 index 000000000..3f26feb2a --- /dev/null +++ b/src/util/StringHash.cpp @@ -0,0 +1,46 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "util/StringHash.hpp" + +#include +#include +#include + +namespace util { + +size_t +StringHash::operator()(char const* str) const +{ + return hash_type{}(str); +} + +size_t +StringHash::operator()(std::string_view str) const +{ + return hash_type{}(str); +} + +size_t +StringHash::operator()(std::string const& str) const +{ + return hash_type{}(str); +} + +} // namespace util diff --git a/src/util/StringHash.hpp b/src/util/StringHash.hpp new file mode 100644 index 000000000..862c8ed86 --- /dev/null +++ b/src/util/StringHash.hpp @@ -0,0 +1,65 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include +#include +#include +#include + +namespace util { + +/** + * @brief A string hash functor that provides transparent hash operations for various string types. + * + * This hash functor can be used with unordered containers to enable heterogeneous lookups + * for different string-like types without unnecessary conversions. It supports C-style strings, + * string views, and standard strings. + */ +struct StringHash { + using hash_type = std::hash; + using is_transparent = void; ///< Enables heterogeneous lookup + + /** + * @brief Computes the hash of a C-style string. + * @param str Null-terminated C-style string to hash + * @return Size_t hash value + */ + std::size_t + operator()(char const* str) const; + + /** + * @brief Computes the hash of a string_view. + * @param str String view to hash + * @return Size_t hash value + */ + std::size_t + operator()(std::string_view str) const; + + /** + * @brief Computes the hash of a standard string. + * @param str String to hash + * @return Size_t hash value + */ + std::size_t + operator()(std::string const& str) const; +}; + +} // namespace util diff --git a/src/util/newconfig/ConfigConstraints.cpp b/src/util/newconfig/ConfigConstraints.cpp index 2a0f8fb6e..41d5210ba 100644 --- a/src/util/newconfig/ConfigConstraints.cpp +++ b/src/util/newconfig/ConfigConstraints.cpp @@ -19,6 +19,7 @@ #include "util/newconfig/ConfigConstraints.hpp" +#include "rpc/RPCCenter.hpp" #include "util/newconfig/Error.hpp" #include "util/newconfig/Types.hpp" @@ -103,4 +104,22 @@ PositiveDouble::checkValueImpl(Value const& num) const return Error{"Double number must be greater than or equal to 0"}; } +std::optional +RpcNameConstraint::checkTypeImpl(Value const& value) const +{ + if (not std::holds_alternative(value)) + return Error{"RPC command name must be a string"}; + return std::nullopt; +} + +std::optional +RpcNameConstraint::checkValueImpl(Value const& value) const +{ + auto const str = std::get(value); + if (not rpc::RPCCenter::isRpcName(str)) + return Error{"Invalid RPC command name"}; + + return std::nullopt; +} + } // namespace util::config diff --git a/src/util/newconfig/ConfigConstraints.hpp b/src/util/newconfig/ConfigConstraints.hpp index 910dff1e5..d9685ac51 100644 --- a/src/util/newconfig/ConfigConstraints.hpp +++ b/src/util/newconfig/ConfigConstraints.hpp @@ -421,6 +421,41 @@ private: } }; +/** + * @brief A constraint to ensure the value is a valid RPC command name. + */ +class RpcNameConstraint final : public Constraint { +private: + /** + * @brief Check if the type of the value is correct for this specific constraint. + * + * @param value The type to be checked + * @return An Error object if the constraint is not met, nullopt otherwise + */ + [[nodiscard]] std::optional + checkTypeImpl(Value const& value) const override; + + /** + * @brief Check if the value is a valid RPC command name. + * + * @param value The value to check + * @return An Error object if the constraint is not met, nullopt otherwise + */ + [[nodiscard]] std::optional + checkValueImpl(Value const& value) const override; + + /** + * @brief Prints to the output stream for this specific constraint. + * + * @param stream The output stream + */ + void + print(std::ostream& stream) const override + { + stream << "Checks whether provided RPC name is valid"; + } +}; + static constinit PortConstraint gValidatePort{}; static constinit ValidIPConstraint gValidateIp{}; @@ -448,6 +483,8 @@ static constinit NumberValueConstraint gValidateUint32{ std::numeric_limits::min(), std::numeric_limits::max() }; +static constinit NumberValueConstraint gValidateNonNegativeUint32{0, std::numeric_limits::max()}; static constinit NumberValueConstraint gValidateApiVersion{rpc::kAPI_VERSION_MIN, rpc::kAPI_VERSION_MAX}; +static constinit RpcNameConstraint gRpcNameConstraint{}; } // namespace util::config diff --git a/src/util/newconfig/ConfigDefinition.hpp b/src/util/newconfig/ConfigDefinition.hpp index 030b8d160..16867f9b1 100644 --- a/src/util/newconfig/ConfigDefinition.hpp +++ b/src/util/newconfig/ConfigDefinition.hpp @@ -293,7 +293,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - + {"__ng_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}}, {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, @@ -314,6 +314,15 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(gValidateUint32)}, {"dos_guard.sweep_interval", ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(gValidatePositiveDouble)}, + {"dos_guard.__ng_default_weight", + ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(gValidateNonNegativeUint32)}, + {"dos_guard.__ng_weights.[].method", Array{ConfigValue{ConfigType::String}.withConstraint(gRpcNameConstraint)}}, + {"dos_guard.__ng_weights.[].weight", + Array{ConfigValue{ConfigType::Integer}.withConstraint(gValidateNonNegativeUint32)}}, + {"dos_guard.__ng_weights.[].weight_ledger_current", + Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}}, + {"dos_guard.__ng_weights.[].weight_ledger_validated", + Array{ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNonNegativeUint32)}}, {"workers", ConfigValue{ConfigType::Integer} diff --git a/src/web/CMakeLists.txt b/src/web/CMakeLists.txt index 083eae90c..2b62bbe41 100644 --- a/src/web/CMakeLists.txt +++ b/src/web/CMakeLists.txt @@ -5,12 +5,12 @@ target_sources( PRIVATE AdminVerificationStrategy.cpp dosguard/DOSGuard.cpp dosguard/IntervalSweepHandler.cpp + dosguard/Weights.cpp dosguard/WhitelistHandler.cpp ng/Connection.cpp ng/impl/ErrorHandling.cpp ng/impl/ConnectionHandler.cpp ng/impl/ServerSslContext.cpp - ng/impl/WsConnection.cpp ng/Request.cpp ng/Response.cpp ng/Server.cpp diff --git a/src/web/RPCServerHandler.hpp b/src/web/RPCServerHandler.hpp index 632d9551e..34ab0394a 100644 --- a/src/web/RPCServerHandler.hpp +++ b/src/web/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -30,6 +31,7 @@ #include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "util/newconfig/ConfigDefinition.hpp" +#include "web/dosguard/DOSGuardInterface.hpp" #include "web/impl/ErrorHandling.hpp" #include "web/interface/ConnectionBase.hpp" @@ -58,13 +60,14 @@ namespace web { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed + std::reference_wrapper dosguard_; util::Logger log_{"RPC"}; util::Logger perfLog_{"Performance"}; @@ -77,18 +80,21 @@ public: * @param backend The backend to use * @param rpcEngine The RPC engine to use * @param etl The ETL to use + * @param dosguard The DOS guard service to use for request rate limiting */ RPCServerHandler( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl, + web::dosguard::DOSGuardInterface& dosguard ) : backend_(backend) , rpcEngine_(rpcEngine) , etl_(etl) , tagFactory_(config) , apiVersionParser_(config.getObject("api_version")) + , dosguard_(dosguard) { } @@ -101,6 +107,11 @@ public: void operator()(std::string const& request, std::shared_ptr const& connection) { + if (not dosguard_.get().isOk(connection->clientIp)) { + connection->sendSlowDown(request); + return; + } + try { auto req = boost::json::parse(request).as_object(); LOG(perfLog_.debug()) << connection->tag() << "Adding to work queue"; @@ -108,6 +119,11 @@ public: if (not connection->upgraded and shouldReplaceParams(req)) req[JS(params)] = boost::json::array({boost::json::object{}}); + if (not dosguard_.get().request(connection->clientIp, req)) { + connection->sendSlowDown(request); + return; + } + if (!rpcEngine_->post( [this, request = std::move(req), connection](boost::asio::yield_context yield) mutable { handleRequest(yield, std::move(request), connection); @@ -195,8 +211,8 @@ private: auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); }); - auto us = std::chrono::duration(timeDiff); - rpc::logDuration(*context, us); + auto const us = std::chrono::duration(timeDiff); + rpc::logDuration(request, context->tag(), us); boost::json::object response; diff --git a/src/web/dosguard/DOSGuard.cpp b/src/web/dosguard/DOSGuard.cpp index 40a28610a..7cf2ca6b6 100644 --- a/src/web/dosguard/DOSGuard.cpp +++ b/src/web/dosguard/DOSGuard.cpp @@ -24,8 +24,11 @@ #include "util/newconfig/ArrayView.hpp" #include "util/newconfig/ConfigDefinition.hpp" #include "util/newconfig/ValueView.hpp" +#include "web/dosguard/WeightsInterface.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" +#include + #include #include #include @@ -37,8 +40,13 @@ using namespace util::config; namespace web::dosguard { -DOSGuard::DOSGuard(ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler) +DOSGuard::DOSGuard( + ClioConfigDefinition const& config, + WhitelistHandlerInterface const& whitelistHandler, + WeightsInterface const& weights +) : whitelistHandler_{std::cref(whitelistHandler)} + , weights_(weights) , maxFetches_{config.get("dos_guard.max_fetches")} , maxConnCount_{config.get("dos_guard.max_connections")} , maxRequestCount_{config.get("dos_guard.max_requests")} @@ -59,8 +67,8 @@ DOSGuard::isOk(std::string const& ip) const noexcept { auto lock = mtx_.lock(); - if (lock->ipState.find(ip) != lock->ipState.end()) { - auto [transferredByte, requests] = lock->ipState.at(ip); + if (auto const it = lock->ipState.find(ip); it != lock->ipState.end()) { + auto const [transferredByte, requests] = it->second; if (transferredByte > maxFetches_ || requests > maxRequestCount_) { LOG(log_.warn()) << "Dosguard: Client surpassed the rate limit. ip = " << ip << " Transfered Byte: " << transferredByte << "; Requests: " << requests; @@ -115,14 +123,16 @@ DOSGuard::add(std::string const& ip, uint32_t numObjects) noexcept } [[maybe_unused]] bool -DOSGuard::request(std::string const& ip) noexcept +DOSGuard::request(std::string const& ip, boost::json::object const& request) { if (whitelistHandler_.get().isWhiteListed(ip)) return true; + auto const weight = weights_.get().requestWeight(request); + { auto lock = mtx_.lock(); - lock->ipState[ip].requestsCount++; + lock->ipState[ip].requestsCount += weight; } return isOk(ip); diff --git a/src/web/dosguard/DOSGuard.hpp b/src/web/dosguard/DOSGuard.hpp index 0fd83bfda..260e48df0 100644 --- a/src/web/dosguard/DOSGuard.hpp +++ b/src/web/dosguard/DOSGuard.hpp @@ -23,10 +23,12 @@ #include "util/log/Logger.hpp" #include "util/newconfig/ConfigDefinition.hpp" #include "web/dosguard/DOSGuardInterface.hpp" +#include "web/dosguard/WeightsInterface.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" #include #include +#include #include #include @@ -59,6 +61,7 @@ class DOSGuard : public DOSGuardInterface { util::Mutex mtx_; std::reference_wrapper whitelistHandler_; + std::reference_wrapper weights_; std::uint32_t const maxFetches_; std::uint32_t const maxConnCount_; @@ -71,8 +74,13 @@ public: * * @param config Clio config * @param whitelistHandler Whitelist handler that checks whitelist for IP addresses + * @param weights API methods weights */ - DOSGuard(util::config::ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler); + DOSGuard( + util::config::ClioConfigDefinition const& config, + WhitelistHandlerInterface const& whitelistHandler, + WeightsInterface const& weights + ); /** * @brief Check whether an ip address is in the whitelist or not. @@ -133,11 +141,12 @@ public: * returned otherwise. * * @param ip + * @param request The request as json object * @return true * @return false */ [[maybe_unused]] bool - request(std::string const& ip) noexcept override; + request(std::string const& ip, boost::json::object const& request) override; /** * @brief Instantly clears all fetch counters added by @see add(std::string const&, uint32_t). diff --git a/src/web/dosguard/DOSGuardInterface.hpp b/src/web/dosguard/DOSGuardInterface.hpp index 1eee27b65..684087c9f 100644 --- a/src/web/dosguard/DOSGuardInterface.hpp +++ b/src/web/dosguard/DOSGuardInterface.hpp @@ -19,6 +19,8 @@ #pragma once +#include + #include #include #include @@ -99,12 +101,13 @@ public: * * * @param ip + * @param request The request as json object * @return If the total sums up to a value equal or larger than maxRequestCount_ * the operation is no longer allowed and false is returned; true is * returned otherwise. */ [[maybe_unused]] virtual bool - request(std::string const& ip) noexcept = 0; + request(std::string const& ip, boost::json::object const& request) = 0; }; } // namespace web::dosguard diff --git a/src/web/dosguard/Weights.cpp b/src/web/dosguard/Weights.cpp new file mode 100644 index 000000000..bed67ffea --- /dev/null +++ b/src/web/dosguard/Weights.cpp @@ -0,0 +1,106 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "web/dosguard/Weights.hpp" + +#include "rpc/JS.hpp" +#include "util/Assert.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace web::dosguard { + +Weights::Weights(size_t defaultWeight, std::unordered_map weights) + : defaultWeight_(defaultWeight), weights_(std::move_iterator(weights.begin()), std::move_iterator(weights.end())) +{ +} + +Weights +Weights::make(util::config::ClioConfigDefinition const& config) +{ + std::unordered_map weights; + auto const configWeights = config.getArray("dos_guard.__ng_weights"); + for (size_t i = 0; i < configWeights.size(); ++i) { + auto const w = configWeights.objectAt(i); + Weights::Entry const entry{ + .weight = w.get("weight"), + .weightLedgerCurrent = w.maybeValue("weight_ledger_current"), + .weightLedgerValidated = w.maybeValue("weight_ledger_validated"), + }; + weights.emplace(w.get("method"), entry); + } + return Weights{config.get("dos_guard.__ng_default_weight"), std::move(weights)}; +} + +size_t +Weights::requestWeight(boost::json::object const& request) const +{ + if (not((request.contains(JS(method)) and request.at(JS(method)).is_string()) or + (request.contains(JS(command)) and request.at(JS(command)).is_string()))) { + return defaultWeight_; + } + + std::string_view cmd = + request.contains(JS(method)) ? request.at(JS(method)).as_string() : request.at(JS(command)).as_string(); + + auto it = weights_.find(cmd); + if (it == weights_.end()) { + return defaultWeight_; + } + + auto const& entry = it->second; + + boost::json::value const* ledgerIndex = nullptr; + if (request.contains(JS(ledger_index))) { + ledgerIndex = &request.at(JS(ledger_index)); + } else if (request.contains(JS(params))) { + ASSERT( + request.at(JS(params)).is_array() and not request.at(JS(params)).as_array().empty() and + request.at(JS(params)).as_array().at(0).is_object(), + "params should be [{{}}]" + ); + if (auto const& params = request.at(JS(params)).as_array().at(0).as_object(); + params.contains(JS(ledger_index))) { + ledgerIndex = ¶ms.at(JS(ledger_index)); + } + } + + if (ledgerIndex != nullptr and ledgerIndex->is_string()) { + auto const& ledgerIndexString = ledgerIndex->as_string(); + if (ledgerIndexString == JS(validated)) { + return entry.weightLedgerValidated.value_or(entry.weight); + } + if (ledgerIndexString == JS(current)) { + return entry.weightLedgerCurrent.value_or(entry.weight); + } + } + return entry.weight; +} + +} // namespace web::dosguard diff --git a/src/web/dosguard/Weights.hpp b/src/web/dosguard/Weights.hpp new file mode 100644 index 000000000..da041ec8f --- /dev/null +++ b/src/web/dosguard/Weights.hpp @@ -0,0 +1,88 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "util/StringHash.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "web/dosguard/WeightsInterface.hpp" + +#include + +#include +#include +#include +#include +#include + +namespace web::dosguard { + +/** + * @brief Implementation of WeightsInterface that manages command weights for DosGuard. + * + * This class provides a mechanism to assign different weights to API commands + * for the purpose of DOS protection calculations. Commands can have specific weights, + * or fall back to a default weight. + */ +class Weights : public WeightsInterface { +public: + /** + * @brief Structure representing weight configuration for a command. + * + * Contains the base weight and optional specialized weights for different ledger specifications. + */ + struct Entry { + size_t weight; + std::optional weightLedgerCurrent; + std::optional weightLedgerValidated; + }; + +private: + size_t defaultWeight_; + std::unordered_map> weights_; + +public: + /** + * @brief Construct a new Weights object + * + * @param defaultWeight The default weight to use when a command-specific weight is not defined + * @param weights Map of command names to their specific weights + */ + Weights(size_t defaultWeight, std::unordered_map weights); + + /** + * @brief Create a Weights object from configuration + * + * @param config The application configuration + * @return Weights instance initialized with values from configuration + */ + static Weights + make(util::config::ClioConfigDefinition const& config); + + /** + * @brief Get the weight assigned to a specific command + * + * @param request Json request + * @return size_t The weight value (specific weight if defined, otherwise default weight) + */ + size_t + requestWeight(boost::json::object const& request) const override; +}; + +} // namespace web::dosguard diff --git a/src/web/dosguard/WeightsInterface.hpp b/src/web/dosguard/WeightsInterface.hpp new file mode 100644 index 000000000..916a25f49 --- /dev/null +++ b/src/web/dosguard/WeightsInterface.hpp @@ -0,0 +1,48 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include + +#include + +namespace web::dosguard { + +/** + * @brief Interface for determining request weights in DOS protection. + * + * This interface defines the contract for classes that calculate weights for incoming + * requests, which is used for DOS protection mechanisms. + */ +class WeightsInterface { +public: + virtual ~WeightsInterface() = default; + + /** + * @brief Calculate the weight of a request. + * + * @param request The JSON object representing the request + * @return The calculated weight of the request + */ + virtual size_t + requestWeight(boost::json::object const& request) const = 0; +}; + +} // namespace web::dosguard diff --git a/src/web/impl/HttpBase.hpp b/src/web/impl/HttpBase.hpp index cfc257e80..5f3fa26cd 100644 --- a/src/web/impl/HttpBase.hpp +++ b/src/web/impl/HttpBase.hpp @@ -240,19 +240,7 @@ public: return sender_(httpResponse(http::status::bad_request, "text/html", "Expected a POST request")); } - // to avoid overwhelm work queue, the request limit check should be - // before posting to queue the web socket creation will be guarded via - // connection limit - if (!dosGuard_.get().request(clientIp)) { - // TODO: this looks like it could be useful to count too in the future - return sender_(httpResponse( - http::status::service_unavailable, - "text/plain", - boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN)) - )); - } - - LOG(log_.info()) << tag() << "Received request from ip = " << clientIp << " - posting to WorkQueue"; + LOG(log_.info()) << tag() << "Received request from ip = " << clientIp; try { (*handler_)(req_.body(), derived().shared_from_this()); @@ -265,6 +253,16 @@ public: } } + void + sendSlowDown(std::string const&) override + { + sender_(httpResponse( + http::status::service_unavailable, + "text/plain", + boost::json::serialize(rpc::makeError(rpc::RippledError::rpcSLOW_DOWN)) + )); + } + /** * @brief Send a response to the client * The message length will be added to the DOSGuard, if the limit is reached, a warning will be added to the diff --git a/src/web/impl/WsBase.hpp b/src/web/impl/WsBase.hpp index 9be6b334a..195ead02a 100644 --- a/src/web/impl/WsBase.hpp +++ b/src/web/impl/WsBase.hpp @@ -164,6 +164,12 @@ public: doWrite(); } + void + sendSlowDown(std::string const& request) override + { + sendError(rpc::RippledError::rpcSLOW_DOWN, request); + } + /** * @brief Send a message to the client * @param msg The message to send, it will keep the string alive until it is sent. It is useful when we have @@ -173,7 +179,8 @@ public: void send(std::shared_ptr msg) override { - boost::asio::dispatch( + // Note: post used instead of dispatch to guarantee async behavior of wsFail and maybeSendNext + boost::asio::post( derived().ws().get_executor(), [this, self = derived().shared_from_this(), msg = std::move(msg)]() { if (messages_.size() > maxSendingQueueSize_) { @@ -279,36 +286,33 @@ public: LOG(perfLog_.info()) << tag() << "Received request from ip = " << this->clientIp; - auto sendError = [this](auto error, std::string&& requestStr) { - auto e = rpc::makeError(error); - - try { - auto request = boost::json::parse(requestStr); - if (request.is_object() && request.as_object().contains("id")) - e["id"] = request.as_object().at("id"); - e["request"] = std::move(request); - } catch (std::exception const&) { - e["request"] = std::move(requestStr); - } - - this->send(std::make_shared(boost::json::serialize(e))); - }; - std::string requestStr{static_cast(buffer_.data().data()), buffer_.size()}; - // dosGuard served request++ and check ip address - if (!dosGuard_.get().request(clientIp)) { - // TODO: could be useful to count in counters in the future too - sendError(rpc::RippledError::rpcSLOW_DOWN, std::move(requestStr)); - } else { - try { - (*handler_)(requestStr, shared_from_this()); - } catch (std::exception const&) { - sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr)); - } + try { + (*handler_)(requestStr, shared_from_this()); + } catch (std::exception const&) { + sendError(rpc::RippledError::rpcINTERNAL, std::move(requestStr)); } doRead(); } + +private: + void + sendError(rpc::RippledError error, std::string requestStr) + { + auto e = rpc::makeError(error); + + try { + auto request = boost::json::parse(requestStr); + if (request.is_object() && request.as_object().contains("id")) + e["id"] = request.as_object().at("id"); + e["request"] = std::move(request); + } catch (std::exception const&) { + e["request"] = std::move(requestStr); + } + + this->send(std::make_shared(boost::json::serialize(e))); + } }; } // namespace web::impl diff --git a/src/web/interface/ConnectionBase.hpp b/src/web/interface/ConnectionBase.hpp index 839b1d9b0..894ce0dec 100644 --- a/src/web/interface/ConnectionBase.hpp +++ b/src/web/interface/ConnectionBase.hpp @@ -82,6 +82,13 @@ public: throw std::logic_error("web server can not send the shared payload"); } + /** + * @brief Send a "slow down" error response to the client. + * + * @param request The original request that triggered the rate limiting + */ + virtual void + sendSlowDown(std::string const& request) = 0; /** * @brief Get the subscription context for this connection. * diff --git a/src/web/ng/RPCServerHandler.hpp b/src/web/ng/RPCServerHandler.hpp index f8dbcb183..ec44089e3 100644 --- a/src/web/ng/RPCServerHandler.hpp +++ b/src/web/ng/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -32,6 +33,7 @@ #include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "web/SubscriptionContextInterface.hpp" +#include "web/dosguard/DOSGuardInterface.hpp" #include "web/ng/Connection.hpp" #include "web/ng/Request.hpp" #include "web/ng/Response.hpp" @@ -64,11 +66,12 @@ namespace web::ng { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; + std::reference_wrapper dosguard_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed @@ -83,16 +86,19 @@ public: * @param backend The backend to use * @param rpcEngine The RPC engine to use * @param etl The ETL to use + * @param dosguard The DOS guard service to use for request rate limiting */ RPCServerHandler( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl, + dosguard::DOSGuardInterface& dosguard ) : backend_(backend) , rpcEngine_(rpcEngine) , etl_(etl) + , dosguard_(dosguard) , tagFactory_(config) , apiVersionParser_(config.getObject("api_version")) { @@ -115,6 +121,10 @@ public: boost::asio::yield_context yield ) { + if (not dosguard_.get().isOk(connectionMetadata.ip())) { + return makeSlowDownResponse(request, std::nullopt); + } + std::optional response; util::CoroutineGroup coroutineGroup{yield, 1}; auto const onTaskComplete = coroutineGroup.registerForeign(yield); @@ -141,18 +151,23 @@ public: } } else { auto parsedObject = std::move(parsedRequest).as_object(); - LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue"; - if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject)) - parsedObject[JS(params)] = boost::json::array({boost::json::object{}}); + if (not dosguard_.get().request(connectionMetadata.ip(), parsedObject)) { + response = makeSlowDownResponse(request, parsedObject); + } else { + LOG(perfLog_.debug()) << connectionMetadata.tag() << "Adding to work queue"; - response = handleRequest( - innerYield, - request, - std::move(parsedObject), - connectionMetadata, - std::move(subscriptionContext) - ); + if (not connectionMetadata.wasUpgraded() and shouldReplaceParams(parsedObject)) + parsedObject[JS(params)] = boost::json::array({boost::json::object{}}); + + response = handleRequest( + innerYield, + request, + std::move(parsedObject), + connectionMetadata, + std::move(subscriptionContext) + ); + } } } catch (std::exception const& ex) { LOG(perfLog_.error()) << connectionMetadata.tag() << "Caught exception: " << ex.what(); @@ -176,6 +191,11 @@ public: // Put the coroutine to sleep until the foreign task is done coroutineGroup.asyncWait(yield); ASSERT(response.has_value(), "Woke up coroutine without setting response"); + + if (not dosguard_.get().add(connectionMetadata.ip(), response->message().size())) { + response->setMessage(makeLoadWarning(*response)); + } + return std::move(response).value(); } @@ -240,7 +260,7 @@ private: auto [result, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); }); auto us = std::chrono::duration(timeDiff); - rpc::logDuration(*context, us); + rpc::logDuration(request, context->tag(), us); boost::json::object response; @@ -315,6 +335,39 @@ private: } } + static Response + makeSlowDownResponse(Request const& request, std::optional requestJson) + { + auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN); + + if (not request.isHttp()) { + try { + if (not requestJson.has_value()) { + requestJson = boost::json::parse(request.message()); + } + if (requestJson->is_object() && requestJson->as_object().contains("id")) + error["id"] = requestJson->as_object().at("id"); + error["request"] = request.message(); + } catch (std::exception const&) { + error["request"] = request.message(); + } + } + return web::ng::Response{boost::beast::http::status::service_unavailable, error, request}; + } + + static boost::json::object + makeLoadWarning(Response const& response) + { + auto jsonResponse = boost::json::parse(response.message()).as_object(); + jsonResponse["warning"] = "load"; + if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) { + jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit)); + } else { + jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)}; + } + return jsonResponse; + } + bool shouldReplaceParams(boost::json::object const& req) const { diff --git a/src/web/ng/Server.cpp b/src/web/ng/Server.cpp index a209fea35..be3fe1064 100644 --- a/src/web/ng/Server.cpp +++ b/src/web/ng/Server.cpp @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -136,13 +137,19 @@ makeConnection( if (not sslContext.has_value()) return std::unexpected{"Error creating a connection: SSL is not supported by this server"}; - connection = std::make_unique( + auto sslConnection = std::make_unique( std::move(sslDetectionResult.socket), std::move(ip), std::move(sslDetectionResult.buffer), *sslContext, tagDecoratorFactory ); + sslConnection->setTimeout(std::chrono::seconds{10}); + auto const maybeError = sslConnection->sslHandshake(yield); + if (maybeError.has_value()) + return std::unexpected{fmt::format("SSL handshake error: {}", maybeError->message())}; + + connection = std::move(sslConnection); } else { connection = std::make_unique( std::move(sslDetectionResult.socket), @@ -164,7 +171,6 @@ makeConnection( std::expected tryUpgradeConnection( impl::UpgradableConnectionPtr connection, - std::optional& sslContext, util::TagDecoratorFactory& tagDecoratorFactory, boost::asio::yield_context yield ) @@ -177,7 +183,7 @@ tryUpgradeConnection( } if (*expectedIsUpgrade) { - auto expectedUpgradedConnection = connection->upgrade(sslContext, tagDecoratorFactory, yield); + auto expectedUpgradedConnection = connection->upgrade(tagDecoratorFactory, yield); if (expectedUpgradedConnection.has_value()) return std::move(expectedUpgradedConnection).value(); @@ -316,8 +322,7 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield return; } - auto connection = - tryUpgradeConnection(std::move(connectionExpected).value(), sslContext_, tagDecoratorFactory_, yield); + auto connection = tryUpgradeConnection(std::move(connectionExpected).value(), tagDecoratorFactory_, yield); if (not connection.has_value()) { LOG(log_.info()) << connection.error(); return; diff --git a/src/web/ng/impl/ConnectionHandler.cpp b/src/web/ng/impl/ConnectionHandler.cpp index adfd8bc12..2bcbfb651 100644 --- a/src/web/ng/impl/ConnectionHandler.cpp +++ b/src/web/ng/impl/ConnectionHandler.cpp @@ -86,24 +86,6 @@ handleWsRequest( } // namespace -size_t -ConnectionHandler::StringHash::operator()(char const* str) const -{ - return hash_type{}(str); -} - -size_t -ConnectionHandler::StringHash::operator()(std::string_view str) const -{ - return hash_type{}(str); -} - -size_t -ConnectionHandler::StringHash::operator()(std::string const& str) const -{ - return hash_type{}(str); -} - ConnectionHandler::ConnectionHandler( ProcessingPolicy processingPolicy, std::optional maxParallelRequests, diff --git a/src/web/ng/impl/ConnectionHandler.hpp b/src/web/ng/impl/ConnectionHandler.hpp index cc1fc6417..e21b07bdd 100644 --- a/src/web/ng/impl/ConnectionHandler.hpp +++ b/src/web/ng/impl/ConnectionHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "util/StopHelper.hpp" +#include "util/StringHash.hpp" #include "util/Taggable.hpp" #include "util/log/Logger.hpp" #include "util/prometheus/Gauge.hpp" @@ -44,7 +45,6 @@ #include #include #include -#include #include namespace web::ng::impl { @@ -52,20 +52,7 @@ namespace web::ng::impl { class ConnectionHandler { public: using OnDisconnectHook = std::function; - - struct StringHash { - using hash_type = std::hash; - using is_transparent = void; - - std::size_t - operator()(char const* str) const; - std::size_t - operator()(std::string_view str) const; - std::size_t - operator()(std::string const& str) const; - }; - - using TargetToHandlerMap = std::unordered_map>; + using TargetToHandlerMap = std::unordered_map>; private: util::Logger log_{"WebServer"}; diff --git a/src/web/ng/impl/HttpConnection.hpp b/src/web/ng/impl/HttpConnection.hpp index b7bdfae6a..0cad34383 100644 --- a/src/web/ng/impl/HttpConnection.hpp +++ b/src/web/ng/impl/HttpConnection.hpp @@ -28,10 +28,12 @@ #include "web/ng/impl/Concepts.hpp" #include "web/ng/impl/WsConnection.hpp" +#include #include #include #include #include +#include #include #include #include @@ -57,11 +59,7 @@ public: isUpgradeRequested(boost::asio::yield_context yield) = 0; virtual std::expected - upgrade( - std::optional& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield - ) = 0; + upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) = 0; virtual std::optional sendRaw( @@ -104,6 +102,22 @@ public: { } + std::optional + sslHandshake(boost::asio::yield_context yield) + requires IsSslTcpStream + { + boost::system::error_code error; + boost::beast::get_lowest_layer(stream_).expires_after(timeout_); + auto const bytesUsed = + stream_.async_handshake(boost::asio::ssl::stream_base::server, buffer_.cdata(), yield[error]); + if (error) + return error; + + buffer_.consume(bytesUsed); + + return std::nullopt; + } + bool wasUpgraded() const override { @@ -183,35 +197,18 @@ public: } std::expected - upgrade( - [[maybe_unused]] std::optional& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield - ) override + upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) override { ASSERT(request_.has_value(), "Request must be present to upgrade the connection"); - if constexpr (IsSslTcpStream) { - ASSERT(sslContext.has_value(), "SSL context must be present to upgrade the connection"); - return makeSslWsConnection( - boost::beast::get_lowest_layer(stream_).release_socket(), - std::move(ip_), - std::move(buffer_), - std::move(request_).value(), - sslContext.value(), - tagDecoratorFactory, - yield - ); - } else { - return makePlainWsConnection( - stream_.release_socket(), - std::move(ip_), - std::move(buffer_), - std::move(request_).value(), - tagDecoratorFactory, - yield - ); - } + return makeWsConnection( + std::move(stream_), + std::move(ip_), + std::move(buffer_), + std::move(request_).value(), + tagDecoratorFactory, + yield + ); } private: diff --git a/src/web/ng/impl/WsConnection.cpp b/src/web/ng/impl/WsConnection.cpp deleted file mode 100644 index 89c47ba05..000000000 --- a/src/web/ng/impl/WsConnection.cpp +++ /dev/null @@ -1,77 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2024, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include "web/ng/impl/WsConnection.hpp" - -#include "util/Taggable.hpp" -#include "web/ng/Error.hpp" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace web::ng::impl { - -std::expected, Error> -makePlainWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -) -{ - auto connection = std::make_unique( - std::move(socket), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory - ); - auto maybeError = connection->performHandshake(yield); - if (maybeError.has_value()) - return std::unexpected{maybeError.value()}; - return connection; -} - -std::expected, Error> -makeSslWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - boost::asio::ssl::context& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -) -{ - auto connection = std::make_unique( - std::move(socket), std::move(ip), std::move(buffer), sslContext, std::move(request), tagDecoratorFactory - ); - auto maybeError = connection->performHandshake(yield); - if (maybeError.has_value()) - return std::unexpected{maybeError.value()}; - return connection; -} - -} // namespace web::ng::impl diff --git a/src/web/ng/impl/WsConnection.hpp b/src/web/ng/impl/WsConnection.hpp index e13e51b66..9d48d392d 100644 --- a/src/web/ng/impl/WsConnection.hpp +++ b/src/web/ng/impl/WsConnection.hpp @@ -68,31 +68,14 @@ class WsConnection : public WsConnectionBase { public: WsConnection( - boost::asio::ip::tcp::socket socket, + StreamType&& stream, std::string ip, boost::beast::flat_buffer buffer, boost::beast::http::request initialRequest, util::TagDecoratorFactory const& tagDecoratorFactory ) - requires IsTcpStream : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory) - , stream_(std::move(socket)) - , initialRequest_(std::move(initialRequest)) - { - setupWsStream(); - } - - WsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::asio::ssl::context& sslContext, - boost::beast::http::request initialRequest, - util::TagDecoratorFactory const& tagDecoratorFactory - ) - requires IsSslTcpStream - : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory) - , stream_(std::move(socket), sslContext) + , stream_(std::move(stream)) , initialRequest_(std::move(initialRequest)) { setupWsStream(); @@ -189,25 +172,24 @@ private: using PlainWsConnection = WsConnection; using SslWsConnection = WsConnection>; -std::expected, Error> -makePlainWsConnection( - boost::asio::ip::tcp::socket socket, +template +std::expected>, Error> +makeWsConnection( + StreamType&& stream, std::string ip, boost::beast::flat_buffer buffer, boost::beast::http::request request, util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield -); - -std::expected, Error> -makeSslWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - boost::asio::ssl::context& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -); +) +{ + auto connection = std::make_unique>( + std::forward(stream), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory + ); + auto maybeError = connection->performHandshake(yield); + if (maybeError.has_value()) + return std::unexpected{maybeError.value()}; + return connection; +} } // namespace web::ng::impl diff --git a/tests/common/util/BinaryTestObject.cpp b/tests/common/util/BinaryTestObject.cpp index b9082e7d0..30cd07a2c 100644 --- a/tests/common/util/BinaryTestObject.cpp +++ b/tests/common/util/BinaryTestObject.cpp @@ -19,134 +19,31 @@ #include "util/BinaryTestObject.hpp" +#include "data/DBHelpers.hpp" #include "etlng/Models.hpp" #include "etlng/impl/Extraction.hpp" #include "util/StringUtils.hpp" +#include "util/TestObject.hpp" #include #include +#include +#include #include #include +#include #include #include #include #include +#include #include #include namespace { constinit auto const kSEQ = 30; - -constinit auto const kTXN_HEX = - "1200192200000008240011CC9B201B001F71D6202A0000000168400000" - "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" - "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" - "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" - "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" - "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" - "DE3538F248662FC73C"; - -constinit auto const kTXN_META = - "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" - "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" - "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" - "C200000028751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "9808B6B90000001D751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" - "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" - "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" - "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" - "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" - "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" - "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" - "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" - "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" - "00001F751868747470733A2F2F677265677765697362726F642E636F6D" - "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" - "5DAE00000014751868747470733A2F2F677265677765697362726F642E" - "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" - "3CCE1462A500000009751868747470733A2F2F67726567776569736272" - "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" - "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" - "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" - "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" - "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" - "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" - "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" - "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" - "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" - "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" - "00000029751868747470733A2F2F677265677765697362726F642E636F" - "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" - "EE87B80000001E751868747470733A2F2F677265677765697362726F64" - "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" - "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" - "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" - "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" - "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" - "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" - "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" - "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" - "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" - "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" - "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" - "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" - "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" - "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" - "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" - "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" - "B100000015751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" - "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" - "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" - "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" - "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" - "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" - "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" - "2FC73CE1E1F1031000"; - constinit auto const kRAW_HEADER = "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E" "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A" @@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER = namespace util { std::pair -createNftTxAndMetaBlobs() +createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr) { - return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)}; + return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)}; } std::pair -createNftTxAndMeta() +createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr) { ripple::uint256 hash; - EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E")); + EXPECT_TRUE(hash.parseHex(hashStr)); - auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(); + auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr); ripple::SerialIter it{txnBlob.data(), txnBlob.size()}; return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}}; } etlng::model::Transaction -createTransaction(ripple::TxType type) +createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr) { - auto const [sttx, meta] = createNftTxAndMeta(); + auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr); return { .raw = "", .metaRaw = "", @@ -192,10 +89,9 @@ createTransaction(ripple::TxType type) } etlng::model::Object -createObject() +createObject(etlng::model::Object::ModType modType, std::string key) { // random object taken from initial ledger load - static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; static constinit auto const kOBJ_BLOB = @@ -205,12 +101,62 @@ createObject() "8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56"; return { - .key = {}, - .keyRaw = hexStringToBinaryString(kOBJ_KEY), - .data = {}, - .dataRaw = hexStringToBinaryString(kOBJ_BLOB), + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), .successor = hexStringToBinaryString(kOBJ_SUCC), .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key) +{ + // random object taken from initial ledger load + static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; + static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; + static constinit auto const kOBJ_BLOB = + "11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE" + "3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000" + "0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3"; + + return { + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), + .successor = hexStringToBinaryString(kOBJ_SUCC), + .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithTwoNFTs() +{ + std::string const url1 = "abcd1"; + std::string const url2 = "abcd2"; + ripple::Blob const uri1Blob(url1.begin(), url1.end()); + ripple::Blob const uri2Blob(url2.begin(), url2.end()); + + constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q"; + constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000"; + constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA"; + + auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt); + auto const serializerNftPage = nftPage.getSerializer(); + + auto const account = getAccountIdWithString(kACCOUNT); + return { + .key = {}, + .keyRaw = std::string(reinterpret_cast(account.data()), ripple::AccountID::size()), + .data = {}, + .dataRaw = + std::string(static_cast(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()), + .successor = "", + .predecessor = "", .type = etlng::model::Object::ModType::Created, }; } @@ -219,8 +165,10 @@ etlng::model::BookSuccessor createSuccessor() { return { - .firstBook = "A000000000000000000000000000000000000000000000000000000000000000", - .bookBase = "A000000000000000000000000000000000000000000000000000000000000001", + .firstBook = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}), + .bookBase = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}), }; } diff --git a/tests/common/util/BinaryTestObject.hpp b/tests/common/util/BinaryTestObject.hpp index ad5fba055..41d5029ef 100644 --- a/tests/common/util/BinaryTestObject.hpp +++ b/tests/common/util/BinaryTestObject.hpp @@ -32,17 +32,149 @@ namespace util { +static constexpr auto kDEFAULT_TXN_HEX = + "1200192200000008240011CC9B201B001F71D6202A0000000168400000" + "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" + "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" + "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" + "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" + "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" + "DE3538F248662FC73C"; + +static constexpr auto kDEFAULT_TXN_META = + "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" + "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" + "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" + "C200000028751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "9808B6B90000001D751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" + "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" + "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" + "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" + "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" + "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" + "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" + "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" + "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" + "00001F751868747470733A2F2F677265677765697362726F642E636F6D" + "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" + "5DAE00000014751868747470733A2F2F677265677765697362726F642E" + "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" + "3CCE1462A500000009751868747470733A2F2F67726567776569736272" + "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" + "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" + "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" + "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" + "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" + "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" + "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" + "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" + "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" + "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" + "00000029751868747470733A2F2F677265677765697362726F642E636F" + "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" + "EE87B80000001E751868747470733A2F2F677265677765697362726F64" + "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" + "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" + "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" + "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" + "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" + "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" + "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" + "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" + "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" + "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" + "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" + "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" + "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" + "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" + "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" + "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" + "B100000015751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" + "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" + "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" + "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" + "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" + "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" + "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" + "2FC73CE1E1F1031000"; + +static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"; +static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + [[maybe_unused, nodiscard]] std::pair -createNftTxAndMetaBlobs(); +createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX); [[maybe_unused, nodiscard]] std::pair -createNftTxAndMeta(); +createNftTxAndMeta( + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Transaction -createTransaction(ripple::TxType type); +createTransaction( + ripple::TxType type, + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Object -createObject(); +createObject( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithBookBase( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithTwoNFTs(); [[maybe_unused, nodiscard]] etlng::model::BookSuccessor createSuccessor(); diff --git a/tests/common/util/MockBackend.hpp b/tests/common/util/MockBackend.hpp index af16b490f..9aab18ee8 100644 --- a/tests/common/util/MockBackend.hpp +++ b/tests/common/util/MockBackend.hpp @@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface { MOCK_METHOD(void, writeAccountTransactions, (std::vector), (override)); + MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override)); + MOCK_METHOD(void, writeNFTTransactions, (std::vector const&), (override)); MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override)); diff --git a/tests/common/util/MockETLService.hpp b/tests/common/util/MockETLService.hpp index 618e5ab57..f398ad9cd 100644 --- a/tests/common/util/MockETLService.hpp +++ b/tests/common/util/MockETLService.hpp @@ -20,21 +20,21 @@ #pragma once #include "etl/ETLState.hpp" +#include "etlng/ETLServiceInterface.hpp" #include #include #include -#include #include #include -struct MockETLService { - MOCK_METHOD(boost::json::object, getInfo, (), (const)); - MOCK_METHOD(std::chrono::time_point, getLastPublish, (), (const)); - MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const)); - MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const)); - MOCK_METHOD(bool, isAmendmentBlocked, (), (const)); - MOCK_METHOD(bool, isCorruptionDetected, (), (const)); - MOCK_METHOD(std::optional, getETLState, (), (const)); +struct MockETLService : etlng::ETLServiceInterface { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (), (override)); + MOCK_METHOD(boost::json::object, getInfo, (), (const, override)); + MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const, override)); + MOCK_METHOD(bool, isAmendmentBlocked, (), (const, override)); + MOCK_METHOD(bool, isCorruptionDetected, (), (const, override)); + MOCK_METHOD(std::optional, getETLState, (), (const, override)); }; diff --git a/tests/common/util/MockLedgerCache.hpp b/tests/common/util/MockLedgerCache.hpp index 67473b6e9..07fb67856 100644 --- a/tests/common/util/MockLedgerCache.hpp +++ b/tests/common/util/MockLedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface { MOCK_METHOD(std::optional, get, (ripple::uint256 const& a, uint32_t b), (const, override)); + MOCK_METHOD(void, update, (std::vector const&, uint32_t), (override)); + + MOCK_METHOD(std::optional, getDeleted, (ripple::uint256 const&, uint32_t), (const, override)); + MOCK_METHOD( std::optional, getSuccessor, diff --git a/tests/common/util/MockLoadBalancer.hpp b/tests/common/util/MockLoadBalancer.hpp index 12bcfea78..f0b9dd313 100644 --- a/tests/common/util/MockLoadBalancer.hpp +++ b/tests/common/util/MockLoadBalancer.hpp @@ -38,22 +38,6 @@ #include #include -struct MockLoadBalancer { - using RawLedgerObjectType = FakeLedgerObject; - - MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ()); - MOCK_METHOD(std::optional, fetchLedger, (uint32_t, bool, bool), ()); - MOCK_METHOD(boost::json::value, toJson, (), (const)); - - using ForwardToRippledReturnType = std::expected; - MOCK_METHOD( - ForwardToRippledReturnType, - forwardToRippled, - (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), - (const) - ); -}; - struct MockNgLoadBalancer : etlng::LoadBalancerInterface { using RawLedgerObjectType = FakeLedgerObject; @@ -85,4 +69,7 @@ struct MockNgLoadBalancer : etlng::LoadBalancerInterface { (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), (override) ); + MOCK_METHOD(void, stop, (boost::asio::yield_context), ()); }; + +using MockLoadBalancer = MockNgLoadBalancer; diff --git a/tests/common/util/MockSource.hpp b/tests/common/util/MockSource.hpp index 58984d485..a982f3aef 100644 --- a/tests/common/util/MockSource.hpp +++ b/tests/common/util/MockSource.hpp @@ -60,7 +60,7 @@ struct MockSource : etl::SourceBase { (uint32_t, bool, bool), (override) ); - MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t, bool), (override)); + MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t), (override)); using ForwardToRippledReturnType = std::expected; MOCK_METHOD( @@ -132,9 +132,9 @@ public: } std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t maxLedger, bool getObjects) override + loadInitialLedger(uint32_t sequence, uint32_t maxLedger) override { - return mock_->loadInitialLedger(sequence, maxLedger, getObjects); + return mock_->loadInitialLedger(sequence, maxLedger); } std::expected diff --git a/tests/common/util/MockSourceNg.hpp b/tests/common/util/MockSourceNg.hpp new file mode 100644 index 000000000..7137e47ec --- /dev/null +++ b/tests/common/util/MockSourceNg.hpp @@ -0,0 +1,245 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== +#pragma once + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct MockSourceNg : etlng::SourceBase { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (boost::asio::yield_context), (override)); + MOCK_METHOD(bool, isConnected, (), (const, override)); + MOCK_METHOD(void, setForwarding, (bool), (override)); + MOCK_METHOD(boost::json::object, toJson, (), (const, override)); + MOCK_METHOD(std::string, toString, (), (const, override)); + MOCK_METHOD(bool, hasLedger, (uint32_t), (const, override)); + MOCK_METHOD( + (std::pair), + fetchLedger, + (uint32_t, bool, bool), + (override) + ); + MOCK_METHOD( + (std::pair, bool>), + loadInitialLedger, + (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&), + (override) + ); + + using ForwardToRippledReturnType = std::expected; + MOCK_METHOD( + ForwardToRippledReturnType, + forwardToRippled, + (boost::json::object const&, std::optional const&, std::string_view, boost::asio::yield_context), + (const, override) + ); +}; + +template