From df3f1865aec7abc509d6d7788cf6ad89866583af Mon Sep 17 00:00:00 2001 From: Peter Chen <34582813+PeterChen13579@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:26:04 -0400 Subject: [PATCH 01/13] fix: ripple_flag logic in account lines (#1969) fixes #1968 --- src/rpc/handlers/AccountLines.cpp | 15 ++++++++--- src/rpc/handlers/AccountLines.hpp | 4 +-- tests/unit/rpc/handlers/AccountLinesTests.cpp | 26 +++++-------------- 3 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/rpc/handlers/AccountLines.cpp b/src/rpc/handlers/AccountLines.cpp index a5ab872d4..5644e7daf 100644 --- a/src/rpc/handlers/AccountLines.cpp +++ b/src/rpc/handlers/AccountLines.cpp @@ -102,6 +102,12 @@ AccountLinesHandler::addLine( line.qualityIn = lineQualityIn; line.qualityOut = lineQualityOut; + if (lineNoRipple) + line.noRipple = true; + + if (lineNoRipplePeer) + line.noRipplePeer = true; + if (lineAuth) line.authorized = true; @@ -120,8 +126,6 @@ AccountLinesHandler::addLine( if (lineDeepFreezePeer) line.deepFreezePeer = true; - line.noRipple = lineNoRipple; - line.noRipplePeer = lineNoRipplePeer; lines.push_back(line); } @@ -257,8 +261,11 @@ tag_invoke( {JS(quality_out), line.qualityOut}, }; - obj[JS(no_ripple)] = line.noRipple; - obj[JS(no_ripple_peer)] = line.noRipplePeer; + if (line.noRipple) + obj[JS(no_ripple)] = *(line.noRipple); + + if (line.noRipplePeer) + obj[JS(no_ripple_peer)] = *(line.noRipplePeer); if (line.authorized) obj[JS(authorized)] = *(line.authorized); diff --git a/src/rpc/handlers/AccountLines.hpp b/src/rpc/handlers/AccountLines.hpp index 3388f3166..38d6f5231 100644 --- a/src/rpc/handlers/AccountLines.hpp +++ b/src/rpc/handlers/AccountLines.hpp @@ -70,8 +70,8 @@ public: std::string limitPeer; uint32_t qualityIn{}; uint32_t qualityOut{}; - bool noRipple{}; - bool noRipplePeer{}; + std::optional noRipple; + std::optional noRipplePeer; std::optional authorized; std::optional peerAuthorized; std::optional freeze; diff --git a/tests/unit/rpc/handlers/AccountLinesTests.cpp b/tests/unit/rpc/handlers/AccountLinesTests.cpp index 1fa5e00fb..2d0b72aa8 100644 --- a/tests/unit/rpc/handlers/AccountLinesTests.cpp +++ b/tests/unit/rpc/handlers/AccountLinesTests.cpp @@ -522,9 +522,7 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest) "limit": "100", "limit_peer": "200", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 }, { "account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun", @@ -533,9 +531,7 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest) "limit": "200", "limit_peer": "100", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 } ] })"; @@ -737,7 +733,6 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField) "limit_peer": "200", "quality_in": 0, "quality_out": 0, - "no_ripple": false, "no_ripple_peer": true, "peer_authorized": true, "freeze_peer": true, @@ -752,7 +747,6 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField) "quality_in": 0, "quality_out": 0, "no_ripple": true, - "no_ripple_peer": false, "authorized": true, "freeze": true, "deep_freeze": true @@ -992,9 +986,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin) "limit": "100", "limit_peer": "200", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 }}, {{ "account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun", @@ -1003,9 +995,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin) "limit": "200", "limit_peer": "100", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 }} ] }})", @@ -1073,9 +1063,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax) "limit": "100", "limit_peer": "200", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 }}, {{ "account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun", @@ -1084,9 +1072,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax) "limit": "200", "limit_peer": "100", "quality_in": 0, - "quality_out": 0, - "no_ripple": false, - "no_ripple_peer": false + "quality_out": 0 }} ] }})", From 702ee0324eddeda4be9fb7cc6c6bce6a2cf8253c Mon Sep 17 00:00:00 2001 From: Alex Kremer Date: Fri, 21 Mar 2025 16:41:29 +0000 Subject: [PATCH 02/13] feat: ETLng extensions (#1967) For #1599 #1597 --- src/data/BackendInterface.hpp | 10 +- src/data/CassandraBackend.hpp | 41 +- src/data/DBHelpers.hpp | 2 +- src/data/LedgerCache.cpp | 60 ++ src/data/LedgerCache.hpp | 11 +- src/data/LedgerCacheInterface.hpp | 20 + src/etlng/CMakeLists.txt | 4 + src/etlng/impl/ext/Cache.cpp | 59 ++ src/etlng/impl/ext/Cache.hpp | 51 ++ src/etlng/impl/ext/Core.cpp | 85 +++ src/etlng/impl/ext/Core.hpp | 58 ++ src/etlng/impl/ext/NFT.cpp | 77 +++ src/etlng/impl/ext/NFT.hpp | 56 ++ src/etlng/impl/ext/Successor.cpp | 222 ++++++ src/etlng/impl/ext/Successor.hpp | 82 +++ tests/common/util/BinaryTestObject.cpp | 196 ++---- tests/common/util/BinaryTestObject.hpp | 140 +++- tests/common/util/MockBackend.hpp | 2 + tests/common/util/MockLedgerCache.hpp | 5 + tests/unit/CMakeLists.txt | 4 + tests/unit/etlng/ext/CacheTests.cpp | 92 +++ tests/unit/etlng/ext/CoreTests.cpp | 107 +++ tests/unit/etlng/ext/NFTTests.cpp | 287 ++++++++ tests/unit/etlng/ext/SuccessorTests.cpp | 641 ++++++++++++++++++ .../rpc/handlers/AccountCurrenciesTests.cpp | 2 +- 25 files changed, 2167 insertions(+), 147 deletions(-) create mode 100644 src/etlng/impl/ext/Cache.cpp create mode 100644 src/etlng/impl/ext/Cache.hpp create mode 100644 src/etlng/impl/ext/Core.cpp create mode 100644 src/etlng/impl/ext/Core.hpp create mode 100644 src/etlng/impl/ext/NFT.cpp create mode 100644 src/etlng/impl/ext/NFT.hpp create mode 100644 src/etlng/impl/ext/Successor.cpp create mode 100644 src/etlng/impl/ext/Successor.hpp create mode 100644 tests/unit/etlng/ext/CacheTests.cpp create mode 100644 tests/unit/etlng/ext/CoreTests.cpp create mode 100644 tests/unit/etlng/ext/NFTTests.cpp create mode 100644 tests/unit/etlng/ext/SuccessorTests.cpp diff --git a/src/data/BackendInterface.hpp b/src/data/BackendInterface.hpp index 387cd41dd..5c376d798 100644 --- a/src/data/BackendInterface.hpp +++ b/src/data/BackendInterface.hpp @@ -154,7 +154,7 @@ public: } virtual ~BackendInterface() = default; - // TODO: Remove this hack once old ETL is removed. + // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed. // Cache should not be exposed thru BackendInterface /** @@ -648,6 +648,14 @@ public: virtual void writeAccountTransactions(std::vector data) = 0; + /** + * @brief Write a new account transaction. + * + * @param record An object representing the account transaction + */ + virtual void + writeAccountTransaction(AccountTransactionsData record) = 0; + /** * @brief Write NFTs transactions. * diff --git a/src/data/CassandraBackend.hpp b/src/data/CassandraBackend.hpp index 4bba53301..7ea02475c 100644 --- a/src/data/CassandraBackend.hpp +++ b/src/data/CassandraBackend.hpp @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -906,30 +907,42 @@ public: statements.reserve(data.size() * 10); // assume 10 transactions avg for (auto& record : data) { - std::transform( - std::begin(record.accounts), - std::end(record.accounts), - std::back_inserter(statements), - [this, &record](auto&& account) { - return schema_->insertAccountTx.bind( - std::forward(account), - std::make_tuple(record.ledgerSequence, record.transactionIndex), - record.txHash - ); - } - ); + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); } executor_.write(std::move(statements)); } + void + writeAccountTransaction(AccountTransactionsData record) override + { + std::vector statements; + statements.reserve(record.accounts.size()); + + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); + + executor_.write(std::move(statements)); + } + void writeNFTTransactions(std::vector const& data) override { std::vector statements; statements.reserve(data.size()); - std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) { + std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) { return schema_->insertNFTTx.bind( record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash ); @@ -999,7 +1012,7 @@ public: std::vector statements; statements.reserve(data.size()); for (auto [mptId, holder] : data) - statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder))); + statements.push_back(schema_->insertMPTHolder.bind(mptId, holder)); executor_.write(std::move(statements)); } diff --git a/src/data/DBHelpers.hpp b/src/data/DBHelpers.hpp index cba1ec1ac..76b5acedf 100644 --- a/src/data/DBHelpers.hpp +++ b/src/data/DBHelpers.hpp @@ -54,7 +54,7 @@ struct AccountTransactionsData { * @param meta The transaction metadata * @param txHash The transaction hash */ - AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash) + AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash) : accounts(meta.getAffectedAccounts()) , ledgerSequence(meta.getLgrSeq()) , transactionIndex(meta.getIndex()) diff --git a/src/data/LedgerCache.cpp b/src/data/LedgerCache.cpp index 11f0b8162..7180c4e9d 100644 --- a/src/data/LedgerCache.cpp +++ b/src/data/LedgerCache.cpp @@ -20,6 +20,7 @@ #include "data/LedgerCache.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/Assert.hpp" #include @@ -87,6 +88,42 @@ LedgerCache::update(std::vector const& objs, uint32_t seq, bool is } } +void +LedgerCache::update(std::vector const& objs, uint32_t seq) +{ + if (disabled_) + return; + + std::scoped_lock const lck{mtx_}; + if (seq > latestSeq_) { + ASSERT( + seq == latestSeq_ + 1 || latestSeq_ == 0, + "New sequence must be either next or first. seq = {}, latestSeq_ = {}", + seq, + latestSeq_ + ); + latestSeq_ = seq; + } + + deleted_.clear(); // previous update's deletes no longer needed + + for (auto const& obj : objs) { + if (!obj.data.empty()) { + auto& e = map_[obj.key]; + if (seq > e.seq) + e = {.seq = seq, .blob = obj.data}; + } else { + if (map_.contains(obj.key)) + deleted_[obj.key] = map_[obj.key]; + + map_.erase(obj.key); + if (!full_) + deletes_.insert(obj.key); + } + } + cv_.notify_all(); +} + std::optional LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const { @@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const return {e->second.blob}; } +std::optional +LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const +{ + if (disabled_) + return std::nullopt; + + std::shared_lock const lck{mtx_}; + if (seq > latestSeq_) + return std::nullopt; + + ++objectReqCounter_.get(); + + auto e = deleted_.find(key); + if (e == deleted_.end()) + return std::nullopt; + + if (seq < e->second.seq) + return std::nullopt; + + ++objectHitCounter_.get(); + return {e->second.blob}; +} + void LedgerCache::setDisabled() { diff --git a/src/data/LedgerCache.hpp b/src/data/LedgerCache.hpp index 8a3c40688..bf957af28 100644 --- a/src/data/LedgerCache.hpp +++ b/src/data/LedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Label.hpp" @@ -29,7 +30,6 @@ #include #include -#include #include #include #include @@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface { )}; std::map map_; + std::map deleted_; mutable std::shared_mutex mtx_; std::condition_variable_any cv_; @@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface { public: void - update(std::vector const& objs, uint32_t seq, bool isBackground = false) override; + update(std::vector const& objs, uint32_t seq, bool isBackground) override; + + void + update(std::vector const& objs, uint32_t seq) override; std::optional get(ripple::uint256 const& key, uint32_t seq) const override; + std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const override; + std::optional getSuccessor(ripple::uint256 const& key, uint32_t seq) const override; diff --git a/src/data/LedgerCacheInterface.hpp b/src/data/LedgerCacheInterface.hpp index e5ecee49d..850403bcc 100644 --- a/src/data/LedgerCacheInterface.hpp +++ b/src/data/LedgerCacheInterface.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -55,6 +56,15 @@ public: virtual void update(std::vector const& objs, uint32_t seq, bool isBackground = false) = 0; + /** + * @brief Update the cache with new ledger objects. + * + * @param objs The ledger objects to update cache with + * @param seq The sequence to update cache for + */ + virtual void + update(std::vector const& objs, uint32_t seq) = 0; + /** * @brief Fetch a cached object by its key and sequence number. * @@ -65,6 +75,16 @@ public: virtual std::optional get(ripple::uint256 const& key, uint32_t seq) const = 0; + /** + * @brief Fetch a recently deleted object by its key and sequence number. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned + */ + virtual std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0; + /** * @brief Gets a cached successor. * diff --git a/src/etlng/CMakeLists.txt b/src/etlng/CMakeLists.txt index 4b97c43ab..d0f2854e9 100644 --- a/src/etlng/CMakeLists.txt +++ b/src/etlng/CMakeLists.txt @@ -9,6 +9,10 @@ target_sources( impl/Loading.cpp impl/Monitor.cpp impl/TaskManager.cpp + impl/ext/Cache.cpp + impl/ext/Core.cpp + impl/ext/NFT.cpp + impl/ext/Successor.cpp ) target_link_libraries(clio_etlng PUBLIC clio_data) diff --git a/src/etlng/impl/ext/Cache.cpp b/src/etlng/impl/ext/Cache.cpp new file mode 100644 index 000000000..d7c2e5f51 --- /dev/null +++ b/src/etlng/impl/ext/Cache.cpp @@ -0,0 +1,59 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Cache.hpp" + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include + +namespace etlng::impl { + +CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache) +{ +} + +void +CacheExt::onLedgerData(model::LedgerData const& data) const +{ + cache_.get().update(data.objects, data.seq); + LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size(); +} + +void +CacheExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size(); + cache_.get().update(data.objects, data.seq); + cache_.get().setFull(); +} + +void +CacheExt::onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) + const +{ + LOG(log_.trace()) << "got initial objects cnt = " << objs.size(); + cache_.get().update(objs, seq); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Cache.hpp b/src/etlng/impl/ext/Cache.hpp new file mode 100644 index 000000000..fc9fbbeb4 --- /dev/null +++ b/src/etlng/impl/ext/Cache.hpp @@ -0,0 +1,51 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +class CacheExt { + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + CacheExt(data::LedgerCacheInterface& cache); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.cpp b/src/etlng/impl/ext/Core.cpp new file mode 100644 index 000000000..2c3e141a0 --- /dev/null +++ b/src/etlng/impl/ext/Core.cpp @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Core.hpp" + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include + +#include +#include +#include + +namespace etlng::impl { + +CoreExt::CoreExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +CoreExt::onLedgerData(model::LedgerData const& data) const +{ + LOG(log_.debug()) << "Loading ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.info()) << "Loading initial ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::onObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::insertTransactions(model::LedgerData const& data) const +{ + for (auto const& txn : data.transactions) { + LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID(); + + backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()}); + backend_->writeTransaction( + auto{txn.key}, + data.seq, + data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction' + auto{txn.raw}, + auto{txn.metaRaw} + ); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.hpp b/src/etlng/impl/ext/Core.hpp new file mode 100644 index 000000000..7913c15ef --- /dev/null +++ b/src/etlng/impl/ext/Core.hpp @@ -0,0 +1,58 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include + +#include +#include + +namespace etlng::impl { + +class CoreExt { + std::shared_ptr backend_; + + util::Logger log_{"ETL"}; + +public: + CoreExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onObject(uint32_t seq, model::Object const& obj) const; + +private: + void + insertTransactions(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.cpp b/src/etlng/impl/ext/NFT.cpp new file mode 100644 index 000000000..30566acf9 --- /dev/null +++ b/src/etlng/impl/ext/NFT.cpp @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/NFT.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "etl/NFTHelpers.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +NFTExt::NFTExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +NFTExt::onLedgerData(model::LedgerData const& data) const +{ + writeNFTs(data); +} + +void +NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial object with key = " << obj.key; + backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw)); +} + +void +NFTExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size(); + writeNFTs(data); +} + +void +NFTExt::writeNFTs(model::LedgerData const& data) const +{ + std::vector nfts; + std::vector nftTxs; + + for (auto const& tx : data.transactions) { + auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx); + nftTxs.insert(nftTxs.end(), txs.begin(), txs.end()); + if (maybeNFT) + nfts.push_back(*maybeNFT); + } + + // This is uniqued so that we only write latest modification (as in previous implementation) + backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts)); + backend_->writeNFTTransactions(nftTxs); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.hpp b/src/etlng/impl/ext/NFT.hpp new file mode 100644 index 000000000..65e45233c --- /dev/null +++ b/src/etlng/impl/ext/NFT.hpp @@ -0,0 +1,56 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "etl/NFTHelpers.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +class NFTExt { + std::shared_ptr backend_; + util::Logger log_{"ETL"}; + +public: + NFTExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onInitialData(model::LedgerData const& data) const; + +private: + void + writeNFTs(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.cpp b/src/etlng/impl/ext/Successor.cpp new file mode 100644 index 000000000..a46d3158b --- /dev/null +++ b/src/etlng/impl/ext/Successor.cpp @@ -0,0 +1,222 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Successor.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/Assert.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +SuccessorExt::SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache) + : backend_(std::move(backend)), cache_(cache) +{ +} + +void +SuccessorExt::onInitialData(model::LedgerData const& data) const +{ + ASSERT(cache_.get().isFull(), "Cache must be full at this point"); + ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load"); + ASSERT(data.objects.empty(), "Should not have objects from initial data"); + writeSuccessors(data.seq); + writeEdgeKeys(data.seq, data.edgeKeys.value()); +} + +void +SuccessorExt::onInitialObjects( + uint32_t seq, + [[maybe_unused]] std::vector const& objs, + std::string lastKey +) const +{ + for (auto const& obj : objs) { + if (!lastKey.empty()) + backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw}); + lastKey = obj.keyRaw; + } +} + +void +SuccessorExt::onLedgerData(model::LedgerData const& data) const +{ + namespace vs = std::views; + + LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size() + << "; got successors = " << data.successors.has_value() << "; cache is " + << (cache_.get().isFull() ? "FULL" : "Not full"); + + auto filteredObjects = data.objects // + | vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; }); + + if (data.successors.has_value()) { + for (auto const& successor : data.successors.value()) + writeIncludedSuccessor(data.seq, successor); + + for (auto const& obj : filteredObjects) + writeIncludedSuccessor(data.seq, obj); + } else { + if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq) + throw std::logic_error("Cache is not full, but object neighbors were not included"); + + for (auto const& obj : filteredObjects) + updateSuccessorFromCache(data.seq, obj); + } +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const +{ + auto firstBook = succ.firstBook; + if (firstBook.empty()) + firstBook = uint256ToString(data::kLAST_KEY); + + backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook)); +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const +{ + ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object"); + + // TODO: perhaps make these optionals inside of obj and move value_or here + auto pred = obj.predecessor; + auto succ = obj.successor; + + if (obj.type == model::Object::ModType::Deleted) { + backend_->writeSuccessor(std::move(pred), seq, std::move(succ)); + } else if (obj.type == model::Object::ModType::Created) { + backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw}); + backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ)); + } +} + +void +SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const +{ + auto const lb = + cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}}); + auto const ub = + cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}}); + + auto checkBookBase = false; + auto const isDeleted = obj.data.empty(); + + if (isDeleted) { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key)); + } else { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key)); + backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key)); + } + + if (isDeleted) { + auto const old = cache_.get().getDeleted(obj.key, seq - 1); + ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key)); + + checkBookBase = isBookDir(obj.key, *old); + } else { + checkBookBase = isBookDir(obj.key, obj.data); + } + + if (checkBookBase) { + auto const current = cache_.get().get(obj.key, seq); + auto const bookBase = getBookBase(obj.key); + + if (isDeleted and not current.has_value()) { + updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase); + } else if (current.has_value()) { + auto const successor = cache_.get().getSuccessor(bookBase, seq); + ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq); + + if (successor->key == obj.key) { + updateBookSuccessor(successor, seq, bookBase); + } + } + } +} + +void +SuccessorExt::updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase +) const +{ + if (maybeSuccessor.has_value()) { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key)); + } else { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY)); + } +} + +void +SuccessorExt::writeSuccessors(uint32_t seq) const +{ + ripple::uint256 prev = data::kFIRST_KEY; + while (auto cur = cache_.get().getSuccessor(prev, seq)) { + if (prev == data::kFIRST_KEY) + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key)); + + if (isBookDir(cur->key, cur->blob)) { + auto base = getBookBase(cur->key); + + // make sure the base is not an actual object + if (not cache_.get().get(base, seq)) { + auto succ = cache_.get().getSuccessor(base, seq); + ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base)); + + if (succ->key == cur->key) + backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key)); + } + } + + prev = cur->key; + } + + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY)); +} + +void +SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const +{ + for (auto const& key : edgeKeys) { + auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq); + if (succ) + backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key)); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.hpp b/src/etlng/impl/ext/Successor.hpp new file mode 100644 index 000000000..5ba8d7022 --- /dev/null +++ b/src/etlng/impl/ext/Successor.hpp @@ -0,0 +1,82 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class SuccessorExt { + std::shared_ptr backend_; + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache); + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector const& objs, std::string lastKey) const; + + void + onLedgerData(model::LedgerData const& data) const; + +private: + void + writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const; + + void + writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const; + + void + updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const; + + void + updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase + ) const; + + void + writeSuccessors(uint32_t seq) const; + + void + writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const; +}; + +} // namespace etlng::impl diff --git a/tests/common/util/BinaryTestObject.cpp b/tests/common/util/BinaryTestObject.cpp index b9082e7d0..30cd07a2c 100644 --- a/tests/common/util/BinaryTestObject.cpp +++ b/tests/common/util/BinaryTestObject.cpp @@ -19,134 +19,31 @@ #include "util/BinaryTestObject.hpp" +#include "data/DBHelpers.hpp" #include "etlng/Models.hpp" #include "etlng/impl/Extraction.hpp" #include "util/StringUtils.hpp" +#include "util/TestObject.hpp" #include #include +#include +#include #include #include +#include #include #include #include #include +#include #include #include namespace { constinit auto const kSEQ = 30; - -constinit auto const kTXN_HEX = - "1200192200000008240011CC9B201B001F71D6202A0000000168400000" - "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" - "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" - "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" - "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" - "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" - "DE3538F248662FC73C"; - -constinit auto const kTXN_META = - "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" - "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" - "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" - "C200000028751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "9808B6B90000001D751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" - "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" - "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" - "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" - "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" - "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" - "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" - "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" - "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" - "00001F751868747470733A2F2F677265677765697362726F642E636F6D" - "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" - "5DAE00000014751868747470733A2F2F677265677765697362726F642E" - "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" - "3CCE1462A500000009751868747470733A2F2F67726567776569736272" - "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" - "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" - "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" - "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" - "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" - "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" - "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" - "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" - "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" - "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" - "00000029751868747470733A2F2F677265677765697362726F642E636F" - "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" - "EE87B80000001E751868747470733A2F2F677265677765697362726F64" - "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" - "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" - "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" - "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" - "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" - "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" - "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" - "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" - "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" - "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" - "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" - "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" - "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" - "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" - "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" - "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" - "B100000015751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" - "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" - "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" - "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" - "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" - "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" - "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" - "2FC73CE1E1F1031000"; - constinit auto const kRAW_HEADER = "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E" "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A" @@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER = namespace util { std::pair -createNftTxAndMetaBlobs() +createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr) { - return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)}; + return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)}; } std::pair -createNftTxAndMeta() +createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr) { ripple::uint256 hash; - EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E")); + EXPECT_TRUE(hash.parseHex(hashStr)); - auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(); + auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr); ripple::SerialIter it{txnBlob.data(), txnBlob.size()}; return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}}; } etlng::model::Transaction -createTransaction(ripple::TxType type) +createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr) { - auto const [sttx, meta] = createNftTxAndMeta(); + auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr); return { .raw = "", .metaRaw = "", @@ -192,10 +89,9 @@ createTransaction(ripple::TxType type) } etlng::model::Object -createObject() +createObject(etlng::model::Object::ModType modType, std::string key) { // random object taken from initial ledger load - static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; static constinit auto const kOBJ_BLOB = @@ -205,12 +101,62 @@ createObject() "8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56"; return { - .key = {}, - .keyRaw = hexStringToBinaryString(kOBJ_KEY), - .data = {}, - .dataRaw = hexStringToBinaryString(kOBJ_BLOB), + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), .successor = hexStringToBinaryString(kOBJ_SUCC), .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key) +{ + // random object taken from initial ledger load + static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; + static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; + static constinit auto const kOBJ_BLOB = + "11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE" + "3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000" + "0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3"; + + return { + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), + .successor = hexStringToBinaryString(kOBJ_SUCC), + .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithTwoNFTs() +{ + std::string const url1 = "abcd1"; + std::string const url2 = "abcd2"; + ripple::Blob const uri1Blob(url1.begin(), url1.end()); + ripple::Blob const uri2Blob(url2.begin(), url2.end()); + + constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q"; + constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000"; + constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA"; + + auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt); + auto const serializerNftPage = nftPage.getSerializer(); + + auto const account = getAccountIdWithString(kACCOUNT); + return { + .key = {}, + .keyRaw = std::string(reinterpret_cast(account.data()), ripple::AccountID::size()), + .data = {}, + .dataRaw = + std::string(static_cast(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()), + .successor = "", + .predecessor = "", .type = etlng::model::Object::ModType::Created, }; } @@ -219,8 +165,10 @@ etlng::model::BookSuccessor createSuccessor() { return { - .firstBook = "A000000000000000000000000000000000000000000000000000000000000000", - .bookBase = "A000000000000000000000000000000000000000000000000000000000000001", + .firstBook = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}), + .bookBase = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}), }; } diff --git a/tests/common/util/BinaryTestObject.hpp b/tests/common/util/BinaryTestObject.hpp index ad5fba055..41d5029ef 100644 --- a/tests/common/util/BinaryTestObject.hpp +++ b/tests/common/util/BinaryTestObject.hpp @@ -32,17 +32,149 @@ namespace util { +static constexpr auto kDEFAULT_TXN_HEX = + "1200192200000008240011CC9B201B001F71D6202A0000000168400000" + "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" + "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" + "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" + "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" + "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" + "DE3538F248662FC73C"; + +static constexpr auto kDEFAULT_TXN_META = + "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" + "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" + "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" + "C200000028751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "9808B6B90000001D751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" + "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" + "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" + "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" + "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" + "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" + "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" + "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" + "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" + "00001F751868747470733A2F2F677265677765697362726F642E636F6D" + "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" + "5DAE00000014751868747470733A2F2F677265677765697362726F642E" + "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" + "3CCE1462A500000009751868747470733A2F2F67726567776569736272" + "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" + "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" + "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" + "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" + "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" + "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" + "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" + "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" + "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" + "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" + "00000029751868747470733A2F2F677265677765697362726F642E636F" + "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" + "EE87B80000001E751868747470733A2F2F677265677765697362726F64" + "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" + "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" + "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" + "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" + "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" + "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" + "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" + "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" + "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" + "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" + "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" + "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" + "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" + "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" + "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" + "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" + "B100000015751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" + "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" + "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" + "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" + "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" + "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" + "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" + "2FC73CE1E1F1031000"; + +static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"; +static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + [[maybe_unused, nodiscard]] std::pair -createNftTxAndMetaBlobs(); +createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX); [[maybe_unused, nodiscard]] std::pair -createNftTxAndMeta(); +createNftTxAndMeta( + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Transaction -createTransaction(ripple::TxType type); +createTransaction( + ripple::TxType type, + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Object -createObject(); +createObject( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithBookBase( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithTwoNFTs(); [[maybe_unused, nodiscard]] etlng::model::BookSuccessor createSuccessor(); diff --git a/tests/common/util/MockBackend.hpp b/tests/common/util/MockBackend.hpp index af16b490f..9aab18ee8 100644 --- a/tests/common/util/MockBackend.hpp +++ b/tests/common/util/MockBackend.hpp @@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface { MOCK_METHOD(void, writeAccountTransactions, (std::vector), (override)); + MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override)); + MOCK_METHOD(void, writeNFTTransactions, (std::vector const&), (override)); MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override)); diff --git a/tests/common/util/MockLedgerCache.hpp b/tests/common/util/MockLedgerCache.hpp index 67473b6e9..07fb67856 100644 --- a/tests/common/util/MockLedgerCache.hpp +++ b/tests/common/util/MockLedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface { MOCK_METHOD(std::optional, get, (ripple::uint256 const& a, uint32_t b), (const, override)); + MOCK_METHOD(void, update, (std::vector const&, uint32_t), (override)); + + MOCK_METHOD(std::optional, getDeleted, (ripple::uint256 const&, uint32_t), (const, override)); + MOCK_METHOD( std::optional, getSuccessor, diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 39ee9cdad..1f97643fe 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -45,6 +45,10 @@ target_sources( etlng/LoadingTests.cpp etlng/NetworkValidatedLedgersTests.cpp etlng/MonitorTests.cpp + etlng/ext/CoreTests.cpp + etlng/ext/CacheTests.cpp + etlng/ext/NFTTests.cpp + etlng/ext/SuccessorTests.cpp # Feed util/BytesConverterTests.cpp feed/BookChangesFeedTests.cpp diff --git a/tests/unit/etlng/ext/CacheTests.cpp b/tests/unit/etlng/ext/CacheTests.cpp new file mode 100644 index 000000000..bdd15c747 --- /dev/null +++ b/tests/unit/etlng/ext/CacheTests.cpp @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Cache.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockLedgerCache.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; +constinit auto const kUNUSED_LAST_KEY = "unused"; + +auto +createTestData() +{ + auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()}; + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = {}, + .objects = std::move(objects), + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct CacheExtTests : util::prometheus::WithPrometheus { +protected: + MockLedgerCache cache_; + etlng::impl::CacheExt ext_{cache_}; +}; + +TEST_F(CacheExtTests, OnLedgerDataUpdatesCache) +{ + auto const data = createTestData(); + + EXPECT_CALL(cache_, update(data.objects, data.seq)); + + ext_.onLedgerData(data); +} + +TEST_F(CacheExtTests, OnInitialDataUpdatesCacheAndSetsFull) +{ + auto const data = createTestData(); + + EXPECT_CALL(cache_, update(data.objects, data.seq)); + EXPECT_CALL(cache_, setFull); + + ext_.onInitialData(data); +} + +TEST_F(CacheExtTests, OnInitialObjectsUpdateCache) +{ + auto const objects = std::vector{util::createObject(), util::createObject()}; + + EXPECT_CALL(cache_, update(objects, kSEQ)); + + ext_.onInitialObjects(kSEQ, objects, kUNUSED_LAST_KEY); +} diff --git a/tests/unit/etlng/ext/CoreTests.cpp b/tests/unit/etlng/ext/CoreTests.cpp new file mode 100644 index 000000000..bcdf5ff88 --- /dev/null +++ b/tests/unit/etlng/ext/CoreTests.cpp @@ -0,0 +1,107 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Core.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +auto +createTestData() +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER), + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = {}, + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + etlng::impl::CoreExt ext_{backend_}; +}; + +TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader})); + EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size()); + EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size()); + + ext_.onLedgerData(data); +} + +TEST_F(CoreExtTests, OnInitialDataWritesLedgerAndTransactions) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader})); + EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size()); + EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size()); + + ext_.onInitialData(data); +} + +TEST_F(CoreExtTests, OnInitialObjectWritesLedgerObject) +{ + auto const data = util::createObject(); + + EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw})); + + ext_.onInitialObject(kSEQ, data); +} + +TEST_F(CoreExtTests, OnObjectWritesLedgerObject) +{ + auto const data = util::createObject(); + + EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw})); + + ext_.onObject(kSEQ, data); +} diff --git a/tests/unit/etlng/ext/NFTTests.cpp b/tests/unit/etlng/ext/NFTTests.cpp new file mode 100644 index 000000000..647316460 --- /dev/null +++ b/tests/unit/etlng/ext/NFTTests.cpp @@ -0,0 +1,287 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/NFT.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +constinit auto const kTXN_HEX2 = + "12001D230606B58324048A8B6F501C50E8EBCD412E6CF9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487E501DA2A1BC9A62AAEB2A" + "2A70F895587A3FB752514AA03F8C6E7C84864653B8673E0368400000000000001E60134000000000001F09732103B8C234E0598BC26D8A3E1B" + "FF53EB252EC0F15EA6800E4D85AA5D7CD15D76B01E744730450221009C0AFF5F3298E10ABE42894717DA46B59529A366527AA5DFC1577ADEA9" + "B20FA70220494D1D9BFEF2AB09F4D6403AAC5B9DCC5B859DCEC1380C6D817A6EDFE7E68FD581141565EED165BA79999425204A8491C73B1301" + "E34FF9EA7D0F7872702E63616665202D2073616C65E1F1"; + +constinit auto const kTXN_META2 = + "201C00000040F8E51100502505A59E11552ABC2FD74D879BE58489A588838AA2BA59E1E05A48A574226CD8B6CE77998971560B639A808E3B97" + "42A25334E5CF68EEDEDE52F54E50F4E63921C9F3C40588E426E6FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C" + "052AB2897542697066733A2F2F6261666B726569683673796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E71673472" + "6E747037653769757469766C796275E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D526950" + "5679335464654A6170697974576B686141576B4D716D39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60F" + "DDC47DABE4D52E4AD1F0A50A85CBBC00000022752E516D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169" + "316B4C316E5255797131E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C457664644634" + "5442427374455138484C66694867456B6A4465514D56676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4" + "D52E4AD1F0A5608E298B000000EF752E516D4E6E56566F4250426755783848334641486F436162593647347577445758767738757455636976" + "73764D7142E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646" + "777A526D517472694841663277437A3567786161355256446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0" + "A5B09C35A40000020A752E516D5168584346325738535571657633333657703572357935326141464D59564A667570534A536B6B425047786E" + "E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B6968756000001BC752E516D5359647963353276657A7478365338673231" + "70544651724E6E624762676A5A4B5047753836464231394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C" + "000000D2752E516D506645624A38446D624E69794C55637441796A3473625453515937314177597A32776A595238703838416432E1EC5A0008" + "27107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37" + "773338734147397469725867326D626D4262596771E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F75" + "2E516D505A733255393159764538374277635677587A63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E6" + "4C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E21584000001EA752E516D656366646769555A6B37355574667152763474696644557257345439" + "4571616F733231757036706E75345663E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573" + "614345725175396F48596A71765172564B31473155504657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE" + "11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A6854653776" + "43624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A2F2F6261" + "667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E6965" + "2F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F62616679" + "62656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E69652F36" + "342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D616D397436" + "5A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0FE30354A4" + "3281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D79756E697736" + "647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A43281629A" + "0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D6575326934" + "65726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A00082710" + "AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E61776833" + "6F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B7320" + "2331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F516D63436D" + "526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E72200000000501A0B639A808E" + "3B9742A25334E5CF68EEDEDE52F54E4A69498F5B79952C052AB289501B0B639A808E3B9742A25334E5CF68EEDEDE52F54E50F4E639664EC7E5" + "0588E658FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C052AB2897542697066733A2F2F6261666B7265696836" + "73796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E716734726E747037653769757469766C796275E1EC5A00082710" + "7B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D5269505679335464654A6170697974576B686141576B4D716D" + "39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A50A85CBBC00000022752E51" + "6D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169316B4C316E5255797131E1EC5A000827107B87E64C88" + "4BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C4576646446345442427374455138484C66694867456B6A4465514D56" + "676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5608E298B000000EF752E516D4E6E5656" + "6F4250426755783848334641486F43616259364734757744575876773875745563697673764D7142E1EC5A000827107B87E64C884BBFB60FDD" + "C47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646777A526D517472694841663277437A35677861613552" + "56446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B09C35A40000020A752E516D516858434632573853" + "5571657633333657703572357935326141464D59564A667570534A536B6B425047786EE1EC5A000827107B87E64C884BBFB60FDDC47DABE4D5" + "2E4AD1F0A5B6968756000001BC752E516D5359647963353276657A747836533867323170544651724E6E624762676A5A4B5047753836464231" + "394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C000000D2752E516D506645624A38446D624E69794C55" + "637441796A3473625453515937314177597A32776A595238703838416432E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5" + "CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37773338734147397469725867326D626D4262596771E1" + "EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F752E516D505A733255393159764538374277635677587A" + "63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E2158400" + "0001EA752E516D656366646769555A6B373555746671527634746966445572573454394571616F733231757036706E75345663E1EC5A000827" + "107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573614345725175396F48596A71765172564B3147315550" + "4657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051752E" + "516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44668A" + "2B96DFDE11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A68" + "5465377643624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A" + "2F2F6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A3670717136356635" + "6F6E69652F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F" + "6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E" + "69652F36342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D61" + "6D3974365A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0F" + "E30354A43281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D7975" + "6E697736647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A4" + "3281629A0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D65" + "7532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A" + "00082710AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E" + "617768336F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B73205075" + "6E6B73202331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F51" + "6D63436D526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E1E51100502505A4C9" + "C9557DF04CC68DE7C5EF20DBF705221EEDB05FE3806BC3F6A35240652C3E9C97BA5A56246B3E06AB367AB9614566B6F90C718B52A4440852A4" + "440804D409E004C90E52E6FAEC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478B72AD46100000008755E68747470733A2F2F69" + "7066732E696F2F697066732F62616679626569656F6C7667696F71766F737436346367646873797876726962336D6265797278773477626F61" + "617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10" + "276700000009755E68747470733A2F2F697066732E696F2F697066732F62616679626569686D6374376A766B7236366E67337975676D33706D" + "70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA70" + "69E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E68747470733A2F2F697066732E696F2F697066732F6261667962656968626D6D" + "6E626C687736656D776E6A733766787778376736376B6E6A626B6966636674787A6B66777632767162357A6C727575612F6D65746164617461" + "2E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478FBE441630000000B755E68747470733A2F2F697066732E696F" + "2F697066732F62616679626569686E6C77656C7965357270706963646761617678756869376D61636E697879627077667133796734626C3366" + "6A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051" + "752E516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44" + "668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B535470" + "31514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D51" + "674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE58" + "EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A7661" + "6D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC753569" + "7066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A0008" + "2710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F62" + "6D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC24F" + "1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C783579693375" + "706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408" + "005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D7547724839" + "4C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D555744524C" + "425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB9614566" + "B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E346958597233487341715771" + "673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD7535697066733A" + "2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A00081388246B" + "3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A51567957" + "4B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204C9" + "12F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E47" + "E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E357163334E" + "753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52" + "A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E654474" + "6656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D5779" + "6D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367AB9" + "614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A743147454564" + "4C787161513466714E637375363854677153E1F1E1E72200000000501A246B3E06AB367AB9614566B6F90C718B52A444084B237478B72AD461" + "00000008501B246B3E06AB367AB9614566B6F90C718B52A4440852A444080D2641FC04C91315FAEC5A00081388DAA8A3AA7069E65EC1E3E557" + "1D9B6F274B237478B72AD46100000008755E68747470733A2F2F697066732E696F2F697066732F62616679626569656F6C7667696F71766F73" + "7436346367646873797876726962336D6265797278773477626F61617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC" + "5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10276700000009755E68747470733A2F2F697066732E696F2F697066732F62" + "616679626569686D6374376A766B7236366E67337975676D33706D70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69" + "792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E6874747073" + "3A2F2F697066732E696F2F697066732F6261667962656968626D6D6E626C687736656D776E6A733766787778376736376B6E6A626B69666366" + "74787A6B66777632767162357A6C727575612F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B23" + "7478FBE441630000000B755E68747470733A2F2F697066732E696F2F697066732F62616679626569686E6C77656C7965357270706963646761" + "617678756869376D61636E697879627077667133796734626C33666A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A" + "44668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B5354" + "7031514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D" + "51674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE" + "58EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A76" + "616D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC7535" + "697066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A00" + "082710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F" + "626D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC2" + "4F1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C7835796933" + "75706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A444" + "08005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D75477248" + "394C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D55574452" + "4C425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB96145" + "66B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E3469585972334873417157" + "71673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD753569706673" + "3A2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A0008138824" + "6B3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A515679" + "574B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204" + "C912F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E" + "47E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E35716333" + "4E753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B" + "52A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E6544" + "746656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D57" + "796D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367A" + "B9614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A7431474545" + "644C787161513466714E637375363854677153E1F1E1E1E4110064562AED34CB796DF0E82AAC7EB958158EEBF99F51AA8C96B85654DDE05206" + "C18BBCE7220000000225059FB7B755172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235BEBBE582AED34CB796DF0E82A" + "AC7EB958158EEBF99F51AA8C96B85654DDE05206C18BBC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051E1" + "E1E51100612505A58BFB555F1D177245DF64F1BAC04B3EFF72C9B448710F95E86355339305264B0C6450345643CECFECC44660B31BEB70A4AE" + "78ED0BB4B31A754CE746848150EFFC03418847E62D0000010E624000000004FF530BE1E722000000002404C9421F2D0000010D202B00000712" + "202C00000493203204C90C7062400000000506A60B8114246B3E06AB367AB9614566B6F90C718B52A44408E1E1E41100375650E8EBCD412E6C" + "F9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487EE722000000002505A5B1C23400000000000000463C0000000000000000558F56" + "F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB7" + "2AED0000005161400000000007C02982140B639A808E3B9742A25334E5CF68EEDEDE52F54E83141565EED165BA79999425204A8491C73B1301" + "E34FE1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC4565183948C67127DAA598D" + "1197F3DEDC4552F64034F3B8213060B8903FD6C8A561E62D0000039C62400000000D7F518BE1E7220000000024057A8B032D0000039B202B00" + "000290202C000000082032057A6CBC62400000000D77916281140B639A808E3B9742A25334E5CF68EEDEDE52F54E8914CEAECC5B87EA043BD9" + "8E1B4FE8663AC59D5C3518E1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45658" + "7E28972F3B63D2260C0671E59593EE6C4D27AB857D1AF230F5CAA959BB3EACE624048A8B6F624000001B84AABF6AE1E7220000000024048A8B" + "702D00000000624000001B84AADE5581141565EED165BA79999425204A8491C73B1301E34FE1E1E41100645698929E4419455BBB551BF09C08" + "EEBEE19021E6B7E3D1D989968EF49970F10130E722000000012505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1F" + "D7B40F710AC45898929E4419455BBB551BF09C08EEBEE19021E6B7E3D1D989968EF49970F101305A00080FA03A44668A2B96DFDE11BF0817CC" + "6DF60C4E3508D43EB72AED00000051E1E1E411003756A2A1BC9A62AAEB2A2A70F895587A3FB752514AA03F8C6E7C84864653B8673E03E72200" + "00000125059FB7B734000000000000003C3C000000000000000055172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235B" + "EBBE5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED0000005161400000000007A1208214246B3E06AB367AB9614566" + "B6F90C718B52A4440883141565EED165BA79999425204A8491C73B1301E34FE1E1E511006125059DE0B155C0F457C1104D45881194D53DBB40" + "B81F9D812FC507637C4294527C06FDD69A3856A969AD4B14312DFA8739258A6A94C0868F1A3E7D1A1EF4839983521A4F6953FCE66240000000" + "0011B75AE1E7220000000024046044C82D00000000202B00001836202C00000F3F62400000000012057A81143A44668A2B96DFDE11BF0817CC" + "6DF60C4E3508D48914651B85AF14BE4F60AFBF5ADAA4F367061C2DA1D4E1E1E51100642505A5B1C2558F56F16F29177518F3DD6CF827085D7B" + "9E2806CD5EBDE810DD1FD7B40F710AC456CE29B8547FC486F45704C1DE539B748587872AA803FB53FB938E282547DF2CFEE722000000003200" + "0000000000004558DE4F51B35BE5A98D0C97BE07378E9CB56FFE3F861E544E0ABAC5ED765E2F781982140B639A808E3B9742A25334E5CF68EE" + "DEDE52F54EE1E1E51100642505A4EA55556022061889BB4DA5242B7AB048C3D751ACEE5788ED4120B2E70F68E9A322A1E956D7A8E1C70CD8A0" + "3AE9BAA41BC0898471FC26DA3712748A803B2F32007CDCB0DCE7220000000031000000000000003D32000000000000003B58B6629B8F178A18" + "C2926F1ADA669262B5D362BFC2DB1417D837CBF382AA37F2D88214246B3E06AB367AB9614566B6F90C718B52A44408E1E1F1031000"; + +constinit auto const kHASH2 = "D7604B124D5D9C89EC1854A6CBD5A1FFD92502E945411B9C8DE397E7F19A74F8"; + +auto +createTestData() +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN, kHASH2, kTXN_META2, kTXN_HEX2), + util::createTransaction(ripple::TxType::ttAMM_CREATE), // not NFT - will be filtered + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), // not unique - will be filtered + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = {}, + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + etlng::impl::NFTExt ext_{backend_}; +}; + +TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { + EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique + }); + EXPECT_CALL(*backend_, writeNFTTransactions); + + ext_.onLedgerData(data); +} + +TEST_F(NFTExtTests, OnInitialDataFiltersAndWritesNFTs) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { + EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique + }); + EXPECT_CALL(*backend_, writeNFTTransactions); + + ext_.onInitialData(data); +} + +TEST_F(NFTExtTests, OnInitialObjectExtractsAndWritesNFTData) +{ + auto const data = util::createObjectWithTwoNFTs(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { EXPECT_EQ(nfts.size(), 2); }); + + ext_.onInitialObject(kSEQ, data); +} diff --git a/tests/unit/etlng/ext/SuccessorTests.cpp b/tests/unit/etlng/ext/SuccessorTests.cpp new file mode 100644 index 000000000..6828dd801 --- /dev/null +++ b/tests/unit/etlng/ext/SuccessorTests.cpp @@ -0,0 +1,641 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "data/DBHelpers.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Successor.hpp" +#include "util/Assert.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockAssert.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockLedgerCache.hpp" +#include "util/MockPrometheus.hpp" +#include "util/StringUtils.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +auto +createTestData(std::vector objects) +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER), + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = std::move(objects), + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +[[maybe_unused]] auto +createInitialTestData(std::vector edgeKeys) +{ + // initial data expects objects to be empty as well as non-empty edgeKeys + ASSERT(not edgeKeys.empty(), "Initial data requires edgeKeys"); + + auto ret = createTestData({}); + ret.edgeKeys = std::make_optional>(); + std::ranges::transform(edgeKeys, std::back_inserter(ret.edgeKeys.value()), &uint256ToString); + + return ret; +} + +} // namespace + +struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + MockLedgerCache cache_; + etlng::impl::SuccessorExt ext_{backend_, cache_}; +}; + +TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent) +{ + auto const data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(false)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_THROW(ext_.onLedgerData(data), std::logic_error); +} + +TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffersAndSuccessorsNotPresent) +{ + auto const data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ - 1)); + + EXPECT_THROW(ext_.onLedgerData(data), std::logic_error); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(Blob{'0'})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillRepeatedly(testing::Return(LedgerObject{})); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(LedgerObject{.key = createdObj.key, .blob = {}})); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_)); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(deletedObj.key); + auto const oldCachedObj = createdObj.data; + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj)); + + EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt)); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(data::kLAST_KEY))); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(deletedObj.key); + auto const oldCachedObj = createdObj.data; + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj)); + + EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(LedgerObject{.key = deletedObj.key, .blob = {}})); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(deletedObj.key))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const predKey = + binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C" + )); + auto const succKey = + binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + )); + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)) + .WillOnce(testing::Return(data::LedgerObject{.key = predKey, .blob = {}})); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)) + .WillOnce(testing::Return(data::LedgerObject{.key = succKey, .blob = {}})); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(predKey), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(succKey))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const succ = util::createSuccessor(); + data.successors = {succ, succ, succ}; + + EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, auto{succ.firstBook})) + .Times(data.successors->size()); + + EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.predecessor}, kSEQ, auto{createdObj.keyRaw})); + EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.keyRaw}, kSEQ, auto{createdObj.successor})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto succ = util::createSuccessor(); + succ.firstBook = {}; // empty will be transformed into kLAST_KEY + data.successors = {succ, succ}; + + EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, uint256ToString(data::kLAST_KEY))) + .Times(data.successors->size()); + + EXPECT_CALL(*backend_, writeSuccessor(auto{deletedObj.predecessor}, kSEQ, auto{deletedObj.successor})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = {'0'}}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(std::nullopt)); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = {'0'}}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {}})); + EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created); + auto const bookBase = getBookBase(bookBaseObj.key); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = bookBaseObj.data}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(bookBase, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(data::LedgerObject{.key = firstKey, .blob = data::Blob{'1'}})); + EXPECT_CALL( + *backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_) + ); // Called once because firstKey returned repeatedly above + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {'1'}})); + EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))).InSequence(inSeq); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey) +{ + using namespace etlng::model; + + auto const lastKey = std::string{}; + auto const data = std::vector{ + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610" + ), + }; + + std::string lk = lastKey; + for (auto const& obj : data) { + if (not lk.empty()) + EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key))); + lk = uint256ToString(obj.key); + } + + ext_.onInitialObjects(kSEQ, data, lastKey); +} + +TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey) +{ + using namespace etlng::model; + + auto const lastKey = + uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D")); + auto const data = std::vector{ + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610" + ), + }; + + std::string lk = lastKey; + for (auto const& obj : data) { + EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key))); + lk = uint256ToString(obj.key); + } + + ext_.onInitialObjects(kSEQ, data, lastKey); +} + +struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests {}; + +TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); }); +} + +TEST_F( + SuccessorExtAssertTests, + OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt)); + + EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull) +{ + using namespace etlng::model; + + auto const data = createTestData({ + util::createObject(Object::ModType::Modified), + util::createObject(Object::ModType::Created), + }); + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(false)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys) +{ + using namespace etlng::model; + + auto data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto data = createInitialTestData({firstKey, secondKey}); + data.objects = {util::createObject()}; + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} diff --git a/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp b/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp index bc1f63539..f2032fa42 100644 --- a/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp +++ b/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp @@ -41,9 +41,9 @@ using namespace rpc; using namespace data; +using namespace testing; namespace json = boost::json; -using namespace testing; namespace { From e029a9b3dfef7326fcb42ca3542b3f5e467f019a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 12:14:38 +0000 Subject: [PATCH 03/13] style: clang-tidy auto fixes (#1972) Fixes #1971. Co-authored-by: godexsoft <385326+godexsoft@users.noreply.github.com> --- src/etl/impl/LedgerLoader.hpp | 2 +- src/etlng/impl/ext/Core.cpp | 1 - tests/integration/data/cassandra/BackendTests.cpp | 2 +- tests/unit/etlng/ext/CacheTests.cpp | 1 - tests/unit/etlng/ext/SuccessorTests.cpp | 14 ++++++-------- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/etl/impl/LedgerLoader.hpp b/src/etl/impl/LedgerLoader.hpp index e6d88ee8e..716fa8718 100644 --- a/src/etl/impl/LedgerLoader.hpp +++ b/src/etl/impl/LedgerLoader.hpp @@ -121,7 +121,7 @@ public: LOG(log_.trace()) << "Inserting transaction = " << sttx.getTransactionID(); - ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; + ripple::TxMeta const txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()}; auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx); result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end()); diff --git a/src/etlng/impl/ext/Core.cpp b/src/etlng/impl/ext/Core.cpp index 2c3e141a0..fa2c3c9fa 100644 --- a/src/etlng/impl/ext/Core.cpp +++ b/src/etlng/impl/ext/Core.cpp @@ -23,7 +23,6 @@ #include "etlng/Models.hpp" #include "util/log/Logger.hpp" -#include #include #include diff --git a/tests/integration/data/cassandra/BackendTests.cpp b/tests/integration/data/cassandra/BackendTests.cpp index ffc03a48d..3fa0edc04 100644 --- a/tests/integration/data/cassandra/BackendTests.cpp +++ b/tests/integration/data/cassandra/BackendTests.cpp @@ -402,7 +402,7 @@ TEST_F(BackendCassandraTest, Basic) ripple::uint256 hash256; EXPECT_TRUE(hash256.parseHex(hashHex)); - ripple::TxMeta txMeta{hash256, lgrInfoNext.seq, metaBlob}; + ripple::TxMeta const txMeta{hash256, lgrInfoNext.seq, metaBlob}; auto accountsSet = txMeta.getAffectedAccounts(); for (auto& a : accountsSet) { affectedAccounts.push_back(a); diff --git a/tests/unit/etlng/ext/CacheTests.cpp b/tests/unit/etlng/ext/CacheTests.cpp index bdd15c747..c1697b6d1 100644 --- a/tests/unit/etlng/ext/CacheTests.cpp +++ b/tests/unit/etlng/ext/CacheTests.cpp @@ -26,7 +26,6 @@ #include #include -#include #include #include diff --git a/tests/unit/etlng/ext/SuccessorTests.cpp b/tests/unit/etlng/ext/SuccessorTests.cpp index 6828dd801..981afac5f 100644 --- a/tests/unit/etlng/ext/SuccessorTests.cpp +++ b/tests/unit/etlng/ext/SuccessorTests.cpp @@ -32,10 +32,8 @@ #include #include -#include #include #include -#include #include #include @@ -203,7 +201,7 @@ TEST_F( }); auto const bookBase = getBookBase(createdObj.key); - [[maybe_unused]] testing::InSequence inSeq; + [[maybe_unused]] testing::InSequence const inSeq; EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); @@ -239,7 +237,7 @@ TEST_F( auto const bookBase = getBookBase(deletedObj.key); auto const oldCachedObj = createdObj.data; - [[maybe_unused]] testing::InSequence inSeq; + [[maybe_unused]] testing::InSequence const inSeq; EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); @@ -273,7 +271,7 @@ TEST_F( auto const bookBase = getBookBase(deletedObj.key); auto const oldCachedObj = createdObj.data; - [[maybe_unused]] testing::InSequence inSeq; + [[maybe_unused]] testing::InSequence const inSeq; EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); @@ -378,7 +376,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessor successorChain.push(firstKey); successorChain.push(secondKey); - [[maybe_unused]] testing::Sequence inSeq; + [[maybe_unused]] testing::Sequence const inSeq; EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) @@ -417,7 +415,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsF successorChain.push(firstKey); successorChain.push(secondKey); - [[maybe_unused]] testing::Sequence inSeq; + [[maybe_unused]] testing::Sequence const inSeq; EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) @@ -460,7 +458,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForE auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created); auto const bookBase = getBookBase(bookBaseObj.key); - [[maybe_unused]] testing::Sequence inSeq; + [[maybe_unused]] testing::Sequence const inSeq; EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) From f5d494be23a6e3ea2cf64c86346274243d1e8fa4 Mon Sep 17 00:00:00 2001 From: Sergey Kuznetsov Date: Tue, 1 Apr 2025 16:12:16 +0100 Subject: [PATCH 04/13] fix: Fix ssl in new webserver (#1981) Fixes #1980. An SSL handshake was missing and WsConnection should be build from stream not socket because in case SSL connection stream already completed SSL handshake. --- src/etlng/impl/ext/Core.cpp | 1 - src/web/CMakeLists.txt | 1 - src/web/ng/Server.cpp | 15 ++-- src/web/ng/impl/HttpConnection.hpp | 59 +++++++------- src/web/ng/impl/WsConnection.cpp | 77 ------------------- src/web/ng/impl/WsConnection.hpp | 50 ++++-------- .../common/web/ng/impl/MockHttpConnection.hpp | 4 +- .../unit/web/ng/impl/HttpConnectionTests.cpp | 3 +- tests/unit/web/ng/impl/WsConnectionTests.cpp | 3 +- 9 files changed, 57 insertions(+), 156 deletions(-) delete mode 100644 src/web/ng/impl/WsConnection.cpp diff --git a/src/etlng/impl/ext/Core.cpp b/src/etlng/impl/ext/Core.cpp index fa2c3c9fa..3049961fd 100644 --- a/src/etlng/impl/ext/Core.cpp +++ b/src/etlng/impl/ext/Core.cpp @@ -23,7 +23,6 @@ #include "etlng/Models.hpp" #include "util/log/Logger.hpp" - #include #include #include diff --git a/src/web/CMakeLists.txt b/src/web/CMakeLists.txt index 083eae90c..251fda1bd 100644 --- a/src/web/CMakeLists.txt +++ b/src/web/CMakeLists.txt @@ -10,7 +10,6 @@ target_sources( ng/impl/ErrorHandling.cpp ng/impl/ConnectionHandler.cpp ng/impl/ServerSslContext.cpp - ng/impl/WsConnection.cpp ng/Request.cpp ng/Response.cpp ng/Server.cpp diff --git a/src/web/ng/Server.cpp b/src/web/ng/Server.cpp index a209fea35..be3fe1064 100644 --- a/src/web/ng/Server.cpp +++ b/src/web/ng/Server.cpp @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -136,13 +137,19 @@ makeConnection( if (not sslContext.has_value()) return std::unexpected{"Error creating a connection: SSL is not supported by this server"}; - connection = std::make_unique( + auto sslConnection = std::make_unique( std::move(sslDetectionResult.socket), std::move(ip), std::move(sslDetectionResult.buffer), *sslContext, tagDecoratorFactory ); + sslConnection->setTimeout(std::chrono::seconds{10}); + auto const maybeError = sslConnection->sslHandshake(yield); + if (maybeError.has_value()) + return std::unexpected{fmt::format("SSL handshake error: {}", maybeError->message())}; + + connection = std::move(sslConnection); } else { connection = std::make_unique( std::move(sslDetectionResult.socket), @@ -164,7 +171,6 @@ makeConnection( std::expected tryUpgradeConnection( impl::UpgradableConnectionPtr connection, - std::optional& sslContext, util::TagDecoratorFactory& tagDecoratorFactory, boost::asio::yield_context yield ) @@ -177,7 +183,7 @@ tryUpgradeConnection( } if (*expectedIsUpgrade) { - auto expectedUpgradedConnection = connection->upgrade(sslContext, tagDecoratorFactory, yield); + auto expectedUpgradedConnection = connection->upgrade(tagDecoratorFactory, yield); if (expectedUpgradedConnection.has_value()) return std::move(expectedUpgradedConnection).value(); @@ -316,8 +322,7 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield return; } - auto connection = - tryUpgradeConnection(std::move(connectionExpected).value(), sslContext_, tagDecoratorFactory_, yield); + auto connection = tryUpgradeConnection(std::move(connectionExpected).value(), tagDecoratorFactory_, yield); if (not connection.has_value()) { LOG(log_.info()) << connection.error(); return; diff --git a/src/web/ng/impl/HttpConnection.hpp b/src/web/ng/impl/HttpConnection.hpp index b7bdfae6a..0cad34383 100644 --- a/src/web/ng/impl/HttpConnection.hpp +++ b/src/web/ng/impl/HttpConnection.hpp @@ -28,10 +28,12 @@ #include "web/ng/impl/Concepts.hpp" #include "web/ng/impl/WsConnection.hpp" +#include #include #include #include #include +#include #include #include #include @@ -57,11 +59,7 @@ public: isUpgradeRequested(boost::asio::yield_context yield) = 0; virtual std::expected - upgrade( - std::optional& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield - ) = 0; + upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) = 0; virtual std::optional sendRaw( @@ -104,6 +102,22 @@ public: { } + std::optional + sslHandshake(boost::asio::yield_context yield) + requires IsSslTcpStream + { + boost::system::error_code error; + boost::beast::get_lowest_layer(stream_).expires_after(timeout_); + auto const bytesUsed = + stream_.async_handshake(boost::asio::ssl::stream_base::server, buffer_.cdata(), yield[error]); + if (error) + return error; + + buffer_.consume(bytesUsed); + + return std::nullopt; + } + bool wasUpgraded() const override { @@ -183,35 +197,18 @@ public: } std::expected - upgrade( - [[maybe_unused]] std::optional& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield - ) override + upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) override { ASSERT(request_.has_value(), "Request must be present to upgrade the connection"); - if constexpr (IsSslTcpStream) { - ASSERT(sslContext.has_value(), "SSL context must be present to upgrade the connection"); - return makeSslWsConnection( - boost::beast::get_lowest_layer(stream_).release_socket(), - std::move(ip_), - std::move(buffer_), - std::move(request_).value(), - sslContext.value(), - tagDecoratorFactory, - yield - ); - } else { - return makePlainWsConnection( - stream_.release_socket(), - std::move(ip_), - std::move(buffer_), - std::move(request_).value(), - tagDecoratorFactory, - yield - ); - } + return makeWsConnection( + std::move(stream_), + std::move(ip_), + std::move(buffer_), + std::move(request_).value(), + tagDecoratorFactory, + yield + ); } private: diff --git a/src/web/ng/impl/WsConnection.cpp b/src/web/ng/impl/WsConnection.cpp deleted file mode 100644 index 89c47ba05..000000000 --- a/src/web/ng/impl/WsConnection.cpp +++ /dev/null @@ -1,77 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2024, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include "web/ng/impl/WsConnection.hpp" - -#include "util/Taggable.hpp" -#include "web/ng/Error.hpp" - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -namespace web::ng::impl { - -std::expected, Error> -makePlainWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -) -{ - auto connection = std::make_unique( - std::move(socket), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory - ); - auto maybeError = connection->performHandshake(yield); - if (maybeError.has_value()) - return std::unexpected{maybeError.value()}; - return connection; -} - -std::expected, Error> -makeSslWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - boost::asio::ssl::context& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -) -{ - auto connection = std::make_unique( - std::move(socket), std::move(ip), std::move(buffer), sslContext, std::move(request), tagDecoratorFactory - ); - auto maybeError = connection->performHandshake(yield); - if (maybeError.has_value()) - return std::unexpected{maybeError.value()}; - return connection; -} - -} // namespace web::ng::impl diff --git a/src/web/ng/impl/WsConnection.hpp b/src/web/ng/impl/WsConnection.hpp index e13e51b66..9d48d392d 100644 --- a/src/web/ng/impl/WsConnection.hpp +++ b/src/web/ng/impl/WsConnection.hpp @@ -68,31 +68,14 @@ class WsConnection : public WsConnectionBase { public: WsConnection( - boost::asio::ip::tcp::socket socket, + StreamType&& stream, std::string ip, boost::beast::flat_buffer buffer, boost::beast::http::request initialRequest, util::TagDecoratorFactory const& tagDecoratorFactory ) - requires IsTcpStream : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory) - , stream_(std::move(socket)) - , initialRequest_(std::move(initialRequest)) - { - setupWsStream(); - } - - WsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::asio::ssl::context& sslContext, - boost::beast::http::request initialRequest, - util::TagDecoratorFactory const& tagDecoratorFactory - ) - requires IsSslTcpStream - : WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory) - , stream_(std::move(socket), sslContext) + , stream_(std::move(stream)) , initialRequest_(std::move(initialRequest)) { setupWsStream(); @@ -189,25 +172,24 @@ private: using PlainWsConnection = WsConnection; using SslWsConnection = WsConnection>; -std::expected, Error> -makePlainWsConnection( - boost::asio::ip::tcp::socket socket, +template +std::expected>, Error> +makeWsConnection( + StreamType&& stream, std::string ip, boost::beast::flat_buffer buffer, boost::beast::http::request request, util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield -); - -std::expected, Error> -makeSslWsConnection( - boost::asio::ip::tcp::socket socket, - std::string ip, - boost::beast::flat_buffer buffer, - boost::beast::http::request request, - boost::asio::ssl::context& sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield -); +) +{ + auto connection = std::make_unique>( + std::forward(stream), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory + ); + auto maybeError = connection->performHandshake(yield); + if (maybeError.has_value()) + return std::unexpected{maybeError.value()}; + return connection; +} } // namespace web::ng::impl diff --git a/tests/common/web/ng/impl/MockHttpConnection.hpp b/tests/common/web/ng/impl/MockHttpConnection.hpp index 8f4ab0885..7288a5f8a 100644 --- a/tests/common/web/ng/impl/MockHttpConnection.hpp +++ b/tests/common/web/ng/impl/MockHttpConnection.hpp @@ -67,9 +67,7 @@ struct MockHttpConnectionImpl : web::ng::impl::UpgradableConnection { MOCK_METHOD( UpgradeReturnType, upgrade, - (OptionalSslContext & sslContext, - util::TagDecoratorFactory const& tagDecoratorFactory, - boost::asio::yield_context yield), + (util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield), (override) ); }; diff --git a/tests/unit/web/ng/impl/HttpConnectionTests.cpp b/tests/unit/web/ng/impl/HttpConnectionTests.cpp index eb03f1ffd..4c6e5d4f4 100644 --- a/tests/unit/web/ng/impl/HttpConnectionTests.cpp +++ b/tests/unit/web/ng/impl/HttpConnectionTests.cpp @@ -300,8 +300,7 @@ TEST_F(HttpConnectionTests, Upgrade) [&]() { ASSERT_TRUE(expectedResult.has_value()) << expectedResult.error().message(); }(); [&]() { ASSERT_TRUE(expectedResult.value()); }(); - std::optional sslContext; - auto expectedWsConnection = connection.upgrade(sslContext, tagDecoratorFactory_, yield); + auto expectedWsConnection = connection.upgrade(tagDecoratorFactory_, yield); [&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }(); }); } diff --git a/tests/unit/web/ng/impl/WsConnectionTests.cpp b/tests/unit/web/ng/impl/WsConnectionTests.cpp index 122ad746e..b449919a4 100644 --- a/tests/unit/web/ng/impl/WsConnectionTests.cpp +++ b/tests/unit/web/ng/impl/WsConnectionTests.cpp @@ -74,8 +74,7 @@ struct WebWsConnectionTests : SyncAsioContextTest { ASSERT_TRUE(expectedTrue.value()) << "Expected upgrade request"; }(); - std::optional sslContext; - auto expectedWsConnection = httpConnection.upgrade(sslContext, tagDecoratorFactory_, yield); + auto expectedWsConnection = httpConnection.upgrade(tagDecoratorFactory_, yield); [&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }(); auto connection = std::move(expectedWsConnection).value(); auto wsConnectionPtr = dynamic_cast(connection.release()); From 9a392ca072e3c5b5c7c1edcd5cbd0545137930a7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 12:40:38 +0100 Subject: [PATCH 05/13] style: clang-tidy auto fixes (#1985) Fixes #1984. Co-authored-by: godexsoft <385326+godexsoft@users.noreply.github.com> --- tests/unit/web/ng/impl/HttpConnectionTests.cpp | 1 - tests/unit/web/ng/impl/WsConnectionTests.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/unit/web/ng/impl/HttpConnectionTests.cpp b/tests/unit/web/ng/impl/HttpConnectionTests.cpp index 4c6e5d4f4..b2d42f2dc 100644 --- a/tests/unit/web/ng/impl/HttpConnectionTests.cpp +++ b/tests/unit/web/ng/impl/HttpConnectionTests.cpp @@ -31,7 +31,6 @@ #include #include -#include #include #include #include diff --git a/tests/unit/web/ng/impl/WsConnectionTests.cpp b/tests/unit/web/ng/impl/WsConnectionTests.cpp index b449919a4..2eb3e0349 100644 --- a/tests/unit/web/ng/impl/WsConnectionTests.cpp +++ b/tests/unit/web/ng/impl/WsConnectionTests.cpp @@ -36,7 +36,6 @@ #include #include #include -#include #include #include #include From 12a3feddb78f3a3cf806c1f0a47de1cbea6aa514 Mon Sep 17 00:00:00 2001 From: Peter Chen <34582813+PeterChen13579@users.noreply.github.com> Date: Wed, 2 Apr 2025 09:53:35 -0400 Subject: [PATCH 06/13] fix: incorrect set HighDeepFreeze flag (#1978) fixes #1977 --- src/rpc/handlers/AccountLines.cpp | 5 +- tests/unit/rpc/RPCHelpersTests.cpp | 6 +- tests/unit/rpc/handlers/AccountLinesTests.cpp | 83 ++++++++++++++++++- 3 files changed, 88 insertions(+), 6 deletions(-) diff --git a/src/rpc/handlers/AccountLines.cpp b/src/rpc/handlers/AccountLines.cpp index 5644e7daf..6459e27d6 100644 --- a/src/rpc/handlers/AccountLines.cpp +++ b/src/rpc/handlers/AccountLines.cpp @@ -86,8 +86,9 @@ AccountLinesHandler::addLine( bool const lineNoRipplePeer = (flags & (not viewLowest ? ripple::lsfLowNoRipple : ripple::lsfHighNoRipple)) != 0u; bool const lineFreeze = (flags & (viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; bool const lineFreezePeer = (flags & (not viewLowest ? ripple::lsfLowFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; - bool const lineDeepFreezePeer = (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighFreeze)) != 0u; + bool const lineDeepFreeze = (flags & (viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; + bool const lineDeepFreezePeer = + (flags & (not viewLowest ? ripple::lsfLowDeepFreeze : ripple::lsfHighDeepFreeze)) != 0u; ripple::STAmount const& saBalance = balance; ripple::STAmount const& saLimit = lineLimit; diff --git a/tests/unit/rpc/RPCHelpersTests.cpp b/tests/unit/rpc/RPCHelpersTests.cpp index cfc0c7e03..0dd500d34 100644 --- a/tests/unit/rpc/RPCHelpersTests.cpp +++ b/tests/unit/rpc/RPCHelpersTests.cpp @@ -581,7 +581,7 @@ TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExists_BlobDoesNotExist) }); } -TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExistsBlobExists_AccountWithCorrectFlag) +TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExists_AccountWithCorrectFlag) { auto const account = getAccountIdWithString(kACCOUNT); auto const issuerKey = ripple::keylet::account(account); @@ -600,12 +600,12 @@ TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExistsBlobExists_AccountWithCorrectFl }); } -TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExistsBlobExists_AccountFlagDoesNotExist) +TEST_F(RPCHelpersTest, FetchAndCheckAnyFlagExists_TrustLineIsFrozenAndCheckFreezeFlag) { auto const account = getAccountIdWithString(kACCOUNT); auto const issuerKey = ripple::keylet::account(account); - // create account with highDeepFreeze Flag + // create account with lowDeepFreeze Flag auto const accountObject = createAccountRootObject(kACCOUNT, ripple::lsfLowDeepFreeze, 1, 10, 2, kTXN_ID, 3); ON_CALL(*backend_, doFetchLedgerObject(issuerKey.key, kLEDGER_SEQ_OBJECT, _)) diff --git a/tests/unit/rpc/handlers/AccountLinesTests.cpp b/tests/unit/rpc/handlers/AccountLinesTests.cpp index 2d0b72aa8..1b774fd49 100644 --- a/tests/unit/rpc/handlers/AccountLinesTests.cpp +++ b/tests/unit/rpc/handlers/AccountLinesTests.cpp @@ -19,6 +19,7 @@ #include "data/Types.hpp" #include "rpc/Errors.hpp" +#include "rpc/RPCHelpers.hpp" #include "rpc/common/AnyHandler.hpp" #include "rpc/common/Types.hpp" #include "rpc/handlers/AccountLines.hpp" @@ -716,7 +717,7 @@ TEST_F(RPCAccountLinesHandlerTest, EmptyChannel) }); } -TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField) +TEST_F(RPCAccountLinesHandlerTest, OptionalResponseFieldWithDeepFreeze) { static constexpr auto kCORRECT_OUTPUT = R"({ "account": "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", @@ -806,6 +807,86 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField) }); } +TEST_F(RPCAccountLinesHandlerTest, FrozenTrustLineResponse) +{ + static constexpr auto kCORRECT_OUTPUT = R"({ + "account": "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", + "ledger_hash": "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652", + "ledger_index": 30, + "validated": true, + "limit": 200, + "lines": [ + { + "account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun", + "balance": "10", + "currency": "USD", + "limit": "100", + "limit_peer": "200", + "quality_in": 0, + "quality_out": 0, + "peer_authorized": true, + "freeze_peer": true + }, + { + "account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun", + "balance": "20", + "currency": "USD", + "limit": "200", + "limit_peer": "400", + "quality_in": 0, + "quality_out": 0, + "authorized": true, + "freeze": true + } + ] + })"; + + auto ledgerHeader = createLedgerHeader(kLEDGER_HASH, 30); + EXPECT_CALL(*backend_, fetchLedgerBySequence).WillOnce(Return(ledgerHeader)); + + // fetch account object return something + auto account = getAccountIdWithString(kACCOUNT); + auto accountKk = ripple::keylet::account(account).key; + auto owneDirKk = ripple::keylet::ownerDir(account).key; + auto fake = Blob{'f', 'a', 'k', 'e'}; + + // return a non empty account + ON_CALL(*backend_, doFetchLedgerObject(accountKk, testing::_, testing::_)).WillByDefault(Return(fake)); + + // return owner index + ripple::STObject const ownerDir = + createOwnerDirLedgerObject({ripple::uint256{kINDEX1}, ripple::uint256{kINDEX2}}, kINDEX1); + + ON_CALL(*backend_, doFetchLedgerObject(owneDirKk, testing::_, testing::_)) + .WillByDefault(Return(ownerDir.getSerializer().peekData())); + + // return few trust lines + std::vector bbs; + auto line1 = createRippleStateLedgerObject("USD", kACCOUNT2, 10, kACCOUNT, 100, kACCOUNT2, 200, kTXN_ID, 0); + line1.setFlag(ripple::lsfHighAuth); + line1.setFlag(ripple::lsfHighFreeze); + bbs.push_back(line1.getSerializer().peekData()); + + auto line2 = createRippleStateLedgerObject("USD", kACCOUNT2, 20, kACCOUNT, 200, kACCOUNT2, 400, kTXN_ID, 0); + line2.setFlag(ripple::lsfLowAuth); + line2.setFlag(ripple::lsfLowFreeze); + bbs.push_back(line2.getSerializer().peekData()); + + EXPECT_CALL(*backend_, doFetchLedgerObjects).WillOnce(Return(bbs)); + auto const input = json::parse(fmt::format( + R"({{ + "account": "{}" + }})", + kACCOUNT + )); + runSpawn([&, this](auto yield) { + auto handler = AnyHandler{AccountLinesHandler{this->backend_}}; + auto const output = handler.process(input, Context{yield}); + ASSERT_TRUE(output); + EXPECT_EQ(json::parse(kCORRECT_OUTPUT), *output.result); + }); +} + // normal case : test marker output correct TEST_F(RPCAccountLinesHandlerTest, MarkerOutput) { From 409dcd106c9d25fd2b70d4b18b02a2db9ed81465 Mon Sep 17 00:00:00 2001 From: Alex Kremer Date: Fri, 4 Apr 2025 15:52:22 +0100 Subject: [PATCH 07/13] feat: ETLng integration (#1986) For #1594 --- src/app/ClioApplication.cpp | 16 +- src/app/Stopper.hpp | 13 +- src/etl/ETLService.cpp | 4 +- src/etl/ETLService.hpp | 64 +- src/etl/LoadBalancer.cpp | 9 +- src/etl/LoadBalancer.hpp | 45 +- src/etl/NetworkValidatedLedgersInterface.hpp | 1 + src/etl/Source.hpp | 5 +- src/etl/impl/GrpcSource.cpp | 4 +- src/etl/impl/GrpcSource.hpp | 3 +- src/etl/impl/LedgerFetcher.hpp | 16 +- src/etl/impl/LedgerLoader.hpp | 13 +- src/etl/impl/SourceImpl.hpp | 4 +- src/etlng/CMakeLists.txt | 5 +- src/etlng/ETLService.hpp | 283 +++++++ src/etlng/ETLServiceInterface.hpp | 92 +++ src/etlng/LoadBalancer.cpp | 371 +++++++++ src/etlng/LoadBalancer.hpp | 271 ++++++ src/etlng/LoadBalancerInterface.hpp | 9 + src/etlng/LoaderInterface.hpp | 2 +- src/etlng/Source.cpp | 73 ++ src/etlng/Source.hpp | 194 +++++ src/etlng/impl/ForwardingSource.cpp | 116 +++ src/etlng/impl/ForwardingSource.hpp | 70 ++ src/etlng/impl/Loading.hpp | 1 - src/etlng/impl/Registry.hpp | 9 +- src/etlng/impl/SourceImpl.hpp | 232 ++++++ src/etlng/impl/ext/NFT.hpp | 4 - src/rpc/RPCEngine.hpp | 9 +- src/rpc/common/impl/ForwardingProxy.hpp | 8 +- src/rpc/common/impl/HandlerProvider.cpp | 7 +- src/rpc/common/impl/HandlerProvider.hpp | 10 +- src/rpc/handlers/ServerInfo.hpp | 20 +- src/rpc/handlers/Tx.hpp | 26 +- src/util/newconfig/ConfigDefinition.hpp | 2 +- src/web/RPCServerHandler.hpp | 7 +- src/web/ng/RPCServerHandler.hpp | 7 +- tests/common/util/MockETLService.hpp | 18 +- tests/common/util/MockLoadBalancer.hpp | 19 +- tests/common/util/MockSource.hpp | 6 +- tests/common/util/MockSourceNg.hpp | 245 ++++++ tests/unit/CMakeLists.txt | 3 + tests/unit/app/StopperTests.cpp | 14 +- tests/unit/etl/ETLStateTests.cpp | 1 + tests/unit/etl/GrpcSourceTests.cpp | 7 +- tests/unit/etl/LoadBalancerTests.cpp | 29 +- tests/unit/etl/SourceImplTests.cpp | 4 +- tests/unit/etlng/ForwardingSourceTests.cpp | 196 +++++ tests/unit/etlng/LoadBalancerTests.cpp | 821 +++++++++++++++++++ tests/unit/etlng/SourceImplTests.cpp | 223 +++++ tests/unit/rpc/ForwardingProxyTests.cpp | 6 +- tests/unit/rpc/RPCEngineTests.cpp | 61 +- tests/unit/rpc/handlers/AllHandlerTests.cpp | 2 +- tests/unit/rpc/handlers/ServerInfoTests.cpp | 2 +- tests/unit/rpc/handlers/TxTests.cpp | 56 +- tests/unit/web/RPCServerHandlerTests.cpp | 10 +- tests/unit/web/ng/RPCServerHandlerTests.cpp | 2 +- 57 files changed, 3473 insertions(+), 277 deletions(-) create mode 100644 src/etlng/ETLService.hpp create mode 100644 src/etlng/ETLServiceInterface.hpp create mode 100644 src/etlng/LoadBalancer.cpp create mode 100644 src/etlng/LoadBalancer.hpp create mode 100644 src/etlng/Source.cpp create mode 100644 src/etlng/Source.hpp create mode 100644 src/etlng/impl/ForwardingSource.cpp create mode 100644 src/etlng/impl/ForwardingSource.hpp create mode 100644 src/etlng/impl/SourceImpl.hpp create mode 100644 tests/common/util/MockSourceNg.hpp create mode 100644 tests/unit/etlng/ForwardingSourceTests.cpp create mode 100644 tests/unit/etlng/LoadBalancerTests.cpp create mode 100644 tests/unit/etlng/SourceImplTests.cpp diff --git a/src/app/ClioApplication.cpp b/src/app/ClioApplication.cpp index e68a2797a..6c40900ac 100644 --- a/src/app/ClioApplication.cpp +++ b/src/app/ClioApplication.cpp @@ -27,6 +27,8 @@ #include "etl/ETLService.hpp" #include "etl/LoadBalancer.hpp" #include "etl/NetworkValidatedLedgers.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManager.hpp" #include "migration/MigrationInspectorFactory.hpp" #include "rpc/Counters.hpp" @@ -130,7 +132,12 @@ ClioApplication::run(bool const useNgWebServer) // ETL uses the balancer to extract data. // The server uses the balancer to forward RPCs to a rippled node. // The balancer itself publishes to streams (transactions_proposed and accounts_proposed) - auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + auto balancer = [&] -> std::shared_ptr { + if (config_.get("__ng_etl")) + return etlng::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + + return etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers); + }(); // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers); @@ -142,12 +149,12 @@ ClioApplication::run(bool const useNgWebServer) config_, backend, subscriptions, balancer, etl, amendmentCenter, counters ); - using RPCEngineType = rpc::RPCEngine; + using RPCEngineType = rpc::RPCEngine; auto const rpcEngine = RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider); if (useNgWebServer or config_.get("server.__ng_web_server")) { - web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl}; + web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl}; auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_); if (not expectedAdminVerifier.has_value()) { @@ -188,8 +195,7 @@ ClioApplication::run(bool const useNgWebServer) } // Init the web server - auto handler = - std::make_shared>(config_, backend, rpcEngine, etl); + auto handler = std::make_shared>(config_, backend, rpcEngine, etl); auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler); diff --git a/src/app/Stopper.hpp b/src/app/Stopper.hpp index b4faa1377..daba1164e 100644 --- a/src/app/Stopper.hpp +++ b/src/app/Stopper.hpp @@ -20,8 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" -#include "etl/LoadBalancer.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/CoroutineGroup.hpp" #include "util/log/Logger.hpp" @@ -74,15 +74,12 @@ public: * @param ioc The io_context to stop. * @return The callback to be called on application stop. */ - template < - web::ng::SomeServer ServerType, - etl::SomeLoadBalancer LoadBalancerType, - etl::SomeETLService ETLServiceType> + template static std::function makeOnStopCallback( ServerType& server, - LoadBalancerType& balancer, - ETLServiceType& etl, + etlng::LoadBalancerInterface& balancer, + etlng::ETLServiceInterface& etl, feed::SubscriptionManagerInterface& subscriptions, data::BackendInterface& backend, boost::asio::io_context& ioc diff --git a/src/etl/ETLService.cpp b/src/etl/ETLService.cpp index 6eca76339..a7207c99b 100644 --- a/src/etl/ETLService.cpp +++ b/src/etl/ETLService.cpp @@ -22,6 +22,7 @@ #include "data/BackendInterface.hpp" #include "etl/CorruptionDetector.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "util/Assert.hpp" #include "util/Constants.hpp" @@ -43,6 +44,7 @@ #include namespace etl { + // Database must be populated when this starts std::optional ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors) @@ -265,7 +267,7 @@ ETLService::ETLService( boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) : backend_(backend) diff --git a/src/etl/ETLService.hpp b/src/etl/ETLService.hpp index d1d865475..ea8b7852d 100644 --- a/src/etl/ETLService.hpp +++ b/src/etl/ETLService.hpp @@ -32,7 +32,12 @@ #include "etl/impl/LedgerLoader.hpp" #include "etl/impl/LedgerPublisher.hpp" #include "etl/impl/Transformer.hpp" +#include "etlng/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancer.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" #include "util/log/Logger.hpp" #include @@ -41,7 +46,6 @@ #include #include -#include #include #include #include @@ -81,14 +85,13 @@ concept SomeETLService = std::derived_from; * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring * to writing and from writing to monitoring, based on the activity of other processes running on different machines. */ -class ETLService : public ETLServiceTag { +class ETLService : public etlng::ETLServiceInterface, ETLServiceTag { // TODO: make these template parameters in ETLService - using LoadBalancerType = LoadBalancer; using DataPipeType = etl::impl::ExtractionDataPipe; using CacheLoaderType = etl::CacheLoader<>; - using LedgerFetcherType = etl::impl::LedgerFetcher; + using LedgerFetcherType = etl::impl::LedgerFetcher; using ExtractorType = etl::impl::Extractor; - using LedgerLoaderType = etl::impl::LedgerLoader; + using LedgerLoaderType = etl::impl::LedgerLoader; using LedgerPublisherType = etl::impl::LedgerPublisher; using AmendmentBlockHandlerType = etl::impl::AmendmentBlockHandler; using TransformerType = @@ -97,7 +100,7 @@ class ETLService : public ETLServiceTag { util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::shared_ptr networkValidatedLedgers_; std::uint32_t extractorThreads_ = 1; @@ -132,7 +135,7 @@ public: boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ); @@ -154,20 +157,33 @@ public: * @param ledgers The network validated ledgers datastructure * @return A shared pointer to a new instance of ETLService */ - static std::shared_ptr + static std::shared_ptr makeETLService( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr balancer, + std::shared_ptr balancer, std::shared_ptr ledgers ) { - auto etl = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); - etl->run(); + std::shared_ptr ret; - return etl; + if (config.get("__ng_etl")) { + ASSERT( + std::dynamic_pointer_cast(balancer), + "LoadBalancer type must be etlng::LoadBalancer" + ); + ret = std::make_shared(config, backend, subscriptions, balancer, ledgers); + } else { + ASSERT( + std::dynamic_pointer_cast(balancer), "LoadBalancer type must be etl::LoadBalancer" + ); + ret = std::make_shared(config, ioc, backend, subscriptions, balancer, ledgers); + } + + ret->run(); + return ret; } /** @@ -184,7 +200,7 @@ public: * @note This method blocks until the ETL service has stopped. */ void - stop() + stop() override { LOG(log_.info()) << "Stop called"; @@ -203,7 +219,7 @@ public: * @return Time passed since last ledger close */ std::uint32_t - lastCloseAgeSeconds() const + lastCloseAgeSeconds() const override { return ledgerPublisher_.lastCloseAgeSeconds(); } @@ -214,7 +230,7 @@ public: * @return true if currently amendment blocked; false otherwise */ bool - isAmendmentBlocked() const + isAmendmentBlocked() const override { return state_.isAmendmentBlocked; } @@ -225,7 +241,7 @@ public: * @return true if corruption of DB was detected and cache was stopped. */ bool - isCorruptionDetected() const + isCorruptionDetected() const override { return state_.isCorruptionDetected; } @@ -236,7 +252,7 @@ public: * @return The state of ETL as a JSON object */ boost::json::object - getInfo() const + getInfo() const override { boost::json::object result; @@ -254,11 +270,17 @@ public: * @return The etl nodes' state, nullopt if etl nodes are not connected */ std::optional - getETLState() const noexcept + getETLState() const noexcept override { return loadBalancer_->getETLState(); } + /** + * @brief Start all components to run ETL service. + */ + void + run() override; + private: /** * @brief Run the ETL pipeline. @@ -325,12 +347,6 @@ private: return numMarkers_; } - /** - * @brief Start all components to run ETL service. - */ - void - run(); - /** * @brief Spawn the worker thread and start monitoring. */ diff --git a/src/etl/LoadBalancer.cpp b/src/etl/LoadBalancer.cpp index 7dc0d1925..a26c5bb58 100644 --- a/src/etl/LoadBalancer.cpp +++ b/src/etl/LoadBalancer.cpp @@ -23,6 +23,7 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "util/Assert.hpp" @@ -59,7 +60,7 @@ using namespace util::config; namespace etl { -std::shared_ptr +std::shared_ptr LoadBalancer::makeLoadBalancer( ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -175,12 +176,12 @@ LoadBalancer::~LoadBalancer() } std::vector -LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly, std::chrono::steady_clock::duration retryAfter) +LoadBalancer::loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter) { std::vector response; execute( - [this, &response, &sequence, cacheOnly](auto& source) { - auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, cacheOnly); + [this, &response, &sequence](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_); if (!res) { LOG(log_.error()) << "Failed to download initial ledger." diff --git a/src/etl/LoadBalancer.hpp b/src/etl/LoadBalancer.hpp index cfe01d019..169636685 100644 --- a/src/etl/LoadBalancer.hpp +++ b/src/etl/LoadBalancer.hpp @@ -23,8 +23,11 @@ #include "etl/ETLState.hpp" #include "etl/NetworkValidatedLedgersInterface.hpp" #include "etl/Source.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" +#include "util/Assert.hpp" #include "util/Mutex.hpp" #include "util/ResponseExpirationCache.hpp" #include "util/log/Logger.hpp" @@ -41,13 +44,13 @@ #include #include -#include #include #include #include #include #include #include +#include #include namespace etl { @@ -69,7 +72,7 @@ concept SomeLoadBalancer = std::derived_from; * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also * allows requests for ledger data to be load balanced across all possible ETL sources. */ -class LoadBalancer : public LoadBalancerTag { +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { public: using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; @@ -133,7 +136,7 @@ public: * @param sourceFactory A factory function to create a source * @return A shared pointer to a new instance of LoadBalancer */ - static std::shared_ptr + static std::shared_ptr makeLoadBalancer( util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, @@ -150,16 +153,32 @@ public: * @note This function will retry indefinitely until the ledger is downloaded. * * @param sequence Sequence of ledger to download - * @param cacheOnly Whether to only write to cache and not to the DB; defaults to false + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}) + override; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param observer The observer to notify of progress * @param retryAfter Time to wait between retries (2 seconds by default) * @return A std::vector The ledger data */ std::vector loadInitialLedger( - uint32_t sequence, - bool cacheOnly = false, - std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] etlng::InitialLoadObserverInterface& observer, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for old ETL"); + std::unreachable(); + } /** * @brief Fetch data for a specific ledger. @@ -180,7 +199,7 @@ public: bool getObjects, bool getObjectNeighbors, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} - ); + ) override; /** * @brief Represent the state of this load balancer as a JSON object @@ -188,7 +207,7 @@ public: * @return JSON representation of the state of this load balancer. */ boost::json::value - toJson() const; + toJson() const override; /** * @brief Forward a JSON RPC request to a randomly selected rippled node. @@ -205,14 +224,14 @@ public: std::optional const& clientIp, bool isAdmin, boost::asio::yield_context yield - ); + ) override; /** * @brief Return state of ETL nodes. * @return ETL state, nullopt if etl nodes not available */ std::optional - getETLState() noexcept; + getETLState() noexcept override; /** * @brief Stop the load balancer. This will stop all subscription sources. @@ -221,7 +240,7 @@ public: * @param yield The coroutine context */ void - stop(boost::asio::yield_context yield); + stop(boost::asio::yield_context yield) override; private: /** diff --git a/src/etl/NetworkValidatedLedgersInterface.hpp b/src/etl/NetworkValidatedLedgersInterface.hpp index d2ab9d12b..067407c8c 100644 --- a/src/etl/NetworkValidatedLedgersInterface.hpp +++ b/src/etl/NetworkValidatedLedgersInterface.hpp @@ -26,6 +26,7 @@ #include #include + namespace etl { /** diff --git a/src/etl/Source.hpp b/src/etl/Source.hpp index 91d9bf817..6ff7e7285 100644 --- a/src/etl/Source.hpp +++ b/src/etl/Source.hpp @@ -23,8 +23,6 @@ #include "etl/NetworkValidatedLedgersInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" -#include "util/log/Logger.hpp" -#include "util/newconfig/ConfigDefinition.hpp" #include "util/newconfig/ObjectView.hpp" #include @@ -130,11 +128,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ virtual std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0; + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) = 0; /** * @brief Forward a request to rippled. diff --git a/src/etl/impl/GrpcSource.cpp b/src/etl/impl/GrpcSource.cpp index 29be5b224..539cdf481 100644 --- a/src/etl/impl/GrpcSource.cpp +++ b/src/etl/impl/GrpcSource.cpp @@ -98,7 +98,7 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb } std::pair, bool> -GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers, bool const cacheOnly) +GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers) { if (!stub_) return {{}, false}; @@ -130,7 +130,7 @@ GrpcSource::loadInitialLedger(uint32_t const sequence, uint32_t const numMarkers LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix(); - auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly); + auto result = ptr->process(stub_, cq, *backend_, abort); if (result != etl::impl::AsyncCallData::CallStatus::MORE) { ++numFinished; LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished; diff --git a/src/etl/impl/GrpcSource.hpp b/src/etl/impl/GrpcSource.hpp index 5d41f193d..248f4191d 100644 --- a/src/etl/impl/GrpcSource.hpp +++ b/src/etl/impl/GrpcSource.hpp @@ -60,11 +60,10 @@ public: * * @param sequence Sequence of the ledger to download * @param numMarkers Number of markers to generate for async calls - * @param cacheOnly Only insert into cache, not the DB; defaults to false * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t numMarkers, bool cacheOnly = false); + loadInitialLedger(uint32_t sequence, uint32_t numMarkers); }; } // namespace etl::impl diff --git a/src/etl/impl/LedgerFetcher.hpp b/src/etl/impl/LedgerFetcher.hpp index 9d8a0df88..acb792ac2 100644 --- a/src/etl/impl/LedgerFetcher.hpp +++ b/src/etl/impl/LedgerFetcher.hpp @@ -20,6 +20,8 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/log/Logger.hpp" #include @@ -34,22 +36,18 @@ namespace etl::impl { /** * @brief GRPC Ledger data fetcher */ -template -class LedgerFetcher { -public: - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - +class LedgerFetcher : public LedgerFetcherInterface { private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; public: /** * @brief Create an instance of the fetcher */ - LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) + LedgerFetcher(std::shared_ptr backend, std::shared_ptr balancer) : backend_(std::move(backend)), loadBalancer_(std::move(balancer)) { } @@ -64,7 +62,7 @@ public: * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchData(uint32_t sequence) + fetchData(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; @@ -84,7 +82,7 @@ public: * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down */ [[nodiscard]] OptionalGetLedgerResponseType - fetchDataAndDiff(uint32_t sequence) + fetchDataAndDiff(uint32_t sequence) override { LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence; diff --git a/src/etl/impl/LedgerLoader.hpp b/src/etl/impl/LedgerLoader.hpp index 716fa8718..6be0f30e3 100644 --- a/src/etl/impl/LedgerLoader.hpp +++ b/src/etl/impl/LedgerLoader.hpp @@ -26,6 +26,7 @@ #include "etl/NFTHelpers.hpp" #include "etl/SystemState.hpp" #include "etl/impl/LedgerFetcher.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "util/Assert.hpp" #include "util/LedgerUtils.hpp" #include "util/Profiler.hpp" @@ -65,18 +66,18 @@ namespace etl::impl { /** * @brief Loads ledger data into the DB */ -template +template class LedgerLoader { public: - using GetLedgerResponseType = typename LoadBalancerType::GetLedgerResponseType; - using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType; - using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType; + using GetLedgerResponseType = etlng::LoadBalancerInterface::GetLedgerResponseType; + using OptionalGetLedgerResponseType = etlng::LoadBalancerInterface::OptionalGetLedgerResponseType; + using RawLedgerObjectType = etlng::LoadBalancerInterface::RawLedgerObjectType; private: util::Logger log_{"ETL"}; std::shared_ptr backend_; - std::shared_ptr loadBalancer_; + std::shared_ptr loadBalancer_; std::reference_wrapper fetcher_; std::reference_wrapper state_; // shared state for ETL @@ -86,7 +87,7 @@ public: */ LedgerLoader( std::shared_ptr backend, - std::shared_ptr balancer, + std::shared_ptr balancer, LedgerFetcherType& fetcher, SystemState const& state ) diff --git a/src/etl/impl/SourceImpl.hpp b/src/etl/impl/SourceImpl.hpp index 3f2f25a33..a2b4175e0 100644 --- a/src/etl/impl/SourceImpl.hpp +++ b/src/etl/impl/SourceImpl.hpp @@ -202,9 +202,9 @@ public: * @return A std::pair of the data and a bool indicating whether the download was successful */ std::pair, bool> - loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) final + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers) final { - return grpcSource_.loadInitialLedger(sequence, numMarkers, cacheOnly); + return grpcSource_.loadInitialLedger(sequence, numMarkers); } /** diff --git a/src/etlng/CMakeLists.txt b/src/etlng/CMakeLists.txt index d0f2854e9..49e443a34 100644 --- a/src/etlng/CMakeLists.txt +++ b/src/etlng/CMakeLists.txt @@ -2,10 +2,13 @@ add_library(clio_etlng) target_sources( clio_etlng - PRIVATE impl/AmendmentBlockHandler.cpp + PRIVATE LoadBalancer.cpp + Source.cpp + impl/AmendmentBlockHandler.cpp impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp + impl/ForwardingSource.cpp impl/Loading.cpp impl/Monitor.cpp impl/TaskManager.cpp diff --git a/src/etlng/ETLService.hpp b/src/etlng/ETLService.hpp new file mode 100644 index 000000000..960d6e456 --- /dev/null +++ b/src/etlng/ETLService.hpp @@ -0,0 +1,283 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCache.hpp" +#include "data/Types.hpp" +#include "etl/CacheLoader.hpp" +#include "etl/ETLState.hpp" +#include "etl/LedgerFetcherInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/SystemState.hpp" +#include "etl/impl/AmendmentBlockHandler.hpp" +#include "etl/impl/LedgerFetcher.hpp" +#include "etl/impl/LedgerPublisher.hpp" +#include "etlng/AmendmentBlockHandlerInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/ExtractorInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/impl/AmendmentBlockHandler.hpp" +#include "etlng/impl/Extraction.hpp" +#include "etlng/impl/Loading.hpp" +#include "etlng/impl/Monitor.hpp" +#include "etlng/impl/Registry.hpp" +#include "etlng/impl/Scheduling.hpp" +#include "etlng/impl/TaskManager.hpp" +#include "etlng/impl/ext/Cache.hpp" +#include "etlng/impl/ext/Core.hpp" +#include "etlng/impl/ext/NFT.hpp" +#include "etlng/impl/ext/Successor.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/Assert.hpp" +#include "util/Profiler.hpp" +#include "util/async/context/BasicExecutionContext.hpp" +#include "util/config/Config.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + * + * Usually, multiple different processes share access to the same network accessible databases, in which case only one + * such process is performing ETL and writing to the database. The other processes simply monitor the database for new + * ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the + * ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer. + * + * If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and + * the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring + * to writing and from writing to monitoring, based on the activity of other processes running on different machines. + */ +class ETLService : public ETLServiceInterface { + util::Logger log_{"ETL"}; + + std::shared_ptr backend_; + std::shared_ptr subscriptions_; + std::shared_ptr balancer_; + std::shared_ptr ledgers_; + std::shared_ptr> cacheLoader_; + + std::shared_ptr fetcher_; + std::shared_ptr extractor_; + + etl::SystemState state_; + util::async::CoroExecutionContext ctx_{8}; + + std::shared_ptr amendmentBlockHandler_; + std::shared_ptr loader_; + + std::optional> mainLoop_; + +public: + /** + * @brief Create an instance of ETLService. + * + * @param config The configuration to use + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param balancer Load balancer to use + * @param ledgers The network validated ledgers datastructure + */ + ETLService( + util::config::ClioConfigDefinition const& config, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr balancer, + std::shared_ptr ledgers + ) + : backend_(std::move(backend)) + , subscriptions_(std::move(subscriptions)) + , balancer_(std::move(balancer)) + , ledgers_(std::move(ledgers)) + , cacheLoader_(std::make_shared>(config, backend_, backend_->cache())) + , fetcher_(std::make_shared(backend_, balancer_)) + , extractor_(std::make_shared(fetcher_)) + , amendmentBlockHandler_(std::make_shared(ctx_, state_)) + , loader_(std::make_shared( + backend_, + fetcher_, + impl::makeRegistry( + impl::CacheExt{backend_->cache()}, + impl::CoreExt{backend_}, + impl::SuccessorExt{backend_, backend_->cache()}, + impl::NFTExt{backend_} + ), + amendmentBlockHandler_ + )) + { + LOG(log_.info()) << "Creating ETLng..."; + } + + ~ETLService() override + { + LOG(log_.debug()) << "Stopping ETLng"; + } + + void + run() override + { + LOG(log_.info()) << "run() in ETLng..."; + + mainLoop_.emplace(ctx_.execute([this] { + auto const rng = loadInitialLedgerIfNeeded(); + + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + std::optional mostRecentValidated = ledgers_->getMostRecent(); + + if (not mostRecentValidated) { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + return; + } + + ASSERT(rng.has_value(), "Ledger range can't be null"); + auto const nextSequence = rng->maxSequence + 1; + + LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence; + + auto scheduler = impl::makeScheduler(impl::ForwardScheduler{*ledgers_, nextSequence} + // impl::BackfillScheduler{nextSequence - 1, nextSequence - 1000}, + // TODO lift limit and start with rng.minSeq + ); + + auto man = impl::TaskManager(ctx_, *scheduler, *extractor_, *loader_); + + // TODO: figure out this: std::make_shared(backend_, ledgers_, nextSequence) + man.run({}); // TODO: needs to be interruptable and fill out settings + })); + } + + void + stop() override + { + LOG(log_.info()) << "Stop called"; + // TODO: stop the service correctly + } + + boost::json::object + getInfo() const override + { + // TODO + return {{"ok", true}}; + } + + bool + isAmendmentBlocked() const override + { + // TODO + return false; + } + + bool + isCorruptionDetected() const override + { + // TODO + return false; + } + + std::optional + getETLState() const override + { + // TODO + return std::nullopt; + } + + std::uint32_t + lastCloseAgeSeconds() const override + { + // TODO + return 0; + } + +private: + // TODO: this better be std::expected + std::optional + loadInitialLedgerIfNeeded() + { + if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); not rng.has_value()) { + LOG(log_.info()) << "Database is empty. Will download a ledger from the network."; + + try { + LOG(log_.info()) << "Waiting for next ledger to be validated by network..."; + if (auto const mostRecentValidated = ledgers_->getMostRecent(); mostRecentValidated.has_value()) { + auto const seq = *mostRecentValidated; + LOG(log_.info()) << "Ledger " << seq << " has been validated. Downloading... "; + + auto [ledger, timeDiff] = ::util::timed>([this, seq]() { + return extractor_->extractLedgerOnly(seq).and_then([this, seq](auto&& data) { + // TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar + data.edgeKeys = balancer_->loadInitialLedger(seq, *loader_); + + // TODO: this should be interruptable for graceful shutdown + return loader_->loadInitialLedger(data); + }); + }); + + LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff; + LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size(); + + if (ledger.has_value()) + return backend_->hardFetchLedgerRangeNoThrow(); + + LOG(log_.error()) << "Failed to load initial ledger. Exiting monitor loop"; + } else { + LOG(log_.info()) << "The wait for the next validated ledger has been aborted. " + "Exiting monitor loop"; + } + } catch (std::runtime_error const& e) { + LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what(); + amendmentBlockHandler_->notifyAmendmentBlocked(); + } + } else { + LOG(log_.info()) << "Database already populated. Picking up from the tip of history"; + cacheLoader_->load(rng->maxSequence); + + return rng; + } + + return std::nullopt; + } +}; +} // namespace etlng diff --git a/src/etlng/ETLServiceInterface.hpp b/src/etlng/ETLServiceInterface.hpp new file mode 100644 index 000000000..7f0b2fc2b --- /dev/null +++ b/src/etlng/ETLServiceInterface.hpp @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/ETLState.hpp" + +#include + +#include +#include + +namespace etlng { + +/** + * @brief This is a base class for any ETL service implementations. + * @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the + * databases. + */ +struct ETLServiceInterface { + virtual ~ETLServiceInterface() = default; + + /** + * @brief Start all components to run ETL service. + */ + virtual void + run() = 0; + + /** + * @brief Stop the ETL service. + * @note This method blocks until the ETL service has stopped. + */ + virtual void + stop() = 0; + + /** + * @brief Get state of ETL as a JSON object + * + * @return The state of ETL as a JSON object + */ + [[nodiscard]] virtual boost::json::object + getInfo() const = 0; + + /** + * @brief Check for the amendment blocked state. + * + * @return true if currently amendment blocked; false otherwise + */ + [[nodiscard]] virtual bool + isAmendmentBlocked() const = 0; + + /** + * @brief Check whether Clio detected DB corruptions. + * + * @return true if corruption of DB was detected and cache was stopped. + */ + [[nodiscard]] virtual bool + isCorruptionDetected() const = 0; + + /** + * @brief Get the etl nodes' state + * @return The etl nodes' state, nullopt if etl nodes are not connected + */ + [[nodiscard]] virtual std::optional + getETLState() const = 0; + + /** + * @brief Get time passed since last ledger close, in seconds. + * + * @return Time passed since last ledger close + */ + [[nodiscard]] virtual std::uint32_t + lastCloseAgeSeconds() const = 0; +}; + +} // namespace etlng diff --git a/src/etlng/LoadBalancer.cpp b/src/etlng/LoadBalancer.cpp new file mode 100644 index 000000000..fa34af78e --- /dev/null +++ b/src/etlng/LoadBalancer.cpp @@ -0,0 +1,371 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/LoadBalancer.hpp" + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/CoroutineGroup.hpp" +#include "util/Random.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace util::config; + +namespace etlng { + +std::shared_ptr +LoadBalancer::makeLoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) +{ + return std::make_shared( + config, ioc, std::move(backend), std::move(subscriptions), std::move(validatedLedgers), std::move(sourceFactory) + ); +} + +LoadBalancer::LoadBalancer( + ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory +) +{ + auto const forwardingCacheTimeout = config.get("forwarding.cache_timeout"); + if (forwardingCacheTimeout > 0.f) { + forwardingCache_ = util::ResponseExpirationCache{ + util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout), + {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"} + }; + } + + auto const numMarkers = config.getValueView("num_markers"); + if (numMarkers.hasValue()) { + auto const value = numMarkers.asIntType(); + downloadRanges_ = value; + } else if (backend->fetchLedgerRange()) { + downloadRanges_ = 4; + } + + auto const allowNoEtl = config.get("allow_no_etl"); + + auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) { + LOG(log_.warn()) << log; + + if (!allowNoEtl) { + LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources."; + throw std::logic_error("ETL configuration error."); + } + }; + + auto const forwardingTimeout = + ClioConfigDefinition::toMilliseconds(config.get("forwarding.request_timeout")); + auto const etlArray = config.getArray("etl_sources"); + for (auto it = etlArray.begin(); it != etlArray.end(); ++it) { + auto source = sourceFactory( + *it, + ioc, + subscriptions, + validatedLedgers, + forwardingTimeout, + [this]() { + if (not hasForwardingSource_.lock().get()) + chooseForwardingSource(); + }, + [this](bool wasForwarding) { + if (wasForwarding) + chooseForwardingSource(); + }, + [this]() { + if (forwardingCache_.has_value()) + forwardingCache_->invalidate(); + } + ); + + // checking etl node validity + auto const stateOpt = etl::ETLState::fetchETLStateFromSource(*source); + + if (!stateOpt) { + LOG(log_.warn()) << "Failed to fetch ETL state from source = " << source->toString() + << " Please check the configuration and network"; + } else if (etlState_ && etlState_->networkID && stateOpt->networkID && + etlState_->networkID != stateOpt->networkID) { + checkOnETLFailure(fmt::format( + "ETL sources must be on the same network. Source network id = {} does not match others network id = {}", + *(stateOpt->networkID), + *(etlState_->networkID) + )); + } else { + etlState_ = stateOpt; + } + + sources_.push_back(std::move(source)); + LOG(log_.info()) << "Added etl source - " << sources_.back()->toString(); + } + + if (!etlState_) + checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network"); + + if (sources_.empty()) + checkOnETLFailure("No ETL sources configured. Please check the configuration"); + + // This is made separate from source creation to prevent UB in case one of the sources will call + // chooseForwardingSource while we are still filling the sources_ vector + for (auto const& source : sources_) { + source->run(); + } +} + +LoadBalancer::~LoadBalancer() +{ + sources_.clear(); +} + +std::vector +LoadBalancer::loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& loadObserver, + std::chrono::steady_clock::duration retryAfter +) +{ + std::vector response; + execute( + [this, &response, &sequence, &loadObserver](auto& source) { + auto [data, res] = source->loadInitialLedger(sequence, downloadRanges_, loadObserver); + + if (!res) { + LOG(log_.error()) << "Failed to download initial ledger." + << " Sequence = " << sequence << " source = " << source->toString(); + } else { + response = std::move(data); + } + + return res; + }, + sequence, + retryAfter + ); + return response; +} + +LoadBalancer::OptionalGetLedgerResponseType +LoadBalancer::fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter +) +{ + GetLedgerResponseType response; + execute( + [&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) { + auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors); + response = std::move(data); + if (status.ok() && response.validated()) { + LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence + << " from source = " << source->toString(); + return true; + } + + LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString() + << ", error_code: " << status.error_code() << ", error_msg: " << status.error_message() + << ", source = " << source->toString(); + return false; + }, + ledgerSequence, + retryAfter + ); + return response; +} + +std::expected +LoadBalancer::forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield +) +{ + if (not request.contains("command")) + return std::unexpected{rpc::ClioError::RpcCommandIsMissing}; + + auto const cmd = boost::json::value_to(request.at("command")); + if (forwardingCache_) { + if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) { + return std::move(cachedResponse).value(); + } + } + + ASSERT(not sources_.empty(), "ETL sources must be configured to forward requests."); + std::size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + auto numAttempts = 0u; + + auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE; + + std::optional response; + rpc::ClioError error = rpc::ClioError::EtlConnectionError; + while (numAttempts < sources_.size()) { + auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); + if (res) { + response = std::move(res).value(); + break; + } + error = std::max(error, res.error()); // Choose the best result between all sources + + sourceIdx = (sourceIdx + 1) % sources_.size(); + ++numAttempts; + } + + if (response) { + if (forwardingCache_ and not response->contains("error")) + forwardingCache_->put(cmd, *response); + return std::move(response).value(); + } + + return std::unexpected{error}; +} + +boost::json::value +LoadBalancer::toJson() const +{ + boost::json::array ret; + for (auto& src : sources_) + ret.push_back(src->toJson()); + + return ret; +} + +template +void +LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter) +{ + ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions."); + size_t sourceIdx = util::Random::uniform(0ul, sources_.size() - 1); + + size_t numAttempts = 0; + + while (true) { + auto& source = sources_[sourceIdx]; + + LOG(log_.debug()) << "Attempting to execute func. ledger sequence = " << ledgerSequence + << " - source = " << source->toString(); + // Originally, it was (source->hasLedger(ledgerSequence) || true) + /* Sometimes rippled has ledger but doesn't actually know. However, + but this does NOT happen in the normal case and is safe to remove + This || true is only needed when loading full history standalone */ + if (source->hasLedger(ledgerSequence)) { + bool const res = f(source); + if (res) { + LOG(log_.debug()) << "Successfully executed func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + break; + } + + LOG(log_.warn()) << "Failed to execute func at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } else { + LOG(log_.warn()) << "Ledger not present at source = " << source->toString() + << " - ledger sequence = " << ledgerSequence; + } + sourceIdx = (sourceIdx + 1) % sources_.size(); + numAttempts++; + if (numAttempts % sources_.size() == 0) { + LOG(log_.info()) << "Ledger sequence " << ledgerSequence + << " is not yet available from any configured sources. Sleeping and trying again"; + std::this_thread::sleep_for(retryAfter); + } + } +} + +std::optional +LoadBalancer::getETLState() noexcept +{ + if (!etlState_) { + // retry ETLState fetch + etlState_ = etl::ETLState::fetchETLStateFromSource(*this); + } + return etlState_; +} + +void +LoadBalancer::stop(boost::asio::yield_context yield) +{ + util::CoroutineGroup group{yield}; + std::ranges::for_each(sources_, [&group, yield](auto& source) { + group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); }); + }); + group.asyncWait(yield); +} + +void +LoadBalancer::chooseForwardingSource() +{ + LOG(log_.info()) << "Choosing a new source to forward subscriptions"; + auto hasForwardingSourceLock = hasForwardingSource_.lock(); + hasForwardingSourceLock.get() = false; + for (auto& source : sources_) { + if (not hasForwardingSourceLock.get() and source->isConnected()) { + source->setForwarding(true); + hasForwardingSourceLock.get() = true; + } else { + source->setForwarding(false); + } + } +} + +} // namespace etlng diff --git a/src/etlng/LoadBalancer.hpp b/src/etlng/LoadBalancer.hpp new file mode 100644 index 000000000..ac01100da --- /dev/null +++ b/src/etlng/LoadBalancer.hpp @@ -0,0 +1,271 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2022, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/ETLState.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/Assert.hpp" +#include "util/Mutex.hpp" +#include "util/ResponseExpirationCache.hpp" +#include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief A tag class to help identify LoadBalancer in templated code. + */ +struct LoadBalancerTag { + virtual ~LoadBalancerTag() = default; +}; + +template +concept SomeLoadBalancer = std::derived_from; + +/** + * @brief This class is used to manage connections to transaction processing processes. + * + * This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of + * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also + * allows requests for ledger data to be load balanced across all possible ETL sources. + */ +class LoadBalancer : public etlng::LoadBalancerInterface, LoadBalancerTag { +public: + using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject; + using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse; + using OptionalGetLedgerResponseType = std::optional; + +private: + static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16; + + util::Logger log_{"ETL"}; + // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache + std::optional forwardingCache_; + std::optional forwardingXUserValue_; + + std::vector sources_; + std::optional etlState_; + std::uint32_t downloadRanges_ = + kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */ + + // Using mutext instead of atomic_bool because choosing a new source to + // forward messages should be done with a mutual exclusion otherwise there will be a race condition + util::Mutex hasForwardingSource_{false}; + +public: + /** + * @brief Value for the X-User header when forwarding admin requests + */ + static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin"; + + /** + * @brief Value for the X-User header when forwarding user requests + */ + static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user"; + + /** + * @brief Create an instance of the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + */ + LoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + /** + * @brief A factory function for the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param sourceFactory A factory function to create a source + * @return A shared pointer to a new instance of LoadBalancer + */ + static std::shared_ptr + makeLoadBalancer( + util::config::ClioConfigDefinition const& config, + boost::asio::io_context& ioc, + std::shared_ptr backend, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + SourceFactory sourceFactory = makeSource + ); + + ~LoadBalancer() override; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + [[maybe_unused]] uint32_t sequence, + [[maybe_unused]] std::chrono::steady_clock::duration retryAfter + ) override + { + ASSERT(false, "Not available for new ETL"); + std::unreachable(); + }; + + /** + * @brief Load the initial ledger, writing data to the queue. + * @note This function will retry indefinitely until the ledger is downloaded. + * + * @param sequence Sequence of ledger to download + * @param observer The observer to notify of progress + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return A std::vector The ledger data + */ + std::vector + loadInitialLedger( + uint32_t sequence, + etlng::InitialLoadObserverInterface& observer, + std::chrono::steady_clock::duration retryAfter + ) override; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param ledgerSequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one + * @param getObjectNeighbors Whether to request object neighbors + * @param retryAfter Time to wait between retries (2 seconds by default) + * @return The extracted data, if extraction was successful. If the ledger was found + * in the database or the server is shutting down, the optional will be empty + */ + OptionalGetLedgerResponseType + fetchLedger( + uint32_t ledgerSequence, + bool getObjects, + bool getObjectNeighbors, + std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2} + ) override; + + /** + * @brief Represent the state of this load balancer as a JSON object + * + * @return JSON representation of the state of this load balancer. + */ + boost::json::value + toJson() const override; + + /** + * @brief Forward a JSON RPC request to a randomly selected rippled node. + * + * @param request JSON-RPC request to forward + * @param clientIp The IP address of the peer, if known + * @param isAdmin Whether the request is from an admin + * @param yield The coroutine context + * @return Response received from rippled node as JSON object on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& clientIp, + bool isAdmin, + boost::asio::yield_context yield + ) override; + + /** + * @brief Return state of ETL nodes. + * @return ETL state, nullopt if etl nodes not available + */ + std::optional + getETLState() noexcept override; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + void + stop(boost::asio::yield_context yield) override; + +private: + /** + * @brief Execute a function on a randomly selected source. + * + * @note f is a function that takes an Source as an argument and returns a bool. + * Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another + * randomly chosen Source is used. The process repeats until f returns true. + * + * @param f Function to execute. This function takes the ETL source as an argument, and returns a bool + * @param ledgerSequence f is executed for each Source that has this ledger + * @param retryAfter Time to wait between retries (2 seconds by default) + * server is shutting down + */ + template + void + execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}); + + /** + * @brief Choose a new source to forward requests + */ + void + chooseForwardingSource(); +}; + +} // namespace etlng diff --git a/src/etlng/LoadBalancerInterface.hpp b/src/etlng/LoadBalancerInterface.hpp index 3466a6e79..20623d456 100644 --- a/src/etlng/LoadBalancerInterface.hpp +++ b/src/etlng/LoadBalancerInterface.hpp @@ -129,6 +129,15 @@ public: */ virtual std::optional getETLState() noexcept = 0; + + /** + * @brief Stop the load balancer. This will stop all subscription sources. + * @note This function will asynchronously wait for all sources to stop. + * + * @param yield The coroutine context + */ + virtual void + stop(boost::asio::yield_context yield) = 0; }; } // namespace etlng diff --git a/src/etlng/LoaderInterface.hpp b/src/etlng/LoaderInterface.hpp index 929d71679..72cad3192 100644 --- a/src/etlng/LoaderInterface.hpp +++ b/src/etlng/LoaderInterface.hpp @@ -45,7 +45,7 @@ struct LoaderInterface { * @param data The data to load * @return Optional ledger header */ - virtual std::optional + [[nodiscard]] virtual std::optional loadInitialLedger(model::LedgerData const& data) = 0; }; diff --git a/src/etlng/Source.cpp b/src/etlng/Source.cpp new file mode 100644 index 000000000..3f9395446 --- /dev/null +++ b/src/etlng/Source.cpp @@ -0,0 +1,73 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Source.hpp" + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "etlng/impl/SourceImpl.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include + +#include +#include +#include +#include + +namespace etlng { + +SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +) +{ + auto const ip = config.get("ip"); + auto const wsPort = config.get("ws_port"); + auto const grpcPort = config.get("grpc_port"); + + etl::impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout}; + impl::GrpcSource grpcSource{ip, grpcPort}; + auto subscriptionSource = std::make_unique( + ioc, + ip, + wsPort, + std::move(validatedLedgers), + std::move(subscriptions), + std::move(onConnect), + std::move(onDisconnect), + std::move(onLedgerClosed) + ); + + return std::make_unique>( + ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource) + ); +} + +} // namespace etlng diff --git a/src/etlng/Source.hpp b/src/etlng/Source.hpp new file mode 100644 index 000000000..52c07ef9b --- /dev/null +++ b/src/etlng/Source.hpp @@ -0,0 +1,194 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng { + +/** + * @brief Provides an implementation of a ETL source + */ +class SourceBase { +public: + using OnConnectHook = std::function; + using OnDisconnectHook = std::function; + using OnLedgerClosedHook = std::function; + + virtual ~SourceBase() = default; + + /** + * @brief Run subscriptions loop of the source + */ + virtual void + run() = 0; + + /** + * @brief Stop Source. + * @note This method will asynchronously wait for source to be stopped. + * + * @param yield The coroutine context. + */ + virtual void + stop(boost::asio::yield_context yield) = 0; + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + [[nodiscard]] virtual bool + isConnected() const = 0; + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + virtual void + setForwarding(bool isForwarding) = 0; + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + [[nodiscard]] virtual boost::json::object + toJson() const = 0; + + /** @return String representation of the source (for debug) */ + [[nodiscard]] virtual std::string + toString() const = 0; + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + [[nodiscard]] virtual bool + hasLedger(uint32_t sequence) const = 0; + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + [[nodiscard]] virtual std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0; + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + virtual std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) = 0; + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Value of the X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + [[nodiscard]] virtual std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const = 0; +}; + +using SourcePtr = std::unique_ptr; + +using SourceFactory = std::function subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +)>; + +/** + * @brief Create a source + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers data structure + * @param forwardingTimeout The timeout for forwarding to rippled + * @param onConnect The hook to call on connect + * @param onDisconnect The hook to call on disconnect + * @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set + * as forwarding. + * @return The created source + */ +[[nodiscard]] SourcePtr +makeSource( + util::config::ObjectView const& config, + boost::asio::io_context& ioc, + std::shared_ptr subscriptions, + std::shared_ptr validatedLedgers, + std::chrono::steady_clock::duration forwardingTimeout, + SourceBase::OnConnectHook onConnect, + SourceBase::OnDisconnectHook onDisconnect, + SourceBase::OnLedgerClosedHook onLedgerClosed +); + +} // namespace etlng diff --git a/src/etlng/impl/ForwardingSource.cpp b/src/etlng/impl/ForwardingSource.cpp new file mode 100644 index 000000000..b4fb024bf --- /dev/null +++ b/src/etlng/impl/ForwardingSource.cpp @@ -0,0 +1,116 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ForwardingSource.hpp" + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +ForwardingSource::ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout +) + : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort)) + , connectionBuilder_(std::move(ip), std::move(wsPort)) + , forwardingTimeout_{forwardingTimeout} +{ + connectionBuilder_.setConnectionTimeout(connTimeout) + .addHeader( + {boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)} + ); +} + +std::expected +ForwardingSource::forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield +) const +{ + auto connectionBuilder = connectionBuilder_; + if (forwardToRippledClientIp) { + connectionBuilder.addHeader( + {boost::beast::http::field::forwarded, fmt::format("for={}", *forwardToRippledClientIp)} + ); + } + + connectionBuilder.addHeader({"X-User", std::string{xUserValue}}); + + auto expectedConnection = connectionBuilder.connect(yield); + if (not expectedConnection) { + LOG(log_.debug()) << "Couldn't connect to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlConnectionError}; + } + auto& connection = expectedConnection.value(); + + auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_); + if (writeError) { + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + auto response = connection->read(yield, forwardingTimeout_); + if (not response) { + if (auto errorCode = response.error().errorCode(); + errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) { + LOG(log_.debug()) << "Request to rippled timed out"; + return std::unexpected{rpc::ClioError::EtlRequestTimeout}; + } + LOG(log_.debug()) << "Error sending request to rippled to forward request."; + return std::unexpected{rpc::ClioError::EtlRequestError}; + } + + boost::json::value parsedResponse; + try { + parsedResponse = boost::json::parse(*response); + if (not parsedResponse.is_object()) + throw std::runtime_error("response is not an object"); + } catch (std::exception const& e) { + LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response; + return std::unexpected{rpc::ClioError::EtlInvalidResponse}; + } + + auto responseObject = parsedResponse.as_object(); + responseObject["forwarded"] = true; + + return responseObject; +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ForwardingSource.hpp b/src/etlng/impl/ForwardingSource.hpp new file mode 100644 index 000000000..8624e4e50 --- /dev/null +++ b/src/etlng/impl/ForwardingSource.hpp @@ -0,0 +1,70 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "rpc/Errors.hpp" +#include "util/log/Logger.hpp" +#include "util/requests/WsConnection.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class ForwardingSource { + util::Logger log_; + util::requests::WsConnectionBuilder connectionBuilder_; + std::chrono::steady_clock::duration forwardingTimeout_; + + static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3}; + +public: + ForwardingSource( + std::string ip, + std::string wsPort, + std::chrono::steady_clock::duration forwardingTimeout, + std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT + ); + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value for X-User header + * @param yield The coroutine context + * @return Response on success or error on failure + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/Loading.hpp b/src/etlng/impl/Loading.hpp index 435dcb4e4..caa677bce 100644 --- a/src/etlng/impl/Loading.hpp +++ b/src/etlng/impl/Loading.hpp @@ -39,7 +39,6 @@ #include #include -#include #include #include #include diff --git a/src/etlng/impl/Registry.hpp b/src/etlng/impl/Registry.hpp index d5d262388..921b70811 100644 --- a/src/etlng/impl/Registry.hpp +++ b/src/etlng/impl/Registry.hpp @@ -24,7 +24,6 @@ #include -#include #include #include #include @@ -81,7 +80,7 @@ concept ContainsValidHook = HasLedgerDataHook or HasInitialDataHook or template concept NoTwoOfKind = not(HasLedgerDataHook and HasTransactionHook) and - not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialDataHook and HasObjectHook) and + not(HasInitialDataHook and HasInitialTransactionHook) and not(HasInitialObjectsHook and HasInitialObjectHook); template @@ -216,4 +215,10 @@ public: } }; +static auto +makeRegistry(auto&&... exts) +{ + return std::make_unique...>>(std::forward(exts)...); +} + } // namespace etlng::impl diff --git a/src/etlng/impl/SourceImpl.hpp b/src/etlng/impl/SourceImpl.hpp new file mode 100644 index 000000000..1c99d973b --- /dev/null +++ b/src/etlng/impl/SourceImpl.hpp @@ -0,0 +1,232 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2024, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "etl/impl/ForwardingSource.hpp" +#include "etl/impl/SubscriptionSource.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "etlng/impl/GrpcSource.hpp" +#include "rpc/Errors.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +/** + * @brief Provides an implementation of a ETL source + * + * @tparam GrpcSourceType The type of the gRPC source + * @tparam SubscriptionSourceTypePtr The type of the subscription source + * @tparam ForwardingSourceType The type of the forwarding source + */ +template < + typename GrpcSourceType = GrpcSource, + typename SubscriptionSourceTypePtr = std::unique_ptr, + typename ForwardingSourceType = etl::impl::ForwardingSource> +class SourceImpl : public SourceBase { + std::string ip_; + std::string wsPort_; + std::string grpcPort_; + + GrpcSourceType grpcSource_; + SubscriptionSourceTypePtr subscriptionSource_; + ForwardingSourceType forwardingSource_; + +public: + /** + * @brief Construct a new SourceImpl object + * + * @param ip The IP of the source + * @param wsPort The web socket port of the source + * @param grpcPort The gRPC port of the source + * @param grpcSource The gRPC source + * @param subscriptionSource The subscription source + * @param forwardingSource The forwarding source + */ + template + requires std::is_same_v and + std::is_same_v + SourceImpl( + std::string ip, + std::string wsPort, + std::string grpcPort, + SomeGrpcSourceType&& grpcSource, + SubscriptionSourceTypePtr subscriptionSource, + SomeForwardingSourceType&& forwardingSource + ) + : ip_(std::move(ip)) + , wsPort_(std::move(wsPort)) + , grpcPort_(std::move(grpcPort)) + , grpcSource_(std::forward(grpcSource)) + , subscriptionSource_(std::move(subscriptionSource)) + , forwardingSource_(std::forward(forwardingSource)) + { + } + + /** + * @brief Run subscriptions loop of the source + */ + void + run() final + { + subscriptionSource_->run(); + } + + void + stop(boost::asio::yield_context yield) final + { + subscriptionSource_->stop(yield); + } + + /** + * @brief Check if source is connected + * + * @return true if source is connected; false otherwise + */ + bool + isConnected() const final + { + return subscriptionSource_->isConnected(); + } + + /** + * @brief Set the forwarding state of the source. + * + * @param isForwarding Whether to forward or not + */ + void + setForwarding(bool isForwarding) final + { + subscriptionSource_->setForwarding(isForwarding); + } + + /** + * @brief Represent the source as a JSON object + * + * @return JSON representation of the source + */ + boost::json::object + toJson() const final + { + boost::json::object res; + + res["validated_range"] = subscriptionSource_->validatedRange(); + res["is_connected"] = std::to_string(static_cast(subscriptionSource_->isConnected())); + res["ip"] = ip_; + res["ws_port"] = wsPort_; + res["grpc_port"] = grpcPort_; + + auto last = subscriptionSource_->lastMessageTime(); + if (last.time_since_epoch().count() != 0) { + res["last_msg_age_seconds"] = std::to_string( + std::chrono::duration_cast(std::chrono::steady_clock::now() - last).count() + ); + } + + return res; + } + + /** @return String representation of the source (for debug) */ + std::string + toString() const final + { + return "{validated range: " + subscriptionSource_->validatedRange() + ", ip: " + ip_ + + ", web socket port: " + wsPort_ + ", grpc port: " + grpcPort_ + "}"; + } + + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ + bool + hasLedger(uint32_t sequence) const final + { + return subscriptionSource_->hasLedger(sequence); + } + + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ + std::pair + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) final + { + return grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors); + } + + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param loader InitialLoadObserverInterface implementation + * @return A std::pair of the data and a bool indicating whether the download was successful + */ + std::pair, bool> + loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, etlng::InitialLoadObserverInterface& loader) final + { + return grpcSource_.loadInitialLedger(sequence, numMarkers, loader); + } + + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param forwardToRippledClientIp IP of the client forwarding this request if known + * @param xUserValue Optional value of the X-User header + * @param yield The coroutine context + * @return Response or ClioError + */ + std::expected + forwardToRippled( + boost::json::object const& request, + std::optional const& forwardToRippledClientIp, + std::string_view xUserValue, + boost::asio::yield_context yield + ) const final + { + return forwardingSource_.forwardToRippled(request, forwardToRippledClientIp, xUserValue, yield); + } +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.hpp b/src/etlng/impl/ext/NFT.hpp index 65e45233c..6d6a146cf 100644 --- a/src/etlng/impl/ext/NFT.hpp +++ b/src/etlng/impl/ext/NFT.hpp @@ -20,15 +20,11 @@ #pragma once #include "data/BackendInterface.hpp" -#include "data/DBHelpers.hpp" -#include "etl/NFTHelpers.hpp" #include "etlng/Models.hpp" #include "util/log/Logger.hpp" #include #include -#include -#include namespace etlng::impl { diff --git a/src/rpc/RPCEngine.hpp b/src/rpc/RPCEngine.hpp index 13216c20d..fe46db104 100644 --- a/src/rpc/RPCEngine.hpp +++ b/src/rpc/RPCEngine.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/WorkQueue.hpp" @@ -55,7 +56,7 @@ namespace rpc { /** * @brief The RPC engine that ties all RPC-related functionality together. */ -template +template class RPCEngine { util::Logger perfLog_{"Performance"}; util::Logger log_{"RPC"}; @@ -67,7 +68,7 @@ class RPCEngine { std::shared_ptr handlerProvider_; - impl::ForwardingProxy forwardingProxy_; + impl::ForwardingProxy forwardingProxy_; std::optional responseCache_; @@ -86,7 +87,7 @@ public: RPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, @@ -128,7 +129,7 @@ public: makeRPCEngine( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, WorkQueue& workQueue, CountersType& counters, diff --git a/src/rpc/common/impl/ForwardingProxy.hpp b/src/rpc/common/impl/ForwardingProxy.hpp index c60773df1..2c07b79ab 100644 --- a/src/rpc/common/impl/ForwardingProxy.hpp +++ b/src/rpc/common/impl/ForwardingProxy.hpp @@ -19,6 +19,7 @@ #pragma once +#include "etlng/LoadBalancerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/RPCHelpers.hpp" #include "rpc/common/Types.hpp" @@ -31,20 +32,21 @@ #include #include #include +#include namespace rpc::impl { -template +template class ForwardingProxy { util::Logger log_{"RPC"}; - std::shared_ptr balancer_; + std::shared_ptr balancer_; std::reference_wrapper counters_; std::shared_ptr handlerProvider_; public: ForwardingProxy( - std::shared_ptr const& balancer, + std::shared_ptr const& balancer, CountersType& counters, std::shared_ptr const& handlerProvider ) diff --git a/src/rpc/common/impl/HandlerProvider.cpp b/src/rpc/common/impl/HandlerProvider.cpp index cf4485cc2..6707cca07 100644 --- a/src/rpc/common/impl/HandlerProvider.cpp +++ b/src/rpc/common/impl/HandlerProvider.cpp @@ -21,7 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" -#include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Counters.hpp" #include "rpc/common/AnyHandler.hpp" @@ -72,8 +73,8 @@ ProductionHandlerProvider::ProductionHandlerProvider( util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ) diff --git a/src/rpc/common/impl/HandlerProvider.hpp b/src/rpc/common/impl/HandlerProvider.hpp index 89ea5661f..2c07428cf 100644 --- a/src/rpc/common/impl/HandlerProvider.hpp +++ b/src/rpc/common/impl/HandlerProvider.hpp @@ -21,6 +21,8 @@ #include "data/AmendmentCenterInterface.hpp" #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/common/AnyHandler.hpp" #include "rpc/common/HandlerProvider.hpp" @@ -32,10 +34,6 @@ #include #include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -55,8 +53,8 @@ public: util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, std::shared_ptr const& amendmentCenter, Counters const& counters ); diff --git a/src/rpc/handlers/ServerInfo.hpp b/src/rpc/handlers/ServerInfo.hpp index 008450721..085066f7c 100644 --- a/src/rpc/handlers/ServerInfo.hpp +++ b/src/rpc/handlers/ServerInfo.hpp @@ -21,6 +21,8 @@ #include "data/BackendInterface.hpp" #include "data/DBHelpers.hpp" +#include "etlng/ETLServiceInterface.hpp" +#include "etlng/LoadBalancerInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" @@ -49,10 +51,6 @@ #include #include -namespace etl { -class ETLService; -class LoadBalancer; -} // namespace etl namespace rpc { class Counters; } // namespace rpc @@ -62,18 +60,16 @@ namespace rpc { /** * @brief Contains common functionality for handling the `server_info` command * - * @tparam LoadBalancerType The type of the load balancer - * @tparam ETLServiceType The type of the ETL service * @tparam CountersType The type of the counters */ -template +template class BaseServerInfoHandler { static constexpr auto kBACKEND_COUNTERS_KEY = "backend_counters"; std::shared_ptr backend_; std::shared_ptr subscriptions_; - std::shared_ptr balancer_; - std::shared_ptr etl_; + std::shared_ptr balancer_; + std::shared_ptr etl_; std::reference_wrapper counters_; public: @@ -158,8 +154,8 @@ public: BaseServerInfoHandler( std::shared_ptr const& backend, std::shared_ptr const& subscriptions, - std::shared_ptr const& balancer, - std::shared_ptr const& etl, + std::shared_ptr const& balancer, + std::shared_ptr const& etl, CountersType const& counters ) : backend_(backend) @@ -352,6 +348,6 @@ private: * * For more details see: https://xrpl.org/server_info-clio.html */ -using ServerInfoHandler = BaseServerInfoHandler; +using ServerInfoHandler = BaseServerInfoHandler; } // namespace rpc diff --git a/src/rpc/handlers/Tx.hpp b/src/rpc/handlers/Tx.hpp index b05a64c1e..acc5de4a3 100644 --- a/src/rpc/handlers/Tx.hpp +++ b/src/rpc/handlers/Tx.hpp @@ -22,6 +22,7 @@ #include "data/BackendInterface.hpp" #include "data/Types.hpp" #include "etl/ETLService.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/JS.hpp" #include "rpc/RPCHelpers.hpp" @@ -52,14 +53,13 @@ namespace rpc { /** - * @brief Contains common functionality for handling the `tx` command + * @brief The tx method retrieves information on a single transaction, by its identifying hash. * - * @tparam ETLServiceType The type of the ETL service to use + * For more details see: https://xrpl.org/tx.html */ -template -class BaseTxHandler { +class TxHandler { std::shared_ptr sharedPtrBackend_; - std::shared_ptr etl_; + std::shared_ptr etl_; public: /** @@ -95,14 +95,14 @@ public: using Result = HandlerReturnType; /** - * @brief Construct a new BaseTxHandler object + * @brief Construct a new TxHandler object * * @param sharedPtrBackend The backend to use * @param etl The ETL service to use */ - BaseTxHandler( + TxHandler( std::shared_ptr const& sharedPtrBackend, - std::shared_ptr const& etl + std::shared_ptr const& etl ) : sharedPtrBackend_(sharedPtrBackend), etl_(etl) { @@ -183,7 +183,7 @@ public: dbResponse = sharedPtrBackend_->fetchTransaction(ripple::uint256{input.transaction->c_str()}, ctx.yield); } - auto output = BaseTxHandler::Output{.apiVersion = ctx.apiVersion}; + auto output = TxHandler::Output{.apiVersion = ctx.apiVersion}; if (!dbResponse) { if (rangeSupplied && input.transaction) // ranges not for ctid @@ -320,7 +320,7 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv) { - auto input = BaseTxHandler::Input{}; + auto input = TxHandler::Input{}; auto const& jsonObject = jv.as_object(); if (jsonObject.contains(JS(transaction))) @@ -344,10 +344,4 @@ private: } }; -/** - * @brief The tx method retrieves information on a single transaction, by its identifying hash. - * - * For more details see: https://xrpl.org/tx.html - */ -using TxHandler = BaseTxHandler; } // namespace rpc diff --git a/src/util/newconfig/ConfigDefinition.hpp b/src/util/newconfig/ConfigDefinition.hpp index 030b8d160..b00cbd1cb 100644 --- a/src/util/newconfig/ConfigDefinition.hpp +++ b/src/util/newconfig/ConfigDefinition.hpp @@ -293,7 +293,7 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{ {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - + {"__ng_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}}, {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}}, diff --git a/src/web/RPCServerHandler.hpp b/src/web/RPCServerHandler.hpp index 632d9551e..a0abacb7c 100644 --- a/src/web/RPCServerHandler.hpp +++ b/src/web/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -58,11 +59,11 @@ namespace web { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed @@ -82,7 +83,7 @@ public: util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl ) : backend_(backend) , rpcEngine_(rpcEngine) diff --git a/src/web/ng/RPCServerHandler.hpp b/src/web/ng/RPCServerHandler.hpp index f8dbcb183..fec8a22f7 100644 --- a/src/web/ng/RPCServerHandler.hpp +++ b/src/web/ng/RPCServerHandler.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/BackendInterface.hpp" +#include "etlng/ETLServiceInterface.hpp" #include "rpc/Errors.hpp" #include "rpc/Factories.hpp" #include "rpc/JS.hpp" @@ -64,11 +65,11 @@ namespace web::ng { * * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const etl_; util::TagDecoratorFactory const tagFactory_; rpc::impl::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed @@ -88,7 +89,7 @@ public: util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl + std::shared_ptr const& etl ) : backend_(backend) , rpcEngine_(rpcEngine) diff --git a/tests/common/util/MockETLService.hpp b/tests/common/util/MockETLService.hpp index 618e5ab57..f398ad9cd 100644 --- a/tests/common/util/MockETLService.hpp +++ b/tests/common/util/MockETLService.hpp @@ -20,21 +20,21 @@ #pragma once #include "etl/ETLState.hpp" +#include "etlng/ETLServiceInterface.hpp" #include #include #include -#include #include #include -struct MockETLService { - MOCK_METHOD(boost::json::object, getInfo, (), (const)); - MOCK_METHOD(std::chrono::time_point, getLastPublish, (), (const)); - MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const)); - MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const)); - MOCK_METHOD(bool, isAmendmentBlocked, (), (const)); - MOCK_METHOD(bool, isCorruptionDetected, (), (const)); - MOCK_METHOD(std::optional, getETLState, (), (const)); +struct MockETLService : etlng::ETLServiceInterface { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (), (override)); + MOCK_METHOD(boost::json::object, getInfo, (), (const, override)); + MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const, override)); + MOCK_METHOD(bool, isAmendmentBlocked, (), (const, override)); + MOCK_METHOD(bool, isCorruptionDetected, (), (const, override)); + MOCK_METHOD(std::optional, getETLState, (), (const, override)); }; diff --git a/tests/common/util/MockLoadBalancer.hpp b/tests/common/util/MockLoadBalancer.hpp index 12bcfea78..f0b9dd313 100644 --- a/tests/common/util/MockLoadBalancer.hpp +++ b/tests/common/util/MockLoadBalancer.hpp @@ -38,22 +38,6 @@ #include #include -struct MockLoadBalancer { - using RawLedgerObjectType = FakeLedgerObject; - - MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ()); - MOCK_METHOD(std::optional, fetchLedger, (uint32_t, bool, bool), ()); - MOCK_METHOD(boost::json::value, toJson, (), (const)); - - using ForwardToRippledReturnType = std::expected; - MOCK_METHOD( - ForwardToRippledReturnType, - forwardToRippled, - (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), - (const) - ); -}; - struct MockNgLoadBalancer : etlng::LoadBalancerInterface { using RawLedgerObjectType = FakeLedgerObject; @@ -85,4 +69,7 @@ struct MockNgLoadBalancer : etlng::LoadBalancerInterface { (boost::json::object const&, std::optional const&, bool, boost::asio::yield_context), (override) ); + MOCK_METHOD(void, stop, (boost::asio::yield_context), ()); }; + +using MockLoadBalancer = MockNgLoadBalancer; diff --git a/tests/common/util/MockSource.hpp b/tests/common/util/MockSource.hpp index 58984d485..a982f3aef 100644 --- a/tests/common/util/MockSource.hpp +++ b/tests/common/util/MockSource.hpp @@ -60,7 +60,7 @@ struct MockSource : etl::SourceBase { (uint32_t, bool, bool), (override) ); - MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t, bool), (override)); + MOCK_METHOD((std::pair, bool>), loadInitialLedger, (uint32_t, uint32_t), (override)); using ForwardToRippledReturnType = std::expected; MOCK_METHOD( @@ -132,9 +132,9 @@ public: } std::pair, bool> - loadInitialLedger(uint32_t sequence, uint32_t maxLedger, bool getObjects) override + loadInitialLedger(uint32_t sequence, uint32_t maxLedger) override { - return mock_->loadInitialLedger(sequence, maxLedger, getObjects); + return mock_->loadInitialLedger(sequence, maxLedger); } std::expected diff --git a/tests/common/util/MockSourceNg.hpp b/tests/common/util/MockSourceNg.hpp new file mode 100644 index 000000000..7137e47ec --- /dev/null +++ b/tests/common/util/MockSourceNg.hpp @@ -0,0 +1,245 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== +#pragma once + +#include "etl/NetworkValidatedLedgersInterface.hpp" +#include "etlng/InitialLoadObserverInterface.hpp" +#include "etlng/Source.hpp" +#include "feed/SubscriptionManagerInterface.hpp" +#include "rpc/Errors.hpp" +#include "util/newconfig/ObjectView.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct MockSourceNg : etlng::SourceBase { + MOCK_METHOD(void, run, (), (override)); + MOCK_METHOD(void, stop, (boost::asio::yield_context), (override)); + MOCK_METHOD(bool, isConnected, (), (const, override)); + MOCK_METHOD(void, setForwarding, (bool), (override)); + MOCK_METHOD(boost::json::object, toJson, (), (const, override)); + MOCK_METHOD(std::string, toString, (), (const, override)); + MOCK_METHOD(bool, hasLedger, (uint32_t), (const, override)); + MOCK_METHOD( + (std::pair), + fetchLedger, + (uint32_t, bool, bool), + (override) + ); + MOCK_METHOD( + (std::pair, bool>), + loadInitialLedger, + (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&), + (override) + ); + + using ForwardToRippledReturnType = std::expected; + MOCK_METHOD( + ForwardToRippledReturnType, + forwardToRippled, + (boost::json::object const&, std::optional const&, std::string_view, boost::asio::yield_context), + (const, override) + ); +}; + +template