diff --git a/src/data/BackendInterface.hpp b/src/data/BackendInterface.hpp index 387cd41d..5c376d79 100644 --- a/src/data/BackendInterface.hpp +++ b/src/data/BackendInterface.hpp @@ -154,7 +154,7 @@ public: } virtual ~BackendInterface() = default; - // TODO: Remove this hack once old ETL is removed. + // TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed. // Cache should not be exposed thru BackendInterface /** @@ -648,6 +648,14 @@ public: virtual void writeAccountTransactions(std::vector data) = 0; + /** + * @brief Write a new account transaction. + * + * @param record An object representing the account transaction + */ + virtual void + writeAccountTransaction(AccountTransactionsData record) = 0; + /** * @brief Write NFTs transactions. * diff --git a/src/data/CassandraBackend.hpp b/src/data/CassandraBackend.hpp index 4bba5330..7ea02475 100644 --- a/src/data/CassandraBackend.hpp +++ b/src/data/CassandraBackend.hpp @@ -46,6 +46,7 @@ #include #include +#include #include #include #include @@ -906,30 +907,42 @@ public: statements.reserve(data.size() * 10); // assume 10 transactions avg for (auto& record : data) { - std::transform( - std::begin(record.accounts), - std::end(record.accounts), - std::back_inserter(statements), - [this, &record](auto&& account) { - return schema_->insertAccountTx.bind( - std::forward(account), - std::make_tuple(record.ledgerSequence, record.transactionIndex), - record.txHash - ); - } - ); + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); } executor_.write(std::move(statements)); } + void + writeAccountTransaction(AccountTransactionsData record) override + { + std::vector statements; + statements.reserve(record.accounts.size()); + + std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) { + return schema_->insertAccountTx.bind( + std::forward(account), + std::make_tuple(record.ledgerSequence, record.transactionIndex), + record.txHash + ); + }); + + executor_.write(std::move(statements)); + } + void writeNFTTransactions(std::vector const& data) override { std::vector statements; statements.reserve(data.size()); - std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) { + std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) { return schema_->insertNFTTx.bind( record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash ); @@ -999,7 +1012,7 @@ public: std::vector statements; statements.reserve(data.size()); for (auto [mptId, holder] : data) - statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder))); + statements.push_back(schema_->insertMPTHolder.bind(mptId, holder)); executor_.write(std::move(statements)); } diff --git a/src/data/DBHelpers.hpp b/src/data/DBHelpers.hpp index cba1ec1a..76b5aced 100644 --- a/src/data/DBHelpers.hpp +++ b/src/data/DBHelpers.hpp @@ -54,7 +54,7 @@ struct AccountTransactionsData { * @param meta The transaction metadata * @param txHash The transaction hash */ - AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash) + AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash) : accounts(meta.getAffectedAccounts()) , ledgerSequence(meta.getLgrSeq()) , transactionIndex(meta.getIndex()) diff --git a/src/data/LedgerCache.cpp b/src/data/LedgerCache.cpp index 11f0b816..7180c4e9 100644 --- a/src/data/LedgerCache.cpp +++ b/src/data/LedgerCache.cpp @@ -20,6 +20,7 @@ #include "data/LedgerCache.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/Assert.hpp" #include @@ -87,6 +88,42 @@ LedgerCache::update(std::vector const& objs, uint32_t seq, bool is } } +void +LedgerCache::update(std::vector const& objs, uint32_t seq) +{ + if (disabled_) + return; + + std::scoped_lock const lck{mtx_}; + if (seq > latestSeq_) { + ASSERT( + seq == latestSeq_ + 1 || latestSeq_ == 0, + "New sequence must be either next or first. seq = {}, latestSeq_ = {}", + seq, + latestSeq_ + ); + latestSeq_ = seq; + } + + deleted_.clear(); // previous update's deletes no longer needed + + for (auto const& obj : objs) { + if (!obj.data.empty()) { + auto& e = map_[obj.key]; + if (seq > e.seq) + e = {.seq = seq, .blob = obj.data}; + } else { + if (map_.contains(obj.key)) + deleted_[obj.key] = map_[obj.key]; + + map_.erase(obj.key); + if (!full_) + deletes_.insert(obj.key); + } + } + cv_.notify_all(); +} + std::optional LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const { @@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const return {e->second.blob}; } +std::optional +LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const +{ + if (disabled_) + return std::nullopt; + + std::shared_lock const lck{mtx_}; + if (seq > latestSeq_) + return std::nullopt; + + ++objectReqCounter_.get(); + + auto e = deleted_.find(key); + if (e == deleted_.end()) + return std::nullopt; + + if (seq < e->second.seq) + return std::nullopt; + + ++objectHitCounter_.get(); + return {e->second.blob}; +} + void LedgerCache::setDisabled() { diff --git a/src/data/LedgerCache.hpp b/src/data/LedgerCache.hpp index 8a3c4068..bf957af2 100644 --- a/src/data/LedgerCache.hpp +++ b/src/data/LedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Label.hpp" @@ -29,7 +30,6 @@ #include #include -#include #include #include #include @@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface { )}; std::map map_; + std::map deleted_; mutable std::shared_mutex mtx_; std::condition_variable_any cv_; @@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface { public: void - update(std::vector const& objs, uint32_t seq, bool isBackground = false) override; + update(std::vector const& objs, uint32_t seq, bool isBackground) override; + + void + update(std::vector const& objs, uint32_t seq) override; std::optional get(ripple::uint256 const& key, uint32_t seq) const override; + std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const override; + std::optional getSuccessor(ripple::uint256 const& key, uint32_t seq) const override; diff --git a/src/data/LedgerCacheInterface.hpp b/src/data/LedgerCacheInterface.hpp index e5ecee49..850403bc 100644 --- a/src/data/LedgerCacheInterface.hpp +++ b/src/data/LedgerCacheInterface.hpp @@ -20,6 +20,7 @@ #pragma once #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -55,6 +56,15 @@ public: virtual void update(std::vector const& objs, uint32_t seq, bool isBackground = false) = 0; + /** + * @brief Update the cache with new ledger objects. + * + * @param objs The ledger objects to update cache with + * @param seq The sequence to update cache for + */ + virtual void + update(std::vector const& objs, uint32_t seq) = 0; + /** * @brief Fetch a cached object by its key and sequence number. * @@ -65,6 +75,16 @@ public: virtual std::optional get(ripple::uint256 const& key, uint32_t seq) const = 0; + /** + * @brief Fetch a recently deleted object by its key and sequence number. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned + */ + virtual std::optional + getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0; + /** * @brief Gets a cached successor. * diff --git a/src/etlng/CMakeLists.txt b/src/etlng/CMakeLists.txt index 4b97c43a..d0f2854e 100644 --- a/src/etlng/CMakeLists.txt +++ b/src/etlng/CMakeLists.txt @@ -9,6 +9,10 @@ target_sources( impl/Loading.cpp impl/Monitor.cpp impl/TaskManager.cpp + impl/ext/Cache.cpp + impl/ext/Core.cpp + impl/ext/NFT.cpp + impl/ext/Successor.cpp ) target_link_libraries(clio_etlng PUBLIC clio_data) diff --git a/src/etlng/impl/ext/Cache.cpp b/src/etlng/impl/ext/Cache.cpp new file mode 100644 index 00000000..d7c2e5f5 --- /dev/null +++ b/src/etlng/impl/ext/Cache.cpp @@ -0,0 +1,59 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Cache.hpp" + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include + +namespace etlng::impl { + +CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache) +{ +} + +void +CacheExt::onLedgerData(model::LedgerData const& data) const +{ + cache_.get().update(data.objects, data.seq); + LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size(); +} + +void +CacheExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size(); + cache_.get().update(data.objects, data.seq); + cache_.get().setFull(); +} + +void +CacheExt::onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) + const +{ + LOG(log_.trace()) << "got initial objects cnt = " << objs.size(); + cache_.get().update(objs, seq); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Cache.hpp b/src/etlng/impl/ext/Cache.hpp new file mode 100644 index 00000000..fc9fbbeb --- /dev/null +++ b/src/etlng/impl/ext/Cache.hpp @@ -0,0 +1,51 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/LedgerCacheInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +class CacheExt { + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + CacheExt(data::LedgerCacheInterface& cache); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, std::vector const& objs, [[maybe_unused]] std::string lastKey) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.cpp b/src/etlng/impl/ext/Core.cpp new file mode 100644 index 00000000..2c3e141a --- /dev/null +++ b/src/etlng/impl/ext/Core.cpp @@ -0,0 +1,85 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Core.hpp" + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include + +#include +#include +#include + +namespace etlng::impl { + +CoreExt::CoreExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +CoreExt::onLedgerData(model::LedgerData const& data) const +{ + LOG(log_.debug()) << "Loading ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.info()) << "Loading initial ledger data for " << data.seq; + backend_->writeLedger(data.header, auto{data.rawHeader}); + insertTransactions(data); +} + +void +CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::onObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq; + backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw}); +} + +void +CoreExt::insertTransactions(model::LedgerData const& data) const +{ + for (auto const& txn : data.transactions) { + LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID(); + + backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()}); + backend_->writeTransaction( + auto{txn.key}, + data.seq, + data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction' + auto{txn.raw}, + auto{txn.metaRaw} + ); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Core.hpp b/src/etlng/impl/ext/Core.hpp new file mode 100644 index 00000000..7913c15e --- /dev/null +++ b/src/etlng/impl/ext/Core.hpp @@ -0,0 +1,58 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include + +#include +#include + +namespace etlng::impl { + +class CoreExt { + std::shared_ptr backend_; + + util::Logger log_{"ETL"}; + +public: + CoreExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onObject(uint32_t seq, model::Object const& obj) const; + +private: + void + insertTransactions(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.cpp b/src/etlng/impl/ext/NFT.cpp new file mode 100644 index 00000000..30566acf --- /dev/null +++ b/src/etlng/impl/ext/NFT.cpp @@ -0,0 +1,77 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/NFT.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "etl/NFTHelpers.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +NFTExt::NFTExt(std::shared_ptr backend) : backend_(std::move(backend)) +{ +} + +void +NFTExt::onLedgerData(model::LedgerData const& data) const +{ + writeNFTs(data); +} + +void +NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const +{ + LOG(log_.trace()) << "got initial object with key = " << obj.key; + backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw)); +} + +void +NFTExt::onInitialData(model::LedgerData const& data) const +{ + LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size(); + writeNFTs(data); +} + +void +NFTExt::writeNFTs(model::LedgerData const& data) const +{ + std::vector nfts; + std::vector nftTxs; + + for (auto const& tx : data.transactions) { + auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx); + nftTxs.insert(nftTxs.end(), txs.begin(), txs.end()); + if (maybeNFT) + nfts.push_back(*maybeNFT); + } + + // This is uniqued so that we only write latest modification (as in previous implementation) + backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts)); + backend_->writeNFTTransactions(nftTxs); +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/NFT.hpp b/src/etlng/impl/ext/NFT.hpp new file mode 100644 index 00000000..65e45233 --- /dev/null +++ b/src/etlng/impl/ext/NFT.hpp @@ -0,0 +1,56 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "etl/NFTHelpers.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include +#include +#include + +namespace etlng::impl { + +class NFTExt { + std::shared_ptr backend_; + util::Logger log_{"ETL"}; + +public: + NFTExt(std::shared_ptr backend); + + void + onLedgerData(model::LedgerData const& data) const; + + void + onInitialObject(uint32_t seq, model::Object const& obj) const; + + void + onInitialData(model::LedgerData const& data) const; + +private: + void + writeNFTs(model::LedgerData const& data) const; +}; + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.cpp b/src/etlng/impl/ext/Successor.cpp new file mode 100644 index 00000000..a46d3158 --- /dev/null +++ b/src/etlng/impl/ext/Successor.cpp @@ -0,0 +1,222 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/impl/ext/Successor.hpp" + +#include "data/BackendInterface.hpp" +#include "data/DBHelpers.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/Assert.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +SuccessorExt::SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache) + : backend_(std::move(backend)), cache_(cache) +{ +} + +void +SuccessorExt::onInitialData(model::LedgerData const& data) const +{ + ASSERT(cache_.get().isFull(), "Cache must be full at this point"); + ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load"); + ASSERT(data.objects.empty(), "Should not have objects from initial data"); + writeSuccessors(data.seq); + writeEdgeKeys(data.seq, data.edgeKeys.value()); +} + +void +SuccessorExt::onInitialObjects( + uint32_t seq, + [[maybe_unused]] std::vector const& objs, + std::string lastKey +) const +{ + for (auto const& obj : objs) { + if (!lastKey.empty()) + backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw}); + lastKey = obj.keyRaw; + } +} + +void +SuccessorExt::onLedgerData(model::LedgerData const& data) const +{ + namespace vs = std::views; + + LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size() + << "; got successors = " << data.successors.has_value() << "; cache is " + << (cache_.get().isFull() ? "FULL" : "Not full"); + + auto filteredObjects = data.objects // + | vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; }); + + if (data.successors.has_value()) { + for (auto const& successor : data.successors.value()) + writeIncludedSuccessor(data.seq, successor); + + for (auto const& obj : filteredObjects) + writeIncludedSuccessor(data.seq, obj); + } else { + if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq) + throw std::logic_error("Cache is not full, but object neighbors were not included"); + + for (auto const& obj : filteredObjects) + updateSuccessorFromCache(data.seq, obj); + } +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const +{ + auto firstBook = succ.firstBook; + if (firstBook.empty()) + firstBook = uint256ToString(data::kLAST_KEY); + + backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook)); +} + +void +SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const +{ + ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object"); + + // TODO: perhaps make these optionals inside of obj and move value_or here + auto pred = obj.predecessor; + auto succ = obj.successor; + + if (obj.type == model::Object::ModType::Deleted) { + backend_->writeSuccessor(std::move(pred), seq, std::move(succ)); + } else if (obj.type == model::Object::ModType::Created) { + backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw}); + backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ)); + } +} + +void +SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const +{ + auto const lb = + cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}}); + auto const ub = + cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}}); + + auto checkBookBase = false; + auto const isDeleted = obj.data.empty(); + + if (isDeleted) { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key)); + } else { + backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key)); + backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key)); + } + + if (isDeleted) { + auto const old = cache_.get().getDeleted(obj.key, seq - 1); + ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key)); + + checkBookBase = isBookDir(obj.key, *old); + } else { + checkBookBase = isBookDir(obj.key, obj.data); + } + + if (checkBookBase) { + auto const current = cache_.get().get(obj.key, seq); + auto const bookBase = getBookBase(obj.key); + + if (isDeleted and not current.has_value()) { + updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase); + } else if (current.has_value()) { + auto const successor = cache_.get().getSuccessor(bookBase, seq); + ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq); + + if (successor->key == obj.key) { + updateBookSuccessor(successor, seq, bookBase); + } + } + } +} + +void +SuccessorExt::updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase +) const +{ + if (maybeSuccessor.has_value()) { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key)); + } else { + backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY)); + } +} + +void +SuccessorExt::writeSuccessors(uint32_t seq) const +{ + ripple::uint256 prev = data::kFIRST_KEY; + while (auto cur = cache_.get().getSuccessor(prev, seq)) { + if (prev == data::kFIRST_KEY) + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key)); + + if (isBookDir(cur->key, cur->blob)) { + auto base = getBookBase(cur->key); + + // make sure the base is not an actual object + if (not cache_.get().get(base, seq)) { + auto succ = cache_.get().getSuccessor(base, seq); + ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base)); + + if (succ->key == cur->key) + backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key)); + } + } + + prev = cur->key; + } + + backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY)); +} + +void +SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const +{ + for (auto const& key : edgeKeys) { + auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq); + if (succ) + backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key)); + } +} + +} // namespace etlng::impl diff --git a/src/etlng/impl/ext/Successor.hpp b/src/etlng/impl/ext/Successor.hpp new file mode 100644 index 00000000..5ba8d702 --- /dev/null +++ b/src/etlng/impl/ext/Successor.hpp @@ -0,0 +1,82 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#pragma once + +#include "data/BackendInterface.hpp" +#include "data/LedgerCacheInterface.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "util/log/Logger.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace etlng::impl { + +class SuccessorExt { + std::shared_ptr backend_; + std::reference_wrapper cache_; + + util::Logger log_{"ETL"}; + +public: + SuccessorExt(std::shared_ptr backend, data::LedgerCacheInterface& cache); + + void + onInitialData(model::LedgerData const& data) const; + + void + onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector const& objs, std::string lastKey) const; + + void + onLedgerData(model::LedgerData const& data) const; + +private: + void + writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const; + + void + writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const; + + void + updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const; + + void + updateBookSuccessor( + std::optional const& maybeSuccessor, + auto seq, + ripple::uint256 const& bookBase + ) const; + + void + writeSuccessors(uint32_t seq) const; + + void + writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const; +}; + +} // namespace etlng::impl diff --git a/tests/common/util/BinaryTestObject.cpp b/tests/common/util/BinaryTestObject.cpp index b9082e7d..30cd07a2 100644 --- a/tests/common/util/BinaryTestObject.cpp +++ b/tests/common/util/BinaryTestObject.cpp @@ -19,134 +19,31 @@ #include "util/BinaryTestObject.hpp" +#include "data/DBHelpers.hpp" #include "etlng/Models.hpp" #include "etlng/impl/Extraction.hpp" #include "util/StringUtils.hpp" +#include "util/TestObject.hpp" #include #include +#include +#include #include #include +#include #include #include #include #include +#include #include #include namespace { constinit auto const kSEQ = 30; - -constinit auto const kTXN_HEX = - "1200192200000008240011CC9B201B001F71D6202A0000000168400000" - "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" - "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" - "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" - "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" - "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" - "DE3538F248662FC73C"; - -constinit auto const kTXN_META = - "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" - "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" - "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" - "C200000028751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "9808B6B90000001D751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" - "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" - "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" - "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" - "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" - "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" - "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" - "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" - "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" - "00001F751868747470733A2F2F677265677765697362726F642E636F6D" - "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" - "5DAE00000014751868747470733A2F2F677265677765697362726F642E" - "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" - "3CCE1462A500000009751868747470733A2F2F67726567776569736272" - "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" - "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" - "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" - "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" - "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" - "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" - "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" - "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" - "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" - "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" - "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" - "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" - "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" - "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" - "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" - "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" - "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" - "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" - "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" - "00000029751868747470733A2F2F677265677765697362726F642E636F" - "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" - "EE87B80000001E751868747470733A2F2F677265677765697362726F64" - "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" - "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" - "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" - "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" - "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" - "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" - "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" - "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" - "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" - "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" - "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" - "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" - "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" - "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" - "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" - "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" - "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" - "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" - "B100000015751868747470733A2F2F677265677765697362726F642E63" - "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" - "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" - "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" - "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" - "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" - "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" - "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" - "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" - "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" - "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" - "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" - "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" - "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" - "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" - "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" - "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" - "2FC73CE1E1F1031000"; - constinit auto const kRAW_HEADER = "03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E" "DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A" @@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER = namespace util { std::pair -createNftTxAndMetaBlobs() +createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr) { - return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)}; + return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)}; } std::pair -createNftTxAndMeta() +createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr) { ripple::uint256 hash; - EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E")); + EXPECT_TRUE(hash.parseHex(hashStr)); - auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(); + auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr); ripple::SerialIter it{txnBlob.data(), txnBlob.size()}; return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}}; } etlng::model::Transaction -createTransaction(ripple::TxType type) +createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr) { - auto const [sttx, meta] = createNftTxAndMeta(); + auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr); return { .raw = "", .metaRaw = "", @@ -192,10 +89,9 @@ createTransaction(ripple::TxType type) } etlng::model::Object -createObject() +createObject(etlng::model::Object::ModType modType, std::string key) { // random object taken from initial ledger load - static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; static constinit auto const kOBJ_BLOB = @@ -205,12 +101,62 @@ createObject() "8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56"; return { - .key = {}, - .keyRaw = hexStringToBinaryString(kOBJ_KEY), - .data = {}, - .dataRaw = hexStringToBinaryString(kOBJ_BLOB), + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), .successor = hexStringToBinaryString(kOBJ_SUCC), .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key) +{ + // random object taken from initial ledger load + static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A"; + static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"; + static constinit auto const kOBJ_BLOB = + "11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE" + "3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000" + "0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3"; + + return { + .key = binaryStringToUint256(hexStringToBinaryString(key)), + .keyRaw = hexStringToBinaryString(key), + .data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB), + .dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB), + .successor = hexStringToBinaryString(kOBJ_SUCC), + .predecessor = hexStringToBinaryString(kOBJ_PRED), + .type = modType, + }; +} + +etlng::model::Object +createObjectWithTwoNFTs() +{ + std::string const url1 = "abcd1"; + std::string const url2 = "abcd2"; + ripple::Blob const uri1Blob(url1.begin(), url1.end()); + ripple::Blob const uri2Blob(url2.begin(), url2.end()); + + constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q"; + constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000"; + constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA"; + + auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt); + auto const serializerNftPage = nftPage.getSerializer(); + + auto const account = getAccountIdWithString(kACCOUNT); + return { + .key = {}, + .keyRaw = std::string(reinterpret_cast(account.data()), ripple::AccountID::size()), + .data = {}, + .dataRaw = + std::string(static_cast(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()), + .successor = "", + .predecessor = "", .type = etlng::model::Object::ModType::Created, }; } @@ -219,8 +165,10 @@ etlng::model::BookSuccessor createSuccessor() { return { - .firstBook = "A000000000000000000000000000000000000000000000000000000000000000", - .bookBase = "A000000000000000000000000000000000000000000000000000000000000001", + .firstBook = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}), + .bookBase = + uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}), }; } diff --git a/tests/common/util/BinaryTestObject.hpp b/tests/common/util/BinaryTestObject.hpp index ad5fba05..41d5029e 100644 --- a/tests/common/util/BinaryTestObject.hpp +++ b/tests/common/util/BinaryTestObject.hpp @@ -32,17 +32,149 @@ namespace util { +static constexpr auto kDEFAULT_TXN_HEX = + "1200192200000008240011CC9B201B001F71D6202A0000000168400000" + "000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E" + "4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F" + "C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA" + "34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F" + "677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16" + "DE3538F248662FC73C"; + +static constexpr auto kDEFAULT_TXN_META = + "201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E" + "07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000" + "0006751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1" + "C200000028751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "9808B6B90000001D751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CA048C0A300000007751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F" + "2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21" + "D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470" + "733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203" + "F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868" + "747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800" + "006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A" + "751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A" + "000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00" + "00001F751868747470733A2F2F677265677765697362726F642E636F6D" + "E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4" + "5DAE00000014751868747470733A2F2F677265677765697362726F642E" + "636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7" + "3CCE1462A500000009751868747470733A2F2F67726567776569736272" + "6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248" + "662FC73CD89A24C70000002B751868747470733A2F2F67726567776569" + "7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35" + "38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567" + "7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000" + "0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1" + "E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C" + "662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB" + "16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67" + "7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6" + "E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A" + "2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C" + "21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474" + "70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062" + "03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518" + "68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008" + "00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000" + "07751868747470733A2F2F677265677765697362726F642E636F6DE1EC" + "5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5" + "00000029751868747470733A2F2F677265677765697362726F642E636F" + "6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE" + "EE87B80000001E751868747470733A2F2F677265677765697362726F64" + "2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F" + "C73CB30E8CAF00000013751868747470733A2F2F677265677765697362" + "726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2" + "48662FC73CB72E91A200000008751868747470733A2F2F677265677765" + "697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE" + "3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265" + "677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022" + "CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F" + "677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5" + "D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073" + "3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4" + "9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874" + "7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000" + "6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75" + "1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00" + "0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000" + "0020751868747470733A2F2F677265677765697362726F642E636F6DE1" + "EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E" + "B100000015751868747470733A2F2F677265677765697362726F642E63" + "6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C" + "E4FA33A40000000A751868747470733A2F2F677265677765697362726F" + "642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866" + "2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973" + "62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538" + "F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777" + "65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16" + "DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772" + "65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0" + "22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F" + "2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71" + "B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435" + "40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94" + "883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3" + "ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000" + "066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866" + "2FC73CE1E1F1031000"; + +static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"; +static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + [[maybe_unused, nodiscard]] std::pair -createNftTxAndMetaBlobs(); +createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX); [[maybe_unused, nodiscard]] std::pair -createNftTxAndMeta(); +createNftTxAndMeta( + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Transaction -createTransaction(ripple::TxType type); +createTransaction( + ripple::TxType type, + std::string hashStr = kDEFAULT_HASH, + std::string metaStr = kDEFAULT_TXN_META, + std::string txnStr = kDEFAULT_TXN_HEX +); [[maybe_unused, nodiscard]] etlng::model::Object -createObject(); +createObject( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithBookBase( + etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created, + std::string key = kDEFAULT_OBJ_KEY +); + +[[maybe_unused, nodiscard]] etlng::model::Object +createObjectWithTwoNFTs(); [[maybe_unused, nodiscard]] etlng::model::BookSuccessor createSuccessor(); diff --git a/tests/common/util/MockBackend.hpp b/tests/common/util/MockBackend.hpp index af16b490..9aab18ee 100644 --- a/tests/common/util/MockBackend.hpp +++ b/tests/common/util/MockBackend.hpp @@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface { MOCK_METHOD(void, writeAccountTransactions, (std::vector), (override)); + MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override)); + MOCK_METHOD(void, writeNFTTransactions, (std::vector const&), (override)); MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override)); diff --git a/tests/common/util/MockLedgerCache.hpp b/tests/common/util/MockLedgerCache.hpp index 67473b6e..07fb6785 100644 --- a/tests/common/util/MockLedgerCache.hpp +++ b/tests/common/util/MockLedgerCache.hpp @@ -21,6 +21,7 @@ #include "data/LedgerCacheInterface.hpp" #include "data/Types.hpp" +#include "etlng/Models.hpp" #include #include @@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface { MOCK_METHOD(std::optional, get, (ripple::uint256 const& a, uint32_t b), (const, override)); + MOCK_METHOD(void, update, (std::vector const&, uint32_t), (override)); + + MOCK_METHOD(std::optional, getDeleted, (ripple::uint256 const&, uint32_t), (const, override)); + MOCK_METHOD( std::optional, getSuccessor, diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 39ee9cda..1f97643f 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -45,6 +45,10 @@ target_sources( etlng/LoadingTests.cpp etlng/NetworkValidatedLedgersTests.cpp etlng/MonitorTests.cpp + etlng/ext/CoreTests.cpp + etlng/ext/CacheTests.cpp + etlng/ext/NFTTests.cpp + etlng/ext/SuccessorTests.cpp # Feed util/BytesConverterTests.cpp feed/BookChangesFeedTests.cpp diff --git a/tests/unit/etlng/ext/CacheTests.cpp b/tests/unit/etlng/ext/CacheTests.cpp new file mode 100644 index 00000000..bdd15c74 --- /dev/null +++ b/tests/unit/etlng/ext/CacheTests.cpp @@ -0,0 +1,92 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Cache.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockLedgerCache.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; +constinit auto const kUNUSED_LAST_KEY = "unused"; + +auto +createTestData() +{ + auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()}; + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = {}, + .objects = std::move(objects), + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct CacheExtTests : util::prometheus::WithPrometheus { +protected: + MockLedgerCache cache_; + etlng::impl::CacheExt ext_{cache_}; +}; + +TEST_F(CacheExtTests, OnLedgerDataUpdatesCache) +{ + auto const data = createTestData(); + + EXPECT_CALL(cache_, update(data.objects, data.seq)); + + ext_.onLedgerData(data); +} + +TEST_F(CacheExtTests, OnInitialDataUpdatesCacheAndSetsFull) +{ + auto const data = createTestData(); + + EXPECT_CALL(cache_, update(data.objects, data.seq)); + EXPECT_CALL(cache_, setFull); + + ext_.onInitialData(data); +} + +TEST_F(CacheExtTests, OnInitialObjectsUpdateCache) +{ + auto const objects = std::vector{util::createObject(), util::createObject()}; + + EXPECT_CALL(cache_, update(objects, kSEQ)); + + ext_.onInitialObjects(kSEQ, objects, kUNUSED_LAST_KEY); +} diff --git a/tests/unit/etlng/ext/CoreTests.cpp b/tests/unit/etlng/ext/CoreTests.cpp new file mode 100644 index 00000000..bcdf5ff8 --- /dev/null +++ b/tests/unit/etlng/ext/CoreTests.cpp @@ -0,0 +1,107 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Core.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +auto +createTestData() +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER), + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = {}, + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + etlng::impl::CoreExt ext_{backend_}; +}; + +TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader})); + EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size()); + EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size()); + + ext_.onLedgerData(data); +} + +TEST_F(CoreExtTests, OnInitialDataWritesLedgerAndTransactions) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader})); + EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size()); + EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size()); + + ext_.onInitialData(data); +} + +TEST_F(CoreExtTests, OnInitialObjectWritesLedgerObject) +{ + auto const data = util::createObject(); + + EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw})); + + ext_.onInitialObject(kSEQ, data); +} + +TEST_F(CoreExtTests, OnObjectWritesLedgerObject) +{ + auto const data = util::createObject(); + + EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw})); + + ext_.onObject(kSEQ, data); +} diff --git a/tests/unit/etlng/ext/NFTTests.cpp b/tests/unit/etlng/ext/NFTTests.cpp new file mode 100644 index 00000000..64731646 --- /dev/null +++ b/tests/unit/etlng/ext/NFTTests.cpp @@ -0,0 +1,287 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "etlng/Models.hpp" +#include "etlng/impl/ext/NFT.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockPrometheus.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include + +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +constinit auto const kTXN_HEX2 = + "12001D230606B58324048A8B6F501C50E8EBCD412E6CF9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487E501DA2A1BC9A62AAEB2A" + "2A70F895587A3FB752514AA03F8C6E7C84864653B8673E0368400000000000001E60134000000000001F09732103B8C234E0598BC26D8A3E1B" + "FF53EB252EC0F15EA6800E4D85AA5D7CD15D76B01E744730450221009C0AFF5F3298E10ABE42894717DA46B59529A366527AA5DFC1577ADEA9" + "B20FA70220494D1D9BFEF2AB09F4D6403AAC5B9DCC5B859DCEC1380C6D817A6EDFE7E68FD581141565EED165BA79999425204A8491C73B1301" + "E34FF9EA7D0F7872702E63616665202D2073616C65E1F1"; + +constinit auto const kTXN_META2 = + "201C00000040F8E51100502505A59E11552ABC2FD74D879BE58489A588838AA2BA59E1E05A48A574226CD8B6CE77998971560B639A808E3B97" + "42A25334E5CF68EEDEDE52F54E50F4E63921C9F3C40588E426E6FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C" + "052AB2897542697066733A2F2F6261666B726569683673796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E71673472" + "6E747037653769757469766C796275E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D526950" + "5679335464654A6170697974576B686141576B4D716D39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60F" + "DDC47DABE4D52E4AD1F0A50A85CBBC00000022752E516D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169" + "316B4C316E5255797131E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C457664644634" + "5442427374455138484C66694867456B6A4465514D56676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4" + "D52E4AD1F0A5608E298B000000EF752E516D4E6E56566F4250426755783848334641486F436162593647347577445758767738757455636976" + "73764D7142E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646" + "777A526D517472694841663277437A3567786161355256446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0" + "A5B09C35A40000020A752E516D5168584346325738535571657633333657703572357935326141464D59564A667570534A536B6B425047786E" + "E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B6968756000001BC752E516D5359647963353276657A7478365338673231" + "70544651724E6E624762676A5A4B5047753836464231394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C" + "000000D2752E516D506645624A38446D624E69794C55637441796A3473625453515937314177597A32776A595238703838416432E1EC5A0008" + "27107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37" + "773338734147397469725867326D626D4262596771E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F75" + "2E516D505A733255393159764538374277635677587A63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E6" + "4C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E21584000001EA752E516D656366646769555A6B37355574667152763474696644557257345439" + "4571616F733231757036706E75345663E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573" + "614345725175396F48596A71765172564B31473155504657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE" + "11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A6854653776" + "43624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A2F2F6261" + "667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E6965" + "2F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F62616679" + "62656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E69652F36" + "342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D616D397436" + "5A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0FE30354A4" + "3281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D79756E697736" + "647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A43281629A" + "0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D6575326934" + "65726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A00082710" + "AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E61776833" + "6F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B7320" + "2331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F516D63436D" + "526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E72200000000501A0B639A808E" + "3B9742A25334E5CF68EEDEDE52F54E4A69498F5B79952C052AB289501B0B639A808E3B9742A25334E5CF68EEDEDE52F54E50F4E639664EC7E5" + "0588E658FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C052AB2897542697066733A2F2F6261666B7265696836" + "73796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E716734726E747037653769757469766C796275E1EC5A00082710" + "7B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D5269505679335464654A6170697974576B686141576B4D716D" + "39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A50A85CBBC00000022752E51" + "6D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169316B4C316E5255797131E1EC5A000827107B87E64C88" + "4BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C4576646446345442427374455138484C66694867456B6A4465514D56" + "676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5608E298B000000EF752E516D4E6E5656" + "6F4250426755783848334641486F43616259364734757744575876773875745563697673764D7142E1EC5A000827107B87E64C884BBFB60FDD" + "C47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646777A526D517472694841663277437A35677861613552" + "56446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B09C35A40000020A752E516D516858434632573853" + "5571657633333657703572357935326141464D59564A667570534A536B6B425047786EE1EC5A000827107B87E64C884BBFB60FDDC47DABE4D5" + "2E4AD1F0A5B6968756000001BC752E516D5359647963353276657A747836533867323170544651724E6E624762676A5A4B5047753836464231" + "394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C000000D2752E516D506645624A38446D624E69794C55" + "637441796A3473625453515937314177597A32776A595238703838416432E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5" + "CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37773338734147397469725867326D626D4262596771E1" + "EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F752E516D505A733255393159764538374277635677587A" + "63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E2158400" + "0001EA752E516D656366646769555A6B373555746671527634746966445572573454394571616F733231757036706E75345663E1EC5A000827" + "107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573614345725175396F48596A71765172564B3147315550" + "4657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051752E" + "516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44668A" + "2B96DFDE11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A68" + "5465377643624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A" + "2F2F6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A3670717136356635" + "6F6E69652F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F" + "6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E" + "69652F36342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D61" + "6D3974365A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0F" + "E30354A43281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D7975" + "6E697736647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A4" + "3281629A0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D65" + "7532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A" + "00082710AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E" + "617768336F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B73205075" + "6E6B73202331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F51" + "6D63436D526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E1E51100502505A4C9" + "C9557DF04CC68DE7C5EF20DBF705221EEDB05FE3806BC3F6A35240652C3E9C97BA5A56246B3E06AB367AB9614566B6F90C718B52A4440852A4" + "440804D409E004C90E52E6FAEC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478B72AD46100000008755E68747470733A2F2F69" + "7066732E696F2F697066732F62616679626569656F6C7667696F71766F737436346367646873797876726962336D6265797278773477626F61" + "617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10" + "276700000009755E68747470733A2F2F697066732E696F2F697066732F62616679626569686D6374376A766B7236366E67337975676D33706D" + "70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA70" + "69E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E68747470733A2F2F697066732E696F2F697066732F6261667962656968626D6D" + "6E626C687736656D776E6A733766787778376736376B6E6A626B6966636674787A6B66777632767162357A6C727575612F6D65746164617461" + "2E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478FBE441630000000B755E68747470733A2F2F697066732E696F" + "2F697066732F62616679626569686E6C77656C7965357270706963646761617678756869376D61636E697879627077667133796734626C3366" + "6A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051" + "752E516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44" + "668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B535470" + "31514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D51" + "674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE58" + "EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A7661" + "6D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC753569" + "7066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A0008" + "2710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F62" + "6D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC24F" + "1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C783579693375" + "706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408" + "005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D7547724839" + "4C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D555744524C" + "425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB9614566" + "B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E346958597233487341715771" + "673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD7535697066733A" + "2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A00081388246B" + "3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A51567957" + "4B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204C9" + "12F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E47" + "E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E357163334E" + "753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52" + "A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E654474" + "6656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D5779" + "6D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367AB9" + "614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A743147454564" + "4C787161513466714E637375363854677153E1F1E1E72200000000501A246B3E06AB367AB9614566B6F90C718B52A444084B237478B72AD461" + "00000008501B246B3E06AB367AB9614566B6F90C718B52A4440852A444080D2641FC04C91315FAEC5A00081388DAA8A3AA7069E65EC1E3E557" + "1D9B6F274B237478B72AD46100000008755E68747470733A2F2F697066732E696F2F697066732F62616679626569656F6C7667696F71766F73" + "7436346367646873797876726962336D6265797278773477626F61617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC" + "5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10276700000009755E68747470733A2F2F697066732E696F2F697066732F62" + "616679626569686D6374376A766B7236366E67337975676D33706D70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69" + "792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E6874747073" + "3A2F2F697066732E696F2F697066732F6261667962656968626D6D6E626C687736656D776E6A733766787778376736376B6E6A626B69666366" + "74787A6B66777632767162357A6C727575612F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B23" + "7478FBE441630000000B755E68747470733A2F2F697066732E696F2F697066732F62616679626569686E6C77656C7965357270706963646761" + "617678756869376D61636E697879627077667133796734626C33666A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A" + "44668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B5354" + "7031514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D" + "51674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE" + "58EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A76" + "616D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC7535" + "697066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A00" + "082710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F" + "626D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC2" + "4F1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C7835796933" + "75706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A444" + "08005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D75477248" + "394C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D55574452" + "4C425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB96145" + "66B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E3469585972334873417157" + "71673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD753569706673" + "3A2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A0008138824" + "6B3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A515679" + "574B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204" + "C912F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E" + "47E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E35716333" + "4E753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B" + "52A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E6544" + "746656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D57" + "796D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367A" + "B9614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A7431474545" + "644C787161513466714E637375363854677153E1F1E1E1E4110064562AED34CB796DF0E82AAC7EB958158EEBF99F51AA8C96B85654DDE05206" + "C18BBCE7220000000225059FB7B755172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235BEBBE582AED34CB796DF0E82A" + "AC7EB958158EEBF99F51AA8C96B85654DDE05206C18BBC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051E1" + "E1E51100612505A58BFB555F1D177245DF64F1BAC04B3EFF72C9B448710F95E86355339305264B0C6450345643CECFECC44660B31BEB70A4AE" + "78ED0BB4B31A754CE746848150EFFC03418847E62D0000010E624000000004FF530BE1E722000000002404C9421F2D0000010D202B00000712" + "202C00000493203204C90C7062400000000506A60B8114246B3E06AB367AB9614566B6F90C718B52A44408E1E1E41100375650E8EBCD412E6C" + "F9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487EE722000000002505A5B1C23400000000000000463C0000000000000000558F56" + "F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB7" + "2AED0000005161400000000007C02982140B639A808E3B9742A25334E5CF68EEDEDE52F54E83141565EED165BA79999425204A8491C73B1301" + "E34FE1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC4565183948C67127DAA598D" + "1197F3DEDC4552F64034F3B8213060B8903FD6C8A561E62D0000039C62400000000D7F518BE1E7220000000024057A8B032D0000039B202B00" + "000290202C000000082032057A6CBC62400000000D77916281140B639A808E3B9742A25334E5CF68EEDEDE52F54E8914CEAECC5B87EA043BD9" + "8E1B4FE8663AC59D5C3518E1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45658" + "7E28972F3B63D2260C0671E59593EE6C4D27AB857D1AF230F5CAA959BB3EACE624048A8B6F624000001B84AABF6AE1E7220000000024048A8B" + "702D00000000624000001B84AADE5581141565EED165BA79999425204A8491C73B1301E34FE1E1E41100645698929E4419455BBB551BF09C08" + "EEBEE19021E6B7E3D1D989968EF49970F10130E722000000012505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1F" + "D7B40F710AC45898929E4419455BBB551BF09C08EEBEE19021E6B7E3D1D989968EF49970F101305A00080FA03A44668A2B96DFDE11BF0817CC" + "6DF60C4E3508D43EB72AED00000051E1E1E411003756A2A1BC9A62AAEB2A2A70F895587A3FB752514AA03F8C6E7C84864653B8673E03E72200" + "00000125059FB7B734000000000000003C3C000000000000000055172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235B" + "EBBE5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED0000005161400000000007A1208214246B3E06AB367AB9614566" + "B6F90C718B52A4440883141565EED165BA79999425204A8491C73B1301E34FE1E1E511006125059DE0B155C0F457C1104D45881194D53DBB40" + "B81F9D812FC507637C4294527C06FDD69A3856A969AD4B14312DFA8739258A6A94C0868F1A3E7D1A1EF4839983521A4F6953FCE66240000000" + "0011B75AE1E7220000000024046044C82D00000000202B00001836202C00000F3F62400000000012057A81143A44668A2B96DFDE11BF0817CC" + "6DF60C4E3508D48914651B85AF14BE4F60AFBF5ADAA4F367061C2DA1D4E1E1E51100642505A5B1C2558F56F16F29177518F3DD6CF827085D7B" + "9E2806CD5EBDE810DD1FD7B40F710AC456CE29B8547FC486F45704C1DE539B748587872AA803FB53FB938E282547DF2CFEE722000000003200" + "0000000000004558DE4F51B35BE5A98D0C97BE07378E9CB56FFE3F861E544E0ABAC5ED765E2F781982140B639A808E3B9742A25334E5CF68EE" + "DEDE52F54EE1E1E51100642505A4EA55556022061889BB4DA5242B7AB048C3D751ACEE5788ED4120B2E70F68E9A322A1E956D7A8E1C70CD8A0" + "3AE9BAA41BC0898471FC26DA3712748A803B2F32007CDCB0DCE7220000000031000000000000003D32000000000000003B58B6629B8F178A18" + "C2926F1ADA669262B5D362BFC2DB1417D837CBF382AA37F2D88214246B3E06AB367AB9614566B6F90C718B52A44408E1E1F1031000"; + +constinit auto const kHASH2 = "D7604B124D5D9C89EC1854A6CBD5A1FFD92502E945411B9C8DE397E7F19A74F8"; + +auto +createTestData() +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN, kHASH2, kTXN_META2, kTXN_HEX2), + util::createTransaction(ripple::TxType::ttAMM_CREATE), // not NFT - will be filtered + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), // not unique - will be filtered + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = {}, + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +} // namespace + +struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + etlng::impl::NFTExt ext_{backend_}; +}; + +TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { + EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique + }); + EXPECT_CALL(*backend_, writeNFTTransactions); + + ext_.onLedgerData(data); +} + +TEST_F(NFTExtTests, OnInitialDataFiltersAndWritesNFTs) +{ + auto const data = createTestData(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { + EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique + }); + EXPECT_CALL(*backend_, writeNFTTransactions); + + ext_.onInitialData(data); +} + +TEST_F(NFTExtTests, OnInitialObjectExtractsAndWritesNFTData) +{ + auto const data = util::createObjectWithTwoNFTs(); + + EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { EXPECT_EQ(nfts.size(), 2); }); + + ext_.onInitialObject(kSEQ, data); +} diff --git a/tests/unit/etlng/ext/SuccessorTests.cpp b/tests/unit/etlng/ext/SuccessorTests.cpp new file mode 100644 index 00000000..6828dd80 --- /dev/null +++ b/tests/unit/etlng/ext/SuccessorTests.cpp @@ -0,0 +1,641 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2025, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "data/DBHelpers.hpp" +#include "data/Types.hpp" +#include "etlng/Models.hpp" +#include "etlng/impl/ext/Successor.hpp" +#include "util/Assert.hpp" +#include "util/BinaryTestObject.hpp" +#include "util/MockAssert.hpp" +#include "util/MockBackendTestFixture.hpp" +#include "util/MockLedgerCache.hpp" +#include "util/MockPrometheus.hpp" +#include "util/StringUtils.hpp" +#include "util/TestObject.hpp" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace etlng::impl; +using namespace data; + +namespace { +constinit auto const kSEQ = 123u; +constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; + +auto +createTestData(std::vector objects) +{ + auto transactions = std::vector{ + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), + util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER), + }; + + auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ); + return etlng::model::LedgerData{ + .transactions = std::move(transactions), + .objects = std::move(objects), + .successors = {}, + .edgeKeys = {}, + .header = header, + .rawHeader = {}, + .seq = kSEQ + }; +} + +[[maybe_unused]] auto +createInitialTestData(std::vector edgeKeys) +{ + // initial data expects objects to be empty as well as non-empty edgeKeys + ASSERT(not edgeKeys.empty(), "Initial data requires edgeKeys"); + + auto ret = createTestData({}); + ret.edgeKeys = std::make_optional>(); + std::ranges::transform(edgeKeys, std::back_inserter(ret.edgeKeys.value()), &uint256ToString); + + return ret; +} + +} // namespace + +struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest { +protected: + MockLedgerCache cache_; + etlng::impl::SuccessorExt ext_{backend_, cache_}; +}; + +TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent) +{ + auto const data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(false)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_THROW(ext_.onLedgerData(data), std::logic_error); +} + +TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffersAndSuccessorsNotPresent) +{ + auto const data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ - 1)); + + EXPECT_THROW(ext_.onLedgerData(data), std::logic_error); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(Blob{'0'})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillRepeatedly(testing::Return(LedgerObject{})); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(LedgerObject{.key = createdObj.key, .blob = {}})); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_)); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(deletedObj.key); + auto const oldCachedObj = createdObj.data; + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj)); + + EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt)); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(data::kLAST_KEY))); + + ext_.onLedgerData(data); +} + +TEST_F( + SuccessorExtTests, + OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(deletedObj.key); + auto const oldCachedObj = createdObj.data; + + [[maybe_unused]] testing::InSequence inSeq; + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj)); + + EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(LedgerObject{.key = deletedObj.key, .blob = {}})); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(deletedObj.key))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const predKey = + binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C" + )); + auto const succKey = + binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + )); + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)) + .WillOnce(testing::Return(data::LedgerObject{.key = predKey, .blob = {}})); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)) + .WillOnce(testing::Return(data::LedgerObject{.key = succKey, .blob = {}})); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(predKey), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(succKey))); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObject(Object::ModType::Created, objKey); + auto data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const succ = util::createSuccessor(); + data.successors = {succ, succ, succ}; + + EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, auto{succ.firstBook})) + .Times(data.successors->size()); + + EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.predecessor}, kSEQ, auto{createdObj.keyRaw})); + EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.keyRaw}, kSEQ, auto{createdObj.successor})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + auto succ = util::createSuccessor(); + succ.firstBook = {}; // empty will be transformed into kLAST_KEY + data.successors = {succ, succ}; + + EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, uint256ToString(data::kLAST_KEY))) + .Times(data.successors->size()); + + EXPECT_CALL(*backend_, writeSuccessor(auto{deletedObj.predecessor}, kSEQ, auto{deletedObj.successor})); + + ext_.onLedgerData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = {'0'}}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(std::nullopt)); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = {'0'}}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {}})); + EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto const data = createInitialTestData({firstKey, secondKey}); + + auto successorChain = std::queue(); + successorChain.push(firstKey); + successorChain.push(secondKey); + + auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created); + auto const bookBase = getBookBase(bookBaseObj.key); + + [[maybe_unused]] testing::Sequence inSeq; + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + + EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ)) + .Times(3) + .InSequence(inSeq) + .WillRepeatedly([&](auto&&, auto&&) -> std::optional { + if (successorChain.empty()) + return std::nullopt; + + auto v = successorChain.front(); + successorChain.pop(); + return data::LedgerObject{.key = v, .blob = bookBaseObj.data}; + }); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(bookBase, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)) + .WillRepeatedly(testing::Return(data::LedgerObject{.key = firstKey, .blob = data::Blob{'1'}})); + EXPECT_CALL( + *backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_) + ); // Called once because firstKey returned repeatedly above + + for (auto const& key : data.edgeKeys.value()) { + EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ)) + .InSequence(inSeq) + .WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {'1'}})); + EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))).InSequence(inSeq); + } + + ext_.onInitialData(data); +} + +TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey) +{ + using namespace etlng::model; + + auto const lastKey = std::string{}; + auto const data = std::vector{ + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610" + ), + }; + + std::string lk = lastKey; + for (auto const& obj : data) { + if (not lk.empty()) + EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key))); + lk = uint256ToString(obj.key); + } + + ext_.onInitialObjects(kSEQ, data, lastKey); +} + +TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey) +{ + using namespace etlng::model; + + auto const lastKey = + uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D")); + auto const data = std::vector{ + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F" + ), + util::createObject( + Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610" + ), + }; + + std::string lk = lastKey; + for (auto const& obj : data) { + EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key))); + lk = uint256ToString(obj.key); + } + + ext_.onInitialObjects(kSEQ, data, lastKey); +} + +struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests {}; + +TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey); + auto const data = createTestData({ + deletedObj, + util::createObject(Object::ModType::Modified), + }); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY))); + EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); }); +} + +TEST_F( + SuccessorExtAssertTests, + OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache +) +{ + using namespace etlng::model; + + auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"; + auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey); + auto const data = createTestData({ + createdObj, + util::createObject(Object::ModType::Modified), + }); + auto const bookBase = getBookBase(createdObj.key); + + EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true)); + EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ)); + + EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt)); + + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key))); + EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY))); + + EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'})); + EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt)); + + EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull) +{ + using namespace etlng::model; + + auto const data = createTestData({ + util::createObject(Object::ModType::Modified), + util::createObject(Object::ModType::Created), + }); + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(false)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys) +{ + using namespace etlng::model; + + auto data = createTestData({}); + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} + +TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects) +{ + using namespace etlng::model; + + auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"); + auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"); + auto data = createInitialTestData({firstKey, secondKey}); + data.objects = {util::createObject()}; + + EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true)); + EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); }); +} diff --git a/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp b/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp index bc1f6353..f2032fa4 100644 --- a/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp +++ b/tests/unit/rpc/handlers/AccountCurrenciesTests.cpp @@ -41,9 +41,9 @@ using namespace rpc; using namespace data; +using namespace testing; namespace json = boost::json; -using namespace testing; namespace {