feat: ETLng extensions (#1967)

For #1599 #1597
This commit is contained in:
Alex Kremer
2025-03-21 16:41:29 +00:00
committed by GitHub
parent a36aa3618f
commit b31b7633c9
25 changed files with 2167 additions and 147 deletions

View File

@@ -154,7 +154,7 @@ public:
}
virtual ~BackendInterface() = default;
// TODO: Remove this hack once old ETL is removed.
// TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
// Cache should not be exposed thru BackendInterface
/**
@@ -648,6 +648,14 @@ public:
virtual void
writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
/**
* @brief Write a new account transaction.
*
* @param record An object representing the account transaction
*/
virtual void
writeAccountTransaction(AccountTransactionsData record) = 0;
/**
* @brief Write NFTs transactions.
*

View File

@@ -46,6 +46,7 @@
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/nft.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstddef>
@@ -906,30 +907,42 @@ public:
statements.reserve(data.size() * 10); // assume 10 transactions avg
for (auto& record : data) {
std::transform(
std::begin(record.accounts),
std::end(record.accounts),
std::back_inserter(statements),
[this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
}
);
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
}
executor_.write(std::move(statements));
}
void
writeAccountTransaction(AccountTransactionsData record) override
{
std::vector<Statement> statements;
statements.reserve(record.accounts.size());
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
executor_.write(std::move(statements));
}
void
writeNFTTransactions(std::vector<NFTTransactionsData> const& data) override
{
std::vector<Statement> statements;
statements.reserve(data.size());
std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
);
@@ -999,7 +1012,7 @@ public:
std::vector<Statement> statements;
statements.reserve(data.size());
for (auto [mptId, holder] : data)
statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
statements.push_back(schema_->insertMPTHolder.bind(mptId, holder));
executor_.write(std::move(statements));
}

View File

@@ -54,7 +54,7 @@ struct AccountTransactionsData {
* @param meta The transaction metadata
* @param txHash The transaction hash
*/
AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash)
AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash)
: accounts(meta.getAffectedAccounts())
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())

View File

@@ -20,6 +20,7 @@
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include <xrpl/basics/base_uint.h>
@@ -87,6 +88,42 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
}
}
void
LedgerCache::update(std::vector<etlng::model::Object> const& objs, uint32_t seq)
{
if (disabled_)
return;
std::scoped_lock const lck{mtx_};
if (seq > latestSeq_) {
ASSERT(
seq == latestSeq_ + 1 || latestSeq_ == 0,
"New sequence must be either next or first. seq = {}, latestSeq_ = {}",
seq,
latestSeq_
);
latestSeq_ = seq;
}
deleted_.clear(); // previous update's deletes no longer needed
for (auto const& obj : objs) {
if (!obj.data.empty()) {
auto& e = map_[obj.key];
if (seq > e.seq)
e = {.seq = seq, .blob = obj.data};
} else {
if (map_.contains(obj.key))
deleted_[obj.key] = map_[obj.key];
map_.erase(obj.key);
if (!full_)
deletes_.insert(obj.key);
}
}
cv_.notify_all();
}
std::optional<LedgerObject>
LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
{
@@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
return {e->second.blob};
}
std::optional<Blob>
LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const
{
if (disabled_)
return std::nullopt;
std::shared_lock const lck{mtx_};
if (seq > latestSeq_)
return std::nullopt;
++objectReqCounter_.get();
auto e = deleted_.find(key);
if (e == deleted_.end())
return std::nullopt;
if (seq < e->second.seq)
return std::nullopt;
++objectHitCounter_.get();
return {e->second.blob};
}
void
LedgerCache::setDisabled()
{

View File

@@ -21,6 +21,7 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/prometheus/Bool.hpp"
#include "util/prometheus/Counter.hpp"
#include "util/prometheus/Label.hpp"
@@ -29,7 +30,6 @@
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/hardened_hash.h>
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
@@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface {
)};
std::map<ripple::uint256, CacheEntry> map_;
std::map<ripple::uint256, CacheEntry> deleted_;
mutable std::shared_mutex mtx_;
std::condition_variable_any cv_;
@@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface {
public:
void
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) override;
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground) override;
void
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) override;
std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const override;
std::optional<Blob>
getDeleted(ripple::uint256 const& key, uint32_t seq) const override;
std::optional<LedgerObject>
getSuccessor(ripple::uint256 const& key, uint32_t seq) const override;

View File

@@ -20,6 +20,7 @@
#pragma once
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/hardened_hash.h>
@@ -55,6 +56,15 @@ public:
virtual void
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) = 0;
/**
* @brief Update the cache with new ledger objects.
*
* @param objs The ledger objects to update cache with
* @param seq The sequence to update cache for
*/
virtual void
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) = 0;
/**
* @brief Fetch a cached object by its key and sequence number.
*
@@ -65,6 +75,16 @@ public:
virtual std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const = 0;
/**
* @brief Fetch a recently deleted object by its key and sequence number.
*
* @param key The key to fetch for
* @param seq The sequence to fetch for
* @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned
*/
virtual std::optional<Blob>
getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0;
/**
* @brief Gets a cached successor.
*

View File

@@ -9,6 +9,10 @@ target_sources(
impl/Loading.cpp
impl/Monitor.cpp
impl/TaskManager.cpp
impl/ext/Cache.cpp
impl/ext/Core.cpp
impl/ext/NFT.cpp
impl/ext/Successor.cpp
)
target_link_libraries(clio_etlng PUBLIC clio_data)

View File

@@ -0,0 +1,59 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Cache.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <string>
#include <vector>
namespace etlng::impl {
CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache)
{
}
void
CacheExt::onLedgerData(model::LedgerData const& data) const
{
cache_.get().update(data.objects, data.seq);
LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size();
}
void
CacheExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size();
cache_.get().update(data.objects, data.seq);
cache_.get().setFull();
}
void
CacheExt::onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey)
const
{
LOG(log_.trace()) << "got initial objects cnt = " << objs.size();
cache_.get().update(objs, seq);
}
} // namespace etlng::impl

View File

@@ -0,0 +1,51 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/LedgerCacheInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <functional>
#include <string>
#include <vector>
namespace etlng::impl {
class CacheExt {
std::reference_wrapper<data::LedgerCacheInterface> cache_;
util::Logger log_{"ETL"};
public:
CacheExt(data::LedgerCacheInterface& cache);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,85 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Core.hpp"
#include "data/BackendInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <memory>
#include <utility>
namespace etlng::impl {
CoreExt::CoreExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
{
}
void
CoreExt::onLedgerData(model::LedgerData const& data) const
{
LOG(log_.debug()) << "Loading ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
insertTransactions(data);
}
void
CoreExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.info()) << "Loading initial ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
insertTransactions(data);
}
void
CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::onObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::insertTransactions(model::LedgerData const& data) const
{
for (auto const& txn : data.transactions) {
LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID();
backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()});
backend_->writeTransaction(
auto{txn.key},
data.seq,
data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction'
auto{txn.raw},
auto{txn.metaRaw}
);
}
}
} // namespace etlng::impl

View File

@@ -0,0 +1,58 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <memory>
namespace etlng::impl {
class CoreExt {
std::shared_ptr<BackendInterface> backend_;
util::Logger log_{"ETL"};
public:
CoreExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
void
onObject(uint32_t seq, model::Object const& obj) const;
private:
void
insertTransactions(model::LedgerData const& data) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,77 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/NFT.hpp"
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/NFTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace etlng::impl {
NFTExt::NFTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
{
}
void
NFTExt::onLedgerData(model::LedgerData const& data) const
{
writeNFTs(data);
}
void
NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got initial object with key = " << obj.key;
backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw));
}
void
NFTExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
writeNFTs(data);
}
void
NFTExt::writeNFTs(model::LedgerData const& data) const
{
std::vector<NFTsData> nfts;
std::vector<NFTTransactionsData> nftTxs;
for (auto const& tx : data.transactions) {
auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx);
nftTxs.insert(nftTxs.end(), txs.begin(), txs.end());
if (maybeNFT)
nfts.push_back(*maybeNFT);
}
// This is uniqued so that we only write latest modification (as in previous implementation)
backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts));
backend_->writeNFTTransactions(nftTxs);
}
} // namespace etlng::impl

View File

@@ -0,0 +1,56 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/NFTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace etlng::impl {
class NFTExt {
std::shared_ptr<BackendInterface> backend_;
util::Logger log_{"ETL"};
public:
NFTExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
void
onInitialData(model::LedgerData const& data) const;
private:
void
writeNFTs(model::LedgerData const& data) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,222 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Successor.hpp"
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <cstdint>
#include <memory>
#include <optional>
#include <ranges>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace etlng::impl {
SuccessorExt::SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache)
: backend_(std::move(backend)), cache_(cache)
{
}
void
SuccessorExt::onInitialData(model::LedgerData const& data) const
{
ASSERT(cache_.get().isFull(), "Cache must be full at this point");
ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load");
ASSERT(data.objects.empty(), "Should not have objects from initial data");
writeSuccessors(data.seq);
writeEdgeKeys(data.seq, data.edgeKeys.value());
}
void
SuccessorExt::onInitialObjects(
uint32_t seq,
[[maybe_unused]] std::vector<model::Object> const& objs,
std::string lastKey
) const
{
for (auto const& obj : objs) {
if (!lastKey.empty())
backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw});
lastKey = obj.keyRaw;
}
}
void
SuccessorExt::onLedgerData(model::LedgerData const& data) const
{
namespace vs = std::views;
LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size()
<< "; got successors = " << data.successors.has_value() << "; cache is "
<< (cache_.get().isFull() ? "FULL" : "Not full");
auto filteredObjects = data.objects //
| vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; });
if (data.successors.has_value()) {
for (auto const& successor : data.successors.value())
writeIncludedSuccessor(data.seq, successor);
for (auto const& obj : filteredObjects)
writeIncludedSuccessor(data.seq, obj);
} else {
if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq)
throw std::logic_error("Cache is not full, but object neighbors were not included");
for (auto const& obj : filteredObjects)
updateSuccessorFromCache(data.seq, obj);
}
}
void
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const
{
auto firstBook = succ.firstBook;
if (firstBook.empty())
firstBook = uint256ToString(data::kLAST_KEY);
backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook));
}
void
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const
{
ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object");
// TODO: perhaps make these optionals inside of obj and move value_or here
auto pred = obj.predecessor;
auto succ = obj.successor;
if (obj.type == model::Object::ModType::Deleted) {
backend_->writeSuccessor(std::move(pred), seq, std::move(succ));
} else if (obj.type == model::Object::ModType::Created) {
backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw});
backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ));
}
}
void
SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const
{
auto const lb =
cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}});
auto const ub =
cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}});
auto checkBookBase = false;
auto const isDeleted = obj.data.empty();
if (isDeleted) {
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key));
} else {
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key));
backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key));
}
if (isDeleted) {
auto const old = cache_.get().getDeleted(obj.key, seq - 1);
ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key));
checkBookBase = isBookDir(obj.key, *old);
} else {
checkBookBase = isBookDir(obj.key, obj.data);
}
if (checkBookBase) {
auto const current = cache_.get().get(obj.key, seq);
auto const bookBase = getBookBase(obj.key);
if (isDeleted and not current.has_value()) {
updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase);
} else if (current.has_value()) {
auto const successor = cache_.get().getSuccessor(bookBase, seq);
ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq);
if (successor->key == obj.key) {
updateBookSuccessor(successor, seq, bookBase);
}
}
}
}
void
SuccessorExt::updateBookSuccessor(
std::optional<data::LedgerObject> const& maybeSuccessor,
auto seq,
ripple::uint256 const& bookBase
) const
{
if (maybeSuccessor.has_value()) {
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key));
} else {
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY));
}
}
void
SuccessorExt::writeSuccessors(uint32_t seq) const
{
ripple::uint256 prev = data::kFIRST_KEY;
while (auto cur = cache_.get().getSuccessor(prev, seq)) {
if (prev == data::kFIRST_KEY)
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key));
if (isBookDir(cur->key, cur->blob)) {
auto base = getBookBase(cur->key);
// make sure the base is not an actual object
if (not cache_.get().get(base, seq)) {
auto succ = cache_.get().getSuccessor(base, seq);
ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
if (succ->key == cur->key)
backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key));
}
}
prev = cur->key;
}
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY));
}
void
SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const
{
for (auto const& key : edgeKeys) {
auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq);
if (succ)
backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key));
}
}
} // namespace etlng::impl

View File

@@ -0,0 +1,82 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <vector>
namespace etlng::impl {
class SuccessorExt {
std::shared_ptr<BackendInterface> backend_;
std::reference_wrapper<data::LedgerCacheInterface> cache_;
util::Logger log_{"ETL"};
public:
SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache);
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector<model::Object> const& objs, std::string lastKey) const;
void
onLedgerData(model::LedgerData const& data) const;
private:
void
writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const;
void
writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const;
void
updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const;
void
updateBookSuccessor(
std::optional<data::LedgerObject> const& maybeSuccessor,
auto seq,
ripple::uint256 const& bookBase
) const;
void
writeSuccessors(uint32_t seq) const;
void
writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const;
};
} // namespace etlng::impl

View File

@@ -19,134 +19,31 @@
#include "util/BinaryTestObject.hpp"
#include "data/DBHelpers.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/StringUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/protocol/AccountID.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <optional>
#include <string>
#include <utility>
namespace {
constinit auto const kSEQ = 30;
constinit auto const kTXN_HEX =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
constinit auto const kTXN_META =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
constinit auto const kRAW_HEADER =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
@@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER =
namespace util {
std::pair<std::string, std::string>
createNftTxAndMetaBlobs()
createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr)
{
return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)};
return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)};
}
std::pair<ripple::STTx, ripple::TxMeta>
createNftTxAndMeta()
createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr)
{
ripple::uint256 hash;
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
EXPECT_TRUE(hash.parseHex(hashStr));
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs();
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr);
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}};
}
etlng::model::Transaction
createTransaction(ripple::TxType type)
createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr)
{
auto const [sttx, meta] = createNftTxAndMeta();
auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr);
return {
.raw = "",
.metaRaw = "",
@@ -192,10 +89,9 @@ createTransaction(ripple::TxType type)
}
etlng::model::Object
createObject()
createObject(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
@@ -205,12 +101,62 @@ createObject()
"8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
return {
.key = {},
.keyRaw = hexStringToBinaryString(kOBJ_KEY),
.data = {},
.dataRaw = hexStringToBinaryString(kOBJ_BLOB),
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
"11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE"
"3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000"
"0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3";
return {
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithTwoNFTs()
{
std::string const url1 = "abcd1";
std::string const url2 = "abcd2";
ripple::Blob const uri1Blob(url1.begin(), url1.end());
ripple::Blob const uri2Blob(url2.begin(), url2.end());
constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q";
constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000";
constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA";
auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt);
auto const serializerNftPage = nftPage.getSerializer();
auto const account = getAccountIdWithString(kACCOUNT);
return {
.key = {},
.keyRaw = std::string(reinterpret_cast<char const*>(account.data()), ripple::AccountID::size()),
.data = {},
.dataRaw =
std::string(static_cast<char const*>(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()),
.successor = "",
.predecessor = "",
.type = etlng::model::Object::ModType::Created,
};
}
@@ -219,8 +165,10 @@ etlng::model::BookSuccessor
createSuccessor()
{
return {
.firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
.bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
.firstBook =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}),
.bookBase =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}),
};
}

View File

@@ -32,17 +32,149 @@
namespace util {
static constexpr auto kDEFAULT_TXN_HEX =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
static constexpr auto kDEFAULT_TXN_META =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E";
static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
[[maybe_unused, nodiscard]] std::pair<std::string, std::string>
createNftTxAndMetaBlobs();
createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX);
[[maybe_unused, nodiscard]] std::pair<ripple::STTx, ripple::TxMeta>
createNftTxAndMeta();
createNftTxAndMeta(
std::string hashStr = kDEFAULT_HASH,
std::string metaStr = kDEFAULT_TXN_META,
std::string txnStr = kDEFAULT_TXN_HEX
);
[[maybe_unused, nodiscard]] etlng::model::Transaction
createTransaction(ripple::TxType type);
createTransaction(
ripple::TxType type,
std::string hashStr = kDEFAULT_HASH,
std::string metaStr = kDEFAULT_TXN_META,
std::string txnStr = kDEFAULT_TXN_HEX
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObject();
createObject(
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
std::string key = kDEFAULT_OBJ_KEY
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObjectWithBookBase(
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
std::string key = kDEFAULT_OBJ_KEY
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObjectWithTwoNFTs();
[[maybe_unused, nodiscard]] etlng::model::BookSuccessor
createSuccessor();

View File

@@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface {
MOCK_METHOD(void, writeAccountTransactions, (std::vector<AccountTransactionsData>), (override));
MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override));
MOCK_METHOD(void, writeNFTTransactions, (std::vector<NFTTransactionsData> const&), (override));
MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override));

View File

@@ -21,6 +21,7 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include <gmock/gmock.h>
#include <xrpl/basics/base_uint.h>
@@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface {
MOCK_METHOD(std::optional<data::Blob>, get, (ripple::uint256 const& a, uint32_t b), (const, override));
MOCK_METHOD(void, update, (std::vector<etlng::model::Object> const&, uint32_t), (override));
MOCK_METHOD(std::optional<data::Blob>, getDeleted, (ripple::uint256 const&, uint32_t), (const, override));
MOCK_METHOD(
std::optional<data::LedgerObject>,
getSuccessor,

View File

@@ -45,6 +45,10 @@ target_sources(
etlng/LoadingTests.cpp
etlng/NetworkValidatedLedgersTests.cpp
etlng/MonitorTests.cpp
etlng/ext/CoreTests.cpp
etlng/ext/CacheTests.cpp
etlng/ext/NFTTests.cpp
etlng/ext/SuccessorTests.cpp
# Feed
util/BytesConverterTests.cpp
feed/BookChangesFeedTests.cpp

View File

@@ -0,0 +1,92 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/protocol/TxFormats.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const kUNUSED_LAST_KEY = "unused";
auto
createTestData()
{
auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct CacheExtTests : util::prometheus::WithPrometheus {
protected:
MockLedgerCache cache_;
etlng::impl::CacheExt ext_{cache_};
};
TEST_F(CacheExtTests, OnLedgerDataUpdatesCache)
{
auto const data = createTestData();
EXPECT_CALL(cache_, update(data.objects, data.seq));
ext_.onLedgerData(data);
}
TEST_F(CacheExtTests, OnInitialDataUpdatesCacheAndSetsFull)
{
auto const data = createTestData();
EXPECT_CALL(cache_, update(data.objects, data.seq));
EXPECT_CALL(cache_, setFull);
ext_.onInitialData(data);
}
TEST_F(CacheExtTests, OnInitialObjectsUpdateCache)
{
auto const objects = std::vector{util::createObject(), util::createObject()};
EXPECT_CALL(cache_, update(objects, kSEQ));
ext_.onInitialObjects(kSEQ, objects, kUNUSED_LAST_KEY);
}

View File

@@ -0,0 +1,107 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/protocol/TxFormats.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto
createTestData()
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::CoreExt ext_{backend_};
};
TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
ext_.onLedgerData(data);
}
TEST_F(CoreExtTests, OnInitialDataWritesLedgerAndTransactions)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
ext_.onInitialData(data);
}
TEST_F(CoreExtTests, OnInitialObjectWritesLedgerObject)
{
auto const data = util::createObject();
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
ext_.onInitialObject(kSEQ, data);
}
TEST_F(CoreExtTests, OnObjectWritesLedgerObject)
{
auto const data = util::createObject();
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
ext_.onObject(kSEQ, data);
}

View File

@@ -0,0 +1,287 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/protocol/TxFormats.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const kTXN_HEX2 =
"12001D230606B58324048A8B6F501C50E8EBCD412E6CF9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487E501DA2A1BC9A62AAEB2A"
"2A70F895587A3FB752514AA03F8C6E7C84864653B8673E0368400000000000001E60134000000000001F09732103B8C234E0598BC26D8A3E1B"
"FF53EB252EC0F15EA6800E4D85AA5D7CD15D76B01E744730450221009C0AFF5F3298E10ABE42894717DA46B59529A366527AA5DFC1577ADEA9"
"B20FA70220494D1D9BFEF2AB09F4D6403AAC5B9DCC5B859DCEC1380C6D817A6EDFE7E68FD581141565EED165BA79999425204A8491C73B1301"
"E34FF9EA7D0F7872702E63616665202D2073616C65E1F1";
constinit auto const kTXN_META2 =
"201C00000040F8E51100502505A59E11552ABC2FD74D879BE58489A588838AA2BA59E1E05A48A574226CD8B6CE77998971560B639A808E3B97"
"42A25334E5CF68EEDEDE52F54E50F4E63921C9F3C40588E426E6FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C"
"052AB2897542697066733A2F2F6261666B726569683673796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E71673472"
"6E747037653769757469766C796275E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D526950"
"5679335464654A6170697974576B686141576B4D716D39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60F"
"DDC47DABE4D52E4AD1F0A50A85CBBC00000022752E516D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169"
"316B4C316E5255797131E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C457664644634"
"5442427374455138484C66694867456B6A4465514D56676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4"
"D52E4AD1F0A5608E298B000000EF752E516D4E6E56566F4250426755783848334641486F436162593647347577445758767738757455636976"
"73764D7142E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646"
"777A526D517472694841663277437A3567786161355256446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0"
"A5B09C35A40000020A752E516D5168584346325738535571657633333657703572357935326141464D59564A667570534A536B6B425047786E"
"E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B6968756000001BC752E516D5359647963353276657A7478365338673231"
"70544651724E6E624762676A5A4B5047753836464231394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C"
"000000D2752E516D506645624A38446D624E69794C55637441796A3473625453515937314177597A32776A595238703838416432E1EC5A0008"
"27107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37"
"773338734147397469725867326D626D4262596771E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F75"
"2E516D505A733255393159764538374277635677587A63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E6"
"4C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E21584000001EA752E516D656366646769555A6B37355574667152763474696644557257345439"
"4571616F733231757036706E75345663E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573"
"614345725175396F48596A71765172564B31473155504657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE"
"11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A6854653776"
"43624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A2F2F6261"
"667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E6965"
"2F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F62616679"
"62656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E69652F36"
"342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D616D397436"
"5A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0FE30354A4"
"3281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D79756E697736"
"647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A43281629A"
"0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D6575326934"
"65726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A00082710"
"AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E61776833"
"6F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B7320"
"2331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F516D63436D"
"526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E72200000000501A0B639A808E"
"3B9742A25334E5CF68EEDEDE52F54E4A69498F5B79952C052AB289501B0B639A808E3B9742A25334E5CF68EEDEDE52F54E50F4E639664EC7E5"
"0588E658FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C052AB2897542697066733A2F2F6261666B7265696836"
"73796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E716734726E747037653769757469766C796275E1EC5A00082710"
"7B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D5269505679335464654A6170697974576B686141576B4D716D"
"39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A50A85CBBC00000022752E51"
"6D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169316B4C316E5255797131E1EC5A000827107B87E64C88"
"4BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C4576646446345442427374455138484C66694867456B6A4465514D56"
"676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5608E298B000000EF752E516D4E6E5656"
"6F4250426755783848334641486F43616259364734757744575876773875745563697673764D7142E1EC5A000827107B87E64C884BBFB60FDD"
"C47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646777A526D517472694841663277437A35677861613552"
"56446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B09C35A40000020A752E516D516858434632573853"
"5571657633333657703572357935326141464D59564A667570534A536B6B425047786EE1EC5A000827107B87E64C884BBFB60FDDC47DABE4D5"
"2E4AD1F0A5B6968756000001BC752E516D5359647963353276657A747836533867323170544651724E6E624762676A5A4B5047753836464231"
"394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C000000D2752E516D506645624A38446D624E69794C55"
"637441796A3473625453515937314177597A32776A595238703838416432E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5"
"CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37773338734147397469725867326D626D4262596771E1"
"EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F752E516D505A733255393159764538374277635677587A"
"63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E2158400"
"0001EA752E516D656366646769555A6B373555746671527634746966445572573454394571616F733231757036706E75345663E1EC5A000827"
"107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573614345725175396F48596A71765172564B3147315550"
"4657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051752E"
"516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44668A"
"2B96DFDE11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A68"
"5465377643624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A"
"2F2F6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A3670717136356635"
"6F6E69652F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F"
"6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E"
"69652F36342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D61"
"6D3974365A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0F"
"E30354A43281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D7975"
"6E697736647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A4"
"3281629A0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D65"
"7532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A"
"00082710AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E"
"617768336F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B73205075"
"6E6B73202331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F51"
"6D63436D526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E1E51100502505A4C9"
"C9557DF04CC68DE7C5EF20DBF705221EEDB05FE3806BC3F6A35240652C3E9C97BA5A56246B3E06AB367AB9614566B6F90C718B52A4440852A4"
"440804D409E004C90E52E6FAEC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478B72AD46100000008755E68747470733A2F2F69"
"7066732E696F2F697066732F62616679626569656F6C7667696F71766F737436346367646873797876726962336D6265797278773477626F61"
"617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10"
"276700000009755E68747470733A2F2F697066732E696F2F697066732F62616679626569686D6374376A766B7236366E67337975676D33706D"
"70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA70"
"69E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E68747470733A2F2F697066732E696F2F697066732F6261667962656968626D6D"
"6E626C687736656D776E6A733766787778376736376B6E6A626B6966636674787A6B66777632767162357A6C727575612F6D65746164617461"
"2E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478FBE441630000000B755E68747470733A2F2F697066732E696F"
"2F697066732F62616679626569686E6C77656C7965357270706963646761617678756869376D61636E697879627077667133796734626C3366"
"6A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051"
"752E516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44"
"668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B535470"
"31514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D51"
"674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE58"
"EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A7661"
"6D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC753569"
"7066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A0008"
"2710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F62"
"6D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC24F"
"1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C783579693375"
"706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408"
"005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D7547724839"
"4C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D555744524C"
"425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB9614566"
"B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E346958597233487341715771"
"673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD7535697066733A"
"2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A00081388246B"
"3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A51567957"
"4B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204C9"
"12F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E47"
"E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E357163334E"
"753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52"
"A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E654474"
"6656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D5779"
"6D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367AB9"
"614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A743147454564"
"4C787161513466714E637375363854677153E1F1E1E72200000000501A246B3E06AB367AB9614566B6F90C718B52A444084B237478B72AD461"
"00000008501B246B3E06AB367AB9614566B6F90C718B52A4440852A444080D2641FC04C91315FAEC5A00081388DAA8A3AA7069E65EC1E3E557"
"1D9B6F274B237478B72AD46100000008755E68747470733A2F2F697066732E696F2F697066732F62616679626569656F6C7667696F71766F73"
"7436346367646873797876726962336D6265797278773477626F61617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC"
"5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10276700000009755E68747470733A2F2F697066732E696F2F697066732F62"
"616679626569686D6374376A766B7236366E67337975676D33706D70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69"
"792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E6874747073"
"3A2F2F697066732E696F2F697066732F6261667962656968626D6D6E626C687736656D776E6A733766787778376736376B6E6A626B69666366"
"74787A6B66777632767162357A6C727575612F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B23"
"7478FBE441630000000B755E68747470733A2F2F697066732E696F2F697066732F62616679626569686E6C77656C7965357270706963646761"
"617678756869376D61636E697879627077667133796734626C33666A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A"
"44668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B5354"
"7031514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D"
"51674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE"
"58EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A76"
"616D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC7535"
"697066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A00"
"082710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F"
"626D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC2"
"4F1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C7835796933"
"75706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A444"
"08005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D75477248"
"394C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D55574452"
"4C425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB96145"
"66B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E3469585972334873417157"
"71673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD753569706673"
"3A2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A0008138824"
"6B3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A515679"
"574B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204"
"C912F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E"
"47E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E35716333"
"4E753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B"
"52A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E6544"
"746656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D57"
"796D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367A"
"B9614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A7431474545"
"644C787161513466714E637375363854677153E1F1E1E1E4110064562AED34CB796DF0E82AAC7EB958158EEBF99F51AA8C96B85654DDE05206"
"C18BBCE7220000000225059FB7B755172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235BEBBE582AED34CB796DF0E82A"
"AC7EB958158EEBF99F51AA8C96B85654DDE05206C18BBC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051E1"
"E1E51100612505A58BFB555F1D177245DF64F1BAC04B3EFF72C9B448710F95E86355339305264B0C6450345643CECFECC44660B31BEB70A4AE"
"78ED0BB4B31A754CE746848150EFFC03418847E62D0000010E624000000004FF530BE1E722000000002404C9421F2D0000010D202B00000712"
"202C00000493203204C90C7062400000000506A60B8114246B3E06AB367AB9614566B6F90C718B52A44408E1E1E41100375650E8EBCD412E6C"
"F9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487EE722000000002505A5B1C23400000000000000463C0000000000000000558F56"
"F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB7"
"2AED0000005161400000000007C02982140B639A808E3B9742A25334E5CF68EEDEDE52F54E83141565EED165BA79999425204A8491C73B1301"
"E34FE1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC4565183948C67127DAA598D"
"1197F3DEDC4552F64034F3B8213060B8903FD6C8A561E62D0000039C62400000000D7F518BE1E7220000000024057A8B032D0000039B202B00"
"000290202C000000082032057A6CBC62400000000D77916281140B639A808E3B9742A25334E5CF68EEDEDE52F54E8914CEAECC5B87EA043BD9"
"8E1B4FE8663AC59D5C3518E1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45658"
"7E28972F3B63D2260C0671E59593EE6C4D27AB857D1AF230F5CAA959BB3EACE624048A8B6F624000001B84AABF6AE1E7220000000024048A8B"
"702D00000000624000001B84AADE5581141565EED165BA79999425204A8491C73B1301E34FE1E1E41100645698929E4419455BBB551BF09C08"
"EEBEE19021E6B7E3D1D989968EF49970F10130E722000000012505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1F"
"D7B40F710AC45898929E4419455BBB551BF09C08EEBEE19021E6B7E3D1D989968EF49970F101305A00080FA03A44668A2B96DFDE11BF0817CC"
"6DF60C4E3508D43EB72AED00000051E1E1E411003756A2A1BC9A62AAEB2A2A70F895587A3FB752514AA03F8C6E7C84864653B8673E03E72200"
"00000125059FB7B734000000000000003C3C000000000000000055172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235B"
"EBBE5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED0000005161400000000007A1208214246B3E06AB367AB9614566"
"B6F90C718B52A4440883141565EED165BA79999425204A8491C73B1301E34FE1E1E511006125059DE0B155C0F457C1104D45881194D53DBB40"
"B81F9D812FC507637C4294527C06FDD69A3856A969AD4B14312DFA8739258A6A94C0868F1A3E7D1A1EF4839983521A4F6953FCE66240000000"
"0011B75AE1E7220000000024046044C82D00000000202B00001836202C00000F3F62400000000012057A81143A44668A2B96DFDE11BF0817CC"
"6DF60C4E3508D48914651B85AF14BE4F60AFBF5ADAA4F367061C2DA1D4E1E1E51100642505A5B1C2558F56F16F29177518F3DD6CF827085D7B"
"9E2806CD5EBDE810DD1FD7B40F710AC456CE29B8547FC486F45704C1DE539B748587872AA803FB53FB938E282547DF2CFEE722000000003200"
"0000000000004558DE4F51B35BE5A98D0C97BE07378E9CB56FFE3F861E544E0ABAC5ED765E2F781982140B639A808E3B9742A25334E5CF68EE"
"DEDE52F54EE1E1E51100642505A4EA55556022061889BB4DA5242B7AB048C3D751ACEE5788ED4120B2E70F68E9A322A1E956D7A8E1C70CD8A0"
"3AE9BAA41BC0898471FC26DA3712748A803B2F32007CDCB0DCE7220000000031000000000000003D32000000000000003B58B6629B8F178A18"
"C2926F1ADA669262B5D362BFC2DB1417D837CBF382AA37F2D88214246B3E06AB367AB9614566B6F90C718B52A44408E1E1F1031000";
constinit auto const kHASH2 = "D7604B124D5D9C89EC1854A6CBD5A1FFD92502E945411B9C8DE397E7F19A74F8";
auto
createTestData()
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN, kHASH2, kTXN_META2, kTXN_HEX2),
util::createTransaction(ripple::TxType::ttAMM_CREATE), // not NFT - will be filtered
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), // not unique - will be filtered
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::NFTExt ext_{backend_};
};
TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
});
EXPECT_CALL(*backend_, writeNFTTransactions);
ext_.onLedgerData(data);
}
TEST_F(NFTExtTests, OnInitialDataFiltersAndWritesNFTs)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
});
EXPECT_CALL(*backend_, writeNFTTransactions);
ext_.onInitialData(data);
}
TEST_F(NFTExtTests, OnInitialObjectExtractsAndWritesNFTData)
{
auto const data = util::createObjectWithTwoNFTs();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { EXPECT_EQ(nfts.size(), 2); });
ext_.onInitialObject(kSEQ, data);
}

View File

@@ -0,0 +1,641 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "util/Assert.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockAssert.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/StringUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/UintTypes.h>
#include <algorithm>
#include <iterator>
#include <optional>
#include <queue>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto
createTestData(std::vector<etlng::model::Object> objects)
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = std::move(objects),
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
[[maybe_unused]] auto
createInitialTestData(std::vector<ripple::uint256> edgeKeys)
{
// initial data expects objects to be empty as well as non-empty edgeKeys
ASSERT(not edgeKeys.empty(), "Initial data requires edgeKeys");
auto ret = createTestData({});
ret.edgeKeys = std::make_optional<std::vector<std::string>>();
std::ranges::transform(edgeKeys, std::back_inserter(ret.edgeKeys.value()), &uint256ToString);
return ret;
}
} // namespace
struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
MockLedgerCache cache_;
etlng::impl::SuccessorExt ext_{backend_, cache_};
};
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent)
{
auto const data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(false));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
}
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffersAndSuccessorsNotPresent)
{
auto const data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ - 1));
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(Blob{'0'}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillRepeatedly(testing::Return(LedgerObject{}));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
[[maybe_unused]] testing::InSequence inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(LedgerObject{.key = createdObj.key, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(deletedObj.key);
auto const oldCachedObj = createdObj.data;
[[maybe_unused]] testing::InSequence inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(data::kLAST_KEY)));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(deletedObj.key);
auto const oldCachedObj = createdObj.data;
[[maybe_unused]] testing::InSequence inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(LedgerObject{.key = deletedObj.key, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(deletedObj.key)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const predKey =
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"
));
auto const succKey =
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
));
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ))
.WillOnce(testing::Return(data::LedgerObject{.key = predKey, .blob = {}}));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ))
.WillOnce(testing::Return(data::LedgerObject{.key = succKey, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(predKey), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(succKey)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const succ = util::createSuccessor();
data.successors = {succ, succ, succ};
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, auto{succ.firstBook}))
.Times(data.successors->size());
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.predecessor}, kSEQ, auto{createdObj.keyRaw}));
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.keyRaw}, kSEQ, auto{createdObj.successor}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto succ = util::createSuccessor();
succ.firstBook = {}; // empty will be transformed into kLAST_KEY
data.successors = {succ, succ};
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, uint256ToString(data::kLAST_KEY)))
.Times(data.successors->size());
EXPECT_CALL(*backend_, writeSuccessor(auto{deletedObj.predecessor}, kSEQ, auto{deletedObj.successor}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
[[maybe_unused]] testing::Sequence inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = {'0'}};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(std::nullopt));
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
[[maybe_unused]] testing::Sequence inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = {'0'}};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey)));
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created);
auto const bookBase = getBookBase(bookBaseObj.key);
[[maybe_unused]] testing::Sequence inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = bookBaseObj.data};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(bookBase, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(data::LedgerObject{.key = firstKey, .blob = data::Blob{'1'}}));
EXPECT_CALL(
*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_)
); // Called once because firstKey returned repeatedly above
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {'1'}}));
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))).InSequence(inSeq);
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey)
{
using namespace etlng::model;
auto const lastKey = std::string{};
auto const data = std::vector{
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
),
};
std::string lk = lastKey;
for (auto const& obj : data) {
if (not lk.empty())
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
lk = uint256ToString(obj.key);
}
ext_.onInitialObjects(kSEQ, data, lastKey);
}
TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey)
{
using namespace etlng::model;
auto const lastKey =
uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"));
auto const data = std::vector{
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
),
};
std::string lk = lastKey;
for (auto const& obj : data) {
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
lk = uint256ToString(obj.key);
}
ext_.onInitialObjects(kSEQ, data, lastKey);
}
struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests {};
TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
}
TEST_F(
SuccessorExtAssertTests,
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull)
{
using namespace etlng::model;
auto const data = createTestData({
util::createObject(Object::ModType::Modified),
util::createObject(Object::ModType::Created),
});
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(false));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys)
{
using namespace etlng::model;
auto data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto data = createInitialTestData({firstKey, secondKey});
data.objects = {util::createObject()};
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}

View File

@@ -41,9 +41,9 @@
using namespace rpc;
using namespace data;
using namespace testing;
namespace json = boost::json;
using namespace testing;
namespace {