ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 11:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Extraction basics (#1733)

Browse Source 
For #1596
This commit is contained in:
Alex Kremer
2024-11-15 19:55:13 +00:00
committed by GitHub
parent a4b3877cb2
commit 815dfd672e
16 changed files with 1146 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/data/CassandraBackend.hpp
View File

@@ -949,7 +949,7 @@ public:
{
std::vector<Statement> statements;
statements.reserve(data.size());
for (auto [mptId, holder] : data)
for (auto [mptId, holder] : data)
statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
executor_.write(std::move(statements));

65
src/etl/LedgerFetcherInterface.hpp Normal file
View File

@@ -0,0 +1,65 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
/** @file */
#pragma once
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
#include <cstdint>
#include <optional>
namespace etl {
/**
* @brief An interface for LedgerFetcher
*/
struct LedgerFetcherInterface {
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
virtual ~LedgerFetcherInterface() = default;
/**
* @brief Extract data for a particular ledger from an ETL source
*
* This function continously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
*
* @param seq sequence of the ledger to extract
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
*/
[[nodiscard]] virtual OptionalGetLedgerResponseType
fetchData(uint32_t seq) = 0;
/**
* @brief Extract diff data for a particular ledger from an ETL source.
*
* This function continously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
*
* @param seq sequence of the ledger to extract
* @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
*/
[[nodiscard]] virtual OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t seq) = 0;
};
} // namespace etl

6
src/etl/impl/Extractor.hpp
View File

@@ -121,19 +121,19 @@ private:
pipe_.get().finish(startSequence_);
}
bool
[[nodiscard]] bool
isStopping() const
{
return state_.get().isStopping;
}
bool
[[nodiscard]] bool
hasWriteConflict() const
{
return state_.get().writeConflict;
}
bool
[[nodiscard]] bool
shouldFinish(uint32_t seq) const
{
// Stopping conditions:

4
src/etl/impl/LedgerFetcher.hpp
View File

@@ -63,7 +63,7 @@ public:
* @param sequence sequence of the ledger to extract
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
*/
OptionalGetLedgerResponseType
[[nodiscard]] OptionalGetLedgerResponseType
fetchData(uint32_t sequence)
{
LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
@@ -83,7 +83,7 @@ public:
* @param sequence sequence of the ledger to extract
* @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
*/
OptionalGetLedgerResponseType
[[nodiscard]] OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t sequence)
{
LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;

6
src/etlng/CMakeLists.txt
View File

@@ -1,5 +1,5 @@
add_library(clio_etlng INTERFACE)
add_library(clio_etlng)
# target_sources(clio_etlng PRIVATE )
target_sources(clio_etlng PRIVATE impl/Extraction.cpp)
target_link_libraries(clio_etlng INTERFACE clio_data)
target_link_libraries(clio_etlng PUBLIC clio_data)

54
src/etlng/ExtractorInterface.hpp Normal file
View File

@@ -0,0 +1,54 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etlng/Models.hpp"
#include <cstdint>
#include <optional>
namespace etlng {
/**
* @brief An interface for the Extractor
*/
struct ExtractorInterface {
virtual ~ExtractorInterface() = default;
/**
* @brief Extract diff data for a particular ledger
*
* @param seq sequence of the ledger to extract
* @return Ledger data diff between sequence and parent if available
*/
[[nodiscard]] virtual std::optional<model::LedgerData>
extractLedgerWithDiff(uint32_t seq) = 0;
/**
* @brief Extract data for a particular ledger
*
* @param seq sequence of the ledger to extract
* @return Ledger header and transaction+metadata blobs if available
*/
[[nodiscard]] virtual std::optional<model::LedgerData>
extractLedgerOnly(uint32_t seq) = 0;
};
} // namespace etlng

1
src/etlng/Models.hpp
View File

@@ -120,6 +120,7 @@ struct LedgerData {
std::vector<Transaction> transactions;
std::vector<Object> objects;
std::optional<std::vector<BookSuccessor>> successors;
std::optional<std::vector<std::string>> edgeKeys;
ripple::LedgerHeader header;
std::string rawHeader;

224
src/etlng/impl/Extraction.cpp Normal file
View File

@@ -0,0 +1,224 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/Extraction.hpp"
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include "util/LedgerUtils.hpp"
#include "util/Profiler.hpp"
#include "util/log/Logger.hpp"
#include <google/protobuf/repeated_field.h>
#include <sys/types.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxMeta.h>
#include <algorithm>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <memory>
#include <optional>
#include <ranges>
#include <string>
#include <utility>
#include <vector>
namespace etlng::impl {
model::Object::ModType
extractModType(PBModType type)
{
switch (type) {
case PBObjType::UNSPECIFIED:
return model::Object::ModType::Unspecified;
case PBObjType::CREATED:
return model::Object::ModType::Created;
case PBObjType::MODIFIED:
return model::Object::ModType::Modified;
case PBObjType::DELETED:
return model::Object::ModType::Deleted;
default: // some gRPC system values that we don't care about
ASSERT(false, "Tried to extract bogus mod type '{}'", PBObjType::ModificationType_Name(type));
}
std::unreachable();
}
model::Transaction
extractTx(PBTxType tx, uint32_t seq)
{
auto raw = std::move(*tx.mutable_transaction_blob());
ripple::SerialIter it{raw.data(), raw.size()};
ripple::STTx const sttx{it};
ripple::TxMeta meta{sttx.getTransactionID(), seq, tx.metadata_blob()};
return {
.raw = std::move(raw),
.metaRaw = std::move(*tx.mutable_metadata_blob()),
.sttx = sttx, // trivially copyable
.meta = std::move(meta),
.id = sttx.getTransactionID(),
.key = uint256ToString(sttx.getTransactionID()),
.type = sttx.getTxnType()
};
}
std::vector<model::Transaction>
extractTxs(PBTxListType transactions, uint32_t seq)
{
namespace rg = std::ranges;
namespace vs = std::views;
// TODO: should be simplified with ranges::to<> when available
std::vector<model::Transaction> output;
output.reserve(transactions.size());
rg::move(transactions | vs::transform([seq](auto&& tx) { return extractTx(tx, seq); }), std::back_inserter(output));
return output;
}
model::Object
extractObj(PBObjType obj)
{
auto const key = ripple::uint256::fromVoidChecked(obj.key());
ASSERT(key.has_value(), "Failed to deserialize key from void");
auto const valueOr = [](std::string const& maybe, std::string fallback) -> std::string {
if (maybe.empty())
return fallback;
return maybe;
};
return {
.key = *key, // trivially copyable
.keyRaw = std::move(*obj.mutable_key()),
.data = {obj.mutable_data()->begin(), obj.mutable_data()->end()},
.dataRaw = std::move(*obj.mutable_data()),
.successor = valueOr(obj.successor(), uint256ToString(data::firstKey)),
.predecessor = valueOr(obj.predecessor(), uint256ToString(data::lastKey)),
.type = extractModType(obj.mod_type()),
};
}
std::vector<model::Object>
extractObjs(PBObjListType objects)
{
namespace rg = std::ranges;
namespace vs = std::views;
// TODO: should be simplified with ranges::to<> when available
std::vector<model::Object> output;
output.reserve(objects.size());
rg::move(objects | vs::transform([](auto&& obj) { return extractObj(obj); }), std::back_inserter(output));
return output;
}
model::BookSuccessor
extractSuccessor(PBBookSuccessorType successor)
{
return {
.firstBook = std::move(successor.first_book()),
.bookBase = std::move(successor.book_base()),
};
}
std::optional<std::vector<model::BookSuccessor>>
maybeExtractSuccessors(PBLedgerResponseType const& data)
{
namespace rg = std::ranges;
namespace vs = std::views;
if (not data.object_neighbors_included())
return std::nullopt;
// TODO: should be simplified with ranges::to<> when available
std::vector<model::BookSuccessor> output;
output.reserve(data.book_successors_size());
rg::copy(
data.book_successors() | vs::transform([](auto&& obj) { return extractSuccessor(obj); }),
std::back_inserter(output)
);
return output;
}
auto
Extractor::unpack()
{
return [](auto&& data) {
auto header = ::util::deserializeHeader(ripple::makeSlice(data.ledger_header()));
return std::make_optional<model::LedgerData>({
.transactions =
extractTxs(std::move(*data.mutable_transactions_list()->mutable_transactions()), header.seq),
.objects = extractObjs(std::move(*data.mutable_ledger_objects()->mutable_objects())),
.successors = maybeExtractSuccessors(data),
.edgeKeys = std::nullopt,
.header = header,
.rawHeader = std::move(*data.mutable_ledger_header()),
.seq = header.seq,
});
};
}
std::optional<model::LedgerData>
Extractor::extractLedgerWithDiff(uint32_t seq)
{
LOG(log_.debug()) << "Extracting DIFF " << seq;
auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
return fetcher_->fetchDataAndDiff(seq).and_then(unpack());
});
LOG(log_.debug()) << "Extracted and Transformed diff for " << seq << " in " << time << "ms";
// can be nullopt. this means that either the server is stopping or another node took over ETL writing.
return batch;
}
std::optional<model::LedgerData>
Extractor::extractLedgerOnly(uint32_t seq)
{
LOG(log_.debug()) << "Extracting FULL " << seq;
auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
return fetcher_->fetchData(seq).and_then(unpack());
});
LOG(log_.debug()) << "Extracted and Transformed full ledger for " << seq << " in " << time << "ms";
// can be nullopt. this means that either the server is stopping or another node took over ETL writing.
return batch;
}
} // namespace etlng::impl

100
src/etlng/impl/Extraction.hpp Normal file
View File

@@ -0,0 +1,100 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etl/LedgerFetcherInterface.hpp"
#include "etl/impl/LedgerFetcher.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <google/protobuf/repeated_ptr_field.h>
#include <sys/types.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxMeta.h>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <optional>
#include <string>
#include <utility>
#include <vector>
namespace etlng::impl {
using PBObjType = org::xrpl::rpc::v1::RawLedgerObject;
using PBModType = PBObjType::ModificationType;
using PBTxType = org::xrpl::rpc::v1::TransactionAndMetadata;
using PBTxListType = google::protobuf::RepeatedPtrField<PBTxType>;
using PBObjListType = google::protobuf::RepeatedPtrField<PBObjType>;
using PBBookSuccessorType = org::xrpl::rpc::v1::BookSuccessor;
using PBLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
[[nodiscard]] model::Object::ModType
extractModType(PBModType type);
[[nodiscard]] model::Transaction
extractTx(PBTxType tx, uint32_t seq);
[[nodiscard]] std::vector<model::Transaction>
extractTxs(PBTxListType transactions, uint32_t seq);
[[nodiscard]] model::Object
extractObj(PBObjType obj);
[[nodiscard]] std::vector<model::Object>
extractObjs(PBObjListType objects);
[[nodiscard]] model::BookSuccessor
extractSuccessor(PBBookSuccessorType successor);
[[nodiscard]] std::optional<std::vector<model::BookSuccessor>>
maybeExtractSuccessors(PBLedgerResponseType const& data);
// fetches the data in gRPC and transforms to local representation
class Extractor : public ExtractorInterface {
std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
util::Logger log_{"ETL"};
private:
[[nodiscard]] static auto
unpack();
public:
Extractor(std::shared_ptr<etl::LedgerFetcherInterface> fetcher) : fetcher_(std::move(fetcher))
{
}
[[nodiscard]] std::optional<model::LedgerData>
extractLedgerWithDiff(uint32_t seq) override;
[[nodiscard]] std::optional<model::LedgerData>
extractLedgerOnly(uint32_t seq) override;
};
} // namespace etlng::impl

2
src/rpc/common/Validators.cpp
View File

@@ -271,7 +271,7 @@ CustomValidator CustomValidators::CredentialTypeValidator =
return Error{
Status{ClioError::rpcMALFORMED_AUTHORIZED_CREDENTIALS, std::string(key) + " greater than max length"}
};
}
}
return MaybeError{};
}};

1
tests/common/CMakeLists.txt
View File

@@ -8,6 +8,7 @@ target_sources(
util/TestHttpClient.cpp
util/TestHttpServer.cpp
util/TestObject.cpp
util/BinaryTestObject.cpp
util/TestWebSocketClient.cpp
util/TestWsServer.cpp
)

297
tests/common/util/BinaryTestObject.cpp Normal file
View File

@@ -0,0 +1,297 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "util/BinaryTestObject.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "util/StringUtils.hpp"
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <string>
#include <utility>
namespace {
constinit auto const Seq = 30;
constinit auto const TxnHex =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
constinit auto const TxnMeta =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
constinit auto const RawHeader =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
"6DB6FE30CC5909B285080FCD6773CC883F9FE0EE4D439340AC592AADB973ED3CF5"
"3E2232B33EF57CECAC2816E3122816E31A0A00F8377CD95DFA484CFAE282656A58"
"CE5AA29652EFFD80AC59CD91416E4E13DBBE";
} // namespace
namespace util {
std::pair<std::string, std::string>
CreateNftTxAndMetaBlobs()
{
return {hexStringToBinaryString(TxnMeta), hexStringToBinaryString(TxnHex)};
}
std::pair<ripple::STTx, ripple::TxMeta>
CreateNftTxAndMeta()
{
ripple::uint256 hash;
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
auto const [metaBlob, txnBlob] = CreateNftTxAndMetaBlobs();
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
return {ripple::STTx{it}, ripple::TxMeta{hash, Seq, metaBlob}};
}
etlng::model::Transaction
CreateTransaction(ripple::TxType type)
{
auto const [sttx, meta] = CreateNftTxAndMeta();
return {
.raw = "",
.metaRaw = "",
.sttx = sttx,
.meta = meta,
.id = ripple::uint256{"0000000000000000000000000000000000000000000000000000000000000001"},
.key = "0000000000000000000000000000000000000000000000000000000000000001",
.type = type
};
}
etlng::model::Object
CreateObject()
{
// random object taken from initial ledger load
static constinit auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
static constinit auto const objPred = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const objSucc = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const objBlob =
"11007222002200002504270918370000000000000C4538000000000000000A554D94799200CC37EFAF45DA76704ED3CBEDBB4B4FCD"
"56E9CBA5399EB40A7B3BEC629546DD24CDB4C0004C4A50590000000000000000000000000000000000000000000000000000000000"
"000000000000016680000000000000004C4A505900000000000000000000000000000000368480B7780E3DCF5D062A7BB54129F42F"
"8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
return {
.key = {},
.keyRaw = hexStringToBinaryString(objKey),
.data = {},
.dataRaw = hexStringToBinaryString(objBlob),
.successor = hexStringToBinaryString(objSucc),
.predecessor = hexStringToBinaryString(objPred),
.type = {},
};
}
etlng::model::BookSuccessor
CreateSuccessor()
{
return {
.firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
.bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
};
}
etlng::impl::PBLedgerResponseType
CreateDataAndDiff()
{
auto const rawHeaderBlob = hexStringToBinaryString(RawHeader);
auto res = etlng::impl::PBLedgerResponseType();
res.set_ledger_header(rawHeaderBlob);
res.set_objects_included(true);
res.set_object_neighbors_included(true);
{
auto original = org::xrpl::rpc::v1::TransactionAndMetadata();
auto const [metaRaw, txRaw] = CreateNftTxAndMetaBlobs();
original.set_transaction_blob(txRaw);
original.set_metadata_blob(metaRaw);
for (int i = 0; i < 10; ++i) {
auto* p = res.mutable_transactions_list()->add_transactions();
*p = original;
}
}
{
auto expected = CreateObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
original.set_data(expected.dataRaw);
original.set_key(expected.keyRaw);
for (auto i = 0; i < 10; ++i) {
auto* p = res.mutable_ledger_objects()->add_objects();
*p = original;
}
}
{
auto expected = CreateSuccessor();
auto original = org::xrpl::rpc::v1::BookSuccessor();
original.set_first_book(expected.firstBook);
original.set_book_base(expected.bookBase);
res.set_object_neighbors_included(true);
for (auto i = 0; i < 10; ++i) {
auto* p = res.mutable_book_successors()->Add();
*p = original;
}
}
return res;
}
etlng::impl::PBLedgerResponseType
CreateData()
{
auto const rawHeaderBlob = hexStringToBinaryString(RawHeader);
auto res = etlng::impl::PBLedgerResponseType();
res.set_ledger_header(rawHeaderBlob);
res.set_objects_included(false);
res.set_object_neighbors_included(false);
{
auto original = org::xrpl::rpc::v1::TransactionAndMetadata();
auto const [metaRaw, txRaw] = CreateNftTxAndMetaBlobs();
original.set_transaction_blob(txRaw);
original.set_metadata_blob(metaRaw);
for (int i = 0; i < 10; ++i) {
auto* p = res.mutable_transactions_list()->add_transactions();
*p = original;
}
}
return res;
}
} // namespace util

56
tests/common/util/BinaryTestObject.hpp Normal file
View File

@@ -0,0 +1,56 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include <gtest/gtest.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <string>
#include <utility>
namespace util {
[[maybe_unused, nodiscard]] std::pair<std::string, std::string>
CreateNftTxAndMetaBlobs();
[[maybe_unused, nodiscard]] std::pair<ripple::STTx, ripple::TxMeta>
CreateNftTxAndMeta();
[[maybe_unused, nodiscard]] etlng::model::Transaction
CreateTransaction(ripple::TxType type);
[[maybe_unused, nodiscard]] etlng::model::Object
CreateObject();
[[maybe_unused, nodiscard]] etlng::model::BookSuccessor
CreateSuccessor();
[[maybe_unused, nodiscard]] etlng::impl::PBLedgerResponseType
CreateDataAndDiff();
[[maybe_unused, nodiscard]] etlng::impl::PBLedgerResponseType
CreateData();
} // namespace util

1
tests/unit/CMakeLists.txt
View File

@@ -33,6 +33,7 @@ target_sources(
etl/TransformerTests.cpp
# ETLng
etlng/RegistryTests.cpp
etlng/ExtractionTests.cpp
# Feed
feed/BookChangesFeedTests.cpp
feed/ForwardFeedTests.cpp

293
tests/unit/etlng/ExtractionTests.cpp Normal file
View File

@@ -0,0 +1,293 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSeqUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/LoggerFixtures.hpp"
#include <gmock/gmock.h>
#include <google/protobuf/repeated_ptr_field.h>
#include <gtest/gtest.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/Slice.h>
#include <xrpl/basics/StringUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <cstdint>
#include <memory>
#include <optional>
namespace {
constinit auto const Seq = 30;
} // namespace
struct ExtractionTests : NoLoggerFixture {};
TEST_F(ExtractionTests, ModType)
{
using namespace etlng::impl;
using ModType = etlng::model::Object::ModType;
EXPECT_EQ(extractModType(PBObjType::MODIFIED), ModType::Modified);
EXPECT_EQ(extractModType(PBObjType::CREATED), ModType::Created);
EXPECT_EQ(extractModType(PBObjType::DELETED), ModType::Deleted);
EXPECT_EQ(extractModType(PBObjType::UNSPECIFIED), ModType::Unspecified);
}
TEST_F(ExtractionTests, OneTransaction)
{
using namespace etlng::impl;
auto expected = util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER);
auto original = org::xrpl::rpc::v1::TransactionAndMetadata();
auto [metaRaw, txRaw] = util::CreateNftTxAndMetaBlobs();
original.set_transaction_blob(txRaw);
original.set_metadata_blob(metaRaw);
auto res = extractTx(original, Seq);
EXPECT_EQ(res.meta.getLgrSeq(), Seq);
EXPECT_EQ(res.meta.getLgrSeq(), expected.meta.getLgrSeq());
EXPECT_EQ(res.meta.getTxID(), expected.meta.getTxID());
EXPECT_EQ(res.sttx.getTxnType(), expected.sttx.getTxnType());
}
TEST_F(ExtractionTests, MultipleTransactions)
{
using namespace etlng::impl;
auto expected = util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER);
auto original = org::xrpl::rpc::v1::TransactionAndMetadata();
auto [metaRaw, txRaw] = util::CreateNftTxAndMetaBlobs();
original.set_transaction_blob(txRaw);
original.set_metadata_blob(metaRaw);
auto list = org::xrpl::rpc::v1::TransactionAndMetadataList();
for (auto i = 0; i < 10; ++i) {
auto* p = list.add_transactions();
*p = original;
}
auto res = extractTxs(list.transactions(), Seq);
EXPECT_EQ(res.size(), 10);
for (auto const& tx : res) {
EXPECT_EQ(tx.meta.getLgrSeq(), Seq);
EXPECT_EQ(tx.meta.getLgrSeq(), expected.meta.getLgrSeq());
EXPECT_EQ(tx.meta.getTxID(), expected.meta.getTxID());
EXPECT_EQ(tx.sttx.getTxnType(), expected.sttx.getTxnType());
}
}
TEST_F(ExtractionTests, OneObject)
{
using namespace etlng::impl;
auto expected = util::CreateObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
original.set_data(expected.dataRaw);
original.set_key(expected.keyRaw);
auto res = extractObj(original);
EXPECT_EQ(ripple::strHex(res.key), ripple::strHex(expected.keyRaw));
EXPECT_EQ(ripple::strHex(res.data), ripple::strHex(expected.dataRaw));
EXPECT_EQ(res.predecessor, uint256ToString(data::lastKey));
EXPECT_EQ(res.successor, uint256ToString(data::firstKey));
EXPECT_EQ(res.type, expected.type);
}
TEST_F(ExtractionTests, OneObjectWithSuccessorAndPredecessor)
{
using namespace etlng::impl;
auto expected = util::CreateObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
original.set_data(expected.dataRaw);
original.set_key(expected.keyRaw);
original.set_predecessor(expected.predecessor);
original.set_successor(expected.successor);
auto res = extractObj(original);
EXPECT_EQ(ripple::strHex(res.key), ripple::strHex(expected.keyRaw));
EXPECT_EQ(ripple::strHex(res.data), ripple::strHex(expected.dataRaw));
EXPECT_EQ(res.predecessor, expected.predecessor);
EXPECT_EQ(res.successor, expected.successor);
EXPECT_EQ(res.type, expected.type);
}
TEST_F(ExtractionTests, MultipleObjects)
{
using namespace etlng::impl;
auto expected = util::CreateObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
original.set_data(expected.dataRaw);
original.set_key(expected.keyRaw);
auto list = org::xrpl::rpc::v1::RawLedgerObjects();
for (auto i = 0; i < 10; ++i) {
auto* p = list.add_objects();
*p = original;
}
auto res = extractObjs(list.objects());
EXPECT_EQ(res.size(), 10);
for (auto const& obj : res) {
EXPECT_EQ(ripple::strHex(obj.key), ripple::strHex(expected.keyRaw));
EXPECT_EQ(ripple::strHex(obj.data), ripple::strHex(expected.dataRaw));
EXPECT_EQ(obj.predecessor, uint256ToString(data::lastKey));
EXPECT_EQ(obj.successor, uint256ToString(data::firstKey));
EXPECT_EQ(obj.type, expected.type);
}
}
TEST_F(ExtractionTests, OneSuccessor)
{
using namespace etlng::impl;
auto expected = util::CreateSuccessor();
auto original = org::xrpl::rpc::v1::BookSuccessor();
original.set_first_book(expected.firstBook);
original.set_book_base(expected.bookBase);
auto res = extractSuccessor(original);
EXPECT_EQ(ripple::strHex(res.firstBook), ripple::strHex(expected.firstBook));
EXPECT_EQ(ripple::strHex(res.bookBase), ripple::strHex(expected.bookBase));
}
TEST_F(ExtractionTests, MultipleSuccessors)
{
using namespace etlng::impl;
auto expected = util::CreateSuccessor();
auto original = org::xrpl::rpc::v1::BookSuccessor();
original.set_first_book(expected.firstBook);
original.set_book_base(expected.bookBase);
auto data = PBLedgerResponseType();
data.set_object_neighbors_included(true);
for (auto i = 0; i < 10; ++i) {
auto* el = data.mutable_book_successors()->Add();
*el = original;
}
auto res = maybeExtractSuccessors(data);
ASSERT_TRUE(res.has_value());
EXPECT_EQ(res->size(), 10);
for (auto const& successor : res.value()) {
EXPECT_EQ(successor.firstBook, expected.firstBook);
EXPECT_EQ(successor.bookBase, expected.bookBase);
}
}
TEST_F(ExtractionTests, SuccessorsWithNoNeighborsIncluded)
{
using namespace etlng::impl;
auto data = PBLedgerResponseType();
data.set_object_neighbors_included(false);
auto res = maybeExtractSuccessors(data);
ASSERT_FALSE(res.has_value());
}
struct ExtractionDeathTest : NoLoggerFixture {};
TEST_F(ExtractionDeathTest, InvalidModTypeAsserts)
{
using namespace etlng::impl;
EXPECT_DEATH(
{
[[maybe_unused]] auto _ = extractModType(
PBModType::
RawLedgerObject_ModificationType_RawLedgerObject_ModificationType_INT_MIN_SENTINEL_DO_NOT_USE_
);
},
".*"
);
}
struct MockFetcher : etl::LedgerFetcherInterface {
MOCK_METHOD(std::optional<GetLedgerResponseType>, fetchData, (uint32_t), (override));
MOCK_METHOD(std::optional<GetLedgerResponseType>, fetchDataAndDiff, (uint32_t), (override));
};
struct ExtractorTests : ExtractionTests {
std::shared_ptr<MockFetcher> fetcher = std::make_shared<MockFetcher>();
etlng::impl::Extractor extractor{fetcher};
};
TEST_F(ExtractorTests, ExtractLedgerWithDiffNoResult)
{
EXPECT_CALL(*fetcher, fetchDataAndDiff(Seq)).WillOnce(testing::Return(std::nullopt));
auto res = extractor.extractLedgerWithDiff(Seq);
EXPECT_FALSE(res.has_value());
}
TEST_F(ExtractorTests, ExtractLedgerOnlyNoResult)
{
EXPECT_CALL(*fetcher, fetchData(Seq)).WillOnce(testing::Return(std::nullopt));
auto res = extractor.extractLedgerOnly(Seq);
EXPECT_FALSE(res.has_value());
}
TEST_F(ExtractorTests, ExtractLedgerWithDiffWithResult)
{
auto original = util::CreateDataAndDiff();
EXPECT_CALL(*fetcher, fetchDataAndDiff(Seq)).WillOnce(testing::Return(original));
auto res = extractor.extractLedgerWithDiff(Seq);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res->objects.size(), 10);
EXPECT_EQ(res->transactions.size(), 10);
EXPECT_TRUE(res->successors.has_value());
EXPECT_EQ(res->successors->size(), 10);
EXPECT_FALSE(res->edgeKeys.has_value()); // this is set separately in ETL
}
TEST_F(ExtractorTests, ExtractLedgerOnlyWithResult)
{
auto original = util::CreateData();
EXPECT_CALL(*fetcher, fetchData(Seq)).WillOnce(testing::Return(original));
auto res = extractor.extractLedgerOnly(Seq);
EXPECT_TRUE(res.has_value());
EXPECT_TRUE(res->objects.empty());
EXPECT_EQ(res->transactions.size(), 10);
EXPECT_FALSE(res->successors.has_value());
EXPECT_FALSE(res->edgeKeys.has_value()); // this is set separately in ETL
}

232
tests/unit/etlng/RegistryTests.cpp
View File

@@ -19,8 +19,8 @@
#include "etlng/Models.hpp"
#include "etlng/impl/Registry.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
@@ -131,8 +131,8 @@ static_assert(ContainsSpec<ValidSpec>);
namespace {
auto constinit LEDGERHASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto constinit SEQ = 30;
constinit auto const LedgerHash = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const Seq = 30;
struct MockExtLedgerData {
MOCK_METHOD(void, onLedgerData, (etlng::model::LedgerData const&), (const));
@@ -180,166 +180,16 @@ struct MockExtNftOffer {
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etlng::model::Transaction const&), (const));
};
struct RegistryTest : NoLoggerFixture {
static std::pair<ripple::STTx, ripple::TxMeta>
CreateNftTxAndMeta()
{
ripple::uint256 hash;
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
static constinit auto txnHex =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
static constinit auto txnMeta =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
auto const metaBlob = hexStringToBinaryString(txnMeta);
auto const txnBlob = hexStringToBinaryString(txnHex);
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
return {ripple::STTx{it}, ripple::TxMeta{hash, SEQ, metaBlob}};
}
static auto
CreateTransaction(ripple::TxType type)
{
auto [sttx, meta] = CreateNftTxAndMeta();
return etlng::model::Transaction{
.raw = "",
.metaRaw = "",
.sttx = sttx,
.meta = meta,
.id = ripple::uint256{"0000000000000000000000000000000000000000000000000000000000000001"},
.key = "0000000000000000000000000000000000000000000000000000000000000001",
.type = type
};
}
static auto
CreateObject()
{
return etlng::model::Object{
.key = {},
.keyRaw = {},
.data = {},
.dataRaw = {},
.successor = {},
.predecessor = {},
.type = {},
};
}
};
struct RegistryTest : NoLoggerFixture {};
} // namespace
TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForInitialTransaction)
{
auto transactions = std::vector{
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto extBurn = MockExtNftBurn{};
@@ -348,24 +198,25 @@ TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForInitialTransaction)
EXPECT_CALL(extBurn, onInitialTransaction(testing::_, testing::_)).Times(2); // 2 burn txs
EXPECT_CALL(extOffer, onInitialTransaction(testing::_, testing::_)); // 1 create offer
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtNftBurn&, MockExtNftOffer&>(extBurn, extOffer);
reg.dispatchInitialData(etlng::model::LedgerData{
.transactions = transactions,
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ,
.seq = Seq,
});
}
TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForTransaction)
{
auto transactions = std::vector{
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto extBurn = MockExtTransactionNftBurn{};
@@ -374,15 +225,16 @@ TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForTransaction)
EXPECT_CALL(extBurn, onTransaction(testing::_, testing::_)).Times(2); // 2 burn txs
EXPECT_CALL(extOffer, onTransaction(testing::_, testing::_)); // 1 create offer
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtTransactionNftBurn&, MockExtTransactionNftOffer&>(extBurn, extOffer);
reg.dispatch(etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ
.seq = Seq
});
}
@@ -395,7 +247,7 @@ TEST_F(RegistryTest, InitialObjectsEmpty)
EXPECT_CALL(extObjs, onInitialObjects(testing::_, testing::_, testing::_)); // 1 vector passed as is
auto reg = Registry<MockExtInitialObject&, MockExtInitialObjects&>(extObj, extObjs);
reg.dispatchInitialObjects(SEQ, {}, {});
reg.dispatchInitialObjects(Seq, {}, {});
}
TEST_F(RegistryTest, InitialObjectsDispatched)
@@ -407,7 +259,7 @@ TEST_F(RegistryTest, InitialObjectsDispatched)
EXPECT_CALL(extObjs, onInitialObjects(testing::_, testing::_, testing::_)); // 1 vector passed as is
auto reg = Registry<MockExtInitialObject&, MockExtInitialObjects&>(extObj, extObjs);
reg.dispatchInitialObjects(SEQ, {CreateObject(), CreateObject(), CreateObject()}, {});
reg.dispatchInitialObjects(Seq, {util::CreateObject(), util::CreateObject(), util::CreateObject()}, {});
}
TEST_F(RegistryTest, ObjectsDispatched)
@@ -416,39 +268,41 @@ TEST_F(RegistryTest, ObjectsDispatched)
EXPECT_CALL(extObj, onObject(testing::_, testing::_)).Times(3); // 3 objects sent
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtOnObject&>(extObj);
reg.dispatch(etlng::model::LedgerData{
.transactions = {},
.objects = {CreateObject(), CreateObject(), CreateObject()},
.objects = {util::CreateObject(), util::CreateObject(), util::CreateObject()},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ
.seq = Seq
});
}
TEST_F(RegistryTest, OnLedgerDataForBatch)
{
auto transactions = std::vector{
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto ext = MockExtLedgerData{};
EXPECT_CALL(ext, onLedgerData(testing::_)); // 1 batch (dispatch call)
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtLedgerData&>(ext);
reg.dispatch(etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ
.seq = Seq
});
}
@@ -462,7 +316,7 @@ TEST_F(RegistryTest, InitialObjectsCorrectOrderOfHookCalls)
EXPECT_CALL(extObj, onInitialObject).Times(3);
auto reg = Registry<MockExtInitialObject&, MockExtInitialObjects&>(extObj, extObjs);
reg.dispatchInitialObjects(SEQ, {CreateObject(), CreateObject(), CreateObject()}, {});
reg.dispatchInitialObjects(Seq, {util::CreateObject(), util::CreateObject(), util::CreateObject()}, {});
}
TEST_F(RegistryTest, InitialDataCorrectOrderOfHookCalls)
@@ -471,24 +325,25 @@ TEST_F(RegistryTest, InitialDataCorrectOrderOfHookCalls)
auto extInitialTransaction = MockExtNftBurn{};
auto transactions = std::vector{
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
testing::InSequence const seqGuard;
EXPECT_CALL(extInitialData, onInitialData);
EXPECT_CALL(extInitialTransaction, onInitialTransaction).Times(2);
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtNftBurn&, MockExtInitialData&>(extInitialTransaction, extInitialData);
reg.dispatchInitialData(etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ
.seq = Seq
});
}
@@ -499,14 +354,14 @@ TEST_F(RegistryTest, LedgerDataCorrectOrderOfHookCalls)
auto extOnObject = MockExtOnObject{};
auto transactions = std::vector{
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::CreateTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto objects = std::vector{
CreateObject(),
CreateObject(),
CreateObject(),
util::CreateObject(),
util::CreateObject(),
util::CreateObject(),
};
// testing::Sequence seq;
@@ -515,7 +370,7 @@ TEST_F(RegistryTest, LedgerDataCorrectOrderOfHookCalls)
EXPECT_CALL(extOnTransaction, onTransaction).Times(2);
EXPECT_CALL(extOnObject, onObject).Times(3);
auto const header = CreateLedgerHeader(LEDGERHASH, SEQ);
auto const header = CreateLedgerHeader(LedgerHash, Seq);
auto reg = Registry<MockExtOnObject&, MockExtTransactionNftBurn&, MockExtLedgerData&>(
extOnObject, extOnTransaction, extLedgerData
);
@@ -523,8 +378,9 @@ TEST_F(RegistryTest, LedgerDataCorrectOrderOfHookCalls)
.transactions = std::move(transactions),
.objects = std::move(objects),
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = SEQ
.seq = Seq
});
}