feat: ETLng extensions (#1967)

For #1599 #1597
This commit is contained in:
Alex Kremer
2025-03-21 16:41:29 +00:00
committed by GitHub
parent a36aa3618f
commit b31b7633c9
25 changed files with 2167 additions and 147 deletions

View File

@@ -19,134 +19,31 @@
#include "util/BinaryTestObject.hpp"
#include "data/DBHelpers.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/StringUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/protocol/AccountID.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <optional>
#include <string>
#include <utility>
namespace {
constinit auto const kSEQ = 30;
constinit auto const kTXN_HEX =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
constinit auto const kTXN_META =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
constinit auto const kRAW_HEADER =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
@@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER =
namespace util {
std::pair<std::string, std::string>
createNftTxAndMetaBlobs()
createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr)
{
return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)};
return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)};
}
std::pair<ripple::STTx, ripple::TxMeta>
createNftTxAndMeta()
createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr)
{
ripple::uint256 hash;
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
EXPECT_TRUE(hash.parseHex(hashStr));
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs();
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr);
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}};
}
etlng::model::Transaction
createTransaction(ripple::TxType type)
createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr)
{
auto const [sttx, meta] = createNftTxAndMeta();
auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr);
return {
.raw = "",
.metaRaw = "",
@@ -192,10 +89,9 @@ createTransaction(ripple::TxType type)
}
etlng::model::Object
createObject()
createObject(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
@@ -205,12 +101,62 @@ createObject()
"8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
return {
.key = {},
.keyRaw = hexStringToBinaryString(kOBJ_KEY),
.data = {},
.dataRaw = hexStringToBinaryString(kOBJ_BLOB),
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
"11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE"
"3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000"
"0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3";
return {
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithTwoNFTs()
{
std::string const url1 = "abcd1";
std::string const url2 = "abcd2";
ripple::Blob const uri1Blob(url1.begin(), url1.end());
ripple::Blob const uri2Blob(url2.begin(), url2.end());
constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q";
constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000";
constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA";
auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt);
auto const serializerNftPage = nftPage.getSerializer();
auto const account = getAccountIdWithString(kACCOUNT);
return {
.key = {},
.keyRaw = std::string(reinterpret_cast<char const*>(account.data()), ripple::AccountID::size()),
.data = {},
.dataRaw =
std::string(static_cast<char const*>(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()),
.successor = "",
.predecessor = "",
.type = etlng::model::Object::ModType::Created,
};
}
@@ -219,8 +165,10 @@ etlng::model::BookSuccessor
createSuccessor()
{
return {
.firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
.bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
.firstBook =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}),
.bookBase =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}),
};
}