ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-12-06 17:27:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: ETLng integration (#1986)

Browse Source 
For #1594
This commit is contained in:
Alex Kremer
2025-04-04 15:52:22 +01:00
committed by GitHub
parent 6896a2545a
commit 1d011cf8d9
57 changed files with 3473 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/unit/etl/ETLStateTests.cpp
View File

@@ -18,6 +18,7 @@
//==============================================================================
#include "etl/ETLState.hpp"
#include "etl/Source.hpp"
#include "rpc/Errors.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/MockSource.hpp"

7
tests/unit/etl/GrpcSourceTests.cpp
View File

@@ -92,7 +92,7 @@ TEST_F(GrpcSourceTests, fetchLedgerNoStub)
TEST_F(GrpcSourceTests, loadInitialLedgerNoStub)
{
GrpcSource wrongGrpcSource{"wrong", "wrong", mockBackend_};
auto const [data, success] = wrongGrpcSource.loadInitialLedger(0, 0, false);
auto const [data, success] = wrongGrpcSource.loadInitialLedger(0, 0);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
}
@@ -101,7 +101,6 @@ struct GrpcSourceLoadInitialLedgerTests : GrpcSourceTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 4;
bool const cacheOnly_ = false;
};
TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
@@ -116,7 +115,7 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
return grpc::Status{grpc::StatusCode::NOT_FOUND, "Not found"};
});
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_, cacheOnly_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
}
@@ -147,7 +146,7 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, worksFine)
EXPECT_CALL(*mockBackend_, writeNFTs).Times(numMarkers_);
EXPECT_CALL(*mockBackend_, writeLedgerObject).Times(numMarkers_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_, cacheOnly_);
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
EXPECT_TRUE(success);
EXPECT_EQ(data, std::vector<std::string>(4, keyStr));

29
tests/unit/etl/LoadBalancerTests.cpp
View File

@@ -436,56 +436,50 @@ struct LoadBalancerLoadInitialLedgerTests : LoadBalancerOnConnectHookTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 16;
bool const cacheOnly_ = true;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
};
TEST_F(LoadBalancerLoadInitialLedgerTests, load)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0DoesntHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_bothSourcesDontHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_, std::chrono::milliseconds{1}), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, std::chrono::milliseconds{1}), response_.first);
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0ReturnsStatusFalse)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, false)));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
}
struct LoadBalancerLoadInitialLedgerCustomNumMarkersTests : LoadBalancerConstructorTests {
protected:
uint32_t const numMarkers_ = 16;
uint32_t const sequence_ = 123;
bool const cacheOnly_ = true;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
};
@@ -502,10 +496,9 @@ TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersTests, loadInitialLedger)
util::Random::setSeed(0);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, cacheOnly_))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_, cacheOnly_), response_.first);
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_), response_.first);
}
struct LoadBalancerFetchLegerTests : LoadBalancerOnConnectHookTests {

4
tests/unit/etl/SourceImplTests.cpp
View File

@@ -48,7 +48,7 @@ struct GrpcSourceMock {
MOCK_METHOD(FetchLedgerReturnType, fetchLedger, (uint32_t, bool, bool));
using LoadLedgerReturnType = std::pair<std::vector<std::string>, bool>;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t, bool));
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t));
};
struct SubscriptionSourceMock {
@@ -174,7 +174,7 @@ TEST_F(SourceImplTest, loadInitialLedger)
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, false))
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, true)));
auto const [actualLedgers, actualSuccess] = source_.loadInitialLedger(ledgerSeq, numMarkers);