ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-12-06 17:27:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: New ETL by default (#2752)

Browse Source
This commit is contained in:
Alex Kremer
2025-11-05 13:29:36 +00:00
committed by GitHub
parent 316126746b
commit fcc5a5425e
140 changed files with 1766 additions and 8158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
tests/unit/CMakeLists.txt
View File

@@ -28,36 +28,26 @@ target_sources(
etl/CursorFromFixDiffNumProviderTests.cpp
etl/CorruptionDetectorTests.cpp
etl/ETLStateTests.cpp
etl/ExtractionDataPipeTests.cpp
etl/ExtractorTests.cpp
etl/ETLServiceTests.cpp
etl/ExtractionTests.cpp
etl/ForwardingSourceTests.cpp
etl/GrpcSourceTests.cpp
etl/LedgerPublisherTests.cpp
etl/LoadBalancerTests.cpp
etl/LoadingTests.cpp
etl/MonitorTests.cpp
etl/NetworkValidatedLedgersTests.cpp
etl/NFTHelpersTests.cpp
etl/RegistryTests.cpp
etl/SchedulingTests.cpp
etl/SourceImplTests.cpp
etl/SubscriptionSourceTests.cpp
etl/TransformerTests.cpp
# ETLng
etlng/AmendmentBlockHandlerTests.cpp
etlng/ETLServiceTests.cpp
etlng/ExtractionTests.cpp
etlng/ForwardingSourceTests.cpp
etlng/GrpcSourceTests.cpp
etlng/LedgerPublisherTests.cpp
etlng/RegistryTests.cpp
etlng/SchedulingTests.cpp
etlng/TaskManagerTests.cpp
etlng/LoadingTests.cpp
etlng/LoadBalancerTests.cpp
etlng/NetworkValidatedLedgersTests.cpp
etlng/MonitorTests.cpp
etlng/SourceImplTests.cpp
etlng/ext/CoreTests.cpp
etlng/ext/CacheTests.cpp
etlng/ext/MPTTests.cpp
etlng/ext/NFTTests.cpp
etlng/ext/SuccessorTests.cpp
etl/TaskManagerTests.cpp
etl/ext/CoreTests.cpp
etl/ext/CacheTests.cpp
etl/ext/MPTTests.cpp
etl/ext/NFTTests.cpp
etl/ext/SuccessorTests.cpp
# Feed
feed/BookChangesFeedTests.cpp
feed/ForwardFeedTests.cpp

2
tests/unit/app/StopperTests.cpp
View File

@@ -67,7 +67,7 @@ struct StopperMakeCallbackTest : util::prometheus::WithPrometheus, SyncAsioConte
protected:
testing::StrictMock<ServerMock> serverMock_;
testing::StrictMock<MockNgLoadBalancer> loadBalancerMock_;
testing::StrictMock<MockLoadBalancer> loadBalancerMock_;
testing::StrictMock<MockETLService> etlServiceMock_;
testing::StrictMock<MockSubscriptionManager> subscriptionManagerMock_;
testing::StrictMock<MockBackend> backendMock_{util::config::ClioConfigDefinition{}};

37
tests/unit/etl/AmendmentBlockHandlerTests.cpp
View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
@@ -19,36 +19,45 @@
#include "etl/SystemState.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/MockPrometheus.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <chrono>
#include <cstddef>
#include <semaphore>
using namespace etl::impl;
struct AmendmentBlockHandlerTest : util::prometheus::WithPrometheus, SyncAsioContextTest {
testing::StrictMock<testing::MockFunction<void()>> actionMock;
etl::SystemState state;
struct AmendmentBlockHandlerTests : util::prometheus::WithPrometheus {
protected:
testing::StrictMock<testing::MockFunction<void()>> actionMock_;
etl::SystemState state_;
util::async::CoroExecutionContext ctx_;
};
// Note: This test can be flaky due to the way it was written (depends on time)
// Since the old ETL is going to be replaced by ETLng all tests including this one will be deleted anyway so the fix for
// flakiness is to increase the context runtime to 50ms until then (to not waste time).
TEST_F(AmendmentBlockHandlerTest, CallToNotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction)
TEST_F(AmendmentBlockHandlerTests, CallToNotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction)
{
AmendmentBlockHandler handler{ctx_, state, std::chrono::nanoseconds{1}, actionMock.AsStdFunction()};
static constexpr auto kMAX_ITERATIONS = 10uz;
etl::impl::AmendmentBlockHandler handler{ctx_, state_, std::chrono::nanoseconds{1}, actionMock_.AsStdFunction()};
auto counter = 0uz;
std::binary_semaphore stop{0};
EXPECT_FALSE(state_.isAmendmentBlocked);
EXPECT_CALL(actionMock_, Call()).Times(testing::AtLeast(10)).WillRepeatedly([&]() {
if (++counter; counter > kMAX_ITERATIONS)
stop.release();
});
EXPECT_FALSE(state.isAmendmentBlocked);
EXPECT_CALL(actionMock, Call()).Times(testing::AtLeast(10));
handler.notifyAmendmentBlocked();
EXPECT_TRUE(state.isAmendmentBlocked);
stop.acquire(); // wait for the counter to reach over kMAX_ITERATIONS
handler.stop();
runContextFor(std::chrono::milliseconds{50});
EXPECT_TRUE(state_.isAmendmentBlocked);
}
struct DefaultAmendmentBlockActionTest : LoggerFixture {};

72
tests/unit/etlng/ETLServiceTests.cpp → tests/unit/etl/ETLServiceTests.cpp
View File

@@ -19,21 +19,21 @@
#include "data/BackendInterface.hpp"
#include "data/Types.hpp"
#include "etl/CacheLoaderInterface.hpp"
#include "etl/CacheUpdaterInterface.hpp"
#include "etl/ETLService.hpp"
#include "etl/ETLState.hpp"
#include "etl/ExtractorInterface.hpp"
#include "etl/InitialLoadObserverInterface.hpp"
#include "etl/LoadBalancerInterface.hpp"
#include "etl/LoaderInterface.hpp"
#include "etl/Models.hpp"
#include "etl/MonitorInterface.hpp"
#include "etl/MonitorProviderInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etl/SystemState.hpp"
#include "etlng/CacheLoaderInterface.hpp"
#include "etlng/CacheUpdaterInterface.hpp"
#include "etlng/ETLService.hpp"
#include "etlng/ExtractorInterface.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/MonitorProviderInterface.hpp"
#include "etlng/TaskManagerInterface.hpp"
#include "etlng/TaskManagerProviderInterface.hpp"
#include "etl/TaskManagerInterface.hpp"
#include "etl/TaskManagerProviderInterface.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockLedgerPublisher.hpp"
@@ -75,7 +75,7 @@ namespace {
constinit auto const kSEQ = 100;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
struct MockMonitor : public etlng::MonitorInterface {
struct MockMonitor : public etl::MonitorInterface {
MOCK_METHOD(void, notifySequenceLoaded, (uint32_t), (override));
MOCK_METHOD(void, notifyWriteConflict, (uint32_t), (override));
MOCK_METHOD(
@@ -94,59 +94,59 @@ struct MockMonitor : public etlng::MonitorInterface {
MOCK_METHOD(void, stop, (), (override));
};
struct MockExtractor : etlng::ExtractorInterface {
MOCK_METHOD(std::optional<etlng::model::LedgerData>, extractLedgerWithDiff, (uint32_t), (override));
MOCK_METHOD(std::optional<etlng::model::LedgerData>, extractLedgerOnly, (uint32_t), (override));
struct MockExtractor : etl::ExtractorInterface {
MOCK_METHOD(std::optional<etl::model::LedgerData>, extractLedgerWithDiff, (uint32_t), (override));
MOCK_METHOD(std::optional<etl::model::LedgerData>, extractLedgerOnly, (uint32_t), (override));
};
struct MockLoader : etlng::LoaderInterface {
using ExpectedType = std::expected<void, etlng::LoaderError>;
MOCK_METHOD(ExpectedType, load, (etlng::model::LedgerData const&), (override));
MOCK_METHOD(std::optional<ripple::LedgerHeader>, loadInitialLedger, (etlng::model::LedgerData const&), (override));
struct MockLoader : etl::LoaderInterface {
using ExpectedType = std::expected<void, etl::LoaderError>;
MOCK_METHOD(ExpectedType, load, (etl::model::LedgerData const&), (override));
MOCK_METHOD(std::optional<ripple::LedgerHeader>, loadInitialLedger, (etl::model::LedgerData const&), (override));
};
struct MockCacheLoader : etlng::CacheLoaderInterface {
struct MockCacheLoader : etl::CacheLoaderInterface {
MOCK_METHOD(void, load, (uint32_t), (override));
MOCK_METHOD(void, stop, (), (noexcept, override));
MOCK_METHOD(void, wait, (), (noexcept, override));
};
struct MockCacheUpdater : etlng::CacheUpdaterInterface {
MOCK_METHOD(void, update, (etlng::model::LedgerData const&), (override));
struct MockCacheUpdater : etl::CacheUpdaterInterface {
MOCK_METHOD(void, update, (etl::model::LedgerData const&), (override));
MOCK_METHOD(void, update, (uint32_t, std::vector<data::LedgerObject> const&), (override));
MOCK_METHOD(void, update, (uint32_t, std::vector<etlng::model::Object> const&), (override));
MOCK_METHOD(void, update, (uint32_t, std::vector<etl::model::Object> const&), (override));
MOCK_METHOD(void, setFull, (), (override));
};
struct MockInitialLoadObserver : etlng::InitialLoadObserverInterface {
struct MockInitialLoadObserver : etl::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etlng::model::Object> const&, std::optional<std::string>),
(uint32_t, std::vector<etl::model::Object> const&, std::optional<std::string>),
(override)
);
};
struct MockTaskManager : etlng::TaskManagerInterface {
struct MockTaskManager : etl::TaskManagerInterface {
MOCK_METHOD(void, run, (std::size_t), (override));
MOCK_METHOD(void, stop, (), (override));
};
struct MockTaskManagerProvider : etlng::TaskManagerProviderInterface {
struct MockTaskManagerProvider : etl::TaskManagerProviderInterface {
MOCK_METHOD(
std::unique_ptr<etlng::TaskManagerInterface>,
std::unique_ptr<etl::TaskManagerInterface>,
make,
(util::async::AnyExecutionContext,
std::reference_wrapper<etlng::MonitorInterface>,
std::reference_wrapper<etl::MonitorInterface>,
uint32_t,
std::optional<uint32_t>),
(override)
);
};
struct MockMonitorProvider : etlng::MonitorProviderInterface {
struct MockMonitorProvider : etl::MonitorProviderInterface {
MOCK_METHOD(
std::unique_ptr<etlng::MonitorInterface>,
std::unique_ptr<etl::MonitorInterface>,
make,
(util::async::AnyExecutionContext,
std::shared_ptr<BackendInterface>,
@@ -161,7 +161,7 @@ auto
createTestData(uint32_t seq)
{
auto const header = createLedgerHeader(kLEDGER_HASH, seq);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = {},
.objects = {util::createObject(), util::createObject(), util::createObject()},
.successors = {},
@@ -217,7 +217,7 @@ protected:
std::make_shared<testing::NiceMock<MockMonitorProvider>>();
std::shared_ptr<etl::SystemState> systemState_ = std::make_shared<etl::SystemState>();
etlng::ETLService service_{
etl::ETLService service_{
ctx_,
config_,
backend_,
@@ -313,7 +313,7 @@ TEST_F(ETLServiceTests, RunWithEmptyDatabase)
.WillOnce(testing::Return(data::LedgerRange{.minSequence = 1, .maxSequence = kSEQ}));
EXPECT_CALL(mockTaskManagerRef, run);
EXPECT_CALL(*taskManagerProvider_, make(testing::_, testing::_, kSEQ + 1, testing::_))
.WillOnce(testing::Return(std::unique_ptr<etlng::TaskManagerInterface>(mockTaskManager.release())));
.WillOnce(testing::Return(std::unique_ptr<etl::TaskManagerInterface>(mockTaskManager.release())));
EXPECT_CALL(*monitorProvider_, make(testing::_, testing::_, testing::_, testing::_, testing::_))
.WillOnce([](auto, auto, auto, auto, auto) { return std::make_unique<testing::NiceMock<MockMonitor>>(); });
@@ -531,7 +531,7 @@ TEST_F(ETLServiceTests, RunStopsIfInitialLoadIsCancelledByBalancer)
auto const dummyLedgerData = createTestData(kMOCK_START_SEQUENCE);
EXPECT_CALL(*extractor_, extractLedgerOnly(kMOCK_START_SEQUENCE)).WillOnce(testing::Return(dummyLedgerData));
EXPECT_CALL(*balancer_, loadInitialLedger(testing::_, testing::_, testing::_))
.WillOnce(testing::Return(std::unexpected{etlng::InitialLedgerLoadError::Cancelled}));
.WillOnce(testing::Return(std::unexpected{etl::InitialLedgerLoadError::Cancelled}));
service_.run();

86
tests/unit/etl/ExtractionDataPipeTests.cpp
View File

@@ -1,86 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etl/impl/ExtractionDataPipe.hpp"
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <thread>
namespace {
constexpr auto kSTRIDE = 4;
constexpr auto kSTART_SEQ = 1234;
} // namespace
class ETLExtractionDataPipeTest : public ::testing::Test {
protected:
etl::impl::ExtractionDataPipe<uint32_t> pipe_{kSTRIDE, kSTART_SEQ};
};
TEST_F(ETLExtractionDataPipeTest, StrideMatchesInput)
{
EXPECT_EQ(pipe_.getStride(), kSTRIDE);
}
TEST_F(ETLExtractionDataPipeTest, PushedDataCanBeRetrievedAndMatchesOriginal)
{
for (std::size_t i = 0; i < 8; ++i)
pipe_.push(kSTART_SEQ + i, kSTART_SEQ + i);
for (std::size_t i = 0; i < 8; ++i) {
auto const data = pipe_.popNext(kSTART_SEQ + i);
EXPECT_EQ(data.value(), kSTART_SEQ + i);
}
}
TEST_F(ETLExtractionDataPipeTest, CallingFinishPushesAnEmptyOptional)
{
for (std::size_t i = 0; i < 4; ++i)
pipe_.finish(kSTART_SEQ + i);
for (std::size_t i = 0; i < 4; ++i) {
auto const data = pipe_.popNext(kSTART_SEQ + i);
EXPECT_FALSE(data.has_value());
}
}
TEST_F(ETLExtractionDataPipeTest, CallingCleanupUnblocksOtherThread)
{
std::atomic_bool unblocked = false;
auto bgThread = std::thread([this, &unblocked] {
for (std::size_t i = 0; i < 252; ++i)
pipe_.push(kSTART_SEQ, 1234); // 251st element will block this thread here
unblocked = true;
});
// emulate waiting for above thread to push and get blocked
std::this_thread::sleep_for(std::chrono::milliseconds{100});
EXPECT_FALSE(unblocked);
pipe_.cleanup();
bgThread.join();
EXPECT_TRUE(unblocked);
}

70
tests/unit/etlng/ExtractionTests.cpp → tests/unit/etl/ExtractionTests.cpp
View File

@@ -20,8 +20,8 @@
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/LedgerFetcherInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "etl/Models.hpp"
#include "etl/impl/Extraction.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockAssert.hpp"
#include "util/TestObject.hpp"
@@ -49,17 +49,17 @@ constinit auto const kLEDGER_HASH2 = "1B8590C01B0006EDFA9ED60296DD052DC5E90F9965
constinit auto const kSEQ = 30;
} // namespace
struct ExtractionModelNgTests : virtual public ::testing::Test {};
struct ExtractionModelTests : virtual public ::testing::Test {};
TEST_F(ExtractionModelNgTests, LedgerDataCopyableAndEquatable)
TEST_F(ExtractionModelTests, LedgerDataCopyableAndEquatable)
{
auto const first = etlng::model::LedgerData{
auto const first = etl::model::LedgerData{
.transactions =
{util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER)},
.objects = {util::createObject(), util::createObject(), util::createObject()},
.successors = std::vector<etlng::model::BookSuccessor>{{.firstBook = "first", .bookBase = "base"}},
.successors = std::vector<etl::model::BookSuccessor>{{.firstBook = "first", .bookBase = "base"}},
.edgeKeys = std::vector<std::string>{"key1", "key2"},
.header = createLedgerHeader(kLEDGER_HASH, kSEQ, 1),
.rawHeader = {1, 2, 3},
@@ -81,7 +81,7 @@ TEST_F(ExtractionModelNgTests, LedgerDataCopyableAndEquatable)
}
{
auto third = second;
third.successors = std::vector<etlng::model::BookSuccessor>{{.firstBook = "second", .bookBase = "base"}};
third.successors = std::vector<etl::model::BookSuccessor>{{.firstBook = "second", .bookBase = "base"}};
EXPECT_NE(first, third);
}
{
@@ -106,7 +106,7 @@ TEST_F(ExtractionModelNgTests, LedgerDataCopyableAndEquatable)
}
}
TEST_F(ExtractionModelNgTests, TransactionIsEquatable)
TEST_F(ExtractionModelTests, TransactionIsEquatable)
{
auto const tx = std::vector{util::createTransaction(ripple::TxType::ttNFTOKEN_BURN)};
auto other = tx;
@@ -116,7 +116,7 @@ TEST_F(ExtractionModelNgTests, TransactionIsEquatable)
EXPECT_NE(tx, other);
}
TEST_F(ExtractionModelNgTests, ObjectCopyableAndEquatable)
TEST_F(ExtractionModelTests, ObjectCopyableAndEquatable)
{
auto const obj = util::createObject();
auto const other = obj;
@@ -154,14 +154,14 @@ TEST_F(ExtractionModelNgTests, ObjectCopyableAndEquatable)
}
{
auto third = other;
third.type = etlng::model::Object::ModType::Deleted;
third.type = etl::model::Object::ModType::Deleted;
EXPECT_NE(obj, third);
}
}
TEST_F(ExtractionModelNgTests, BookSuccessorCopyableAndEquatable)
TEST_F(ExtractionModelTests, BookSuccessorCopyableAndEquatable)
{
auto const succ = etlng::model::BookSuccessor{.firstBook = "first", .bookBase = "base"};
auto const succ = etl::model::BookSuccessor{.firstBook = "first", .bookBase = "base"};
auto const other = succ;
EXPECT_EQ(succ, other);
@@ -177,12 +177,12 @@ TEST_F(ExtractionModelNgTests, BookSuccessorCopyableAndEquatable)
}
}
struct ExtractionNgTests : public virtual ::testing::Test {};
struct ExtractionTests : public virtual ::testing::Test {};
TEST_F(ExtractionNgTests, ModType)
TEST_F(ExtractionTests, ModType)
{
using namespace etlng::impl;
using ModType = etlng::model::Object::ModType;
using namespace etl::impl;
using ModType = etl::model::Object::ModType;
EXPECT_EQ(extractModType(PBObjType::MODIFIED), ModType::Modified);
EXPECT_EQ(extractModType(PBObjType::CREATED), ModType::Created);
@@ -190,9 +190,9 @@ TEST_F(ExtractionNgTests, ModType)
EXPECT_EQ(extractModType(PBObjType::UNSPECIFIED), ModType::Unspecified);
}
TEST_F(ExtractionNgTests, OneTransaction)
TEST_F(ExtractionTests, OneTransaction)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER);
@@ -208,9 +208,9 @@ TEST_F(ExtractionNgTests, OneTransaction)
EXPECT_EQ(res.sttx.getTxnType(), expected.sttx.getTxnType());
}
TEST_F(ExtractionNgTests, MultipleTransactions)
TEST_F(ExtractionTests, MultipleTransactions)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER);
@@ -236,9 +236,9 @@ TEST_F(ExtractionNgTests, MultipleTransactions)
}
}
TEST_F(ExtractionNgTests, OneObject)
TEST_F(ExtractionTests, OneObject)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
@@ -256,9 +256,9 @@ TEST_F(ExtractionNgTests, OneObject)
EXPECT_EQ(res.type, expected.type);
}
TEST_F(ExtractionNgTests, OneObjectWithSuccessorAndPredecessor)
TEST_F(ExtractionTests, OneObjectWithSuccessorAndPredecessor)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
@@ -278,9 +278,9 @@ TEST_F(ExtractionNgTests, OneObjectWithSuccessorAndPredecessor)
EXPECT_EQ(res.type, expected.type);
}
TEST_F(ExtractionNgTests, MultipleObjects)
TEST_F(ExtractionTests, MultipleObjects)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createObject();
auto original = org::xrpl::rpc::v1::RawLedgerObject();
@@ -308,9 +308,9 @@ TEST_F(ExtractionNgTests, MultipleObjects)
}
}
TEST_F(ExtractionNgTests, OneSuccessor)
TEST_F(ExtractionTests, OneSuccessor)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createSuccessor();
auto original = org::xrpl::rpc::v1::BookSuccessor();
@@ -322,9 +322,9 @@ TEST_F(ExtractionNgTests, OneSuccessor)
EXPECT_EQ(ripple::strHex(res.bookBase), ripple::strHex(expected.bookBase));
}
TEST_F(ExtractionNgTests, MultipleSuccessors)
TEST_F(ExtractionTests, MultipleSuccessors)
{
using namespace etlng::impl;
using namespace etl::impl;
auto expected = util::createSuccessor();
auto original = org::xrpl::rpc::v1::BookSuccessor();
@@ -348,9 +348,9 @@ TEST_F(ExtractionNgTests, MultipleSuccessors)
}
}
TEST_F(ExtractionNgTests, SuccessorsWithNoNeighborsIncluded)
TEST_F(ExtractionTests, SuccessorsWithNoNeighborsIncluded)
{
using namespace etlng::impl;
using namespace etl::impl;
auto data = PBLedgerResponseType();
data.set_object_neighbors_included(false);
@@ -363,7 +363,7 @@ struct ExtractionAssertTest : common::util::WithMockAssert {};
TEST_F(ExtractionAssertTest, InvalidModTypeAsserts)
{
using namespace etlng::impl;
using namespace etl::impl;
EXPECT_CLIO_ASSERT_FAIL({
[[maybe_unused]] auto _ = extractModType(
@@ -377,9 +377,9 @@ struct MockFetcher : etl::LedgerFetcherInterface {
MOCK_METHOD(std::optional<GetLedgerResponseType>, fetchDataAndDiff, (uint32_t), (override));
};
struct ExtractorTests : ExtractionNgTests {
struct ExtractorTests : ExtractionTests {
std::shared_ptr<MockFetcher> fetcher = std::make_shared<MockFetcher>();
etlng::impl::Extractor extractor{fetcher};
etl::impl::Extractor extractor{fetcher};
};
TEST_F(ExtractorTests, ExtractLedgerWithDiffNoResult)

132
tests/unit/etl/ExtractorTests.cpp
View File

@@ -1,132 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etl/SystemState.hpp"
#include "etl/impl/Extractor.hpp"
#include "util/FakeFetchResponse.hpp"
#include "util/MockExtractionDataPipe.hpp"
#include "util/MockLedgerFetcher.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include "util/MockPrometheus.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <optional>
using namespace testing;
using namespace etl;
struct ETLExtractorTest : util::prometheus::WithPrometheus {
using ExtractionDataPipeType = MockExtractionDataPipe;
using LedgerFetcherType = MockLedgerFetcher;
using ExtractorType = etl::impl::Extractor<ExtractionDataPipeType, LedgerFetcherType>;
ETLExtractorTest()
{
state_.isStopping = false;
state_.writeConflict = false;
state_.isStrictReadonly = false;
state_.isWriting = false;
}
protected:
ExtractionDataPipeType dataPipe_;
MockNetworkValidatedLedgersPtr networkValidatedLedgers_;
LedgerFetcherType ledgerFetcher_;
SystemState state_;
};
TEST_F(ETLExtractorTest, StopsWhenCurrentSequenceExceedsFinishSequence)
{
EXPECT_CALL(*networkValidatedLedgers_, waitUntilValidatedByNetwork).Times(3).WillRepeatedly(Return(true));
EXPECT_CALL(dataPipe_, getStride).Times(3).WillRepeatedly(Return(4));
auto response = FakeFetchResponse{};
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).Times(3).WillRepeatedly(Return(response));
EXPECT_CALL(dataPipe_, push).Times(3);
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// expected to invoke for seq 0, 4, 8 and finally stop as seq will be greater than finishing seq
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 11, state_};
}
TEST_F(ETLExtractorTest, StopsOnWriteConflict)
{
EXPECT_CALL(dataPipe_, finish(0));
state_.writeConflict = true;
// despite finish sequence being far ahead, we set writeConflict and so exit the loop immediately
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_};
}
TEST_F(ETLExtractorTest, StopsOnServerShutdown)
{
EXPECT_CALL(dataPipe_, finish(0));
state_.isStopping = true;
// despite finish sequence being far ahead, we set isStopping and so exit the loop immediately
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_};
}
// stop extractor thread if fetcheResponse is empty
TEST_F(ETLExtractorTest, StopsIfFetchIsUnsuccessful)
{
EXPECT_CALL(*networkValidatedLedgers_, waitUntilValidatedByNetwork).WillOnce(Return(true));
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).WillOnce(Return(std::nullopt));
EXPECT_CALL(dataPipe_, finish(0));
// we break immediately because fetchDataAndDiff returns nullopt
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_};
}
TEST_F(ETLExtractorTest, StopsIfWaitingUntilValidatedByNetworkTimesOut)
{
// note that in actual clio code we don't return false unless a timeout is specified and exceeded
EXPECT_CALL(*networkValidatedLedgers_, waitUntilValidatedByNetwork).WillOnce(Return(false));
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// we emulate waitUntilValidatedByNetwork timing out which would lead to shutdown of the extractor thread
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_};
}
TEST_F(ETLExtractorTest, SendsCorrectResponseToDataPipe)
{
EXPECT_CALL(*networkValidatedLedgers_, waitUntilValidatedByNetwork).WillOnce(Return(true));
EXPECT_CALL(dataPipe_, getStride).WillOnce(Return(4));
auto response = FakeFetchResponse{1234};
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).WillOnce(Return(response));
EXPECT_CALL(dataPipe_, push(_, std::optional{response}));
EXPECT_CALL(dataPipe_, finish(0));
// expect to finish after just one response due to finishSequence set to 1
ExtractorType extractor{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 1, state_};
extractor.waitTillFinished(); // this is what clio does too. waiting for the thread to join
}
TEST_F(ETLExtractorTest, CallsPipeFinishWithInitialSequenceAtExit)
{
EXPECT_CALL(dataPipe_, finish(123));
state_.isStopping = true;
ExtractorType{dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 123, 234, state_};
}

2
tests/unit/etl/ForwardingSourceTests.cpp
View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above

334
tests/unit/etl/GrpcSourceTests.cpp
View File

@@ -17,13 +17,19 @@
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "etl/ETLHelpers.hpp"
#include "etl/InitialLoadObserverInterface.hpp"
#include "etl/LoadBalancerInterface.hpp"
#include "etl/Models.hpp"
#include "etl/impl/GrpcSource.hpp"
#include "util/MockBackend.hpp"
#include "util/MockPrometheus.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/Assert.hpp"
#include "util/MockXrpLedgerAPIService.hpp"
#include "util/Mutex.hpp"
#include "util/TestObject.hpp"
#include "util/config/ConfigDefinition.hpp"
#include <boost/asio/spawn.hpp>
#include <gmock/gmock.h>
#include <grpcpp/server_context.h>
#include <grpcpp/support/status.h>
@@ -31,31 +37,118 @@
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <org/xrpl/rpc/v1/get_ledger_data.pb.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <future>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <queue>
#include <semaphore>
#include <string>
#include <vector>
using namespace etl::impl;
using namespace util::config;
using namespace etl::model;
struct GrpcSourceTests : util::prometheus::WithPrometheus, tests::util::WithMockXrpLedgerAPIService {
namespace {
struct MockLoadObserver : etl::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<Object> const&, std::optional<std::string>),
(override)
);
};
struct GrpcSourceTests : virtual public ::testing::Test, tests::util::WithMockXrpLedgerAPIService {
GrpcSourceTests()
: WithMockXrpLedgerAPIService("localhost:0")
, mockBackend_(std::make_shared<testing::StrictMock<MockBackend>>(ClioConfigDefinition{}))
, grpcSource_("localhost", std::to_string(getXRPLMockPort()), mockBackend_)
: WithMockXrpLedgerAPIService("localhost:0"), grpcSource_("localhost", std::to_string(getXRPLMockPort()))
{
}
class KeyStore {
std::vector<ripple::uint256> keys_;
using Store = std::map<std::string, std::queue<ripple::uint256>, std::greater<>>;
util::Mutex<Store> store_;
public:
KeyStore(std::size_t totalKeys, std::size_t numMarkers) : keys_(etl::getMarkers(totalKeys))
{
auto const totalPerMarker = totalKeys / numMarkers;
auto const markers = etl::getMarkers(numMarkers);
auto store = store_.lock();
for (auto mi = 0uz; mi < markers.size(); ++mi) {
for (auto i = 0uz; i < totalPerMarker; ++i) {
auto const mapKey = ripple::strHex(markers.at(mi)).substr(0, 2);
store->operator[](mapKey).push(keys_.at((mi * totalPerMarker) + i));
}
}
}
std::optional<std::string>
next(std::string const& marker)
{
auto store = store_.lock<std::scoped_lock>();
auto const mapKey = ripple::strHex(marker).substr(0, 2);
auto it = store->lower_bound(mapKey);
ASSERT(it != store->end(), "Lower bound not found for '{}'", mapKey);
auto& queue = it->second;
if (queue.empty())
return std::nullopt;
auto data = queue.front();
queue.pop();
return std::make_optional(uint256ToString(data));
};
std::optional<std::string>
peek(std::string const& marker)
{
auto store = store_.lock<std::scoped_lock>();
auto const mapKey = ripple::strHex(marker).substr(0, 2);
auto it = store->lower_bound(mapKey);
ASSERT(it != store->end(), "Lower bound not found for '{}'", mapKey);
auto& queue = it->second;
if (queue.empty())
return std::nullopt;
auto data = queue.front();
return std::make_optional(uint256ToString(data));
};
};
protected:
std::shared_ptr<testing::StrictMock<MockBackend>> mockBackend_;
GrpcSource grpcSource_;
testing::StrictMock<MockLoadObserver> observer_;
etl::impl::GrpcSource grpcSource_;
};
TEST_F(GrpcSourceTests, fetchLedger)
struct GrpcSourceLoadInitialLedgerTests : GrpcSourceTests {
protected:
uint32_t const sequence_ = 123u;
uint32_t const numMarkers_ = 4u;
bool const cacheOnly_ = false;
};
} // namespace
TEST_F(GrpcSourceTests, BasicFetchLedger)
{
uint32_t const sequence = 123;
uint32_t const sequence = 123u;
bool const getObjects = true;
bool const getObjectNeighbors = false;
@@ -69,11 +162,14 @@ TEST_F(GrpcSourceTests, fetchLedger)
EXPECT_EQ(request->get_objects(), getObjects);
EXPECT_EQ(request->get_object_neighbors(), getObjectNeighbors);
EXPECT_EQ(request->user(), "ETL");
response->set_validated(true);
response->set_is_unlimited(false);
response->set_object_neighbors_included(false);
return grpc::Status{};
});
auto const [status, response] = grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors);
ASSERT_TRUE(status.ok());
EXPECT_TRUE(response.validated());
@@ -81,28 +177,7 @@ TEST_F(GrpcSourceTests, fetchLedger)
EXPECT_FALSE(response.object_neighbors_included());
}
TEST_F(GrpcSourceTests, fetchLedgerNoStub)
{
GrpcSource wrongGrpcSource{"wrong", "wrong", mockBackend_};
auto const [status, _response] = wrongGrpcSource.fetchLedger(0, false, false);
EXPECT_EQ(status.error_code(), grpc::StatusCode::INTERNAL);
}
TEST_F(GrpcSourceTests, loadInitialLedgerNoStub)
{
GrpcSource wrongGrpcSource{"wrong", "wrong", mockBackend_};
auto const [data, success] = wrongGrpcSource.loadInitialLedger(0, 0);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
}
struct GrpcSourceLoadInitialLedgerTests : GrpcSourceTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 4;
};
TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataNotFound)
{
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.Times(numMarkers_)
@@ -111,18 +186,18 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, GetLedgerDataFailed)
org::xrpl::rpc::v1::GetLedgerDataResponse* /*response*/) {
EXPECT_EQ(request->ledger().sequence(), sequence_);
EXPECT_EQ(request->user(), "ETL");
return grpc::Status{grpc::StatusCode::NOT_FOUND, "Not found"};
});
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
EXPECT_TRUE(data.empty());
EXPECT_FALSE(success);
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_FALSE(res.has_value());
}
TEST_F(GrpcSourceLoadInitialLedgerTests, worksFine)
TEST_F(GrpcSourceLoadInitialLedgerTests, ObserverCalledCorrectly)
{
auto const key = ripple::uint256{4};
std::string const keyStr{reinterpret_cast<char const*>(key.data()), ripple::uint256::size()};
auto const keyStr = uint256ToString(key);
auto const object = createTicketLedgerObject("rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", sequence_);
auto const objectData = object.getSerializer().peekData();
@@ -136,17 +211,184 @@ TEST_F(GrpcSourceLoadInitialLedgerTests, worksFine)
response->set_is_unlimited(true);
auto newObject = response->mutable_ledger_objects()->add_objects();
newObject->set_key(key.data(), ripple::uint256::size());
newObject->set_key(uint256ToString(key));
newObject->set_data(objectData.data(), objectData.size());
return grpc::Status{};
});
EXPECT_CALL(*mockBackend_, writeNFTs).Times(numMarkers_);
EXPECT_CALL(*mockBackend_, writeLedgerObject).Times(numMarkers_);
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects)
.Times(numMarkers_)
.WillRepeatedly([&](uint32_t, std::vector<Object> const& data, std::optional<std::string> lastKey) {
EXPECT_FALSE(lastKey.has_value());
EXPECT_EQ(data.size(), 1);
});
auto const [data, success] = grpcSource_.loadInitialLedger(sequence_, numMarkers_);
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_TRUE(success);
EXPECT_EQ(data, std::vector<std::string>(4, keyStr));
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res.value().size(), numMarkers_);
EXPECT_EQ(res.value(), std::vector<std::string>(4, keyStr));
}
TEST_F(GrpcSourceLoadInitialLedgerTests, DataTransferredAndObserverCalledCorrectly)
{
auto const totalKeys = 256uz;
auto const totalPerMarker = totalKeys / numMarkers_;
auto const batchSize = totalPerMarker / 4uz;
auto const batchesPerMarker = totalPerMarker / batchSize;
auto keyStore = KeyStore(totalKeys, numMarkers_);
auto const object = createTicketLedgerObject("rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", sequence_);
auto const objectData = object.getSerializer().peekData();
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.Times(numMarkers_ * batchesPerMarker)
.WillRepeatedly([&](grpc::ServerContext* /*context*/,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence_);
EXPECT_EQ(request->user(), "ETL");
response->set_is_unlimited(true);
auto next = request->marker().empty() ? std::string("00") : request->marker();
for (auto i = 0uz; i < batchSize; ++i) {
if (auto maybeLastKey = keyStore.next(next); maybeLastKey.has_value()) {
next = *maybeLastKey;
auto newObject = response->mutable_ledger_objects()->add_objects();
newObject->set_key(next);
newObject->set_data(objectData.data(), objectData.size());
}
}
if (auto maybeNext = keyStore.peek(next); maybeNext.has_value())
response->set_marker(*maybeNext);
return grpc::Status::OK;
});
std::atomic_size_t total = 0uz;
std::atomic_size_t totalWithLastKey = 0uz;
std::atomic_size_t totalWithoutLastKey = 0uz;
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects)
.Times(numMarkers_ * batchesPerMarker)
.WillRepeatedly([&](uint32_t, std::vector<Object> const& data, std::optional<std::string> lastKey) {
EXPECT_LE(data.size(), batchSize);
if (lastKey.has_value()) {
++totalWithLastKey;
} else {
++totalWithoutLastKey;
}
total += data.size();
});
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res.value().size(), numMarkers_);
EXPECT_EQ(total, totalKeys);
EXPECT_EQ(totalWithLastKey + totalWithoutLastKey, numMarkers_ * batchesPerMarker);
EXPECT_EQ(totalWithoutLastKey, numMarkers_);
EXPECT_EQ(totalWithLastKey, (numMarkers_ - 1) * batchesPerMarker);
}
struct GrpcSourceStopTests : GrpcSourceTests, SyncAsioContextTest {};
TEST_F(GrpcSourceStopTests, LoadInitialLedgerStopsWhenRequested)
{
uint32_t const sequence = 123u;
uint32_t const numMarkers = 1;
std::mutex mtx;
std::condition_variable cvGrpcCallActive;
std::condition_variable cvStopCalled;
bool grpcCallIsActive = false;
bool stopHasBeenCalled = false;
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.WillOnce([&](grpc::ServerContext*,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence);
EXPECT_EQ(request->user(), "ETL");
{
std::unique_lock const lk(mtx);
grpcCallIsActive = true;
}
cvGrpcCallActive.notify_one();
{
std::unique_lock lk(mtx);
cvStopCalled.wait(lk, [&] { return stopHasBeenCalled; });
}
response->set_is_unlimited(true);
return grpc::Status::OK;
});
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects).Times(0);
auto loadTask = std::async(std::launch::async, [&]() {
return grpcSource_.loadInitialLedger(sequence, numMarkers, observer_);
});
{
std::unique_lock lk(mtx);
cvGrpcCallActive.wait(lk, [&] { return grpcCallIsActive; });
}
runSyncOperation([&](boost::asio::yield_context yield) {
grpcSource_.stop(yield);
{
std::unique_lock const lk(mtx);
stopHasBeenCalled = true;
}
cvStopCalled.notify_one();
});
auto const res = loadTask.get();
ASSERT_FALSE(res.has_value());
EXPECT_EQ(res.error(), etl::InitialLedgerLoadError::Cancelled);
}
TEST_F(GrpcSourceTests, DeadlineIsHandledCorrectly)
{
static constexpr auto kDEADLINE = std::chrono::milliseconds{5};
uint32_t const sequence = 123u;
bool const getObjects = true;
bool const getObjectNeighbors = false;
std::binary_semaphore sem(0);
auto grpcSource =
std::make_unique<etl::impl::GrpcSource>("localhost", std::to_string(getXRPLMockPort()), kDEADLINE);
// Note: this may not be called at all if gRPC cancels before it gets a chance to call the stub
EXPECT_CALL(mockXrpLedgerAPIService, GetLedger)
.Times(testing::AtMost(1))
.WillRepeatedly([&](grpc::ServerContext*,
org::xrpl::rpc::v1::GetLedgerRequest const*,
org::xrpl::rpc::v1::GetLedgerResponse*) {
// wait for main thread to discard us and fail the test if unsuccessful within expected timeframe
[&] { ASSERT_TRUE(sem.try_acquire_for(std::chrono::milliseconds{50})); }();
return grpc::Status{};
});
auto const [status, response] = grpcSource->fetchLedger(sequence, getObjects, getObjectNeighbors);
ASSERT_FALSE(status.ok()); // timed out after kDEADLINE
sem.release(); // we don't need to hold GetLedger thread any longer
grpcSource.reset();
shutdown(std::chrono::milliseconds{10});
}

206
tests/unit/etl/LedgerPublisherTests.cpp
View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
@@ -23,7 +23,6 @@
#include "etl/impl/LedgerPublisher.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/TestObject.hpp"
@@ -37,6 +36,7 @@
#include <xrpl/protocol/LedgerHeader.h>
#include <chrono>
#include <optional>
#include <vector>
using namespace testing;
@@ -51,66 +51,82 @@ constexpr auto kACCOUNT2 = "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun";
constexpr auto kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constexpr auto kSEQ = 30;
constexpr auto kAGE = 800;
constexpr auto kAMOUNT = 100;
constexpr auto kFEE = 3;
constexpr auto kFINAL_BALANCE = 110;
constexpr auto kFINAL_BALANCE2 = 30;
MATCHER_P(ledgerHeaderMatcher, expectedHeader, "Headers match")
{
return arg.seq == expectedHeader.seq && arg.hash == expectedHeader.hash &&
arg.closeTime == expectedHeader.closeTime;
}
} // namespace
struct ETLLedgerPublisherTest : util::prometheus::WithPrometheus, MockBackendTestStrict, SyncAsioContextTest {
util::config::ClioConfigDefinition cfg{{}};
MockLedgerCache mockCache;
StrictMockSubscriptionManagerSharedPtr mockSubscriptionManagerPtr;
};
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderIsWritingFalseAndCacheDisabled)
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderSkipDueToAge)
{
SystemState dummyState;
dummyState.isWriting = false;
// Use kAGE (800) which is > MAX_LEDGER_AGE_SECONDS (600) to test skipping
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(mockCache, isDisabled).WillOnce(Return(true));
EXPECT_CALL(*backend_, fetchLedgerDiff(kSEQ, _)).Times(0);
auto dummyState = etl::SystemState{};
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
// setLastPublishedSequence not in strand, should verify before run
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
// Since age > MAX_LEDGER_AGE_SECONDS, these should not be called
EXPECT_CALL(*backend_, doFetchLedgerObject).Times(0);
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
ctx_.run();
EXPECT_TRUE(backend_->fetchLedgerRange());
EXPECT_EQ(backend_->fetchLedgerRange().value().minSequence, kSEQ);
EXPECT_EQ(backend_->fetchLedgerRange().value().maxSequence, kSEQ);
}
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderIsWritingFalseAndCacheEnabled)
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderWithinAgeLimit)
{
SystemState dummyState;
dummyState.isWriting = false;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(mockCache, isDisabled).WillOnce(Return(false));
EXPECT_CALL(*backend_, fetchLedgerDiff(kSEQ, _)).Times(1);
// Use age 0 which is < MAX_LEDGER_AGE_SECONDS to ensure publishing happens
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto dummyState = etl::SystemState{};
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
// setLastPublishedSequence not in strand, should verify before run
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(mockCache, updateImp);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 0));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
ctx_.run();
EXPECT_TRUE(backend_->fetchLedgerRange());
EXPECT_EQ(backend_->fetchLedgerRange().value().minSequence, kSEQ);
EXPECT_EQ(backend_->fetchLedgerRange().value().maxSequence, kSEQ);
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderIsWritingTrue)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
// setLastPublishedSequence not in strand, should verify before run
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
@@ -120,28 +136,28 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderIsWritingTrue)
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderInRange)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// mock fetch fee
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction = createPaymentTransactionObject(kACCOUNT, kACCOUNT2, 100, 3, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, 110, 30).getSerializer().peekData();
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
// mock fetch transactions
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
// setLastPublishedSequence not in strand, should verify before run
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
@@ -151,65 +167,62 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderInRange)
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
// last publish time should be set
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherTest, PublishLedgerHeaderCloseTimeGreaterThanNow)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
ripple::LedgerHeader dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const nowPlus10 = system_clock::now() + seconds(10);
auto const closeTime = duration_cast<seconds>(nowPlus10.time_since_epoch()).count() - kRIPPLE_EPOCH_START;
dummyLedgerHeader.closeTime = ripple::NetClock::time_point{seconds{closeTime}};
backend_->setRange(kSEQ - 1, kSEQ);
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
// mock fetch fee
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction = createPaymentTransactionObject(kACCOUNT, kACCOUNT2, 100, 3, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, 110, 30).getSerializer().peekData();
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
// mock fetch transactions
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
// setLastPublishedSequence not in strand, should verify before run
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 1));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
// mock 1 transaction
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
// last publish time should be set
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqStopIsTrue)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isStopping = true;
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
EXPECT_FALSE(publisher.publish(kSEQ, {}));
}
TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqMaxAttempt)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
static constexpr auto kMAX_ATTEMPT = 2;
@@ -221,18 +234,15 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqMaxAttempt)
TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqStopIsFalse)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
LedgerRange const range{.minSequence = kSEQ, .maxSequence = kSEQ};
EXPECT_CALL(*backend_, hardFetchLedgerRange).WillOnce(Return(range));
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
EXPECT_CALL(*backend_, fetchLedgerBySequence(kSEQ, _)).WillOnce(Return(dummyLedgerHeader));
EXPECT_CALL(mockCache, isDisabled).WillOnce(Return(false));
EXPECT_CALL(*backend_, fetchLedgerDiff(kSEQ, _)).WillOnce(Return(std::vector<LedgerObject>{}));
EXPECT_CALL(mockCache, updateImp);
EXPECT_TRUE(publisher.publish(kSEQ, {}));
ctx_.run();
@@ -240,47 +250,107 @@ TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqStopIsFalse)
TEST_F(ETLLedgerPublisherTest, PublishMultipleTxInOrder)
{
SystemState dummyState;
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
impl::LedgerPublisher publisher(ctx_, backend_, mockCache, mockSubscriptionManagerPtr, dummyState);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// mock fetch fee
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
// t1 index > t2 index
TransactionAndMetadata t1;
t1.transaction = createPaymentTransactionObject(kACCOUNT, kACCOUNT2, 100, 3, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, 110, 30, 2).getSerializer().peekData();
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
t1.date = 1;
TransactionAndMetadata t2;
t2.transaction = createPaymentTransactionObject(kACCOUNT, kACCOUNT2, 100, 3, kSEQ).getSerializer().peekData();
t2.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, 110, 30, 1).getSerializer().peekData();
t2.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t2.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 1)
.getSerializer()
.peekData();
t2.ledgerSequence = kSEQ;
t2.date = 2;
// mock fetch transactions
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1, t2}));
// setLastPublishedSequence not in strand, should verify before run
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 2));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
// should call pubTransaction t2 first (greater tx index)
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t2, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t1, _)).InSequence(s);
ctx_.run();
// last publish time should be set
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherTest, PublishVeryOldLedgerShouldSkip)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
// Create a ledger header with age (800) greater than MAX_LEDGER_AGE_SECONDS (600)
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 800);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
ctx_.run();
}
TEST_F(ETLLedgerPublisherTest, PublishMultipleLedgersInQuickSuccession)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader1 = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const dummyLedgerHeader2 = createLedgerHeader(kLEDGER_HASH, kSEQ + 1, 0);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ + 1);
// Publish two ledgers in quick succession
publisher.publish(dummyLedgerHeader1);
publisher.publish(dummyLedgerHeader2);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ + 1, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ + 1, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader1), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader1), _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader2), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader2), _)).InSequence(s);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ + 1);
ctx_.run();
}

67
tests/unit/etl/LoadBalancerTests.cpp
View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
@@ -17,7 +17,10 @@
*/
//==============================================================================
#include "etl/InitialLoadObserverInterface.hpp"
#include "etl/LoadBalancer.hpp"
#include "etl/LoadBalancerInterface.hpp"
#include "etl/Models.hpp"
#include "etl/Source.hpp"
#include "rpc/Errors.hpp"
#include "util/AsioContextTestFixture.hpp"
@@ -62,7 +65,9 @@ using namespace util::config;
using testing::Return;
using namespace util::prometheus;
static constexpr auto kTWO_SOURCES_LEDGER_RESPONSE = R"JSON({
namespace {
constinit auto const kTWO_SOURCES_LEDGER_RESPONSE = R"JSON({
"etl_sources": [
{
"ip": "127.0.0.1",
@@ -77,7 +82,7 @@ static constexpr auto kTWO_SOURCES_LEDGER_RESPONSE = R"JSON({
]
})JSON";
static constexpr auto kTHREE_SOURCES_LEDGER_RESPONSE = R"JSON({
constinit auto const kTHREE_SOURCES_LEDGER_RESPONSE = R"JSON({
"etl_sources": [
{
"ip": "127.0.0.1",
@@ -97,7 +102,7 @@ static constexpr auto kTHREE_SOURCES_LEDGER_RESPONSE = R"JSON({
]
})JSON";
inline static ClioConfigDefinition
inline ClioConfigDefinition
getParseLoadBalancerConfig(boost::json::value val)
{
ClioConfigDefinition config{
@@ -118,6 +123,23 @@ getParseLoadBalancerConfig(boost::json::value val)
return config;
}
struct InitialLoadObserverMock : etl::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etl::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etl::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct LoadBalancerConstructorTests : util::prometheus::WithPrometheus, MockBackendTestStrict {
std::unique_ptr<LoadBalancer>
makeLoadBalancer()
@@ -168,7 +190,6 @@ TEST_F(LoadBalancerConstructorTests, forwardingTimeoutPassedToSourceFactory)
testing::_,
testing::_,
testing::_,
testing::_,
std::chrono::steady_clock::duration{std::chrono::seconds{forwardingTimeout}},
testing::_,
testing::_,
@@ -439,51 +460,57 @@ struct LoadBalancerLoadInitialLedgerTests : LoadBalancerOnConnectHookTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 16;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
InitialLedgerLoadResult const response_{std::vector<std::string>{"1", "2", "3"}};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerTests, load)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0DoesntHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_bothSourcesDontHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, std::chrono::milliseconds{1}), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerTests, load_source0ReturnsStatusFalse)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, false)));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(std::unexpected{InitialLedgerLoadError::Errored}));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_), response_.first);
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
struct LoadBalancerLoadInitialLedgerCustomNumMarkersTests : LoadBalancerConstructorTests {
protected:
uint32_t const numMarkers_ = 16;
uint32_t const sequence_ = 123;
std::pair<std::vector<std::string>, bool> const response_ = {{"1", "2", "3"}, true};
InitialLedgerLoadResult const response_{std::vector<std::string>{"1", "2", "3"}};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersTests, loadInitialLedger)
@@ -498,9 +525,10 @@ TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersTests, loadInitialLedger)
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_)).WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_), response_.first);
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
struct LoadBalancerFetchLegerTests : LoadBalancerOnConnectHookTests {
@@ -813,6 +841,7 @@ TEST_F(LoadBalancerForwardToRippledTests, onLedgerClosedHookInvalidatesCache)
auto const request = boost::json::object{{"command", "server_info"}};
EXPECT_CALL(*randomGenerator_, uniform(0, 1)).WillOnce(Return(0)).WillOnce(Return(1));
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)

16
tests/unit/etlng/LoadingTests.cpp → tests/unit/etl/LoadingTests.cpp
View File

@@ -18,11 +18,11 @@
//==============================================================================
#include "data/Types.hpp"
#include "etl/InitialLoadObserverInterface.hpp"
#include "etl/Models.hpp"
#include "etl/RegistryInterface.hpp"
#include "etl/SystemState.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/RegistryInterface.hpp"
#include "etlng/impl/Loading.hpp"
#include "etl/impl/Loading.hpp"
#include "rpc/RPCHelpers.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockAssert.hpp"
@@ -41,8 +41,8 @@
#include <string>
#include <vector>
using namespace etlng::model;
using namespace etlng::impl;
using namespace etl::model;
using namespace etl::impl;
using namespace data;
namespace {
@@ -50,13 +50,13 @@ namespace {
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const kSEQ = 30;
struct MockRegistry : etlng::RegistryInterface {
struct MockRegistry : etl::RegistryInterface {
MOCK_METHOD(void, dispatchInitialObjects, (uint32_t, std::vector<Object> const&, std::string), (override));
MOCK_METHOD(void, dispatchInitialData, (LedgerData const&), (override));
MOCK_METHOD(void, dispatch, (LedgerData const&), (override));
};
struct MockLoadObserver : etlng::InitialLoadObserverInterface {
struct MockLoadObserver : etl::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,

9
tests/unit/etlng/MonitorTests.cpp → tests/unit/etl/MonitorTests.cpp
View File

@@ -18,8 +18,8 @@
//==============================================================================
#include "data/Types.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etlng/impl/Monitor.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/Monitor.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include "util/MockPrometheus.hpp"
@@ -36,7 +36,7 @@
#include <optional>
#include <semaphore>
using namespace etlng::impl;
using namespace etl::impl;
using namespace data;
namespace {
@@ -51,8 +51,7 @@ protected:
testing::StrictMock<testing::MockFunction<void(uint32_t)>> actionMock_;
testing::StrictMock<testing::MockFunction<void()>> dbStalledMock_;
etlng::impl::Monitor monitor_ =
etlng::impl::Monitor(ctx_, backend_, ledgers_, kSTART_SEQ, kNO_NEW_LEDGER_REPORT_DELAY);
etl::impl::Monitor monitor_ = etl::impl::Monitor(ctx_, backend_, ledgers_, kSTART_SEQ, kNO_NEW_LEDGER_REPORT_DELAY);
};
TEST_F(MonitorTests, ConsumesAndNotifiesForAllOutstandingSequencesAtOnce)

4
tests/unit/etlng/NetworkValidatedLedgersTests.cpp → tests/unit/etl/NetworkValidatedLedgersTests.cpp
View File

@@ -19,7 +19,7 @@
#include "etl/NetworkValidatedLedgers.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etl/impl/AmendmentBlockHandler.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include <gmock/gmock.h>
@@ -29,7 +29,7 @@
#include <cstdint>
#include <memory>
using namespace etlng::impl;
using namespace etl::impl;
struct NetworkValidatedLedgersTests : virtual public ::testing::Test {
protected:

140
tests/unit/etlng/RegistryTests.cpp → tests/unit/etl/RegistryTests.cpp
View File

@@ -17,10 +17,10 @@
*/
//==============================================================================
#include "etl/Models.hpp"
#include "etl/MonitorInterface.hpp"
#include "etl/SystemState.hpp"
#include "etlng/Models.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/impl/Registry.hpp"
#include "etl/impl/Registry.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
@@ -36,56 +36,56 @@
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace etl::impl;
namespace compiletime::checks {
struct Ext1 {
static void
onLedgerData(etlng::model::LedgerData const&);
onLedgerData(etl::model::LedgerData const&);
};
struct Ext2 {
static void
onInitialObjects(uint32_t, std::vector<etlng::model::Object> const&, std::string);
onInitialObjects(uint32_t, std::vector<etl::model::Object> const&, std::string);
};
struct Ext3 {
static void
onInitialData(etlng::model::LedgerData const&);
onInitialData(etl::model::LedgerData const&);
};
struct Ext4SpecMissing {
static void
onTransaction(uint32_t, etlng::model::Transaction const&);
onTransaction(uint32_t, etl::model::Transaction const&);
};
struct Ext4Fixed {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
static void
onTransaction(uint32_t, etlng::model::Transaction const&);
onTransaction(uint32_t, etl::model::Transaction const&);
};
struct Ext5 {
static void
onInitialObject(uint32_t, etlng::model::Object const&);
onInitialObject(uint32_t, etl::model::Object const&);
};
struct Ext6SpecMissing {
static void
onInitialTransaction(uint32_t, etlng::model::Transaction const&);
onInitialTransaction(uint32_t, etl::model::Transaction const&);
};
struct Ext6Fixed {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
static void
onInitialTransaction(uint32_t, etlng::model::Transaction const&);
onInitialTransaction(uint32_t, etl::model::Transaction const&);
};
struct ExtRealistic {
using spec = etlng::model::Spec<
using spec = etl::model::Spec<
ripple::TxType::ttNFTOKEN_BURN,
ripple::TxType::ttNFTOKEN_ACCEPT_OFFER,
ripple::TxType::ttNFTOKEN_CREATE_OFFER,
@@ -93,11 +93,11 @@ struct ExtRealistic {
ripple::TxType::ttNFTOKEN_MINT>;
static void
onLedgerData(etlng::model::LedgerData const&);
onLedgerData(etl::model::LedgerData const&);
static void
onInitialObject(uint32_t, etlng::model::Object const&);
onInitialObject(uint32_t, etl::model::Object const&);
static void
onInitialTransaction(uint32_t, etlng::model::Transaction const&);
onInitialTransaction(uint32_t, etl::model::Transaction const&);
};
struct ExtCombinesTwoOfKind : Ext2, Ext5 {};
@@ -117,12 +117,12 @@ static_assert(SomeExtension<ExtRealistic>);
static_assert(not SomeExtension<ExtCombinesTwoOfKind>);
struct ValidSpec {
using spec = etlng::model::Spec<ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_MINT>;
using spec = etl::model::Spec<ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_MINT>;
};
// invalid spec does not compile:
// struct DuplicatesSpec {
// using spec = etlng::model::Spec<ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_MINT>;
// using spec = etl::model::Spec<ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_BURN, ripple::ttNFTOKEN_MINT>;
// };
static_assert(ContainsSpec<ValidSpec>);
@@ -135,54 +135,54 @@ constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1
constinit auto const kSEQ = 30;
struct MockExtLedgerData {
MOCK_METHOD(void, onLedgerData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onLedgerData, (etl::model::LedgerData const&), (const));
};
struct MockExtInitialData {
MOCK_METHOD(void, onInitialData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialData, (etl::model::LedgerData const&), (const));
};
struct MockExtOnObject {
MOCK_METHOD(void, onObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onObject, (uint32_t, etl::model::Object const&), (const));
};
struct MockExtTransactionNftBurn {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onTransaction, (uint32_t, etlng::model::Transaction const&), (const));
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onTransaction, (uint32_t, etl::model::Transaction const&), (const));
};
struct MockExtTransactionNftOffer {
using spec = etlng::model::Spec<
using spec = etl::model::Spec<
ripple::TxType::ttNFTOKEN_CREATE_OFFER,
ripple::TxType::ttNFTOKEN_CANCEL_OFFER,
ripple::TxType::ttNFTOKEN_ACCEPT_OFFER>;
MOCK_METHOD(void, onTransaction, (uint32_t, etlng::model::Transaction const&), (const));
MOCK_METHOD(void, onTransaction, (uint32_t, etl::model::Transaction const&), (const));
};
struct MockExtInitialObject {
MOCK_METHOD(void, onInitialObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onInitialObject, (uint32_t, etl::model::Object const&), (const));
};
struct MockExtInitialObjects {
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etlng::model::Object> const&, std::string), (const));
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etl::model::Object> const&, std::string), (const));
};
struct MockExtNftBurn {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etlng::model::Transaction const&), (const));
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etl::model::Transaction const&), (const));
};
struct MockExtNftOffer {
using spec = etlng::model::Spec<
using spec = etl::model::Spec<
ripple::TxType::ttNFTOKEN_CREATE_OFFER,
ripple::TxType::ttNFTOKEN_CANCEL_OFFER,
ripple::TxType::ttNFTOKEN_ACCEPT_OFFER>;
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etlng::model::Transaction const&), (const));
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etl::model::Transaction const&), (const));
};
// Mock extensions with allowInReadonly
struct MockExtLedgerDataReadonly {
MOCK_METHOD(void, onLedgerData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onLedgerData, (etl::model::LedgerData const&), (const));
static bool
allowInReadonly()
@@ -192,7 +192,7 @@ struct MockExtLedgerDataReadonly {
};
struct MockExtInitialDataReadonly {
MOCK_METHOD(void, onInitialData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialData, (etl::model::LedgerData const&), (const));
static bool
allowInReadonly()
@@ -202,7 +202,7 @@ struct MockExtInitialDataReadonly {
};
struct MockExtOnObjectReadonly {
MOCK_METHOD(void, onObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onObject, (uint32_t, etl::model::Object const&), (const));
static bool
allowInReadonly()
@@ -212,8 +212,8 @@ struct MockExtOnObjectReadonly {
};
struct MockExtTransactionNftBurnReadonly {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onTransaction, (uint32_t, etlng::model::Transaction const&), (const));
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onTransaction, (uint32_t, etl::model::Transaction const&), (const));
static bool
allowInReadonly()
@@ -223,7 +223,7 @@ struct MockExtTransactionNftBurnReadonly {
};
struct MockExtInitialObjectReadonly {
MOCK_METHOD(void, onInitialObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onInitialObject, (uint32_t, etl::model::Object const&), (const));
static bool
allowInReadonly()
@@ -233,7 +233,7 @@ struct MockExtInitialObjectReadonly {
};
struct MockExtInitialObjectsReadonly {
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etlng::model::Object> const&, std::string), (const));
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etl::model::Object> const&, std::string), (const));
static bool
allowInReadonly()
@@ -243,8 +243,8 @@ struct MockExtInitialObjectsReadonly {
};
struct MockExtNftBurnReadonly {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etlng::model::Transaction const&), (const));
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etl::model::Transaction const&), (const));
static bool
allowInReadonly()
@@ -282,7 +282,7 @@ TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForInitialTransaction)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtNftBurn&, MockExtNftOffer&>(state_, extBurn, extOffer);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = transactions,
.objects = {},
.successors = {},
@@ -311,7 +311,7 @@ TEST_F(RegistryTest, FilteringOfTxWorksCorrectlyForTransaction)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtTransactionNftBurn&, MockExtTransactionNftOffer&>(state_, extBurn, extOffer);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -356,7 +356,7 @@ TEST_F(RegistryTest, ObjectsDispatched)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtOnObject&>(state_, extObj);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = {},
.objects = {util::createObject(), util::createObject(), util::createObject()},
.successors = {},
@@ -383,7 +383,7 @@ TEST_F(RegistryTest, OnLedgerDataForBatch)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtLedgerData&>(state_, ext);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -426,7 +426,7 @@ TEST_F(RegistryTest, InitialDataCorrectOrderOfHookCalls)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtNftBurn&, MockExtInitialData&>(state_, extInitialTransaction, extInitialData);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -466,7 +466,7 @@ TEST_F(RegistryTest, LedgerDataCorrectOrderOfHookCalls)
state_, extOnObject, extOnTransaction, extLedgerData
);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = std::move(objects),
.successors = {},
@@ -493,7 +493,7 @@ TEST_F(RegistryTest, ReadonlyModeLedgerDataAllowed)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtLedgerDataReadonly&>(state_, ext);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -520,7 +520,7 @@ TEST_F(RegistryTest, ReadonlyModeTransactionAllowed)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtTransactionNftBurnReadonly&>(state_, extTx);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -548,7 +548,7 @@ TEST_F(RegistryTest, ReadonlyModeObjectAllowed)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtOnObjectReadonly&>(state_, extObj);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
@@ -575,7 +575,7 @@ TEST_F(RegistryTest, ReadonlyModeInitialDataAllowed)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtInitialDataReadonly&>(state_, extInitialData);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -602,7 +602,7 @@ TEST_F(RegistryTest, ReadonlyModeInitialTransactionAllowed)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtNftBurnReadonly&>(state_, extTx);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -652,7 +652,7 @@ TEST_F(RegistryTest, ReadonlyModeRegularExtensionsNotCalled)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtLedgerData&>(state_, extLedgerData);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
@@ -682,7 +682,7 @@ TEST_F(RegistryTest, MixedReadonlyAndRegularExtensions)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<MockExtLedgerDataReadonly&, MockExtLedgerData&>(state_, extReadonly, extRegular);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
@@ -696,7 +696,7 @@ TEST_F(RegistryTest, MixedReadonlyAndRegularExtensions)
TEST_F(RegistryTest, MonitorInterfaceExecution)
{
struct MockMonitor : etlng::MonitorInterface {
struct MockMonitor : etl::MonitorInterface {
MOCK_METHOD(void, notifySequenceLoaded, (uint32_t), (override));
MOCK_METHOD(void, notifyWriteConflict, (uint32_t), (override));
MOCK_METHOD(
@@ -724,7 +724,7 @@ TEST_F(RegistryTest, MonitorInterfaceExecution)
TEST_F(RegistryTest, ReadonlyModeWithAllowInReadonlyTest)
{
struct ExtWithAllowInReadonly {
MOCK_METHOD(void, onLedgerData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onLedgerData, (etl::model::LedgerData const&), (const));
static bool
allowInReadonly()
@@ -741,7 +741,7 @@ TEST_F(RegistryTest, ReadonlyModeWithAllowInReadonlyTest)
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
auto reg = Registry<ExtWithAllowInReadonly&>(state_, ext);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = {},
.objects = {},
.successors = {},
@@ -756,9 +756,9 @@ TEST_F(RegistryTest, ReadonlyModeWithAllowInReadonlyTest)
TEST_F(RegistryTest, ReadonlyModeExecutePluralHooksIfAllowedPaths)
{
struct ExtWithBothHooksAndAllowReadonly {
MOCK_METHOD(void, onLedgerData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialData, (etlng::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etlng::model::Object> const&, std::string), (const));
MOCK_METHOD(void, onLedgerData, (etl::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialData, (etl::model::LedgerData const&), (const));
MOCK_METHOD(void, onInitialObjects, (uint32_t, std::vector<etl::model::Object> const&, std::string), (const));
static bool
allowInReadonly()
@@ -785,7 +785,7 @@ TEST_F(RegistryTest, ReadonlyModeExecutePluralHooksIfAllowedPaths)
auto reg = Registry<ExtWithBothHooksAndAllowReadonly&>(state_, ext);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = transactions,
.objects = objects,
.successors = {},
@@ -797,7 +797,7 @@ TEST_F(RegistryTest, ReadonlyModeExecutePluralHooksIfAllowedPaths)
);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -814,12 +814,12 @@ TEST_F(RegistryTest, ReadonlyModeExecutePluralHooksIfAllowedPaths)
TEST_F(RegistryTest, ReadonlyModeExecuteByOneHooksIfAllowedPaths)
{
struct ExtWithBothHooksAndAllowReadonly {
using spec = etlng::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
using spec = etl::model::Spec<ripple::TxType::ttNFTOKEN_BURN>;
MOCK_METHOD(void, onObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onInitialObject, (uint32_t, etlng::model::Object const&), (const));
MOCK_METHOD(void, onTransaction, (uint32_t, etlng::model::Transaction const&), (const));
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etlng::model::Transaction const&), (const));
MOCK_METHOD(void, onObject, (uint32_t, etl::model::Object const&), (const));
MOCK_METHOD(void, onInitialObject, (uint32_t, etl::model::Object const&), (const));
MOCK_METHOD(void, onTransaction, (uint32_t, etl::model::Transaction const&), (const));
MOCK_METHOD(void, onInitialTransaction, (uint32_t, etl::model::Transaction const&), (const));
static bool
allowInReadonly()
@@ -847,7 +847,7 @@ TEST_F(RegistryTest, ReadonlyModeExecuteByOneHooksIfAllowedPaths)
auto reg = Registry<ExtWithBothHooksAndAllowReadonly&>(state_, ext);
reg.dispatch(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = transactions,
.objects = objects,
.successors = {},
@@ -859,7 +859,7 @@ TEST_F(RegistryTest, ReadonlyModeExecuteByOneHooksIfAllowedPaths)
);
reg.dispatchInitialData(
etlng::model::LedgerData{
etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},

12
tests/unit/etlng/SchedulingTests.cpp → tests/unit/etl/SchedulingTests.cpp
View File

@@ -17,10 +17,10 @@
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/SchedulerInterface.hpp"
#include "etlng/impl/Loading.hpp"
#include "etlng/impl/Scheduling.hpp"
#include "etl/Models.hpp"
#include "etl/SchedulerInterface.hpp"
#include "etl/impl/Loading.hpp"
#include "etl/impl/Scheduling.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include <gmock/gmock.h>
@@ -31,8 +31,8 @@
#include <optional>
#include <utility>
using namespace etlng;
using namespace etlng::model;
using namespace etl;
using namespace etl::model;
namespace {
class FakeScheduler : SchedulerInterface {

60
tests/unit/etl/SourceImplTests.cpp
View File

@@ -1,7 +1,7 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
@@ -17,6 +17,9 @@
*/
//==============================================================================
#include "etl/InitialLoadObserverInterface.hpp"
#include "etl/LoadBalancerInterface.hpp"
#include "etl/Models.hpp"
#include "etl/impl/SourceImpl.hpp"
#include "rpc/Errors.hpp"
#include "util/Spawn.hpp"
@@ -32,6 +35,7 @@
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
@@ -44,12 +48,16 @@ using namespace etl::impl;
using testing::Return;
using testing::StrictMock;
namespace {
struct GrpcSourceMock {
using FetchLedgerReturnType = std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>;
MOCK_METHOD(FetchLedgerReturnType, fetchLedger, (uint32_t, bool, bool));
using LoadLedgerReturnType = std::pair<std::vector<std::string>, bool>;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t));
using LoadLedgerReturnType = etl::InitialLedgerLoadResult;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t, etl::InitialLoadObserverInterface&));
MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
};
struct SubscriptionSourceMock {
@@ -75,6 +83,23 @@ struct ForwardingSourceMock {
);
};
struct InitialLoadObserverMock : etl::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etl::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etl::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct SourceImplTest : public ::testing::Test {
protected:
boost::asio::io_context ioc_;
@@ -107,6 +132,7 @@ TEST_F(SourceImplTest, run)
TEST_F(SourceImplTest, stop)
{
EXPECT_CALL(*subscriptionSourceMock_, stop);
EXPECT_CALL(grpcSourceMock_, stop);
boost::asio::io_context ctx;
util::spawn(ctx, [&](boost::asio::yield_context yield) { source_.stop(yield); });
ctx.run();
@@ -170,17 +196,33 @@ TEST_F(SourceImplTest, fetchLedger)
EXPECT_EQ(actualStatus.error_code(), grpc::StatusCode::OK);
}
TEST_F(SourceImplTest, loadInitialLedger)
TEST_F(SourceImplTest, loadInitialLedgerErrorPath)
{
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers))
.WillOnce(Return(std::make_pair(std::vector<std::string>{}, true)));
auto const [actualLedgers, actualSuccess] = source_.loadInitialLedger(ledgerSeq, numMarkers);
auto observerMock = testing::StrictMock<InitialLoadObserverMock>();
EXPECT_TRUE(actualLedgers.empty());
EXPECT_TRUE(actualSuccess);
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, testing::_))
.WillOnce(Return(std::unexpected{etl::InitialLedgerLoadError::Errored}));
auto const res = source_.loadInitialLedger(ledgerSeq, numMarkers, observerMock);
EXPECT_FALSE(res.has_value());
}
TEST_F(SourceImplTest, loadInitialLedgerSuccessPath)
{
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
auto response = etl::InitialLedgerLoadResult{{"1", "2", "3"}};
auto observerMock = testing::StrictMock<InitialLoadObserverMock>();
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, testing::_)).WillOnce(Return(response));
auto const res = source_.loadInitialLedger(ledgerSeq, numMarkers, observerMock);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res, response);
}
TEST_F(SourceImplTest, forwardToRippled)

38
tests/unit/etlng/TaskManagerTests.cpp → tests/unit/etl/TaskManagerTests.cpp
View File

@@ -17,13 +17,13 @@
*/
//==============================================================================
#include "etlng/ExtractorInterface.hpp"
#include "etlng/LoaderInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/MonitorInterface.hpp"
#include "etlng/SchedulerInterface.hpp"
#include "etlng/impl/Loading.hpp"
#include "etlng/impl/TaskManager.hpp"
#include "etl/ExtractorInterface.hpp"
#include "etl/LoaderInterface.hpp"
#include "etl/Models.hpp"
#include "etl/MonitorInterface.hpp"
#include "etl/SchedulerInterface.hpp"
#include "etl/impl/Loading.hpp"
#include "etl/impl/TaskManager.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/TestObject.hpp"
#include "util/async/AnyExecutionContext.hpp"
@@ -43,30 +43,30 @@
#include <semaphore>
#include <vector>
using namespace etlng::model;
using namespace etlng::impl;
using namespace etl::model;
using namespace etl::impl;
namespace {
constinit auto const kSEQ = 30;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
struct MockScheduler : etlng::SchedulerInterface {
struct MockScheduler : etl::SchedulerInterface {
MOCK_METHOD(std::optional<Task>, next, (), (override));
};
struct MockExtractor : etlng::ExtractorInterface {
struct MockExtractor : etl::ExtractorInterface {
MOCK_METHOD(std::optional<LedgerData>, extractLedgerWithDiff, (uint32_t), (override));
MOCK_METHOD(std::optional<LedgerData>, extractLedgerOnly, (uint32_t), (override));
};
struct MockLoader : etlng::LoaderInterface {
using ExpectedType = std::expected<void, etlng::LoaderError>;
struct MockLoader : etl::LoaderInterface {
using ExpectedType = std::expected<void, etl::LoaderError>;
MOCK_METHOD(ExpectedType, load, (LedgerData const&), (override));
MOCK_METHOD(std::optional<ripple::LedgerHeader>, loadInitialLedger, (LedgerData const&), (override));
};
struct MockMonitor : etlng::MonitorInterface {
struct MockMonitor : etl::MonitorInterface {
MOCK_METHOD(void, notifySequenceLoaded, (uint32_t), (override));
MOCK_METHOD(void, notifyWriteConflict, (uint32_t), (override));
MOCK_METHOD(
@@ -141,7 +141,7 @@ TEST_F(TaskManagerTests, LoaderGetsDataIfNextSequenceIsExtracted)
EXPECT_CALL(*mockLoaderPtr_, load(testing::_))
.Times(kTOTAL)
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etlng::LoaderError> {
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etl::LoaderError> {
loaded.push_back(data.seq);
if (loaded.size() == kTOTAL)
done.release();
@@ -185,13 +185,13 @@ TEST_F(TaskManagerTests, WriteConflictHandling)
// First kCONFLICT_AFTER calls succeed, then we get a write conflict
EXPECT_CALL(*mockLoaderPtr_, load(testing::_))
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etlng::LoaderError> {
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etl::LoaderError> {
loaded.push_back(data.seq);
if (loaded.size() == kCONFLICT_AFTER) {
conflictOccurred = true;
done.release();
return std::unexpected(etlng::LoaderError::WriteConflict);
return std::unexpected(etl::LoaderError::WriteConflict);
}
if (loaded.size() == kTOTAL)
@@ -238,13 +238,13 @@ TEST_F(TaskManagerTests, AmendmentBlockedHandling)
});
EXPECT_CALL(*mockLoaderPtr_, load(testing::_))
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etlng::LoaderError> {
.WillRepeatedly([&](LedgerData data) -> std::expected<void, etl::LoaderError> {
loaded.push_back(data.seq);
if (loaded.size() == kAMENDMENT_BLOCKED_AFTER) {
amendmentBlockedOccurred = true;
done.release();
return std::unexpected(etlng::LoaderError::AmendmentBlocked);
return std::unexpected(etl::LoaderError::AmendmentBlocked);
}
if (loaded.size() == kTOTAL)

158
tests/unit/etl/TransformerTests.cpp
View File

@@ -1,158 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etl/SystemState.hpp"
#include "etl/impl/Transformer.hpp"
#include "util/FakeFetchResponse.hpp"
#include "util/MockAmendmentBlockHandler.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockExtractionDataPipe.hpp"
#include "util/MockLedgerLoader.hpp"
#include "util/MockLedgerPublisher.hpp"
#include "util/MockPrometheus.hpp"
#include "util/StringUtils.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <chrono>
#include <memory>
#include <optional>
#include <thread>
using namespace testing;
using namespace etl;
namespace {
// taken from BackendTests
constexpr auto kRAW_HEADER =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
"6DB6FE30CC5909B285080FCD6773CC883F9FE0EE4D439340AC592AADB973ED3CF5"
"3E2232B33EF57CECAC2816E3122816E31A0A00F8377CD95DFA484CFAE282656A58"
"CE5AA29652EFFD80AC59CD91416E4E13DBBE";
} // namespace
struct ETLTransformerTest : util::prometheus::WithPrometheus, MockBackendTest {
using DataType = FakeFetchResponse;
using ExtractionDataPipeType = MockExtractionDataPipe;
using LedgerLoaderType = MockLedgerLoader;
using LedgerPublisherType = MockLedgerPublisher;
using AmendmentBlockHandlerType = MockAmendmentBlockHandler;
using TransformerType = etl::impl::
Transformer<ExtractionDataPipeType, LedgerLoaderType, LedgerPublisherType, AmendmentBlockHandlerType>;
ETLTransformerTest()
{
state_.isStopping = false;
state_.writeConflict = false;
state_.isStrictReadonly = false;
state_.isWriting = false;
}
protected:
ExtractionDataPipeType dataPipe_;
LedgerLoaderType ledgerLoader_;
LedgerPublisherType ledgerPublisher_;
AmendmentBlockHandlerType amendmentBlockHandler_;
SystemState state_;
std::unique_ptr<TransformerType> transformer_;
};
TEST_F(ETLTransformerTest, StopsOnWriteConflict)
{
state_.writeConflict = true;
EXPECT_CALL(dataPipe_, popNext).Times(0);
EXPECT_CALL(ledgerPublisher_, publish(_)).Times(0);
transformer_ = std::make_unique<TransformerType>(
dataPipe_, backend_, ledgerLoader_, ledgerPublisher_, amendmentBlockHandler_, 0, state_
);
transformer_->waitTillFinished(); // explicitly joins the thread
}
TEST_F(ETLTransformerTest, StopsOnEmptyFetchResponse)
{
backend_->cache().setFull(); // to avoid throwing exception in updateCache
auto const blob = hexStringToBinaryString(kRAW_HEADER);
auto const response = std::make_optional<FakeFetchResponse>(blob);
ON_CALL(dataPipe_, popNext).WillByDefault([this, &response](auto) -> std::optional<FakeFetchResponse> {
if (state_.isStopping)
return std::nullopt;
return response; // NOLINT (performance-no-automatic-move)
});
ON_CALL(*backend_, doFinishWrites).WillByDefault(Return(true));
// TODO: most of this should be hidden in a smaller entity that is injected into the transformer thread
EXPECT_CALL(dataPipe_, popNext).Times(AtLeast(1));
EXPECT_CALL(*backend_, startWrites).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeLedger(_, _)).Times(AtLeast(1));
EXPECT_CALL(ledgerLoader_, insertTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeAccountTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeNFTs).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeNFTTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, doFinishWrites).Times(AtLeast(1));
EXPECT_CALL(ledgerPublisher_, publish(_)).Times(AtLeast(1));
transformer_ = std::make_unique<TransformerType>(
dataPipe_, backend_, ledgerLoader_, ledgerPublisher_, amendmentBlockHandler_, 0, state_
);
// after 10ms we start spitting out empty responses which means the extractor is finishing up
// this is normally combined with stopping the entire thing by setting the isStopping flag.
std::this_thread::sleep_for(std::chrono::milliseconds{10});
state_.isStopping = true;
}
TEST_F(ETLTransformerTest, DoesNotPublishIfCanNotBuildNextLedger)
{
backend_->cache().setFull(); // to avoid throwing exception in updateCache
auto const blob = hexStringToBinaryString(kRAW_HEADER);
auto const response = std::make_optional<FakeFetchResponse>(blob);
ON_CALL(dataPipe_, popNext).WillByDefault(Return(response));
ON_CALL(*backend_, doFinishWrites).WillByDefault(Return(false)); // emulate write failure
// TODO: most of this should be hidden in a smaller entity that is injected into the transformer thread
EXPECT_CALL(dataPipe_, popNext).Times(AtLeast(1));
EXPECT_CALL(*backend_, startWrites).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeLedger(_, _)).Times(AtLeast(1));
EXPECT_CALL(ledgerLoader_, insertTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeAccountTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeNFTs).Times(AtLeast(1));
EXPECT_CALL(*backend_, writeNFTTransactions).Times(AtLeast(1));
EXPECT_CALL(*backend_, doFinishWrites).Times(AtLeast(1));
// should not call publish
EXPECT_CALL(ledgerPublisher_, publish(_)).Times(0);
transformer_ = std::make_unique<TransformerType>(
dataPipe_, backend_, ledgerLoader_, ledgerPublisher_, amendmentBlockHandler_, 0, state_
);
}
// TODO: implement tests for amendment block. requires more refactoring

14
tests/unit/etlng/ext/CacheTests.cpp → tests/unit/etl/ext/CacheTests.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/CacheUpdater.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "etl/Models.hpp"
#include "etl/impl/CacheUpdater.hpp"
#include "etl/impl/ext/Cache.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
@@ -32,7 +32,7 @@
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace etl::impl;
using namespace data;
namespace {
@@ -45,7 +45,7 @@ createTestData()
{
auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
@@ -61,8 +61,8 @@ createTestData()
struct CacheExtTests : util::prometheus::WithPrometheus {
protected:
MockLedgerCache cache_;
std::shared_ptr<etlng::impl::CacheUpdater> updater_ = std::make_shared<etlng::impl::CacheUpdater>(cache_);
etlng::impl::CacheExt ext_{updater_};
std::shared_ptr<etl::impl::CacheUpdater> updater_ = std::make_shared<etl::impl::CacheUpdater>(cache_);
etl::impl::CacheExt ext_{updater_};
};
TEST_F(CacheExtTests, OnLedgerDataUpdatesCache)

10
tests/unit/etlng/ext/CoreTests.cpp → tests/unit/etl/ext/CoreTests.cpp
View File

@@ -17,8 +17,8 @@
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "etl/Models.hpp"
#include "etl/impl/ext/Core.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
@@ -31,7 +31,7 @@
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace etl::impl;
using namespace data;
namespace {
@@ -48,7 +48,7 @@ createTestData()
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -63,7 +63,7 @@ createTestData()
struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::CoreExt ext_{backend_};
etl::impl::CoreExt ext_{backend_};
};
TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions)

12
tests/unit/etlng/ext/MPTTests.cpp → tests/unit/etl/ext/MPTTests.cpp
View File

@@ -17,8 +17,8 @@
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/MPT.hpp"
#include "etl/Models.hpp"
#include "etl/impl/ext/MPT.hpp"
#include "rpc/RPCHelpers.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
@@ -33,8 +33,8 @@
#include <utility>
#include <vector>
using namespace etlng;
using namespace etlng::impl;
using namespace etl;
using namespace etl::impl;
using namespace data;
using namespace testing;
@@ -74,7 +74,7 @@ createTestData()
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -95,7 +95,7 @@ createMultipleHoldersTestData()
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},

10
tests/unit/etlng/ext/NFTTests.cpp → tests/unit/etl/ext/NFTTests.cpp
View File

@@ -17,8 +17,8 @@
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "etl/Models.hpp"
#include "etl/impl/ext/NFT.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
@@ -31,7 +31,7 @@
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace etl::impl;
using namespace data;
namespace {
@@ -235,7 +235,7 @@ createTestData()
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
@@ -250,7 +250,7 @@ createTestData()
struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::NFTExt ext_{backend_};
etl::impl::NFTExt ext_{backend_};
};
TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs)

50
tests/unit/etlng/ext/SuccessorTests.cpp → tests/unit/etl/ext/SuccessorTests.cpp
View File

@@ -19,8 +19,8 @@
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "etl/Models.hpp"
#include "etl/impl/ext/Successor.hpp"
#include "util/Assert.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockAssert.hpp"
@@ -44,7 +44,7 @@
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace etl::impl;
using namespace data;
namespace {
@@ -52,7 +52,7 @@ constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto
createTestData(std::vector<etlng::model::Object> objects)
createTestData(std::vector<etl::model::Object> objects)
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
@@ -61,7 +61,7 @@ createTestData(std::vector<etlng::model::Object> objects)
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
return etl::model::LedgerData{
.transactions = std::move(transactions),
.objects = std::move(objects),
.successors = {},
@@ -90,7 +90,7 @@ createInitialTestData(std::vector<ripple::uint256> edgeKeys)
struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
MockLedgerCache cache_;
etlng::impl::SuccessorExt ext_{backend_, cache_};
etl::impl::SuccessorExt ext_{backend_, cache_};
};
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent)
@@ -115,7 +115,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffers
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
@@ -138,7 +138,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecess
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
@@ -161,7 +161,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecess
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
@@ -191,7 +191,7 @@ TEST_F(
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache
)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
@@ -225,7 +225,7 @@ TEST_F(
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache
)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
@@ -259,7 +259,7 @@ TEST_F(
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache
)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
@@ -291,7 +291,7 @@ TEST_F(
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const predKey = binaryStringToUint256(
@@ -322,7 +322,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorA
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
@@ -344,7 +344,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors)
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
@@ -366,7 +366,7 @@ TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWith
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys)
{
using namespace etlng::model;
using namespace etl::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
@@ -405,7 +405,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessor
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
using namespace etl::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
@@ -445,7 +445,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsF
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
using namespace etl::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
@@ -495,7 +495,7 @@ TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForE
TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey)
{
using namespace etlng::model;
using namespace etl::model;
auto const lastKey = std::string{};
auto const data = std::vector{
@@ -522,7 +522,7 @@ TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey)
TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey)
{
using namespace etlng::model;
using namespace etl::model;
auto const lastKey =
uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"));
@@ -551,7 +551,7 @@ struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests
TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
@@ -577,7 +577,7 @@ TEST_F(
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache
)
{
using namespace etlng::model;
using namespace etl::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
@@ -604,7 +604,7 @@ TEST_F(
TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull)
{
using namespace etlng::model;
using namespace etl::model;
auto const data = createTestData({
util::createObject(Object::ModType::Modified),
@@ -617,7 +617,7 @@ TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull)
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys)
{
using namespace etlng::model;
using namespace etl::model;
auto data = createTestData({});
@@ -627,7 +627,7 @@ TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys)
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects)
{
using namespace etlng::model;
using namespace etl::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");

70
tests/unit/etlng/AmendmentBlockHandlerTests.cpp
View File

@@ -1,70 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etl/SystemState.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/MockPrometheus.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <chrono>
#include <cstddef>
#include <semaphore>
using namespace etlng::impl;
struct AmendmentBlockHandlerNgTests : util::prometheus::WithPrometheus {
protected:
testing::StrictMock<testing::MockFunction<void()>> actionMock_;
etl::SystemState state_;
util::async::CoroExecutionContext ctx_;
};
TEST_F(AmendmentBlockHandlerNgTests, CallToNotifyAmendmentBlockedSetsStateAndRepeatedlyCallsAction)
{
static constexpr auto kMAX_ITERATIONS = 10uz;
etlng::impl::AmendmentBlockHandler handler{ctx_, state_, std::chrono::nanoseconds{1}, actionMock_.AsStdFunction()};
auto counter = 0uz;
std::binary_semaphore stop{0};
EXPECT_FALSE(state_.isAmendmentBlocked);
EXPECT_CALL(actionMock_, Call()).Times(testing::AtLeast(10)).WillRepeatedly([&]() {
if (++counter; counter > kMAX_ITERATIONS)
stop.release();
});
handler.notifyAmendmentBlocked();
stop.acquire(); // wait for the counter to reach over kMAX_ITERATIONS
handler.stop();
EXPECT_TRUE(state_.isAmendmentBlocked);
}
struct DefaultAmendmentBlockActionNgTest : LoggerFixture {};
TEST_F(DefaultAmendmentBlockActionNgTest, Call)
{
AmendmentBlockHandler::kDEFAULT_AMENDMENT_BLOCK_ACTION();
auto const loggerString = getLoggerString();
EXPECT_TRUE(loggerString.starts_with("cri:ETL - Can't process new ledgers")) << "LoggerString " << loggerString;
}

197
tests/unit/etlng/ForwardingSourceTests.cpp
View File

@@ -1,197 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ForwardingSource.hpp"
#include "rpc/Errors.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/Spawn.hpp"
#include "util/TestWsServer.hpp"
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/serialize.hpp>
#include <gtest/gtest.h>
#include <algorithm>
#include <chrono>
#include <memory>
#include <optional>
#include <string>
#include <utility>
using namespace etlng::impl;
struct ForwardingSourceNgTests : SyncAsioContextTest {
protected:
TestWsServer server_{ctx_, "0.0.0.0"};
ForwardingSource forwardingSource_{
"127.0.0.1",
server_.port(),
std::chrono::milliseconds{20},
std::chrono::milliseconds{20}
};
};
TEST_F(ForwardingSourceNgTests, ConnectionFailed)
{
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled({}, {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlConnectionError);
});
}
struct ForwardingSourceOperationsNgTests : ForwardingSourceNgTests {
TestWsConnection
serverConnection(boost::asio::yield_context yield)
{
// First connection attempt is SSL handshake so it will fail
auto failedConnection = server_.acceptConnection(yield);
[&]() { ASSERT_FALSE(failedConnection); }();
auto connection = server_.acceptConnection(yield);
[&]() { ASSERT_TRUE(connection) << connection.error().message(); }();
return std::move(connection).value();
}
protected:
std::string const message_ = R"JSON({"data": "some_data"})JSON";
boost::json::object const reply_ = {{"reply", "some_reply"}};
};
TEST_F(ForwardingSourceOperationsNgTests, XUserHeader)
{
std::string const xUserValue = "some_user";
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto headers = connection.headers();
ASSERT_FALSE(headers.empty());
auto it = std::ranges::find_if(headers, [](auto const& header) {
return std::holds_alternative<std::string>(header.name) && std::get<std::string>(header.name) == "X-User";
});
ASSERT_FALSE(it == headers.end());
EXPECT_EQ(std::get<std::string>(it->name), "X-User");
EXPECT_EQ(it->value, xUserValue);
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result =
forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, xUserValue, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestError);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ReadFailed)
{
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestError);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ReadTimeout)
{
TestWsConnectionPtr connection;
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
connection = std::make_unique<TestWsConnection>(serverConnection(yield));
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlRequestTimeout);
});
}
TEST_F(ForwardingSourceOperationsNgTests, ParseFailed)
{
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send("invalid_json", yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlInvalidResponse);
});
}
TEST_F(ForwardingSourceOperationsNgTests, GotNotAnObject)
{
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send(R"(["some_value"])", yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
connection.close(yield);
});
runSpawn([&](boost::asio::yield_context yield) {
auto result = forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), {}, {}, yield);
ASSERT_FALSE(result);
EXPECT_EQ(result.error(), rpc::ClioError::EtlInvalidResponse);
});
}
TEST_F(ForwardingSourceOperationsNgTests, Success)
{
util::spawn(ctx_, [&](boost::asio::yield_context yield) {
auto connection = serverConnection(yield);
auto receivedMessage = connection.receive(yield);
[&]() { ASSERT_TRUE(receivedMessage); }();
EXPECT_EQ(boost::json::parse(*receivedMessage), boost::json::parse(message_)) << *receivedMessage;
auto sendError = connection.send(boost::json::serialize(reply_), yield);
[&]() { ASSERT_FALSE(sendError) << *sendError; }();
});
runSpawn([&](boost::asio::yield_context yield) {
auto result =
forwardingSource_.forwardToRippled(boost::json::parse(message_).as_object(), "some_ip", {}, yield);
[&]() { ASSERT_TRUE(result); }();
auto expectedReply = reply_;
expectedReply["forwarded"] = true;
EXPECT_EQ(*result, expectedReply) << *result;
});
}

395
tests/unit/etlng/GrpcSourceTests.cpp
View File

@@ -1,395 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "etl/ETLHelpers.hpp"
#include "etl/impl/GrpcSource.hpp"
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/GrpcSource.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/Assert.hpp"
#include "util/MockXrpLedgerAPIService.hpp"
#include "util/Mutex.hpp"
#include "util/TestObject.hpp"
#include <boost/asio/spawn.hpp>
#include <gmock/gmock.h>
#include <grpcpp/server_context.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <org/xrpl/rpc/v1/get_ledger_data.pb.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <atomic>
#include <chrono>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <future>
#include <map>
#include <memory>
#include <mutex>
#include <optional>
#include <queue>
#include <semaphore>
#include <string>
#include <vector>
using namespace etlng::model;
namespace {
struct MockLoadObserver : etlng::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<Object> const&, std::optional<std::string>),
(override)
);
};
struct GrpcSourceNgTests : virtual public ::testing::Test, tests::util::WithMockXrpLedgerAPIService {
GrpcSourceNgTests()
: WithMockXrpLedgerAPIService("localhost:0"), grpcSource_("localhost", std::to_string(getXRPLMockPort()))
{
}
class KeyStore {
std::vector<ripple::uint256> keys_;
using Store = std::map<std::string, std::queue<ripple::uint256>, std::greater<>>;
util::Mutex<Store> store_;
public:
KeyStore(std::size_t totalKeys, std::size_t numMarkers) : keys_(etl::getMarkers(totalKeys))
{
auto const totalPerMarker = totalKeys / numMarkers;
auto const markers = etl::getMarkers(numMarkers);
auto store = store_.lock();
for (auto mi = 0uz; mi < markers.size(); ++mi) {
for (auto i = 0uz; i < totalPerMarker; ++i) {
auto const mapKey = ripple::strHex(markers.at(mi)).substr(0, 2);
store->operator[](mapKey).push(keys_.at((mi * totalPerMarker) + i));
}
}
}
std::optional<std::string>
next(std::string const& marker)
{
auto store = store_.lock<std::scoped_lock>();
auto const mapKey = ripple::strHex(marker).substr(0, 2);
auto it = store->lower_bound(mapKey);
ASSERT(it != store->end(), "Lower bound not found for '{}'", mapKey);
auto& queue = it->second;
if (queue.empty())
return std::nullopt;
auto data = queue.front();
queue.pop();
return std::make_optional(uint256ToString(data));
};
std::optional<std::string>
peek(std::string const& marker)
{
auto store = store_.lock<std::scoped_lock>();
auto const mapKey = ripple::strHex(marker).substr(0, 2);
auto it = store->lower_bound(mapKey);
ASSERT(it != store->end(), "Lower bound not found for '{}'", mapKey);
auto& queue = it->second;
if (queue.empty())
return std::nullopt;
auto data = queue.front();
return std::make_optional(uint256ToString(data));
};
};
protected:
testing::StrictMock<MockLoadObserver> observer_;
etlng::impl::GrpcSource grpcSource_;
};
struct GrpcSourceNgLoadInitialLedgerTests : GrpcSourceNgTests {
protected:
uint32_t const sequence_ = 123u;
uint32_t const numMarkers_ = 4u;
bool const cacheOnly_ = false;
};
} // namespace
TEST_F(GrpcSourceNgTests, BasicFetchLedger)
{
uint32_t const sequence = 123u;
bool const getObjects = true;
bool const getObjectNeighbors = false;
EXPECT_CALL(mockXrpLedgerAPIService, GetLedger)
.WillOnce([&](grpc::ServerContext* /*context*/,
org::xrpl::rpc::v1::GetLedgerRequest const* request,
org::xrpl::rpc::v1::GetLedgerResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence);
EXPECT_TRUE(request->transactions());
EXPECT_TRUE(request->expand());
EXPECT_EQ(request->get_objects(), getObjects);
EXPECT_EQ(request->get_object_neighbors(), getObjectNeighbors);
EXPECT_EQ(request->user(), "ETL");
response->set_validated(true);
response->set_is_unlimited(false);
response->set_object_neighbors_included(false);
return grpc::Status{};
});
auto const [status, response] = grpcSource_.fetchLedger(sequence, getObjects, getObjectNeighbors);
ASSERT_TRUE(status.ok());
EXPECT_TRUE(response.validated());
EXPECT_FALSE(response.is_unlimited());
EXPECT_FALSE(response.object_neighbors_included());
}
TEST_F(GrpcSourceNgLoadInitialLedgerTests, GetLedgerDataNotFound)
{
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.Times(numMarkers_)
.WillRepeatedly([&](grpc::ServerContext* /*context*/,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* /*response*/) {
EXPECT_EQ(request->ledger().sequence(), sequence_);
EXPECT_EQ(request->user(), "ETL");
return grpc::Status{grpc::StatusCode::NOT_FOUND, "Not found"};
});
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_FALSE(res.has_value());
}
TEST_F(GrpcSourceNgLoadInitialLedgerTests, ObserverCalledCorrectly)
{
auto const key = ripple::uint256{4};
auto const keyStr = uint256ToString(key);
auto const object = createTicketLedgerObject("rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", sequence_);
auto const objectData = object.getSerializer().peekData();
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.Times(numMarkers_)
.WillRepeatedly([&](grpc::ServerContext* /*context*/,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence_);
EXPECT_EQ(request->user(), "ETL");
response->set_is_unlimited(true);
auto newObject = response->mutable_ledger_objects()->add_objects();
newObject->set_key(uint256ToString(key));
newObject->set_data(objectData.data(), objectData.size());
return grpc::Status{};
});
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects)
.Times(numMarkers_)
.WillRepeatedly([&](uint32_t, std::vector<Object> const& data, std::optional<std::string> lastKey) {
EXPECT_FALSE(lastKey.has_value());
EXPECT_EQ(data.size(), 1);
});
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res.value().size(), numMarkers_);
EXPECT_EQ(res.value(), std::vector<std::string>(4, keyStr));
}
TEST_F(GrpcSourceNgLoadInitialLedgerTests, DataTransferredAndObserverCalledCorrectly)
{
auto const totalKeys = 256uz;
auto const totalPerMarker = totalKeys / numMarkers_;
auto const batchSize = totalPerMarker / 4uz;
auto const batchesPerMarker = totalPerMarker / batchSize;
auto keyStore = KeyStore(totalKeys, numMarkers_);
auto const object = createTicketLedgerObject("rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn", sequence_);
auto const objectData = object.getSerializer().peekData();
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.Times(numMarkers_ * batchesPerMarker)
.WillRepeatedly([&](grpc::ServerContext* /*context*/,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence_);
EXPECT_EQ(request->user(), "ETL");
response->set_is_unlimited(true);
auto next = request->marker().empty() ? std::string("00") : request->marker();
for (auto i = 0uz; i < batchSize; ++i) {
if (auto maybeLastKey = keyStore.next(next); maybeLastKey.has_value()) {
next = *maybeLastKey;
auto newObject = response->mutable_ledger_objects()->add_objects();
newObject->set_key(next);
newObject->set_data(objectData.data(), objectData.size());
}
}
if (auto maybeNext = keyStore.peek(next); maybeNext.has_value())
response->set_marker(*maybeNext);
return grpc::Status::OK;
});
std::atomic_size_t total = 0uz;
std::atomic_size_t totalWithLastKey = 0uz;
std::atomic_size_t totalWithoutLastKey = 0uz;
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects)
.Times(numMarkers_ * batchesPerMarker)
.WillRepeatedly([&](uint32_t, std::vector<Object> const& data, std::optional<std::string> lastKey) {
EXPECT_LE(data.size(), batchSize);
if (lastKey.has_value()) {
++totalWithLastKey;
} else {
++totalWithoutLastKey;
}
total += data.size();
});
auto const res = grpcSource_.loadInitialLedger(sequence_, numMarkers_, observer_);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res.value().size(), numMarkers_);
EXPECT_EQ(total, totalKeys);
EXPECT_EQ(totalWithLastKey + totalWithoutLastKey, numMarkers_ * batchesPerMarker);
EXPECT_EQ(totalWithoutLastKey, numMarkers_);
EXPECT_EQ(totalWithLastKey, (numMarkers_ - 1) * batchesPerMarker);
}
struct GrpcSourceStopTests : GrpcSourceNgTests, SyncAsioContextTest {};
TEST_F(GrpcSourceStopTests, LoadInitialLedgerStopsWhenRequested)
{
uint32_t const sequence = 123u;
uint32_t const numMarkers = 1;
std::mutex mtx;
std::condition_variable cvGrpcCallActive;
std::condition_variable cvStopCalled;
bool grpcCallIsActive = false;
bool stopHasBeenCalled = false;
EXPECT_CALL(mockXrpLedgerAPIService, GetLedgerData)
.WillOnce([&](grpc::ServerContext*,
org::xrpl::rpc::v1::GetLedgerDataRequest const* request,
org::xrpl::rpc::v1::GetLedgerDataResponse* response) {
EXPECT_EQ(request->ledger().sequence(), sequence);
EXPECT_EQ(request->user(), "ETL");
{
std::unique_lock const lk(mtx);
grpcCallIsActive = true;
}
cvGrpcCallActive.notify_one();
{
std::unique_lock lk(mtx);
cvStopCalled.wait(lk, [&] { return stopHasBeenCalled; });
}
response->set_is_unlimited(true);
return grpc::Status::OK;
});
EXPECT_CALL(observer_, onInitialLoadGotMoreObjects).Times(0);
auto loadTask = std::async(std::launch::async, [&]() {
return grpcSource_.loadInitialLedger(sequence, numMarkers, observer_);
});
{
std::unique_lock lk(mtx);
cvGrpcCallActive.wait(lk, [&] { return grpcCallIsActive; });
}
runSyncOperation([&](boost::asio::yield_context yield) {
grpcSource_.stop(yield);
{
std::unique_lock const lk(mtx);
stopHasBeenCalled = true;
}
cvStopCalled.notify_one();
});
auto const res = loadTask.get();
ASSERT_FALSE(res.has_value());
EXPECT_EQ(res.error(), etlng::InitialLedgerLoadError::Cancelled);
}
TEST_F(GrpcSourceNgTests, DeadlineIsHandledCorrectly)
{
static constexpr auto kDEADLINE = std::chrono::milliseconds{5};
uint32_t const sequence = 123u;
bool const getObjects = true;
bool const getObjectNeighbors = false;
std::binary_semaphore sem(0);
auto grpcSource =
std::make_unique<etlng::impl::GrpcSource>("localhost", std::to_string(getXRPLMockPort()), kDEADLINE);
// Note: this may not be called at all if gRPC cancels before it gets a chance to call the stub
EXPECT_CALL(mockXrpLedgerAPIService, GetLedger)
.Times(testing::AtMost(1))
.WillRepeatedly([&](grpc::ServerContext*,
org::xrpl::rpc::v1::GetLedgerRequest const*,
org::xrpl::rpc::v1::GetLedgerResponse*) {
// wait for main thread to discard us and fail the test if unsuccessful within expected timeframe
[&] { ASSERT_TRUE(sem.try_acquire_for(std::chrono::milliseconds{50})); }();
return grpc::Status{};
});
auto const [status, response] = grpcSource->fetchLedger(sequence, getObjects, getObjectNeighbors);
ASSERT_FALSE(status.ok()); // timed out after kDEADLINE
sem.release(); // we don't need to hold GetLedger thread any longer
grpcSource.reset();
shutdown(std::chrono::milliseconds{10});
}

356
tests/unit/etlng/LedgerPublisherTests.cpp
View File

@@ -1,356 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/SystemState.hpp"
#include "etlng/impl/LedgerPublisher.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/TestObject.hpp"
#include "util/config/ConfigDefinition.hpp"
#include <fmt/format.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/protocol/Indexes.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <chrono>
#include <optional>
#include <vector>
using namespace testing;
using namespace etlng;
using namespace data;
using namespace std::chrono;
namespace {
constexpr auto kACCOUNT = "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn";
constexpr auto kACCOUNT2 = "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun";
constexpr auto kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constexpr auto kSEQ = 30;
constexpr auto kAGE = 800;
constexpr auto kAMOUNT = 100;
constexpr auto kFEE = 3;
constexpr auto kFINAL_BALANCE = 110;
constexpr auto kFINAL_BALANCE2 = 30;
MATCHER_P(ledgerHeaderMatcher, expectedHeader, "Headers match")
{
return arg.seq == expectedHeader.seq && arg.hash == expectedHeader.hash &&
arg.closeTime == expectedHeader.closeTime;
}
} // namespace
struct ETLLedgerPublisherNgTest : util::prometheus::WithPrometheus, MockBackendTestStrict, SyncAsioContextTest {
util::config::ClioConfigDefinition cfg{{}};
StrictMockSubscriptionManagerSharedPtr mockSubscriptionManagerPtr;
};
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderSkipDueToAge)
{
// Use kAGE (800) which is > MAX_LEDGER_AGE_SECONDS (600) to test skipping
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
auto dummyState = etl::SystemState{};
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
// Since age > MAX_LEDGER_AGE_SECONDS, these should not be called
EXPECT_CALL(*backend_, doFetchLedgerObject).Times(0);
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderWithinAgeLimit)
{
// Use age 0 which is < MAX_LEDGER_AGE_SECONDS to ensure publishing happens
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto dummyState = etl::SystemState{};
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 0));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderIsWritingTrue)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
ctx_.run();
EXPECT_FALSE(backend_->fetchLedgerRange());
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderInRange)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 1));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
// mock 1 transaction
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderCloseTimeGreaterThanNow)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const nowPlus10 = system_clock::now() + seconds(10);
auto const closeTime = duration_cast<seconds>(nowPlus10.time_since_epoch()).count() - kRIPPLE_EPOCH_START;
dummyLedgerHeader.closeTime = ripple::NetClock::time_point{seconds{closeTime}};
backend_->setRange(kSEQ - 1, kSEQ);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 1));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqStopIsTrue)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = true;
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
EXPECT_FALSE(publisher.publish(kSEQ, {}));
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqMaxAttempt)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
static constexpr auto kMAX_ATTEMPT = 2;
LedgerRange const range{.minSequence = kSEQ - 1, .maxSequence = kSEQ - 1};
EXPECT_CALL(*backend_, hardFetchLedgerRange).Times(kMAX_ATTEMPT).WillRepeatedly(Return(range));
EXPECT_FALSE(publisher.publish(kSEQ, kMAX_ATTEMPT, std::chrono::milliseconds{1}));
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqStopIsFalse)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
LedgerRange const range{.minSequence = kSEQ, .maxSequence = kSEQ};
EXPECT_CALL(*backend_, hardFetchLedgerRange).WillOnce(Return(range));
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
EXPECT_CALL(*backend_, fetchLedgerBySequence(kSEQ, _)).WillOnce(Return(dummyLedgerHeader));
EXPECT_TRUE(publisher.publish(kSEQ, {}));
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishMultipleTxInOrder)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
// t1 index > t2 index
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
t1.date = 1;
TransactionAndMetadata t2;
t2.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t2.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 1)
.getSerializer()
.peekData();
t2.ledgerSequence = kSEQ;
t2.date = 2;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1, t2}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 2));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t2, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t1, _)).InSequence(s);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishVeryOldLedgerShouldSkip)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
// Create a ledger header with age (800) greater than MAX_LEDGER_AGE_SECONDS (600)
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 800);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishMultipleLedgersInQuickSuccession)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader1 = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const dummyLedgerHeader2 = createLedgerHeader(kLEDGER_HASH, kSEQ + 1, 0);
auto publisher = impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ + 1);
// Publish two ledgers in quick succession
publisher.publish(dummyLedgerHeader1);
publisher.publish(dummyLedgerHeader2);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ + 1, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ + 1, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader1), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader1), _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader2), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader2), _)).InSequence(s);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ + 1);
ctx_.run();
}

889
tests/unit/etlng/LoadBalancerTests.cpp
View File

@@ -1,889 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancer.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/Source.hpp"
#include "rpc/Errors.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockRandomGenerator.hpp"
#include "util/MockSourceNg.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/NameGenerator.hpp"
#include "util/config/Array.hpp"
#include "util/config/ConfigConstraints.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/config/ConfigFileJson.hpp"
#include "util/config/ConfigValue.hpp"
#include "util/config/Types.hpp"
#include "util/prometheus/Counter.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/array.hpp>
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp>
#include <gmock/gmock.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
using namespace etlng;
using namespace util::config;
using testing::Return;
using namespace util::prometheus;
namespace {
constinit auto const kTWO_SOURCES_LEDGER_RESPONSE = R"JSON({
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source1"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source2"
}
]
})JSON";
constinit auto const kTHREE_SOURCES_LEDGER_RESPONSE = R"JSON({
"etl_sources": [
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source1"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source2"
},
{
"ip": "127.0.0.1",
"ws_port": "5005",
"grpc_port": "source3"
}
]
})JSON";
inline ClioConfigDefinition
getParseLoadBalancerConfig(boost::json::value val)
{
ClioConfigDefinition config{
{{"forwarding.cache_timeout",
ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(gValidatePositiveDouble)},
{"forwarding.request_timeout",
ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(gValidatePositiveDouble)},
{"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateIp)}},
{"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidatePort)}},
{"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional()}},
{"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(gValidateNumMarkers)}}
};
auto const errors = config.parse(ConfigFileJson{val.as_object()});
[&]() { ASSERT_FALSE(errors.has_value()); }();
return config;
}
struct InitialLoadObserverMock : etlng::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etlng::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etlng::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct LoadBalancerConstructorNgTests : util::prometheus::WithPrometheus, MockBackendTestStrict {
std::unique_ptr<LoadBalancer>
makeLoadBalancer()
{
auto const cfg = getParseLoadBalancerConfig(configJson_);
auto randomGenerator = std::make_unique<MockRandomGenerator>();
randomGenerator_ = randomGenerator.get();
return std::make_unique<LoadBalancer>(
cfg,
ioContext_,
backend_,
subscriptionManager_,
std::move(randomGenerator),
networkManager_,
[this](auto&&... args) -> SourcePtr { return sourceFactory_(std::forward<decltype(args)>(args)...); }
);
}
protected:
MockRandomGenerator* randomGenerator_ = nullptr;
StrictMockSubscriptionManagerSharedPtr subscriptionManager_;
StrictMockNetworkValidatedLedgersPtr networkManager_;
StrictMockSourceNgFactory sourceFactory_{2};
boost::asio::io_context ioContext_;
boost::json::value configJson_ = boost::json::parse(kTWO_SOURCES_LEDGER_RESPONSE);
};
TEST_F(LoadBalancerConstructorNgTests, construct)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, forwardingTimeoutPassedToSourceFactory)
{
auto const forwardingTimeout = 10;
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", float{forwardingTimeout}}};
EXPECT_CALL(
sourceFactory_,
makeSource(
testing::_,
testing::_,
testing::_,
testing::_,
std::chrono::steady_clock::duration{std::chrono::seconds{forwardingTimeout}},
testing::_,
testing::_,
testing::_
)
)
.Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesFail)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesReturnError)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(boost::json::object{{"error", "some error"}}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(boost::json::object{{"error", "some error"}}));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_Source1Fails0OK)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_Source0Fails1OK)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_DifferentNetworkID)
{
auto const source1Json = boost::json::parse(R"JSON({"result": {"info": {"network_id": 0}}})JSON");
auto const source2Json = boost::json::parse(R"JSON({"result": {"info": {"network_id": 1}}})JSON");
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(source1Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(source2Json.as_object()));
EXPECT_THROW({ makeLoadBalancer(); }, std::logic_error);
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_AllSourcesFailButAllowNoEtlIsTrue)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
configJson_.as_object()["allow_no_etl"] = true;
makeLoadBalancer();
}
TEST_F(LoadBalancerConstructorNgTests, fetchETLState_DifferentNetworkIDButAllowNoEtlIsTrue)
{
auto const source1Json = boost::json::parse(R"JSON({"result": {"info": {"network_id": 0}}})JSON");
auto const source2Json = boost::json::parse(R"JSON({"result": {"info": {"network_id": 1}}})JSON");
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(source1Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(source2Json.as_object()));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
configJson_.as_object()["allow_no_etl"] = true;
makeLoadBalancer();
}
struct LoadBalancerOnConnectHookNgTests : LoadBalancerConstructorNgTests {
LoadBalancerOnConnectHookNgTests()
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
loadBalancer_ = makeLoadBalancer();
}
protected:
std::unique_ptr<LoadBalancer> loadBalancer_;
};
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect_Source0IsNotConnected)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect(); // assuming it connects and disconnects immediately
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(1).onConnect();
// Nothing is called on another connect
sourceFactory_.callbacksAt(0).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, sourcesConnect_BothSourcesAreNotConnected)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(1).onConnect();
// Then source 0 got connected
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
}
struct LoadBalancerStopNgTests : LoadBalancerOnConnectHookNgTests, SyncAsioContextTest {};
TEST_F(LoadBalancerStopNgTests, stopCallsSourcesStop)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), stop);
EXPECT_CALL(sourceFactory_.sourceAt(1), stop);
runSyncOperation([this](boost::asio::yield_context yield) { loadBalancer_->stop(yield); });
}
struct LoadBalancerOnDisconnectHookNgTests : LoadBalancerOnConnectHookNgTests {
LoadBalancerOnDisconnectHookNgTests()
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
// nothing happens on source 1 connect
sourceFactory_.callbacksAt(1).onConnect();
}
};
TEST_F(LoadBalancerOnDisconnectHookNgTests, source0Disconnects)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(0).onDisconnect(true);
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source1Disconnects)
{
sourceFactory_.callbacksAt(1).onDisconnect(false);
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source0DisconnectsAndConnectsBack)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(true));
sourceFactory_.callbacksAt(0).onDisconnect(true);
sourceFactory_.callbacksAt(0).onConnect();
}
TEST_F(LoadBalancerOnDisconnectHookNgTests, source1DisconnectsAndConnectsBack)
{
sourceFactory_.callbacksAt(1).onDisconnect(false);
sourceFactory_.callbacksAt(1).onConnect();
}
TEST_F(LoadBalancerOnConnectHookNgTests, bothSourcesDisconnectAndConnectBack)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onDisconnect(true);
sourceFactory_.callbacksAt(1).onDisconnect(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
}
struct LoadBalancer3SourcesNgTests : LoadBalancerConstructorNgTests {
LoadBalancer3SourcesNgTests()
{
sourceFactory_.setSourcesNumber(3);
configJson_ = boost::json::parse(kTHREE_SOURCES_LEDGER_RESPONSE);
EXPECT_CALL(sourceFactory_, makeSource).Times(3);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
EXPECT_CALL(sourceFactory_.sourceAt(2), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(2), run);
loadBalancer_ = makeLoadBalancer();
}
protected:
std::unique_ptr<LoadBalancer> loadBalancer_;
};
TEST_F(LoadBalancer3SourcesNgTests, forwardingUpdate)
{
// Source 2 is connected first
EXPECT_CALL(sourceFactory_.sourceAt(0), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(0), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), isConnected()).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), setForwarding(false));
EXPECT_CALL(sourceFactory_.sourceAt(2), isConnected()).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(2), setForwarding(true));
sourceFactory_.callbacksAt(2).onConnect();
// Then source 0 and 1 are getting connected, but nothing should happen
sourceFactory_.callbacksAt(0).onConnect();
sourceFactory_.callbacksAt(1).onConnect();
// Source 0 got disconnected
sourceFactory_.callbacksAt(0).onDisconnect(false);
}
struct LoadBalancerLoadInitialLedgerNgTests : LoadBalancerOnConnectHookNgTests {
protected:
uint32_t const sequence_ = 123;
uint32_t const numMarkers_ = 16;
InitialLedgerLoadResult const response_{std::vector<std::string>{"1", "2", "3"}};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_source0DoesntHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_bothSourcesDontHaveLedger)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(false));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
TEST_F(LoadBalancerLoadInitialLedgerNgTests, load_source0ReturnsStatusFalse)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(std::unexpected{InitialLedgerLoadError::Errored}));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer_->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
struct LoadBalancerLoadInitialLedgerCustomNumMarkersNgTests : LoadBalancerConstructorNgTests {
protected:
uint32_t const numMarkers_ = 16;
uint32_t const sequence_ = 123;
InitialLedgerLoadResult const response_{std::vector<std::string>{"1", "2", "3"}};
testing::StrictMock<InitialLoadObserverMock> observer_;
};
TEST_F(LoadBalancerLoadInitialLedgerCustomNumMarkersNgTests, loadInitialLedger)
{
configJson_.as_object()["num_markers"] = numMarkers_;
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), loadInitialLedger(sequence_, numMarkers_, testing::_))
.WillOnce(Return(response_));
EXPECT_EQ(loadBalancer->loadInitialLedger(sequence_, observer_, std::chrono::milliseconds{1}), response_.value());
}
struct LoadBalancerFetchLegerNgTests : LoadBalancerOnConnectHookNgTests {
LoadBalancerFetchLegerNgTests()
{
response_.second.set_validated(true);
}
protected:
uint32_t const sequence_ = 123;
bool const getObjects_ = true;
bool const getObjectNeighbors_ = false;
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse> response_ =
std::make_pair(grpc::Status::OK, org::xrpl::rpc::v1::GetLedgerResponse{});
};
TEST_F(LoadBalancerFetchLegerNgTests, fetch)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_Source0ReturnsBadStatus)
{
auto source0Response = response_;
source0Response.first = grpc::Status::CANCELLED;
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(source0Response));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_Source0ReturnsNotValidated)
{
auto source0Response = response_;
source0Response.second.set_validated(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(source0Response));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(response_));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_).has_value());
}
TEST_F(LoadBalancerFetchLegerNgTests, fetch_bothSourcesFail)
{
auto badResponse = response_;
badResponse.second.set_validated(false);
EXPECT_CALL(sourceFactory_.sourceAt(0), hasLedger(sequence_)).Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(0), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(badResponse))
.WillOnce(Return(response_));
EXPECT_CALL(sourceFactory_.sourceAt(1), hasLedger(sequence_)).WillOnce(Return(true));
EXPECT_CALL(sourceFactory_.sourceAt(1), fetchLedger(sequence_, getObjects_, getObjectNeighbors_))
.WillOnce(Return(badResponse));
EXPECT_TRUE(loadBalancer_->fetchLedger(sequence_, getObjects_, getObjectNeighbors_, std::chrono::milliseconds{1})
.has_value());
}
struct LoadBalancerForwardToRippledNgTests : LoadBalancerConstructorNgTests, SyncAsioContextTest {
LoadBalancerForwardToRippledNgTests()
{
EXPECT_CALL(sourceFactory_.sourceAt(0), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(0), run);
EXPECT_CALL(sourceFactory_.sourceAt(1), forwardToRippled).WillOnce(Return(boost::json::object{}));
EXPECT_CALL(sourceFactory_.sourceAt(1), run);
}
protected:
boost::json::object const request_{{"command", "value"}};
std::optional<std::string> const clientIP_ = "some_ip";
boost::json::object const response_{{"response", "other_value"}};
};
TEST_F(LoadBalancerForwardToRippledNgTests, forward)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kADMIN_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, true, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardWithXUserHeader)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, false, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, source0Fails)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, false, yield), response_);
});
}
struct LoadBalancerForwardToRippledPrometheusNgTests : LoadBalancerForwardToRippledNgTests, WithMockPrometheus {};
TEST_F(LoadBalancerForwardToRippledPrometheusNgTests, forwardingCacheEnabled)
{
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", 10.}};
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command", "server_info"}};
auto& cacheHitCounter = makeMock<CounterInt>("forwarding_cache_hit_counter", "");
auto& cacheMissCounter = makeMock<CounterInt>("forwarding_cache_miss_counter", "");
auto& successDurationCounter =
makeMock<CounterInt>("forwarding_duration_milliseconds_counter", "{status=\"success\"}");
EXPECT_CALL(cacheMissCounter, add(1));
EXPECT_CALL(cacheHitCounter, add(1)).Times(3);
EXPECT_CALL(successDurationCounter, add(testing::_));
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledPrometheusNgTests, source0Fails)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto& cacheMissCounter = makeMock<CounterInt>("forwarding_cache_miss_counter", "");
auto& retriesCounter = makeMock<CounterInt>("forwarding_retries_counter", "");
auto& successDurationCounter =
makeMock<CounterInt>("forwarding_duration_milliseconds_counter", "{status=\"success\"}");
auto& failDurationCounter = makeMock<CounterInt>("forwarding_duration_milliseconds_counter", "{status=\"fail\"}");
EXPECT_CALL(cacheMissCounter, add(1));
EXPECT_CALL(retriesCounter, add(1));
EXPECT_CALL(successDurationCounter, add(testing::_));
EXPECT_CALL(failDurationCounter, add(testing::_));
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{rpc::ClioError::EtlConnectionError}));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request_, clientIP_, false, yield), response_);
});
}
struct LoadBalancerForwardToRippledErrorNgTestBundle {
std::string testName;
rpc::ClioError firstSourceError;
rpc::ClioError secondSourceError;
rpc::CombinedError responseExpectedError;
};
struct LoadBalancerForwardToRippledErrorNgTests
: LoadBalancerForwardToRippledNgTests,
testing::WithParamInterface<LoadBalancerForwardToRippledErrorNgTestBundle> {};
INSTANTIATE_TEST_SUITE_P(
LoadBalancerForwardToRippledErrorNgTests,
LoadBalancerForwardToRippledErrorNgTests,
testing::Values(
LoadBalancerForwardToRippledErrorNgTestBundle{
"ConnectionError_RequestError",
rpc::ClioError::EtlConnectionError,
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlRequestError
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"RequestError_RequestTimeout",
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"RequestTimeout_InvalidResponse",
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlInvalidResponse,
rpc::ClioError::EtlInvalidResponse
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"BothRequestTimeout",
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout,
rpc::ClioError::EtlRequestTimeout
},
LoadBalancerForwardToRippledErrorNgTestBundle{
"InvalidResponse_RequestError",
rpc::ClioError::EtlInvalidResponse,
rpc::ClioError::EtlRequestError,
rpc::ClioError::EtlInvalidResponse
}
),
tests::util::kNAME_GENERATOR
);
TEST_P(LoadBalancerForwardToRippledErrorNgTests, bothSourcesFail)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{GetParam().firstSourceError}));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request_, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(std::unexpected{GetParam().secondSourceError}));
runSpawn([&](boost::asio::yield_context yield) {
auto const response = loadBalancer->forwardToRippled(request_, clientIP_, false, yield);
ASSERT_FALSE(response);
EXPECT_EQ(response.error(), GetParam().responseExpectedError);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardingCacheEnabled)
{
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", 10.}};
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command", "server_info"}};
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, forwardingCacheDisabledOnLedgerClosedHookCalled)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
EXPECT_NO_THROW(sourceFactory_.callbacksAt(0).onLedgerClosed());
}
TEST_F(LoadBalancerForwardToRippledNgTests, onLedgerClosedHookInvalidatesCache)
{
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", 10.}};
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command", "server_info"}};
EXPECT_CALL(*randomGenerator_, uniform(0, 1)).WillOnce(Return(0)).WillOnce(Return(1));
EXPECT_CALL(
sourceFactory_.sourceAt(0),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(response_));
EXPECT_CALL(
sourceFactory_.sourceAt(1),
forwardToRippled(request, clientIP_, LoadBalancer::kUSER_FORWARDING_X_USER_VALUE, testing::_)
)
.WillOnce(Return(boost::json::object{}));
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), response_);
sourceFactory_.callbacksAt(0).onLedgerClosed();
EXPECT_EQ(loadBalancer->forwardToRippled(request, clientIP_, false, yield), boost::json::object{});
});
}
TEST_F(LoadBalancerForwardToRippledNgTests, commandLineMissing)
{
EXPECT_CALL(sourceFactory_, makeSource).Times(2);
auto loadBalancer = makeLoadBalancer();
auto const request = boost::json::object{{"command2", "server_info"}};
runSpawn([&](boost::asio::yield_context yield) {
EXPECT_EQ(
loadBalancer->forwardToRippled(request, clientIP_, false, yield).error(),
rpc::CombinedError{rpc::ClioError::RpcCommandIsMissing}
);
});
}
struct LoadBalancerToJsonNgTests : LoadBalancerOnConnectHookNgTests {};
TEST_F(LoadBalancerToJsonNgTests, toJson)
{
EXPECT_CALL(sourceFactory_.sourceAt(0), toJson).WillOnce(Return(boost::json::object{{"source1", "value1"}}));
EXPECT_CALL(sourceFactory_.sourceAt(1), toJson).WillOnce(Return(boost::json::object{{"source2", "value2"}}));
auto const expectedJson =
boost::json::array({boost::json::object{{"source1", "value1"}}, boost::json::object{{"source2", "value2"}}});
EXPECT_EQ(loadBalancer_->toJson(), expectedJson);
}

243
tests/unit/etlng/SourceImplTests.cpp
View File

@@ -1,243 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/InitialLoadObserverInterface.hpp"
#include "etlng/LoadBalancerInterface.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/SourceImpl.hpp"
#include "rpc/Errors.hpp"
#include "util/Spawn.hpp"
#include <boost/asio/io_context.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/json/object.hpp>
#include <boost/json/value_to.hpp>
#include <gmock/gmock.h>
#include <grpcpp/support/status.h>
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/get_ledger.pb.h>
#include <chrono>
#include <cstdint>
#include <expected>
#include <memory>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
using namespace etlng::impl;
using testing::Return;
using testing::StrictMock;
namespace {
struct GrpcSourceMock {
using FetchLedgerReturnType = std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>;
MOCK_METHOD(FetchLedgerReturnType, fetchLedger, (uint32_t, bool, bool));
using LoadLedgerReturnType = etlng::InitialLedgerLoadResult;
MOCK_METHOD(LoadLedgerReturnType, loadInitialLedger, (uint32_t, uint32_t, etlng::InitialLoadObserverInterface&));
MOCK_METHOD(void, stop, (boost::asio::yield_context), ());
};
struct SubscriptionSourceMock {
MOCK_METHOD(void, run, ());
MOCK_METHOD(bool, hasLedger, (uint32_t), (const));
MOCK_METHOD(bool, isConnected, (), (const));
MOCK_METHOD(void, setForwarding, (bool));
MOCK_METHOD(std::chrono::steady_clock::time_point, lastMessageTime, (), (const));
MOCK_METHOD(std::string, validatedRange, (), (const));
MOCK_METHOD(void, stop, (boost::asio::yield_context));
};
struct ForwardingSourceMock {
MOCK_METHOD(void, constructor, (std::string const&, std::string const&, std::chrono::steady_clock::duration));
using ForwardToRippledReturnType = std::expected<boost::json::object, rpc::ClioError>;
using ClientIpOpt = std::optional<std::string>;
MOCK_METHOD(
ForwardToRippledReturnType,
forwardToRippled,
(boost::json::object const&, ClientIpOpt const&, std::string_view, boost::asio::yield_context),
(const)
);
};
struct InitialLoadObserverMock : etlng::InitialLoadObserverInterface {
MOCK_METHOD(
void,
onInitialLoadGotMoreObjects,
(uint32_t, std::vector<etlng::model::Object> const&, std::optional<std::string>),
(override)
);
void
onInitialLoadGotMoreObjects(uint32_t seq, std::vector<etlng::model::Object> const& data)
{
onInitialLoadGotMoreObjects(seq, data, std::nullopt);
}
};
} // namespace
struct SourceImplNgTest : public ::testing::Test {
protected:
boost::asio::io_context ioc_;
StrictMock<GrpcSourceMock> grpcSourceMock_;
std::shared_ptr<StrictMock<SubscriptionSourceMock>> subscriptionSourceMock_ =
std::make_shared<StrictMock<SubscriptionSourceMock>>();
StrictMock<ForwardingSourceMock> forwardingSourceMock_;
SourceImpl<
StrictMock<GrpcSourceMock>&,
std::shared_ptr<StrictMock<SubscriptionSourceMock>>,
StrictMock<ForwardingSourceMock>&>
source_{
"some_ip",
"some_ws_port",
"some_grpc_port",
grpcSourceMock_,
subscriptionSourceMock_,
forwardingSourceMock_
};
};
TEST_F(SourceImplNgTest, run)
{
EXPECT_CALL(*subscriptionSourceMock_, run());
source_.run();
}
TEST_F(SourceImplNgTest, stop)
{
EXPECT_CALL(*subscriptionSourceMock_, stop);
EXPECT_CALL(grpcSourceMock_, stop);
boost::asio::io_context ctx;
util::spawn(ctx, [&](boost::asio::yield_context yield) { source_.stop(yield); });
ctx.run();
}
TEST_F(SourceImplNgTest, isConnected)
{
EXPECT_CALL(*subscriptionSourceMock_, isConnected()).WillOnce(testing::Return(true));
EXPECT_TRUE(source_.isConnected());
}
TEST_F(SourceImplNgTest, setForwarding)
{
EXPECT_CALL(*subscriptionSourceMock_, setForwarding(true));
source_.setForwarding(true);
}
TEST_F(SourceImplNgTest, toJson)
{
EXPECT_CALL(*subscriptionSourceMock_, validatedRange()).WillOnce(Return(std::string("some_validated_range")));
EXPECT_CALL(*subscriptionSourceMock_, isConnected()).WillOnce(Return(true));
auto const lastMessageTime = std::chrono::steady_clock::now();
EXPECT_CALL(*subscriptionSourceMock_, lastMessageTime()).WillOnce(Return(lastMessageTime));
auto const json = source_.toJson();
EXPECT_EQ(boost::json::value_to<std::string>(json.at("validated_range")), "some_validated_range");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("is_connected")), "1");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("ip")), "some_ip");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("ws_port")), "some_ws_port");
EXPECT_EQ(boost::json::value_to<std::string>(json.at("grpc_port")), "some_grpc_port");
auto lastMessageAgeStr = boost::json::value_to<std::string>(json.at("last_msg_age_seconds"));
EXPECT_GE(std::stoi(lastMessageAgeStr), 0);
}
TEST_F(SourceImplNgTest, toString)
{
EXPECT_CALL(*subscriptionSourceMock_, validatedRange()).WillOnce(Return(std::string("some_validated_range")));
auto const str = source_.toString();
EXPECT_EQ(
str,
"{validated range: some_validated_range, ip: some_ip, web socket port: some_ws_port, grpc port: some_grpc_port}"
);
}
TEST_F(SourceImplNgTest, hasLedger)
{
uint32_t const ledgerSeq = 123;
EXPECT_CALL(*subscriptionSourceMock_, hasLedger(ledgerSeq)).WillOnce(Return(true));
EXPECT_TRUE(source_.hasLedger(ledgerSeq));
}
TEST_F(SourceImplNgTest, fetchLedger)
{
uint32_t const ledgerSeq = 123;
EXPECT_CALL(grpcSourceMock_, fetchLedger(ledgerSeq, true, false));
auto const [actualStatus, actualResponse] = source_.fetchLedger(ledgerSeq);
EXPECT_EQ(actualStatus.error_code(), grpc::StatusCode::OK);
}
TEST_F(SourceImplNgTest, loadInitialLedgerErrorPath)
{
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
auto observerMock = testing::StrictMock<InitialLoadObserverMock>();
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, testing::_))
.WillOnce(Return(std::unexpected{etlng::InitialLedgerLoadError::Errored}));
auto const res = source_.loadInitialLedger(ledgerSeq, numMarkers, observerMock);
EXPECT_FALSE(res.has_value());
}
TEST_F(SourceImplNgTest, loadInitialLedgerSuccessPath)
{
uint32_t const ledgerSeq = 123;
uint32_t const numMarkers = 3;
auto response = etlng::InitialLedgerLoadResult{{"1", "2", "3"}};
auto observerMock = testing::StrictMock<InitialLoadObserverMock>();
EXPECT_CALL(grpcSourceMock_, loadInitialLedger(ledgerSeq, numMarkers, testing::_)).WillOnce(Return(response));
auto const res = source_.loadInitialLedger(ledgerSeq, numMarkers, observerMock);
EXPECT_TRUE(res.has_value());
EXPECT_EQ(res, response);
}
TEST_F(SourceImplNgTest, forwardToRippled)
{
boost::json::object const request = {{"some_key", "some_value"}};
std::optional<std::string> const clientIp = "some_client_ip";
std::string_view xUserValue = "some_user";
EXPECT_CALL(forwardingSourceMock_, forwardToRippled(request, clientIp, xUserValue, testing::_))
.WillOnce(Return(request));
boost::asio::io_context ioContext;
util::spawn(ioContext, [&](boost::asio::yield_context yield) {
auto const response = source_.forwardToRippled(request, clientIp, xUserValue, yield);
EXPECT_EQ(response, request);
});
ioContext.run();
}