diff --git a/src/etl/impl/LedgerPublisher.h b/src/etl/impl/LedgerPublisher.h index 59cc1f7b7..ac8ae9fc6 100644 --- a/src/etl/impl/LedgerPublisher.h +++ b/src/etl/impl/LedgerPublisher.h @@ -171,6 +171,16 @@ public: subscriptions_->pubLedger(lgrInfo, *fees, range, transactions.size()); + // order with transaction index + std::sort(transactions.begin(), transactions.end(), [](auto const& t1, auto const& t2) { + ripple::SerialIter iter1{t1.metadata.data(), t1.metadata.size()}; + ripple::STObject const object1(iter1, ripple::sfMetadata); + ripple::SerialIter iter2{t2.metadata.data(), t2.metadata.size()}; + ripple::STObject const object2(iter2, ripple::sfMetadata); + return object1.getFieldU32(ripple::sfTransactionIndex) < + object2.getFieldU32(ripple::sfTransactionIndex); + }); + for (auto& txAndMeta : transactions) subscriptions_->pubTransaction(txAndMeta, lgrInfo); diff --git a/unittests/etl/LedgerPublisherTests.cpp b/unittests/etl/LedgerPublisherTests.cpp new file mode 100644 index 000000000..35c7406f7 --- /dev/null +++ b/unittests/etl/LedgerPublisherTests.cpp @@ -0,0 +1,304 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include +#include +#include +#include + +#include +#include + +#include + +using namespace testing; +using namespace etl; +namespace json = boost::json; +using namespace std::chrono; + +static auto constexpr ACCOUNT = "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn"; +static auto constexpr ACCOUNT2 = "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun"; +static auto constexpr LEDGERHASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652"; +static auto constexpr SEQ = 30; +static auto constexpr AGE = 800; + +class ETLLedgerPublisherTest : public MockBackendTest, public SyncAsioContextTest, public MockSubscriptionManagerTest +{ + void + SetUp() override + { + MockBackendTest::SetUp(); + SyncAsioContextTest::SetUp(); + MockSubscriptionManagerTest::SetUp(); + } + + void + TearDown() override + { + MockSubscriptionManagerTest::TearDown(); + SyncAsioContextTest::TearDown(); + MockBackendTest::TearDown(); + } + +protected: + util::Config cfg{json::parse("{}")}; + MockCache mockCache; +}; + +TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoIsWritingFalse) +{ + SystemState dummyState; + dummyState.isWriting = false; + auto const dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, AGE); + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + publisher.publish(dummyLedgerInfo); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + ASSERT_NE(rawBackendPtr, nullptr); + + ON_CALL(*rawBackendPtr, fetchLedgerDiff(SEQ, _)).WillByDefault(Return(std::vector{})); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(SEQ, _)).Times(1); + + // setLastPublishedSequence not in strand, should verify before run + EXPECT_TRUE(publisher.getLastPublishedSequence()); + EXPECT_EQ(publisher.getLastPublishedSequence().value(), SEQ); + + EXPECT_CALL(mockCache, updateImp).Times(1); + + ctx.run(); + EXPECT_TRUE(rawBackendPtr->fetchLedgerRange()); + EXPECT_EQ(rawBackendPtr->fetchLedgerRange().value().minSequence, SEQ); + EXPECT_EQ(rawBackendPtr->fetchLedgerRange().value().maxSequence, SEQ); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoIsWritingTrue) +{ + SystemState dummyState; + dummyState.isWriting = true; + auto const dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, AGE); + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + publisher.publish(dummyLedgerInfo); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(_, _)).Times(0); + + // setLastPublishedSequence not in strand, should verify before run + EXPECT_TRUE(publisher.getLastPublishedSequence()); + EXPECT_EQ(publisher.getLastPublishedSequence().value(), SEQ); + + ctx.run(); + EXPECT_FALSE(rawBackendPtr->fetchLedgerRange()); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoInRange) +{ + SystemState dummyState; + dummyState.isWriting = true; + + auto const dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, 0); // age is 0 + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + mockBackendPtr->updateRange(SEQ - 1); + mockBackendPtr->updateRange(SEQ); + + publisher.publish(dummyLedgerInfo); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(_, _)).Times(0); + + // mock fetch fee + EXPECT_CALL(*rawBackendPtr, doFetchLedgerObject).Times(1); + ON_CALL(*rawBackendPtr, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _)) + .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0))); + + // mock fetch transactions + EXPECT_CALL(*rawBackendPtr, fetchAllTransactionsInLedger).Times(1); + TransactionAndMetadata t1; + t1.transaction = CreatePaymentTransactionObject(ACCOUNT, ACCOUNT2, 100, 3, SEQ).getSerializer().peekData(); + t1.metadata = CreatePaymentTransactionMetaObject(ACCOUNT, ACCOUNT2, 110, 30).getSerializer().peekData(); + t1.ledgerSequence = SEQ; + ON_CALL(*rawBackendPtr, fetchAllTransactionsInLedger(SEQ, _)) + .WillByDefault(Return(std::vector{t1})); + + // setLastPublishedSequence not in strand, should verify before run + EXPECT_TRUE(publisher.getLastPublishedSequence()); + EXPECT_EQ(publisher.getLastPublishedSequence().value(), SEQ); + + MockSubscriptionManager* rawSubscriptionManagerPtr = + dynamic_cast(mockSubscriptionManagerPtr.get()); + + EXPECT_CALL(*rawSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", SEQ - 1, SEQ), 1)).Times(1); + EXPECT_CALL(*rawSubscriptionManagerPtr, pubBookChanges).Times(1); + // mock 1 transaction + EXPECT_CALL(*rawSubscriptionManagerPtr, pubTransaction).Times(1); + + ctx.run(); + // last publish time should be set + EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerInfoCloseTimeGreaterThanNow) +{ + SystemState dummyState; + dummyState.isWriting = true; + + ripple::LedgerInfo dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, 0); + auto const nowPlus10 = system_clock::now() + seconds(10); + auto const closeTime = duration_cast(nowPlus10.time_since_epoch()).count() - rippleEpochStart; + dummyLedgerInfo.closeTime = ripple::NetClock::time_point{seconds{closeTime}}; + + mockBackendPtr->updateRange(SEQ - 1); + mockBackendPtr->updateRange(SEQ); + + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + publisher.publish(dummyLedgerInfo); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(_, _)).Times(0); + + // mock fetch fee + EXPECT_CALL(*rawBackendPtr, doFetchLedgerObject).Times(1); + ON_CALL(*rawBackendPtr, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _)) + .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0))); + + // mock fetch transactions + EXPECT_CALL(*rawBackendPtr, fetchAllTransactionsInLedger).Times(1); + TransactionAndMetadata t1; + t1.transaction = CreatePaymentTransactionObject(ACCOUNT, ACCOUNT2, 100, 3, SEQ).getSerializer().peekData(); + t1.metadata = CreatePaymentTransactionMetaObject(ACCOUNT, ACCOUNT2, 110, 30).getSerializer().peekData(); + t1.ledgerSequence = SEQ; + ON_CALL(*rawBackendPtr, fetchAllTransactionsInLedger(SEQ, _)) + .WillByDefault(Return(std::vector{t1})); + + // setLastPublishedSequence not in strand, should verify before run + EXPECT_TRUE(publisher.getLastPublishedSequence()); + EXPECT_EQ(publisher.getLastPublishedSequence().value(), SEQ); + + MockSubscriptionManager* rawSubscriptionManagerPtr = + dynamic_cast(mockSubscriptionManagerPtr.get()); + + EXPECT_CALL(*rawSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", SEQ - 1, SEQ), 1)).Times(1); + EXPECT_CALL(*rawSubscriptionManagerPtr, pubBookChanges).Times(1); + // mock 1 transaction + EXPECT_CALL(*rawSubscriptionManagerPtr, pubTransaction).Times(1); + + ctx.run(); + // last publish time should be set + EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqStopIsTrue) +{ + SystemState dummyState; + dummyState.isStopping = true; + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + EXPECT_FALSE(publisher.publish(SEQ, {})); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqMaxAttampt) +{ + SystemState dummyState; + dummyState.isStopping = false; + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + + static auto constexpr MAX_ATTEMPT = 2; + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + EXPECT_CALL(*rawBackendPtr, hardFetchLedgerRange).Times(MAX_ATTEMPT); + + LedgerRange const range{.minSequence = SEQ - 1, .maxSequence = SEQ - 1}; + ON_CALL(*rawBackendPtr, hardFetchLedgerRange(_)).WillByDefault(Return(range)); + EXPECT_FALSE(publisher.publish(SEQ, MAX_ATTEMPT)); +} + +TEST_F(ETLLedgerPublisherTest, PublishLedgerSeqStopIsFalse) +{ + SystemState dummyState; + dummyState.isStopping = false; + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + LedgerRange const range{.minSequence = SEQ, .maxSequence = SEQ}; + ON_CALL(*rawBackendPtr, hardFetchLedgerRange(_)).WillByDefault(Return(range)); + EXPECT_CALL(*rawBackendPtr, hardFetchLedgerRange).Times(1); + + auto const dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, AGE); + ON_CALL(*rawBackendPtr, fetchLedgerBySequence(SEQ, _)).WillByDefault(Return(dummyLedgerInfo)); + EXPECT_CALL(*rawBackendPtr, fetchLedgerBySequence).Times(1); + + ON_CALL(*rawBackendPtr, fetchLedgerDiff(SEQ, _)).WillByDefault(Return(std::vector{})); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(SEQ, _)).Times(1); + EXPECT_CALL(mockCache, updateImp).Times(1); + + EXPECT_TRUE(publisher.publish(SEQ, {})); + ctx.run(); +} + +TEST_F(ETLLedgerPublisherTest, PublishMultipleTxInOrder) +{ + SystemState dummyState; + dummyState.isWriting = true; + + auto const dummyLedgerInfo = CreateLedgerInfo(LEDGERHASH, SEQ, 0); // age is 0 + detail::LedgerPublisher publisher(ctx, mockBackendPtr, mockCache, mockSubscriptionManagerPtr, dummyState); + mockBackendPtr->updateRange(SEQ - 1); + mockBackendPtr->updateRange(SEQ); + + publisher.publish(dummyLedgerInfo); + + MockBackend* rawBackendPtr = dynamic_cast(mockBackendPtr.get()); + EXPECT_CALL(*rawBackendPtr, fetchLedgerDiff(_, _)).Times(0); + + // mock fetch fee + EXPECT_CALL(*rawBackendPtr, doFetchLedgerObject).Times(1); + ON_CALL(*rawBackendPtr, doFetchLedgerObject(ripple::keylet::fees().key, SEQ, _)) + .WillByDefault(Return(CreateFeeSettingBlob(1, 2, 3, 4, 0))); + + // mock fetch transactions + EXPECT_CALL(*rawBackendPtr, fetchAllTransactionsInLedger).Times(1); + // t1 index > t2 index + TransactionAndMetadata t1; + t1.transaction = CreatePaymentTransactionObject(ACCOUNT, ACCOUNT2, 100, 3, SEQ).getSerializer().peekData(); + t1.metadata = CreatePaymentTransactionMetaObject(ACCOUNT, ACCOUNT2, 110, 30, 2).getSerializer().peekData(); + t1.ledgerSequence = SEQ; + t1.date = 1; + TransactionAndMetadata t2; + t2.transaction = CreatePaymentTransactionObject(ACCOUNT, ACCOUNT2, 100, 3, SEQ).getSerializer().peekData(); + t2.metadata = CreatePaymentTransactionMetaObject(ACCOUNT, ACCOUNT2, 110, 30, 1).getSerializer().peekData(); + t2.ledgerSequence = SEQ; + t2.date = 2; + ON_CALL(*rawBackendPtr, fetchAllTransactionsInLedger(SEQ, _)) + .WillByDefault(Return(std::vector{t1, t2})); + + // setLastPublishedSequence not in strand, should verify before run + EXPECT_TRUE(publisher.getLastPublishedSequence()); + EXPECT_EQ(publisher.getLastPublishedSequence().value(), SEQ); + + MockSubscriptionManager* rawSubscriptionManagerPtr = + dynamic_cast(mockSubscriptionManagerPtr.get()); + + EXPECT_CALL(*rawSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", SEQ - 1, SEQ), 2)).Times(1); + EXPECT_CALL(*rawSubscriptionManagerPtr, pubBookChanges).Times(1); + // should call pubTransaction t2 first (greater tx index) + Sequence const s; + EXPECT_CALL(*rawSubscriptionManagerPtr, pubTransaction(t2, _)).InSequence(s); + EXPECT_CALL(*rawSubscriptionManagerPtr, pubTransaction(t1, _)).InSequence(s); + + ctx.run(); + // last publish time should be set + EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1); +} diff --git a/unittests/util/TestObject.cpp b/unittests/util/TestObject.cpp index 1b2acacff..ce9a47372 100644 --- a/unittests/util/TestObject.cpp +++ b/unittests/util/TestObject.cpp @@ -106,7 +106,8 @@ CreatePaymentTransactionMetaObject( std::string_view accountId1, std::string_view accountId2, int finalBalance1, - int finalBalance2) + int finalBalance2, + uint32_t transactionIndex) { ripple::STObject finalFields(ripple::sfFinalFields); finalFields.setAccountID(ripple::sfAccount, GetAccountIDWithString(accountId1)); @@ -128,7 +129,7 @@ CreatePaymentTransactionMetaObject( metaArray.push_back(node2); metaObj.setFieldArray(ripple::sfAffectedNodes, metaArray); metaObj.setFieldU8(ripple::sfTransactionResult, ripple::tesSUCCESS); - metaObj.setFieldU32(ripple::sfTransactionIndex, 0); + metaObj.setFieldU32(ripple::sfTransactionIndex, transactionIndex); return metaObj; } diff --git a/unittests/util/TestObject.h b/unittests/util/TestObject.h index 9f2739fa0..8b1514a7c 100644 --- a/unittests/util/TestObject.h +++ b/unittests/util/TestObject.h @@ -75,7 +75,8 @@ CreatePaymentTransactionMetaObject( std::string_view accountId1, std::string_view accountId2, int finalBalance1, - int finalBalance2); + int finalBalance2, + uint32_t transactionIndex = 0); /* * Create an account root ledger object