ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 20:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement extractor tests (#671)

Browse Source
This commit is contained in:
Alex Kremer
2023-06-07 12:33:46 +01:00
committed by GitHub
parent 244337c5b6
commit 5d2c079f1a
12 changed files with 349 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -116,6 +116,7 @@ if(BUILD_TESTS)
unittests/util/TestObject.cpp
# ETL
unittests/etl/ExtractionDataPipeTest.cpp
unittests/etl/ExtractorTest.cpp
# RPC
unittests/rpc/ErrorTests.cpp
unittests/rpc/BaseTests.cpp

33
src/etl/impl/Extractor.h
View File

@@ -19,12 +19,11 @@
#pragma once
#include <etl/SystemState.h>
#include <log/Logger.h>
#include <util/Profiler.h>
#include <ripple/beast/core/CurrentThreadName.h>
#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
#include <grpcpp/grpcpp.h>
#include <chrono>
#include <mutex>
@@ -89,11 +88,7 @@ private:
double totalTime = 0.0;
auto currentSequence = startSequence_;
// Two stopping conditions:
// - if there is a write conflict in the load thread, the ETL mechanism should stop.
// - if the entire server is shutting down - this can be detected in a variety of ways.
while ((not finishSequence_ || currentSequence <= *finishSequence_) && not state_.get().writeConflict &&
not isStopping() && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence))
while (!shouldFinish(currentSequence) && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence))
{
auto [fetchResponse, time] = util::timed<std::chrono::duration<double>>(
[this, currentSequence]() { return ledgerFetcher_.get().fetchDataAndDiff(currentSequence); });
@@ -101,11 +96,12 @@ private:
// if the fetch is unsuccessful, stop. fetchLedger only returns false if the server is shutting down, or if
// the ledger was found in the database (which means another process already wrote the ledger that this
// process was trying to extract; this is a form of a write conflict). Otherwise, fetchLedgerDataAndDiff
// will keep trying to fetch the specified ledger until successful.
// process was trying to extract; this is a form of a write conflict).
// Otherwise, fetchDataAndDiff will keep trying to fetch the specified ledger until successful.
if (!fetchResponse)
break;
// TODO: extract this part into a strategy perhaps
auto const tps = fetchResponse->transactions_list().transactions_size() / time;
log_.info() << "Extract phase time = " << time << "; Extract phase tps = " << tps
<< "; Avg extract time = " << totalTime / (currentSequence - startSequence_ + 1)
@@ -113,9 +109,6 @@ private:
pipe_.get().push(currentSequence, std::move(fetchResponse));
currentSequence += pipe_.get().getStride();
if (finishSequence_ && currentSequence > *finishSequence_)
break;
}
pipe_.get().finish(startSequence_);
@@ -126,6 +119,22 @@ private:
{
return state_.get().isStopping;
}
bool
hasWriteConflict() const
{
return state_.get().writeConflict;
}
bool
shouldFinish(uint32_t seq) const
{
// Stopping conditions:
// - if there is a write conflict in the load thread, the ETL mechanism should stop.
// - if the entire server is shutting down - this can be detected in a variety of ways.
// - when the given sequence is past the finishSequence in case one is specified
return hasWriteConflict() || isStopping() || (finishSequence_ && seq > *finishSequence_);
}
};
} // namespace clio::detail

170
unittests/etl/ExtractorTest.cpp Normal file
View File

@@ -0,0 +1,170 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include <etl/impl/Extractor.h>
#include <util/FakeFetchResponse.h>
#include <util/Fixtures.h>
#include <util/MockExtractionDataPipe.h>
#include <util/MockLedgerFetcher.h>
#include <util/MockNetworkValidatedLedgers.h>
#include <gtest/gtest.h>
#include <memory>
using namespace testing;
class ETLExtractorTest : public NoLoggerFixture
{
protected:
using DataType = FakeFetchResponse;
using ExtractionDataPipeType = MockExtractionDataPipe<DataType>;
using LedgerFetcherType = MockLedgerFetcher<DataType>;
using ExtractorType =
clio::detail::Extractor<ExtractionDataPipeType, MockNetworkValidatedLedgers, LedgerFetcherType>;
ExtractionDataPipeType dataPipe_;
std::shared_ptr<MockNetworkValidatedLedgers> networkValidatedLedgers_ =
std::make_shared<MockNetworkValidatedLedgers>();
LedgerFetcherType ledgerFetcher_;
SystemState state_;
std::unique_ptr<ExtractorType> extractor_;
public:
void
SetUp() override
{
NoLoggerFixture::SetUp();
state_.isStopping = false;
state_.writeConflict = false;
state_.isReadOnly = false;
state_.isWriting = false;
}
void
TearDown() override
{
extractor_.reset();
NoLoggerFixture::TearDown();
}
};
TEST_F(ETLExtractorTest, StopsWhenCurrentSequenceExceedsFinishSequence)
{
auto const rawNetworkValidatedLedgersPtr =
static_cast<MockNetworkValidatedLedgers*>(networkValidatedLedgers_.get());
ON_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).WillByDefault(Return(true));
EXPECT_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).Times(3);
ON_CALL(dataPipe_, getStride).WillByDefault(Return(4));
EXPECT_CALL(dataPipe_, getStride).Times(3);
auto response = FakeFetchResponse{};
ON_CALL(ledgerFetcher_, fetchDataAndDiff(_)).WillByDefault(Return(response));
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).Times(3);
EXPECT_CALL(dataPipe_, push).Times(3);
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// expected to invoke for seq 0, 4, 8 and finally stop as seq will be greater than finishing seq
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 11, state_);
}
TEST_F(ETLExtractorTest, StopsOnWriteConflict)
{
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
state_.writeConflict = true;
// despite finish sequence being far ahead, we set writeConflict and so exit the loop immediately
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_);
}
TEST_F(ETLExtractorTest, StopsOnServerShutdown)
{
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
state_.isStopping = true;
// despite finish sequence being far ahead, we set isStopping and so exit the loop immediately
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_);
}
// stop extractor thread if fetcheResponse is empty
TEST_F(ETLExtractorTest, StopsIfFetchIsUnsuccessful)
{
auto const rawNetworkValidatedLedgersPtr =
static_cast<MockNetworkValidatedLedgers*>(networkValidatedLedgers_.get());
ON_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).WillByDefault(Return(true));
EXPECT_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).Times(1);
ON_CALL(ledgerFetcher_, fetchDataAndDiff(_)).WillByDefault(Return(std::nullopt));
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).Times(1);
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// we break immediately because fetchDataAndDiff returns nullopt
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_);
}
TEST_F(ETLExtractorTest, StopsIfWaitingUntilValidatedByNetworkTimesOut)
{
auto const rawNetworkValidatedLedgersPtr =
static_cast<MockNetworkValidatedLedgers*>(networkValidatedLedgers_.get());
// note that in actual clio code we don't return false unless a timeout is specified and exceeded
ON_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).WillByDefault(Return(false));
EXPECT_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).Times(1);
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// we emulate waitUntilValidatedByNetwork timing out which would lead to shutdown of the extractor thread
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 64, state_);
}
TEST_F(ETLExtractorTest, SendsCorrectResponseToDataPipe)
{
auto const rawNetworkValidatedLedgersPtr =
static_cast<MockNetworkValidatedLedgers*>(networkValidatedLedgers_.get());
ON_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).WillByDefault(Return(true));
EXPECT_CALL(*rawNetworkValidatedLedgersPtr, waitUntilValidatedByNetwork).Times(1);
ON_CALL(dataPipe_, getStride).WillByDefault(Return(4));
EXPECT_CALL(dataPipe_, getStride).Times(1);
auto response = FakeFetchResponse{1234};
auto optionalResponse = std::optional<FakeFetchResponse>{};
ON_CALL(ledgerFetcher_, fetchDataAndDiff(_)).WillByDefault(Return(response));
EXPECT_CALL(ledgerFetcher_, fetchDataAndDiff).Times(1);
EXPECT_CALL(dataPipe_, push).Times(1).WillOnce(SaveArg<1>(&optionalResponse));
EXPECT_CALL(dataPipe_, finish(0)).Times(1);
// expect to finish after just one response due to finishSequence set to 1
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 0, 1, state_);
extractor_->waitTillFinished(); // this is what clio does too. waiting for the thread to join
EXPECT_TRUE(optionalResponse.has_value());
EXPECT_EQ(optionalResponse.value(), response);
}
TEST_F(ETLExtractorTest, CallsPipeFinishWithInitialSequenceAtExit)
{
EXPECT_CALL(dataPipe_, finish(123)).Times(1);
state_.isStopping = true;
extractor_ = std::make_unique<ExtractorType>(dataPipe_, networkValidatedLedgers_, ledgerFetcher_, 123, 234, state_);
}

55
unittests/util/FakeFetchResponse.h Normal file
View File

@@ -0,0 +1,55 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <cstddef>
class FakeTransactionsList
{
std::size_t size_ = 0;
public:
std::size_t
transactions_size()
{
return size_;
}
};
struct FakeFetchResponse
{
uint32_t id;
FakeFetchResponse(uint32_t id = 0) : id{id}
{
}
bool
operator==(FakeFetchResponse const& other) const
{
return other.id == id;
}
FakeTransactionsList
transactions_list() const
{
return {};
}
};

4
unittests/util/MockBackend.h
View File

@@ -24,12 +24,12 @@
using namespace Backend;
class MockBackend : public BackendInterface
struct MockBackend : public BackendInterface
{
public:
MockBackend(clio::Config)
{
}
MOCK_METHOD(
std::optional<ripple::LedgerInfo>,
fetchLedgerBySequence,

10
unittests/util/MockCounters.h
View File

@@ -24,18 +24,10 @@
#include <chrono>
class MockCounters
struct MockCounters
{
public:
MockCounters()
{
}
MOCK_METHOD(void, rpcErrored, (std::string const&), ());
MOCK_METHOD(void, rpcComplete, (std::string const&, std::chrono::microseconds const&), ());
MOCK_METHOD(void, rpcForwarded, (std::string const&), ());
MOCK_METHOD(boost::json::object, report, (), (const));
};

10
unittests/util/MockETLService.h
View File

@@ -24,18 +24,10 @@
#include <chrono>
class MockETLService
struct MockETLService
{
public:
MockETLService()
{
}
MOCK_METHOD(boost::json::object, getInfo, (), (const));
MOCK_METHOD(std::chrono::time_point<std::chrono::system_clock>, getLastPublish, (), (const));
MOCK_METHOD(std::uint32_t, lastPublishAgeSeconds, (), (const));
MOCK_METHOD(std::uint32_t, lastCloseAgeSeconds, (), (const));
};

34
unittests/util/MockExtractionDataPipe.h Normal file
View File

@@ -0,0 +1,34 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <gmock/gmock.h>
#include <chrono>
template <typename DataType>
struct MockExtractionDataPipe
{
MOCK_METHOD(void, push, (uint32_t, std::optional<DataType>&&), ());
MOCK_METHOD(std::optional<DataType>, popNext, (uint32_t), ());
MOCK_METHOD(uint32_t, getStride, (), (const));
MOCK_METHOD(void, finish, (uint32_t), ());
MOCK_METHOD(void, cleanup, (), ());
};

31
unittests/util/MockLedgerFetcher.h Normal file
View File

@@ -0,0 +1,31 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <gmock/gmock.h>
#include <optional>
template <typename DataType>
struct MockLedgerFetcher
{
MOCK_METHOD(std::optional<DataType>, fetchData, (uint32_t), ());
MOCK_METHOD(std::optional<DataType>, fetchDataAndDiff, (uint32_t), ());
};

11
unittests/util/MockLoadBalancer.h
View File

@@ -30,21 +30,12 @@
#include <optional>
class MockLoadBalancer
struct MockLoadBalancer
{
public:
MockLoadBalancer()
{
}
MOCK_METHOD(void, loadInitialLedger, (std::uint32_t, bool), ());
MOCK_METHOD(std::optional<org::xrpl::rpc::v1::GetLedgerResponse>, fetchLedger, (uint32_t, bool, bool), ());
MOCK_METHOD(bool, shouldPropagateTxnStream, (Source*), (const));
MOCK_METHOD(boost::json::value, toJson, (), (const));
MOCK_METHOD(
std::optional<boost::json::object>,
forwardToRippled,

31
unittests/util/MockNetworkValidatedLedgers.h Normal file
View File

@@ -0,0 +1,31 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <gmock/gmock.h>
#include <optional>
struct MockNetworkValidatedLedgers
{
MOCK_METHOD(void, push, (uint32_t), ());
MOCK_METHOD(std::optional<uint32_t>, getMostRecent, (), ());
MOCK_METHOD(bool, waitUntilValidatedByNetwork, (uint32_t), ());
};

8
unittests/util/MockSubscriptionManager.h
View File

@@ -26,15 +26,9 @@
#include <boost/json.hpp>
#include <gmock/gmock.h>
#include <optional>
class MockSubscriptionManager
struct MockSubscriptionManager
{
public:
using session_ptr = std::shared_ptr<WsBase>;
MockSubscriptionManager()
{
}
MOCK_METHOD(boost::json::object, subLedger, (boost::asio::yield_context&, session_ptr), ());