ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 11:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: ETLng monitor (#1898)

Browse Source 
For #1594
This commit is contained in:
Alex Kremer
2025-02-21 16:10:25 +00:00
committed by GitHub
parent 25296f8ffa
commit 491cd58f93
30 changed files with 756 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/etl/NetworkValidatedLedgers.cpp
View File

@@ -19,6 +19,8 @@
#include "etl/NetworkValidatedLedgers.hpp"
#include <boost/signals2/connection.hpp>
#include <chrono>
#include <cstdint>
#include <memory>
@@ -35,25 +37,27 @@ NetworkValidatedLedgers::makeValidatedLedgers()
void
NetworkValidatedLedgers::push(uint32_t idx)
{
std::lock_guard const lck(m_);
if (!max_ || idx > *max_)
max_ = idx;
std::lock_guard const lck(mtx_);
if (!latest_ || idx > *latest_)
latest_ = idx;
notificationChannel_(idx);
cv_.notify_all();
}
std::optional<uint32_t>
NetworkValidatedLedgers::getMostRecent()
{
std::unique_lock lck(m_);
cv_.wait(lck, [this]() { return max_; });
return max_;
std::unique_lock lck(mtx_);
cv_.wait(lck, [this]() { return latest_; });
return latest_;
}
bool
NetworkValidatedLedgers::waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs)
{
std::unique_lock lck(m_);
auto pred = [sequence, this]() -> bool { return (max_ && sequence <= *max_); };
std::unique_lock lck(mtx_);
auto pred = [sequence, this]() -> bool { return (latest_ && sequence <= *latest_); };
if (maxWaitMs) {
cv_.wait_for(lck, std::chrono::milliseconds(*maxWaitMs));
} else {
@@ -62,4 +66,10 @@ NetworkValidatedLedgers::waitUntilValidatedByNetwork(uint32_t sequence, std::opt
return pred();
}
boost::signals2::scoped_connection
NetworkValidatedLedgers::subscribe(SignalType::slot_type const& subscriber)
{
return notificationChannel_.connect(subscriber);
}
} // namespace etl

14
src/etl/NetworkValidatedLedgers.hpp
View File

@@ -21,6 +21,10 @@
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include <boost/signals2/connection.hpp>
#include <boost/signals2/signal.hpp>
#include <boost/signals2/variadic_signal.hpp>
#include <condition_variable>
#include <cstdint>
#include <memory>
@@ -38,12 +42,13 @@ namespace etl {
* remains stopped for the rest of its lifetime.
*/
class NetworkValidatedLedgers : public NetworkValidatedLedgersInterface {
// max sequence validated by network
std::optional<uint32_t> max_;
std::optional<uint32_t> latest_; // currently known latest sequence validated by network
mutable std::mutex m_;
mutable std::mutex mtx_;
std::condition_variable cv_;
SignalType notificationChannel_;
public:
/**
* @brief A factory function for NetworkValidatedLedgers
@@ -81,6 +86,9 @@ public:
*/
bool
waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs = {}) final;
boost::signals2::scoped_connection
subscribe(SignalType::slot_type const& subscriber) override;
};
} // namespace etl

17
src/etl/NetworkValidatedLedgersInterface.hpp
View File

@@ -20,6 +20,10 @@
/** @file */
#pragma once
#include <boost/signals2/connection.hpp>
#include <boost/signals2/signal.hpp>
#include <boost/signals2/variadic_signal.hpp>
#include <cstdint>
#include <optional>
namespace etl {
@@ -29,6 +33,8 @@ namespace etl {
*/
class NetworkValidatedLedgersInterface {
public:
using SignalType = boost::signals2::signal<void(uint32_t)>;
virtual ~NetworkValidatedLedgersInterface() = default;
/**
@@ -46,7 +52,7 @@ public:
*
* @return Sequence of most recently validated ledger. empty optional if the datastructure has been stopped
*/
virtual std::optional<uint32_t>
[[nodiscard]] virtual std::optional<uint32_t>
getMostRecent() = 0;
/**
@@ -59,6 +65,15 @@ public:
*/
virtual bool
waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs = {}) = 0;
/**
* @brief Allows clients to get notified when a new validated ledger becomes known to Clio
*
* @param subscriber The slot to connect
* @return A connection object that automatically disconnects the subscription once destroyed
*/
[[nodiscard]] virtual boost::signals2::scoped_connection
subscribe(SignalType::slot_type const& subscriber) = 0;
};
} // namespace etl

10
src/etlng/CMakeLists.txt
View File

@@ -1,8 +1,14 @@
add_library(clio_etlng)
target_sources(
clio_etlng PRIVATE impl/AmendmentBlockHandler.cpp impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp
impl/Loading.cpp impl/TaskManager.cpp
clio_etlng
PRIVATE impl/AmendmentBlockHandler.cpp
impl/AsyncGrpcCall.cpp
impl/Extraction.cpp
impl/GrpcSource.cpp
impl/Loading.cpp
impl/Monitor.cpp
impl/TaskManager.cpp
)
target_link_libraries(clio_etlng PUBLIC clio_data)

67
src/etlng/MonitorInterface.hpp Normal file
View File

@@ -0,0 +1,67 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <boost/signals2/connection.hpp>
#include <boost/signals2/signal.hpp>
#include <boost/signals2/variadic_signal.hpp>
#include <chrono>
#include <cstdint>
namespace etlng {
/**
* @brief An interface for the monitor service
* An implementation of this service is responsible for periodically checking various datasources to detect newly
* ingested ledgers.
*/
class MonitorInterface {
public:
static constexpr auto kDEFAULT_REPEAT_INTERVAL = std::chrono::seconds{1};
using SignalType = boost::signals2::signal<void(uint32_t)>;
virtual ~MonitorInterface() = default;
/**
* @brief Allows clients to get notified when a new ledger becomes available in Clio's database
*
* @param subscriber The slot to connect
* @return A connection object that automatically disconnects the subscription once destroyed
*/
[[nodiscard]] virtual boost::signals2::scoped_connection
subscribe(SignalType::slot_type const& subscriber) = 0;
/**
* @brief Run the monitor service
*
* @param repeatInterval The interval between attempts to check the database for new ledgers
*/
virtual void
run(std::chrono::steady_clock::duration repeatInterval = kDEFAULT_REPEAT_INTERVAL) = 0;
/**
* @brief Stops the monitor service
*/
virtual void
stop() = 0;
};
} // namespace etlng

99
src/etlng/impl/Monitor.cpp Normal file
View File

@@ -0,0 +1,99 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/Monitor.hpp"
#include "data/BackendInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "util/Assert.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/async/AnyOperation.hpp"
#include "util/log/Logger.hpp"
#include <boost/signals2/connection.hpp>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <utility>
namespace etlng::impl {
Monitor::Monitor(
util::async::AnyExecutionContext ctx,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
uint32_t startSequence
)
: strand_(ctx.makeStrand())
, backend_(std::move(backend))
, validatedLedgers_(std::move(validatedLedgers))
, nextSequence_(startSequence)
{
}
Monitor::~Monitor()
{
stop();
}
void
Monitor::run(std::chrono::steady_clock::duration repeatInterval)
{
ASSERT(not repeatedTask_.has_value(), "Monitor attempted to run more than once");
LOG(log_.debug()) << "Starting monitor";
repeatedTask_ = strand_.executeRepeatedly(repeatInterval, std::bind_front(&Monitor::doWork, this));
subscription_ = validatedLedgers_->subscribe(std::bind_front(&Monitor::onNextSequence, this));
}
void
Monitor::stop()
{
if (repeatedTask_.has_value())
repeatedTask_->abort();
repeatedTask_ = std::nullopt;
}
boost::signals2::scoped_connection
Monitor::subscribe(SignalType::slot_type const& subscriber)
{
return notificationChannel_.connect(subscriber);
}
void
Monitor::onNextSequence(uint32_t seq)
{
LOG(log_.debug()) << "rippled published sequence " << seq;
repeatedTask_->invoke(); // force-invoke immediately
}
void
Monitor::doWork()
{
if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng) {
while (rng->maxSequence >= nextSequence_)
notificationChannel_(nextSequence_++);
}
}
} // namespace etlng::impl

80
src/etlng/impl/Monitor.hpp Normal file
View File

@@ -0,0 +1,80 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/MonitorInterface.hpp"
#include "util/async/AnyExecutionContext.hpp"
#include "util/async/AnyOperation.hpp"
#include "util/async/AnyStrand.hpp"
#include "util/log/Logger.hpp"
#include <boost/signals2/connection.hpp>
#include <xrpl/protocol/TxFormats.h>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <memory>
#include <optional>
namespace etlng::impl {
class Monitor : public MonitorInterface {
util::async::AnyStrand strand_;
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers_;
uint32_t nextSequence_;
std::optional<util::async::AnyOperation<void>> repeatedTask_;
std::optional<boost::signals2::scoped_connection> subscription_; // network validated ledgers subscription
SignalType notificationChannel_;
util::Logger log_{"ETL"};
public:
Monitor(
util::async::AnyExecutionContext ctx,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<etl::NetworkValidatedLedgersInterface> validatedLedgers,
uint32_t startSequence
);
~Monitor() override;
void
run(std::chrono::steady_clock::duration repeatInterval) override;
void
stop() override;
boost::signals2::scoped_connection
subscribe(SignalType::slot_type const& subscriber) override;
private:
void
onNextSequence(uint32_t seq);
void
doWork();
};
} // namespace etlng::impl

13
src/util/async/AnyOperation.hpp
View File

@@ -19,7 +19,6 @@
#pragma once
#include "util/async/Concepts.hpp"
#include "util/async/Error.hpp"
#include "util/async/impl/ErasedOperation.hpp"
@@ -107,6 +106,18 @@ public:
}
}
/**
* @brief Force-invoke the operation
* @note The action is scheduled on the underlying context/strand
* @warning The code of the user-provided action is expected to take care of thread-safety unless this operation is
* scheduled through a strand
*/
void
invoke()
{
operation_.invoke();
}
private:
impl::ErasedOperation operation_;
};

42
src/util/async/AnyStrand.hpp
View File

@@ -131,14 +131,43 @@ public:
);
}
/**
* @brief Schedule a repeating operation on the execution context
*
* @param interval The interval at which the operation should be repeated
* @param fn The block of code to execute; no args allowed and return type must be void
* @return A repeating stoppable operation that can be used to wait for its cancellation
*/
[[nodiscard]] auto
executeRepeatedly(SomeStdDuration auto interval, SomeHandlerWithoutStopToken auto&& fn)
{
using RetType = std::decay_t<decltype(fn())>;
static_assert(not std::is_same_v<RetType, std::any>);
auto const millis = std::chrono::duration_cast<std::chrono::milliseconds>(interval);
return AnyOperation<RetType>( //
pimpl_->executeRepeatedly(
millis,
[fn = std::forward<decltype(fn)>(fn)] -> std::any {
fn();
return {};
}
)
);
}
private:
struct Concept {
virtual ~Concept() = default;
[[nodiscard]] virtual impl::ErasedOperation
execute(std::function<std::any(AnyStopToken)>, std::optional<std::chrono::milliseconds> timeout = std::nullopt)
const = 0;
execute(
std::function<std::any(AnyStopToken)>,
std::optional<std::chrono::milliseconds> timeout = std::nullopt
) = 0;
[[nodiscard]] virtual impl::ErasedOperation execute(std::function<std::any()>) = 0;
[[nodiscard]] virtual impl::ErasedOperation
executeRepeatedly(std::chrono::milliseconds, std::function<std::any()>) = 0;
};
template <typename StrandType>
@@ -152,8 +181,7 @@ private:
}
[[nodiscard]] impl::ErasedOperation
execute(std::function<std::any(AnyStopToken)> fn, std::optional<std::chrono::milliseconds> timeout)
const override
execute(std::function<std::any(AnyStopToken)> fn, std::optional<std::chrono::milliseconds> timeout) override
{
return strand.execute(std::move(fn), timeout);
}
@@ -163,6 +191,12 @@ private:
{
return strand.execute(std::move(fn));
}
impl::ErasedOperation
executeRepeatedly(std::chrono::milliseconds interval, std::function<std::any()> fn) override
{
return strand.executeRepeatedly(interval, std::move(fn));
}
};
private:

8
src/util/async/Concepts.hpp
View File

@@ -75,6 +75,14 @@ concept SomeOperationWithData = SomeOperation<T> and requires(T v) {
{ v.get() };
};
/**
* @brief Specifies the interface for an operation that can force-invoked
*/
template <typename T>
concept SomeForceInvocableOperation = SomeOperation<T> and requires(T v) {
{ v.invoke() };
};
/**
* @brief Specifies the interface for an operation that can be stopped
*/

21
src/util/async/Operation.hpp
View File

@@ -32,6 +32,7 @@
#include <concepts>
#include <condition_variable>
#include <expected>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
@@ -227,6 +228,7 @@ using ScheduledOperation = impl::BasicScheduledOperation<CtxType, OpType>;
template <typename CtxType>
class RepeatingOperation : public util::MoveTracker {
util::Repeat repeat_;
std::function<void()> action_;
public:
/**
@@ -237,10 +239,11 @@ public:
* @param interval Time to wait before repeating the user-provided block of code
* @param fn The function to execute repeatedly
*/
RepeatingOperation(auto& executor, std::chrono::steady_clock::duration interval, std::invocable auto&& fn)
: repeat_(executor)
template <std::invocable FnType>
RepeatingOperation(auto& executor, std::chrono::steady_clock::duration interval, FnType&& fn)
: repeat_(executor), action_([fn = std::forward<FnType>(fn), &executor] { boost::asio::post(executor, fn); })
{
repeat_.start(interval, std::forward<decltype(fn)>(fn));
repeat_.start(interval, action_);
}
~RepeatingOperation() override
@@ -266,6 +269,18 @@ public:
{
repeat_.stop();
}
/**
* @brief Force-invoke the operation
* @note The action is scheduled on the underlying context/strand
* @warning The code of the user-provided action is expected to take care of thread-safety unless this operation is
* scheduled through a strand
*/
void
invoke()
{
action_();
}
};
} // namespace util::async

5
src/util/async/context/BasicExecutionContext.hpp
View File

@@ -61,8 +61,8 @@ struct AsioPoolStrandContext {
using Executor = boost::asio::strand<boost::asio::thread_pool::executor_type>;
using Timer = SteadyTimer<Executor>;
Executor const&
getExecutor() const
Executor&
getExecutor()
{
return executor;
}
@@ -272,6 +272,7 @@ public:
/**
* @brief Schedule a repeating operation on the execution context
* @warning The code of the user-provided action is expected to be thread-safe
*
* @param interval The interval at which the operation should be repeated
* @param fn The block of code to execute; no args allowed and return type must be void

22
src/util/async/context/impl/Strand.hpp
View File

@@ -20,6 +20,7 @@
#pragma once
#include "util/async/Concepts.hpp"
#include "util/async/Operation.hpp"
#include "util/async/context/impl/Cancellation.hpp"
#include "util/async/context/impl/Execution.hpp"
#include "util/async/context/impl/Timer.hpp"
@@ -52,6 +53,7 @@ public:
using StopToken = typename StopSourceType::Token;
using Timer =
typename ParentContextType::ContextHolderType::Timer; // timers are associated with the parent context
using RepeatedOperation = RepeatingOperation<BasicStrand>;
BasicStrand(ParentContextType& parent, auto&& strand)
: parentContext_{std::ref(parent)}, context_{std::forward<decltype(strand)>(strand)}
@@ -64,8 +66,10 @@ public:
BasicStrand(BasicStrand const&) = delete;
[[nodiscard]] auto
execute(SomeHandlerWith<StopToken> auto&& fn, std::optional<std::chrono::milliseconds> timeout = std::nullopt) const
noexcept(kIS_NOEXCEPT)
execute(
SomeHandlerWith<StopToken> auto&& fn,
std::optional<std::chrono::milliseconds> timeout = std::nullopt
) noexcept(kIS_NOEXCEPT)
{
return DispatcherType::dispatch(
context_,
@@ -89,7 +93,7 @@ public:
}
[[nodiscard]] auto
execute(SomeHandlerWith<StopToken> auto&& fn, SomeStdDuration auto timeout) const noexcept(kIS_NOEXCEPT)
execute(SomeHandlerWith<StopToken> auto&& fn, SomeStdDuration auto timeout) noexcept(kIS_NOEXCEPT)
{
return execute(
std::forward<decltype(fn)>(fn),
@@ -98,7 +102,7 @@ public:
}
[[nodiscard]] auto
execute(SomeHandlerWithoutStopToken auto&& fn) const noexcept(kIS_NOEXCEPT)
execute(SomeHandlerWithoutStopToken auto&& fn) noexcept(kIS_NOEXCEPT)
{
return DispatcherType::dispatch(
context_,
@@ -114,6 +118,16 @@ public:
})
);
}
[[nodiscard]] auto
executeRepeatedly(SomeStdDuration auto interval, SomeHandlerWithoutStopToken auto&& fn) noexcept(kIS_NOEXCEPT)
{
if constexpr (not std::is_same_v<decltype(TimerContextProvider::getContext(*this)), decltype(*this)>) {
return TimerContextProvider::getContext(*this).executeRepeatedly(interval, std::forward<decltype(fn)>(fn));
} else {
return RepeatedOperation(impl::extractAssociatedExecutor(*this), interval, std::forward<decltype(fn)>(fn));
}
}
};
} // namespace util::async::impl

18
src/util/async/impl/ErasedOperation.hpp
View File

@@ -71,6 +71,12 @@ public:
pimpl_->abort();
}
void
invoke()
{
pimpl_->invoke();
}
private:
struct Concept {
virtual ~Concept() = default;
@@ -81,6 +87,8 @@ private:
get() = 0;
virtual void
abort() = 0;
virtual void
invoke() = 0;
};
template <SomeOperation OpType>
@@ -133,6 +141,16 @@ private:
}
}
}
void
invoke() override
{
if constexpr (not SomeForceInvocableOperation<OpType>) {
ASSERT(false, "Called invoke() on an operation that can't be force-invoked");
} else {
operation.invoke();
}
}
};
private:

2
src/util/log/Logger.cpp
View File

@@ -175,7 +175,7 @@ LogService::init(config::ClioConfigDefinition const& config)
for (auto it = overrides.begin<util::config::ObjectView>(); it != overrides.end<util::config::ObjectView>(); ++it) {
auto const& channelConfig = *it;
auto const name = channelConfig.get<std::string>("channel");
if (std::ranges::count(Logger::kCHANNELS, name) == 0) { // TODO: use std::ranges::contains when available
if (std::ranges::count(Logger::kCHANNELS, name) == 0) { // TODO: use std::ranges::contains when available
return std::unexpected{fmt::format("Can't override settings for log channel {}: invalid channel", name)};
}

6
src/util/newconfig/ConfigDefinition.hpp
View File

@@ -345,10 +345,8 @@ static ClioConfigDefinition gClioConfig = ClioConfigDefinition{
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(gValidateUint16)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(gValidateLoadMode)},
{"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.withConstraint(gValidateChannelName)}
},
{"log_channels.[].log_level",
Array{ConfigValue{ConfigType::String}.withConstraint(gValidateLogLevelName)}},
{"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.withConstraint(gValidateChannelName)}},
{"log_channels.[].log_level", Array{ConfigValue{ConfigType::String}.withConstraint(gValidateLogLevelName)}},
{"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(gValidateLogLevelName)},

1
src/util/newconfig/Types.hpp
View File

@@ -81,4 +81,3 @@ getType()
}
} // namespace util::config

9
tests/common/util/MockNetworkValidatedLedgers.hpp
View File

@@ -19,8 +19,9 @@
#pragma once
#include "etl/ETLHelpers.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include <boost/signals2/connection.hpp>
#include <gmock/gmock.h>
#include <cstdint>
@@ -31,6 +32,12 @@ struct MockNetworkValidatedLedgers : public etl::NetworkValidatedLedgersInterfac
MOCK_METHOD(void, push, (uint32_t), (override));
MOCK_METHOD(std::optional<uint32_t>, getMostRecent, (), (override));
MOCK_METHOD(bool, waitUntilValidatedByNetwork, (uint32_t, std::optional<uint32_t>), (override));
MOCK_METHOD(
boost::signals2::scoped_connection,
subscribe,
(etl::NetworkValidatedLedgersInterface::SignalType::slot_type const& subscriber),
(override)
);
};
template <template <typename> typename MockType>

1
tests/common/util/MockOperation.hpp
View File

@@ -48,4 +48,5 @@ struct MockRepeatingOperation {
MOCK_METHOD(void, requestStop, (), (const));
MOCK_METHOD(void, wait, (), (const));
MOCK_METHOD(ValueType, get, (), (const));
MOCK_METHOD(void, invoke, (), (const));
};

9
tests/common/util/MockStrand.hpp
View File

@@ -41,6 +41,9 @@ struct MockStrand {
template <typename T>
using StoppableOperation = MockStoppableOperation<T>;
template <typename T>
using RepeatingOperation = MockRepeatingOperation<T>;
MOCK_METHOD(Operation<std::any> const&, execute, (std::function<std::any()>), (const));
MOCK_METHOD(
Operation<std::any> const&,
@@ -60,4 +63,10 @@ struct MockStrand {
(std::function<std::any(util::async::AnyStopToken)>, std::optional<std::chrono::milliseconds>),
(const)
);
MOCK_METHOD(
RepeatingOperation<std::any> const&,
executeRepeatedly,
(std::chrono::milliseconds, std::function<std::any()>),
(const)
);
};

2
tests/unit/CMakeLists.txt
View File

@@ -42,6 +42,8 @@ target_sources(
etlng/SchedulingTests.cpp
etlng/TaskManagerTests.cpp
etlng/LoadingTests.cpp
etlng/NetworkValidatedLedgersTests.cpp
etlng/MonitorTests.cpp
# Feed
util/BytesConverterTests.cpp
feed/BookChangesFeedTests.cpp

12
tests/unit/LoggerTests.cpp
View File

@@ -215,14 +215,18 @@ TEST_F(LoggerInitTest, LogSizeAndHourRotationCannotBeZero)
"log_rotation_hour_interval", "log_directory_max_size", "log_rotation_size"
};
auto const jsonStr = fmt::format(R"json({{
auto const jsonStr = fmt::format(
R"json({{
"{}": 0,
"{}": 0,
"{}": 0
}})json", keys[0], keys[1], keys[2]);
}})json",
keys[0],
keys[1],
keys[2]
);
auto const parsingErrors =
config_.parse(ConfigFileJson{boost::json::parse(jsonStr).as_object()});
auto const parsingErrors = config_.parse(ConfigFileJson{boost::json::parse(jsonStr).as_object()});
ASSERT_EQ(parsingErrors->size(), 3);
for (std::size_t i = 0; i < parsingErrors->size(); ++i) {
EXPECT_EQ(

112
tests/unit/etlng/MonitorTests.cpp Normal file
View File

@@ -0,0 +1,112 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/Types.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "etlng/impl/Monitor.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockNetworkValidatedLedgers.hpp"
#include "util/MockPrometheus.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include <boost/signals2/connection.hpp>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <semaphore>
using namespace etlng::impl;
namespace {
constexpr auto kSTART_SEQ = 123u;
} // namespace
struct MonitorTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
util::async::CoroExecutionContext ctx_;
StrictMockNetworkValidatedLedgersPtr ledgers_;
testing::StrictMock<testing::MockFunction<void(uint32_t)>> actionMock_;
etlng::impl::Monitor monitor_ = etlng::impl::Monitor(ctx_, backend_, ledgers_, kSTART_SEQ);
};
TEST_F(MonitorTests, ConsumesAndNotifiesForAllOutstandingSequencesAtOnce)
{
uint8_t count = 3;
LedgerRange range(kSTART_SEQ, kSTART_SEQ + count - 1);
std::binary_semaphore unblock(0);
EXPECT_CALL(*ledgers_, subscribe(testing::_));
EXPECT_CALL(*backend_, hardFetchLedgerRange(testing::_)).WillOnce(testing::Return(range));
EXPECT_CALL(actionMock_, Call).Times(count).WillRepeatedly([&] {
if (--count == 0u)
unblock.release();
});
auto subscription = monitor_.subscribe(actionMock_.AsStdFunction());
monitor_.run(std::chrono::milliseconds{1});
unblock.acquire();
}
TEST_F(MonitorTests, NotifiesForEachSequence)
{
uint8_t count = 3;
LedgerRange range(kSTART_SEQ, kSTART_SEQ);
std::binary_semaphore unblock(0);
EXPECT_CALL(*ledgers_, subscribe(testing::_));
EXPECT_CALL(*backend_, hardFetchLedgerRange(testing::_)).Times(count).WillRepeatedly([&] {
auto tmp = range;
++range.maxSequence;
return tmp;
});
EXPECT_CALL(actionMock_, Call).Times(count).WillRepeatedly([&] {
if (--count == 0u)
unblock.release();
});
auto subscription = monitor_.subscribe(actionMock_.AsStdFunction());
monitor_.run(std::chrono::milliseconds{1});
unblock.acquire();
}
TEST_F(MonitorTests, NotifiesWhenForcedByNewSequenceAvailableFromNetwork)
{
LedgerRange range(kSTART_SEQ, kSTART_SEQ);
std::binary_semaphore unblock(0);
std::function<void(uint32_t)> pusher;
EXPECT_CALL(*ledgers_, subscribe(testing::_)).WillOnce([&](auto&& subscriber) {
pusher = subscriber;
return boost::signals2::scoped_connection(); // to keep the compiler happy
});
EXPECT_CALL(*backend_, hardFetchLedgerRange(testing::_)).WillOnce(testing::Return(range));
EXPECT_CALL(actionMock_, Call).WillOnce([&] { unblock.release(); });
auto subscription = monitor_.subscribe(actionMock_.AsStdFunction());
monitor_.run(std::chrono::seconds{10}); // expected to be force-invoked sooner than in 10 sec
pusher(kSTART_SEQ); // pretend network validated a new ledger
unblock.acquire();
}

91
tests/unit/etlng/NetworkValidatedLedgersTests.cpp Normal file
View File

@@ -0,0 +1,91 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etl/NetworkValidatedLedgers.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "etlng/impl/AmendmentBlockHandler.hpp"
#include "util/LoggerFixtures.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <cstddef>
#include <cstdint>
#include <memory>
using namespace etlng::impl;
struct NetworkValidatedLedgersTests : NoLoggerFixture {
protected:
util::async::CoroExecutionContext ctx_{2};
std::shared_ptr<etl::NetworkValidatedLedgersInterface> ledgers_ =
etl::NetworkValidatedLedgers::makeValidatedLedgers();
};
TEST_F(NetworkValidatedLedgersTests, WaitUntilValidatedByNetworkWithoutTimeout)
{
auto awaitable = ctx_.execute([this] { return ledgers_->waitUntilValidatedByNetwork(123u); });
ledgers_->push(122u);
ledgers_->push(123u);
EXPECT_TRUE(awaitable.get().value());
}
TEST_F(NetworkValidatedLedgersTests, WaitUntilValidatedByNetworkWithTimeout)
{
static constexpr auto kTIMEOUT_MILLIS = 10u;
auto awaitable = ctx_.execute([this] { return ledgers_->waitUntilValidatedByNetwork(123u, kTIMEOUT_MILLIS); });
ledgers_->push(122u);
EXPECT_FALSE(awaitable.get().value());
}
TEST_F(NetworkValidatedLedgersTests, GetMostRecent)
{
ledgers_->push(122u);
ledgers_->push(123u);
auto awaitable = ctx_.execute([this] { return ledgers_->getMostRecent(); });
EXPECT_EQ(awaitable.get().value(), 123u);
ledgers_->push(124u);
EXPECT_EQ(ledgers_->getMostRecent(), 124u);
}
TEST_F(NetworkValidatedLedgersTests, SubscribersGetNotifiedWhileConnectionIsAlive)
{
testing::StrictMock<testing::MockFunction<void(uint32_t)>> actionMock1, actionMock2;
EXPECT_CALL(actionMock1, Call).Times(2);
EXPECT_CALL(actionMock2, Call).Times(2);
{
auto connection1 = ledgers_->subscribe(actionMock1.AsStdFunction());
auto connection2 = ledgers_->subscribe(actionMock2.AsStdFunction());
ledgers_->push(123u);
ledgers_->push(124u);
}
ledgers_->push(125u);
ledgers_->push(126u);
}

11
tests/unit/util/async/AnyOperationTests.cpp
View File

@@ -128,7 +128,18 @@ TEST_F(AnyOperationTests, RepeatingOpRequestStopCallPropagated)
repeatingOp.abort();
}
TEST_F(AnyOperationTests, RepeatingOpInvokeCallPropagated)
{
EXPECT_CALL(mockRepeatingOp, invoke());
repeatingOp.invoke();
}
TEST_F(AnyOperationDeathTest, CallAbortOnNonStoppableOrCancellableOperation)
{
EXPECT_DEATH(voidOp.abort(), ".*");
}
TEST_F(AnyOperationDeathTest, CallInvokeOnNonForceInvocableOperation)
{
EXPECT_DEATH(voidOp.invoke(), ".*");
}

15
tests/unit/util/async/AnyStrandTests.cpp
View File

@@ -43,6 +43,9 @@ struct AnyStrandTests : ::testing::Test {
template <typename T>
using StoppableOperationType = ::testing::NiceMock<MockStoppableOperation<T>>;
template <typename T>
using RepeatingOperationType = NiceMock<MockRepeatingOperation<T>>;
::testing::NaggyMock<MockStrand> mockStrand;
AnyStrand strand{static_cast<MockStrand&>(mockStrand)};
};
@@ -146,3 +149,15 @@ TEST_F(AnyStrandTests, ExecuteWithTimoutAndStopTokenAndReturnValueThrowsExceptio
[[maybe_unused]] auto unused = strand.execute([](auto) { return 42; }, std::chrono::milliseconds{1})
);
}
TEST_F(AnyStrandTests, RepeatingOperation)
{
auto mockRepeatingOp = RepeatingOperationType<std::any>{};
EXPECT_CALL(mockRepeatingOp, wait());
EXPECT_CALL(mockStrand, executeRepeatedly(std::chrono::milliseconds{1}, A<std::function<std::any()>>()))
.WillOnce([&mockRepeatingOp] -> RepeatingOperationType<std::any> const& { return mockRepeatingOp; });
auto res = strand.executeRepeatedly(std::chrono::milliseconds{1}, [] -> void { throw 0; });
static_assert(std::is_same_v<decltype(res), AnyOperation<void>>);
res.wait();
}

57
tests/unit/util/async/AsyncExecutionContextTests.cpp
View File

@@ -24,8 +24,10 @@
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <cstddef>
#include <ranges>
#include <semaphore>
#include <stdexcept>
#include <string>
@@ -220,6 +222,24 @@ TYPED_TEST(ExecutionContextTests, repeatingOperation)
EXPECT_LE(callCount, expectedActualCount); // never should be called more times than possible before timeout
}
TYPED_TEST(ExecutionContextTests, repeatingOperationForceInvoke)
{
std::atomic_size_t callCount = 64uz;
std::binary_semaphore unblock(0);
auto res = this->ctx.executeRepeatedly(std::chrono::seconds{10}, [&] {
if (--callCount == 0uz)
unblock.release();
});
for ([[maybe_unused]] auto unused : std::views::iota(0uz, callCount.load()))
res.invoke();
unblock.acquire();
res.abort();
EXPECT_EQ(callCount, 0uz);
}
TYPED_TEST(ExecutionContextTests, strandMove)
{
auto strand = this->ctx.makeStrand();
@@ -273,6 +293,43 @@ TYPED_TEST(ExecutionContextTests, strandWithTimeout)
EXPECT_EQ(res.get().value(), 42);
}
TYPED_TEST(ExecutionContextTests, strandedRepeatingOperation)
{
auto strand = this->ctx.makeStrand();
auto const repeatDelay = std::chrono::milliseconds{1};
auto const timeout = std::chrono::milliseconds{15};
auto callCount = 0uz;
auto res = strand.executeRepeatedly(repeatDelay, [&] { ++callCount; });
auto timeSpent = util::timed([timeout] { std::this_thread::sleep_for(timeout); }); // calculate actual time spent
res.abort(); // outside of the above stopwatch because it blocks and can take arbitrary time
auto const expectedPureCalls = timeout.count() / repeatDelay.count();
auto const expectedActualCount = timeSpent / repeatDelay.count();
EXPECT_GE(callCount, expectedPureCalls / 2u); // expect at least half of the scheduled calls
EXPECT_LE(callCount, expectedActualCount); // never should be called more times than possible before timeout
}
TYPED_TEST(ExecutionContextTests, strandedRepeatingOperationForceInvoke)
{
auto strand = this->ctx.makeStrand();
auto callCount = 64uz; // does not need to be atomic since we are on a strand
std::binary_semaphore unblock(0);
auto res = strand.executeRepeatedly(std::chrono::seconds{10}, [&] {
if (--callCount == 0uz)
unblock.release();
});
for ([[maybe_unused]] auto unused : std::views::iota(0uz, callCount))
res.invoke();
unblock.acquire();
res.abort();
EXPECT_EQ(callCount, 0uz);
}
TYPED_TEST(AsyncExecutionContextTests, executeAutoAborts)
{
auto value = 0;

1
tests/unit/util/newconfig/ArrayTests.cpp
View File

@@ -126,4 +126,3 @@ TEST(ArrayTest, addNullRequired)
auto const error = arr.addNull();
EXPECT_TRUE(error.has_value());
}

35
tests/unit/util/newconfig/ClioConfigDefinitionTests.cpp
View File

@@ -31,8 +31,8 @@
#include <boost/json/object.hpp>
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp>
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <algorithm>
#include <cstdint>
@@ -334,7 +334,8 @@ TEST_F(ClioConfigDefinitionParseArrayTest, emptyArray)
{
auto const configJson = boost::json::parse(R"json({
"array": []
})json").as_object();
})json")
.as_object();
auto const result = config.parse(ConfigFileJson{configJson});
EXPECT_FALSE(result.has_value());
@@ -355,20 +356,23 @@ TEST_F(ClioConfigDefinitionParseArrayTest, fullArray)
{"int": 1, "string": "one"},
{"int": 2, "string": "two"}
]
})json").as_object();
})json")
.as_object();
auto const result = config.parse(ConfigFileJson{configJson});
EXPECT_FALSE(result.has_value());
EXPECT_EQ(config.arraySize("array.[]"), 2);
}
TEST_F(ClioConfigDefinitionParseArrayTest, onlyRequiredFields) {
TEST_F(ClioConfigDefinitionParseArrayTest, onlyRequiredFields)
{
auto const configJson = boost::json::parse(R"json({
"array": [
{"int": 1},
{"int": 2}
]
})json").as_object();
})json")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto const result = config.parse(configFile);
@@ -388,7 +392,8 @@ TEST_F(ClioConfigDefinitionParseArrayTest, someOptionalFieldsMissing)
{"int": 1, "string": "one"},
{"int": 2}
]
})json").as_object();
})json")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto const result = config.parse(configFile);
@@ -401,13 +406,15 @@ TEST_F(ClioConfigDefinitionParseArrayTest, someOptionalFieldsMissing)
EXPECT_FALSE(config.getArray("array.[].string").valueAt(1).hasValue());
}
TEST_F(ClioConfigDefinitionParseArrayTest, optionalFieldMissingAtFirstPosition) {
TEST_F(ClioConfigDefinitionParseArrayTest, optionalFieldMissingAtFirstPosition)
{
auto const configJson = boost::json::parse(R"json({
"array": [
{"int": 1},
{"int": 2, "string": "two"}
]
})json").as_object();
})json")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto const result = config.parse(configFile);
@@ -421,13 +428,15 @@ TEST_F(ClioConfigDefinitionParseArrayTest, optionalFieldMissingAtFirstPosition)
EXPECT_EQ(config.getArray("array.[].string").valueAt(1).asString(), "two");
}
TEST_F(ClioConfigDefinitionParseArrayTest, missingRequiredFields) {
TEST_F(ClioConfigDefinitionParseArrayTest, missingRequiredFields)
{
auto const configJson = boost::json::parse(R"json({
"array": [
{"int": 1},
{"string": "two"}
]
})json").as_object();
})json")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto const result = config.parse(configFile);
@@ -436,13 +445,15 @@ TEST_F(ClioConfigDefinitionParseArrayTest, missingRequiredFields) {
EXPECT_THAT(result->at(0).error, testing::StartsWith("array.[].int"));
}
TEST_F(ClioConfigDefinitionParseArrayTest, missingAllRequiredFields) {
TEST_F(ClioConfigDefinitionParseArrayTest, missingAllRequiredFields)
{
auto const configJson = boost::json::parse(R"json({
"array": [
{"string": "one"},
{"string": "two"}
]
})json").as_object();
})json")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto const result = config.parse(configFile);

3
tests/unit/util/newconfig/ConfigFileJsonTests.cpp
View File

@@ -420,7 +420,8 @@ TEST_F(ConfigFileJsonTest, getArrayObjectInArray)
EXPECT_EQ(std::get<std::string>(strings.at(1).value()), "some string");
}
TEST_F(ConfigFileJsonTest, getArrayOptionalInArray) {
TEST_F(ConfigFileJsonTest, getArrayOptionalInArray)
{
auto const jsonStr = R"json({
"array": [
{ "int": 42 },