ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-08-23 05:30:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

style: Set clang-format width 100 (#2953)

Browse Source
This commit is contained in:
Ayaz Salikhov
2026-02-20 15:56:03 +00:00
committed by GitHub
parent 480264ff8f
commit 6ba58f42f0
575 changed files with 14315 additions and 6552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.clang-format
View File

@@ -22,7 +22,7 @@ BreakBeforeBinaryOperators: false
BreakBeforeBraces: WebKit
BreakBeforeTernaryOperators: true
BreakConstructorInitializersBeforeComma: true
ColumnLimit: 120
ColumnLimit: 100
CommentPragmas: "^ IWYU pragma:"
ConstructorInitializerAllOnOneLineOrOnePerLine: true
ConstructorInitializerIndentWidth: 4

18
benchmarks/rpc/WorkQueueBenchmarks.cpp
View File

@@ -51,12 +51,15 @@ auto const kCONFIG = ClioConfigDefinition{
{"log.channels.[].channel", Array{ConfigValue{ConfigType::String}}},
{"log.channels.[].level", Array{ConfigValue{ConfigType::String}}},
{"log.level", ConfigValue{ConfigType::String}.defaultValue("info")},
{"log.format", ConfigValue{ConfigType::String}.defaultValue(R"(%Y-%m-%d %H:%M:%S.%f %^%3!l:%n%$ - %v)")},
{"log.format",
ConfigValue{ConfigType::String}.defaultValue(R"(%Y-%m-%d %H:%M:%S.%f %^%3!l:%n%$ - %v)")},
{"log.is_async", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"log.enable_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)},
{"log.directory", ConfigValue{ConfigType::String}.optional()},
{"log.rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(gValidateUint32)},
{"log.directory_max_files", ConfigValue{ConfigType::Integer}.defaultValue(25).withConstraint(gValidateUint32)},
{"log.rotation_size",
ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(gValidateUint32)},
{"log.directory_max_files",
ConfigValue{ConfigType::Integer}.defaultValue(25).withConstraint(gValidateUint32)},
{"log.tag_style", ConfigValue{ConfigType::String}.defaultValue("none")},
};
@@ -124,9 +127,14 @@ benchmarkWorkQueue(benchmark::State& state)
ASSERT(totalQueued <= itemsPerClient * clientThreads, "Queued more than requested");
if (maxQueueSize == 0) {
ASSERT(totalQueued == itemsPerClient * clientThreads, "Queued exactly the expected amount");
ASSERT(
totalQueued == itemsPerClient * clientThreads, "Queued exactly the expected amount"
);
} else {
ASSERT(totalQueued >= std::min(maxQueueSize, itemsPerClient * clientThreads), "Queued less than expected");
ASSERT(
totalQueued >= std::min(maxQueueSize, itemsPerClient * clientThreads),
"Queued less than expected"
);
}
}
}

15
benchmarks/util/log/LoggerBenchmark.cpp
View File

@@ -62,8 +62,9 @@ uniqueLogDir()
{
auto const epochTime = std::chrono::high_resolution_clock::now().time_since_epoch();
auto const tmpDir = std::filesystem::temp_directory_path();
std::string const dirName =
fmt::format("logs_{}", std::chrono::duration_cast<std::chrono::microseconds>(epochTime).count());
std::string const dirName = fmt::format(
"logs_{}", std::chrono::duration_cast<std::chrono::microseconds>(epochTime).count()
);
return tmpDir / "clio_benchmark" / dirName;
}
@@ -108,7 +109,8 @@ benchmarkConcurrentFileLogging(benchmark::State& state)
channel, fileSink, spdlog::thread_pool(), spdlog::async_overflow_policy::block
);
spdlog::register_logger(logger);
Logger const threadLogger = BenchmarkLoggingInitializer::getLogger(std::move(logger));
Logger const threadLogger =
BenchmarkLoggingInitializer::getLogger(std::move(logger));
barrier.arrive_and_wait();
@@ -124,13 +126,16 @@ benchmarkConcurrentFileLogging(benchmark::State& state)
spdlog::shutdown();
auto const end = std::chrono::high_resolution_clock::now();
state.SetIterationTime(std::chrono::duration_cast<std::chrono::duration<double>>(end - start).count());
state.SetIterationTime(
std::chrono::duration_cast<std::chrono::duration<double>>(end - start).count()
);
std::filesystem::remove_all(logDir);
}
auto const totalMessages = numThreads * messagesPerThread;
state.counters["TotalMessagesRate"] = benchmark::Counter(totalMessages, benchmark::Counter::kIsRate);
state.counters["TotalMessagesRate"] =
benchmark::Counter(totalMessages, benchmark::Counter::kIsRate);
state.counters["Threads"] = numThreads;
state.counters["MessagesPerThread"] = messagesPerThread;
}

23
src/app/CliArgs.cpp
View File

@@ -58,12 +58,16 @@ CliArgs::parse(int argc, char const* argv[])
positional.add("conf", 1);
auto const printHelp = [&description]() {
std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n"
<< description;
};
po::variables_map parsed;
try {
po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
po::store(
po::command_line_parser(argc, argv).options(description).positional(positional).run(),
parsed
);
po::notify(parsed);
} catch (po::error const& e) {
std::cerr << "Error: " << e.what() << std::endl << std::endl;
@@ -87,7 +91,8 @@ CliArgs::parse(int argc, char const* argv[])
if (parsed.contains("config-description")) {
std::filesystem::path const filePath = parsed["config-description"].as<std::string>();
auto const res = util::config::ClioConfigDescription::generateConfigDescriptionToFile(filePath);
auto const res =
util::config::ClioConfigDescription::generateConfigDescriptionToFile(filePath);
if (res.has_value())
return Action{Action::Exit{EXIT_SUCCESS}};
@@ -100,14 +105,20 @@ CliArgs::parse(int argc, char const* argv[])
if (parsed.contains("migrate")) {
auto const opt = parsed["migrate"].as<std::string>();
if (opt == "status")
return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()}};
return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)}};
return Action{Action::Migrate{
.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()
}};
return Action{Action::Migrate{
.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)
}};
}
if (parsed.contains("verify"))
return Action{Action::VerifyConfig{.configPath = std::move(configPath)}};
return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.contains("ng-web-server")}};
return Action{Action::Run{
.configPath = std::move(configPath), .useNgWebServer = parsed.contains("ng-web-server")
}};
}
} // namespace app

3
src/app/CliArgs.hpp
View File

@@ -79,7 +79,8 @@ public:
/**
* @brief Apply a function to the action.
*
* @tparam Processors Action processors types. Must be callable with the action type and return int.
* @tparam Processors Action processors types. Must be callable with the action type and
* return int.
* @param processors Action processors.
* @return Exit code.
*/

35
src/app/ClioApplication.cpp
View File

@@ -136,14 +136,15 @@ ClioApplication::run(bool const useNgWebServer)
auto const migrationInspector = migration::makeMigrationInspector(config_, backend);
// Check if any migration is blocking Clio server starting.
if (migrationInspector->isBlockingClio() and backend->hardFetchLedgerRangeNoThrow()) {
LOG(util::LogService::error())
<< "Existing Migration is blocking Clio, Please complete the database migration first.";
LOG(util::LogService::error()) << "Existing Migration is blocking Clio, Please "
"complete the database migration first.";
return EXIT_FAILURE;
}
}
// Manages clients subscribed to streams
auto subscriptions = feed::SubscriptionManager::makeSubscriptionManager(config_, backend, amendmentCenter);
auto subscriptions =
feed::SubscriptionManager::makeSubscriptionManager(config_, backend, amendmentCenter);
// Tracks which ledgers have been validated by the network
auto ledgers = etl::NetworkValidatedLedgers::makeValidatedLedgers();
@@ -156,7 +157,8 @@ ClioApplication::run(bool const useNgWebServer)
config_, ioc, backend, subscriptions, std::make_unique<util::MTRandomGenerator>(), ledgers
);
// ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
// ETL is responsible for writing and publishing to streams. In read-only mode, ETL only
// publishes
auto etl = etl::ETLService::makeETLService(
config_, std::move(systemState), ctx, backend, subscriptions, balancer, ledgers
);
@@ -169,15 +171,19 @@ ClioApplication::run(bool const useNgWebServer)
);
using RPCEngineType = rpc::RPCEngine<rpc::Counters>;
auto const rpcEngine =
RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
auto const rpcEngine = RPCEngineType::makeRPCEngine(
config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider
);
if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
web::ng::RPCServerHandler<RPCEngineType> handler{config_, backend, rpcEngine, etl, dosGuard};
web::ng::RPCServerHandler<RPCEngineType> handler{
config_, backend, rpcEngine, etl, dosGuard
};
auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
if (not expectedAdminVerifier.has_value()) {
LOG(util::LogService::error()) << "Error creating admin verifier: " << expectedAdminVerifier.error();
LOG(util::LogService::error())
<< "Error creating admin verifier: " << expectedAdminVerifier.error();
return EXIT_FAILURE;
}
auto const adminVerifier = std::move(expectedAdminVerifier).value();
@@ -226,12 +232,21 @@ ClioApplication::run(bool const useNgWebServer)
}
// Init the web server
auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(config_, backend, rpcEngine, etl, dosGuard);
auto handler = std::make_shared<web::RPCServerHandler<RPCEngineType>>(
config_, backend, rpcEngine, etl, dosGuard
);
auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler, cache);
appStopper_.setOnStop(
Stopper::makeOnStopCallback(
*httpServer, *balancer, *etl, *subscriptions, *backend, cacheSaver, clusterCommunicationService, ioc
*httpServer,
*balancer,
*etl,
*subscriptions,
*backend,
cacheSaver,
clusterCommunicationService,
ioc
)
);

3
src/app/Stopper.hpp
View File

@@ -39,7 +39,8 @@
namespace app {
/**
* @brief Application stopper class. On stop it will create a new thread to run all the shutdown tasks.
* @brief Application stopper class. On stop it will create a new thread to run all the shutdown
* tasks.
*/
class Stopper {
boost::asio::io_context ctx_;

3
src/app/VerifyConfig.hpp
View File

@@ -41,7 +41,8 @@ parseConfig(std::string_view configPath)
auto const json = ConfigFileJson::makeConfigFileJson(configPath);
if (!json.has_value()) {
std::cerr << "Error parsing json from config: " << configPath << "\n" << json.error().error << std::endl;
std::cerr << "Error parsing json from config: " << configPath << "\n"
<< json.error().error << std::endl;
return false;
}
auto const errors = getClioConfig().parse(json.value());

19
src/app/WebHandlers.cpp
View File

@@ -51,9 +51,9 @@ OnConnectCheck::operator()(web::ng::Connection const& connection)
{
dosguard_.get().increment(connection.ip());
if (not dosguard_.get().isOk(connection.ip())) {
return std::unexpected{
web::ng::Response{boost::beast::http::status::too_many_requests, "Too many requests", connection}
};
return std::unexpected{web::ng::Response{
boost::beast::http::status::too_many_requests, "Too many requests", connection
}};
}
return {};
@@ -80,7 +80,10 @@ DisconnectHook::operator()(web::ng::Connection const& connection)
dosguard_.get().decrement(connection.ip());
}
MetricsHandler::MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier, rpc::WorkQueue& workQueue)
MetricsHandler::MetricsHandler(
std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
rpc::WorkQueue& workQueue
)
: adminVerifier_{std::move(adminVerifier)}, workQueue_{std::ref(workQueue)}
{
}
@@ -120,7 +123,9 @@ MetricsHandler::operator()(
if (!postSuccessful) {
return web::ng::Response{
boost::beast::http::status::too_many_requests, rpc::makeError(rpc::RippledError::rpcTOO_BUSY), request
boost::beast::http::status::too_many_requests,
rpc::makeError(rpc::RippledError::rpcTOO_BUSY),
request
};
}
@@ -177,7 +182,9 @@ CacheStateHandler::operator()(
if (cache_.get().isFull())
return web::ng::Response{boost::beast::http::status::ok, kCACHE_CHECK_LOADED_HTML, request};
return web::ng::Response{boost::beast::http::status::service_unavailable, kCACHE_CHECK_NOT_LOADED_HTML, request};
return web::ng::Response{
boost::beast::http::status::service_unavailable, kCACHE_CHECK_NOT_LOADED_HTML, request
};
}
} // namespace app

20
src/app/WebHandlers.hpp
View File

@@ -68,8 +68,8 @@ public:
};
/**
* @brief A function object that is called when the IP of a connection changes (usually if proxy detected).
* This is used to update the DOS guard.
* @brief A function object that is called when the IP of a connection changes (usually if proxy
* detected). This is used to update the DOS guard.
*/
class IpChangeHook {
std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
@@ -126,10 +126,14 @@ public:
/**
* @brief Construct a new MetricsHandler object
*
* @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
* @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for
* admin access.
* @param workQueue The WorkQueue to use for handling the request.
*/
MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier, rpc::WorkQueue& workQueue);
MetricsHandler(
std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
rpc::WorkQueue& workQueue
);
/**
* @brief The call of the function object.
@@ -214,10 +218,14 @@ public:
/**
* @brief Construct a new RequestHandler object
*
* @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
* @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for
* admin access.
* @param rpcHandler The RPC handler to use for handling the request.
*/
RequestHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier, RpcHandlerType& rpcHandler)
RequestHandler(
std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
RpcHandlerType& rpcHandler
)
: adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler)
{
}

4
src/cluster/Backend.cpp
View File

@@ -113,7 +113,9 @@ Backend::doRead(boost::asio::yield_context yield)
auto expectedNodeData = boost::json::try_value_to<ClioNode>(json);
if (expectedNodeData.has_error()) {
return std::unexpected{fmt::format("Error converting json to ClioNode: {}", nodeDataStr)};
return std::unexpected{
fmt::format("Error converting json to ClioNode: {}", nodeDataStr)
};
}
*expectedNodeData->uuid = uuid;
otherNodesData.push_back(std::move(expectedNodeData).value());

7
src/cluster/Backend.hpp
View File

@@ -49,12 +49,13 @@ namespace cluster {
* @brief Backend communication handler for cluster state synchronization.
*
* This class manages reading and writing cluster state information to/from the backend database.
* It periodically reads the state of other nodes in the cluster and writes the current node's state,
* enabling cluster-wide coordination and awareness.
* It periodically reads the state of other nodes in the cluster and writes the current node's
* state, enabling cluster-wide coordination and awareness.
*/
class Backend {
public:
/** @brief Type representing cluster data result - either a vector of nodes or an error message */
/** @brief Type representing cluster data result - either a vector of nodes or an error message
*/
using ClusterData = std::expected<std::vector<ClioNode>, std::string>;
private:

10
src/cluster/ClioNode.cpp
View File

@@ -62,7 +62,9 @@ ClioNode::from(ClioNode::Uuid uuid, etl::WriterStateInterface const& writerState
return writerState.isWriting() ? ClioNode::DbRole::Writer : ClioNode::DbRole::NotWriter;
}();
return ClioNode{.uuid = std::move(uuid), .updateTime = std::chrono::system_clock::now(), .dbRole = dbRole};
return ClioNode{
.uuid = std::move(uuid), .updateTime = std::chrono::system_clock::now(), .dbRole = dbRole
};
}
void
@@ -78,7 +80,8 @@ ClioNode
tag_invoke(boost::json::value_to_tag<ClioNode>, boost::json::value const& jv)
{
auto const& updateTimeStr = jv.as_object().at(JsonFields::kUPDATE_TIME).as_string();
auto const updateTime = util::systemTpFromUtcStr(std::string(updateTimeStr), ClioNode::kTIME_FORMAT);
auto const updateTime =
util::systemTpFromUtcStr(std::string(updateTimeStr), ClioNode::kTIME_FORMAT);
if (!updateTime.has_value()) {
throw std::runtime_error("Failed to parse update time");
}
@@ -88,7 +91,8 @@ tag_invoke(boost::json::value_to_tag<ClioNode>, boost::json::value const& jv)
throw std::runtime_error("Invalid db_role value");
return ClioNode{
// Json data doesn't contain uuid so leaving it empty here. It will be filled outside of this parsing
// Json data doesn't contain uuid so leaving it empty here. It will be filled outside of
// this parsing
.uuid = std::make_shared<boost::uuids::uuid>(),
.updateTime = updateTime.value(),
.dbRole = static_cast<ClioNode::DbRole>(dbRoleValue)

16
src/cluster/ClioNode.hpp
View File

@@ -52,14 +52,22 @@ struct ClioNode {
* from the cluster communication mechanism to the slower but more reliable
* database-based conflict detection mechanism.
*/
enum class DbRole { ReadOnly = 0, LoadingCache = 1, NotWriter = 2, Writer = 3, Fallback = 4, MAX = 4 };
enum class DbRole {
ReadOnly = 0,
LoadingCache = 1,
NotWriter = 2,
Writer = 3,
Fallback = 4,
MAX = 4
};
using Uuid = std::shared_ptr<boost::uuids::uuid>;
using CUuid = std::shared_ptr<boost::uuids::uuid const>;
Uuid uuid; ///< The UUID of the node.
std::chrono::system_clock::time_point updateTime; ///< The time the data about the node was last updated.
DbRole dbRole; ///< The database role of the node
Uuid uuid; ///< The UUID of the node.
std::chrono::system_clock::time_point
updateTime; ///< The time the data about the node was last updated.
DbRole dbRole; ///< The database role of the node
/**
* @brief Create a ClioNode from writer state.

6
src/cluster/ClusterCommunicationService.hpp
View File

@@ -38,10 +38,12 @@
namespace cluster {
/**
* @brief Service to post and read messages to/from the cluster. It uses a backend to communicate with the cluster.
* @brief Service to post and read messages to/from the cluster. It uses a backend to communicate
* with the cluster.
*/
class ClusterCommunicationService : public ClusterCommunicationServiceTag {
// TODO: Use util::async::CoroExecutionContext after https://github.com/XRPLF/clio/issues/1973 is implemented
// TODO: Use util::async::CoroExecutionContext after https://github.com/XRPLF/clio/issues/1973
// is implemented
boost::asio::thread_pool ctx_{1};
Backend backend_;
Metrics metrics_;

6
src/cluster/Metrics.hpp
View File

@@ -48,7 +48,8 @@ class Metrics {
util::prometheus::Bool isHealthy_ = PrometheusService::boolMetric(
"cluster_communication_is_healthy",
{},
"Whether cluster communication service is operating healthy (1 - healthy, 0 - we have a problem)"
"Whether cluster communication service is operating healthy (1 - healthy, 0 - we have a "
"problem)"
);
public:
@@ -67,7 +68,8 @@ public:
* - Node count to reflect the current cluster size
*
* @param uuid The UUID of the node (unused in current implementation)
* @param clusterData Shared pointer to the current cluster data; may be empty if communication failed
* @param clusterData Shared pointer to the current cluster data; may be empty if communication
* failed
*/
void
onNewState(ClioNode::CUuid uuid, std::shared_ptr<Backend::ClusterData const> clusterData);

18
src/cluster/WriterDecider.cpp
View File

@@ -34,13 +34,19 @@
namespace cluster {
WriterDecider::WriterDecider(boost::asio::thread_pool& ctx, std::unique_ptr<etl::WriterStateInterface> writerState)
WriterDecider::WriterDecider(
boost::asio::thread_pool& ctx,
std::unique_ptr<etl::WriterStateInterface> writerState
)
: ctx_(ctx), writerState_(std::move(writerState))
{
}
void
WriterDecider::onNewState(ClioNode::CUuid selfId, std::shared_ptr<Backend::ClusterData const> clusterData)
WriterDecider::onNewState(
ClioNode::CUuid selfId,
std::shared_ptr<Backend::ClusterData const> clusterData
)
{
if (not clusterData->has_value())
return;
@@ -50,8 +56,9 @@ WriterDecider::onNewState(ClioNode::CUuid selfId, std::shared_ptr<Backend::Clust
[writerState = writerState_->clone(),
selfId = std::move(selfId),
clusterData = clusterData->value()](auto&&) mutable {
auto const selfData =
std::ranges::find_if(clusterData, [&selfId](ClioNode const& node) { return node.uuid == selfId; });
auto const selfData = std::ranges::find_if(
clusterData, [&selfId](ClioNode const& node) { return node.uuid == selfId; }
);
ASSERT(selfData != clusterData.end(), "Self data should always be in the cluster data");
if (selfData->dbRole == ClioNode::DbRole::Fallback) {
@@ -78,7 +85,8 @@ WriterDecider::onNewState(ClioNode::CUuid selfId, std::shared_ptr<Backend::Clust
});
auto const it = std::ranges::find_if(clusterData, [](ClioNode const& node) {
return node.dbRole == ClioNode::DbRole::NotWriter or node.dbRole == ClioNode::DbRole::Writer;
return node.dbRole == ClioNode::DbRole::NotWriter or
node.dbRole == ClioNode::DbRole::Writer;
});
if (it == clusterData.end()) {

8
src/cluster/WriterDecider.hpp
View File

@@ -54,7 +54,10 @@ public:
* @param ctx Thread pool for executing asynchronous operations
* @param writerState Writer state interface for controlling write operations
*/
WriterDecider(boost::asio::thread_pool& ctx, std::unique_ptr<etl::WriterStateInterface> writerState);
WriterDecider(
boost::asio::thread_pool& ctx,
std::unique_ptr<etl::WriterStateInterface> writerState
);
/**
* @brief Handles cluster state changes and decides whether this node should be the writer.
@@ -66,7 +69,8 @@ public:
* - Logs a warning if no nodes in the cluster are allowed to write
*
* @param selfId The UUID of the current node
* @param clusterData Shared pointer to current cluster data; may be empty if communication failed
* @param clusterData Shared pointer to current cluster data; may be empty if communication
* failed
*/
void
onNewState(ClioNode::CUuid selfId, std::shared_ptr<Backend::ClusterData const> clusterData);

3
src/cluster/impl/RepeatedTask.hpp
View File

@@ -92,7 +92,8 @@ public:
void
stop()
{
if (auto expected = State::Running; not state_.compare_exchange_strong(expected, State::Stopped))
if (auto expected = State::Running;
not state_.compare_exchange_strong(expected, State::Stopped))
return; // Already stopped or not started
std::binary_semaphore cancelSemaphore{0};

46
src/data/AmendmentCenter.cpp
View File

@@ -57,10 +57,15 @@ supportedAmendments()
}
bool
lookupAmendment(auto const& allAmendments, std::vector<ripple::uint256> const& ledgerAmendments, std::string_view name)
lookupAmendment(
auto const& allAmendments,
std::vector<ripple::uint256> const& ledgerAmendments,
std::string_view name
)
{
namespace rg = std::ranges;
if (auto const am = rg::find(allAmendments, name, &data::Amendment::name); am != rg::end(allAmendments))
if (auto const am = rg::find(allAmendments, name, &data::Amendment::name);
am != rg::end(allAmendments))
return rg::find(ledgerAmendments, am->feature) != rg::end(ledgerAmendments);
return false;
}
@@ -70,9 +75,12 @@ lookupAmendment(auto const& allAmendments, std::vector<ripple::uint256> const& l
namespace data {
namespace impl {
WritingAmendmentKey::WritingAmendmentKey(std::string amendmentName) : AmendmentKey{std::move(amendmentName)}
WritingAmendmentKey::WritingAmendmentKey(std::string amendmentName)
: AmendmentKey{std::move(amendmentName)}
{
ASSERT(not supportedAmendments().contains(name), "Attempt to register the same amendment twice");
ASSERT(
not supportedAmendments().contains(name), "Attempt to register the same amendment twice"
);
supportedAmendments().insert(name);
}
@@ -96,7 +104,8 @@ operator ripple::uint256() const
return Amendment::getAmendmentId(name);
}
AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const& backend) : backend_{backend}
AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const& backend)
: backend_{backend}
{
namespace rg = std::ranges;
namespace vs = std::views;
@@ -108,7 +117,8 @@ AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const&
.name = name,
.feature = Amendment::getAmendmentId(name),
.isSupportedByXRPL = support != ripple::AmendmentSupport::Unsupported,
.isSupportedByClio = rg::find(supportedAmendments(), name) != rg::end(supportedAmendments()),
.isSupportedByClio =
rg::find(supportedAmendments(), name) != rg::end(supportedAmendments()),
.isRetired = support == ripple::AmendmentSupport::Retired
};
}),
@@ -144,19 +154,28 @@ AmendmentCenter::isEnabled(AmendmentKey const& key, uint32_t seq) const
}
bool
AmendmentCenter::isEnabled(boost::asio::yield_context yield, AmendmentKey const& key, uint32_t seq) const
AmendmentCenter::isEnabled(
boost::asio::yield_context yield,
AmendmentKey const& key,
uint32_t seq
) const
{
try {
if (auto const listAmendments = fetchAmendmentsList(yield, seq); listAmendments)
return lookupAmendment(all_, *listAmendments, key);
} catch (std::runtime_error const&) {
return false; // Some old ledger does not contain Amendments ledger object so do best we can for now
return false; // Some old ledger does not contain Amendments ledger object so do best we
// can for now
}
return false;
}
std::vector<bool>
AmendmentCenter::isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const
AmendmentCenter::isEnabled(
boost::asio::yield_context yield,
std::vector<AmendmentKey> const& keys,
uint32_t seq
) const
{
namespace rg = std::ranges;
@@ -181,7 +200,11 @@ AmendmentCenter::isEnabled(boost::asio::yield_context yield, std::vector<Amendme
Amendment const&
AmendmentCenter::getAmendment(AmendmentKey const& key) const
{
ASSERT(supported_.contains(key), "The amendment '{}' must be present in supported amendments list", key.name);
ASSERT(
supported_.contains(key),
"The amendment '{}' must be present in supported amendments list",
key.name
);
return supported_.at(key);
}
@@ -201,7 +224,8 @@ std::optional<std::vector<ripple::uint256>>
AmendmentCenter::fetchAmendmentsList(boost::asio::yield_context yield, uint32_t seq) const
{
// the amendments should always be present on the ledger
auto const amendments = backend_->fetchLedgerObject(ripple::keylet::amendments().key, seq, yield);
auto const amendments =
backend_->fetchLedgerObject(ripple::keylet::amendments().key, seq, yield);
if (not amendments.has_value())
throw std::runtime_error("Amendments ledger object must be present in the database");

12
src/data/AmendmentCenter.hpp
View File

@@ -62,9 +62,9 @@ struct WritingAmendmentKey : AmendmentKey {
*/
struct Amendments {
// NOTE: if Clio wants to report it supports an Amendment it should be listed here.
// Whether an amendment is obsolete and/or supported by libxrpl is extracted directly from libxrpl.
// If an amendment is in the list below it just means Clio did whatever changes needed to support it.
// Most of the time it's going to be no changes at all.
// Whether an amendment is obsolete and/or supported by libxrpl is extracted directly from
// libxrpl. If an amendment is in the list below it just means Clio did whatever changes needed
// to support it. Most of the time it's going to be no changes at all.
/** @cond */
// NOLINTBEGIN(readability-identifier-naming)
@@ -256,7 +256,11 @@ public:
* @return A vector of bools representing enabled state for each of the given keys
*/
[[nodiscard]] std::vector<bool>
isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const final;
isEnabled(
boost::asio::yield_context yield,
std::vector<AmendmentKey> const& keys,
uint32_t seq
) const final;
/**
* @brief Get an amendment

6
src/data/AmendmentCenterInterface.hpp
View File

@@ -92,7 +92,11 @@ public:
* @return A vector of bools representing enabled state for each of the given keys
*/
[[nodiscard]] virtual std::vector<bool>
isEnabled(boost::asio::yield_context yield, std::vector<AmendmentKey> const& keys, uint32_t seq) const = 0;
isEnabled(
boost::asio::yield_context yield,
std::vector<AmendmentKey> const& keys,
uint32_t seq
) const = 0;
/**
* @brief Get an amendment

13
src/data/BackendCounters.cpp
View File

@@ -41,7 +41,10 @@ std::vector<std::int64_t> const kHISTOGRAM_BUCKETS{1, 2, 5, 10, 20, 50, 100, 200
std::int64_t
durationInMillisecondsSince(std::chrono::steady_clock::time_point const startTime)
{
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startTime).count();
return std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - startTime
)
.count();
}
} // namespace
@@ -144,7 +147,10 @@ BackendCounters::registerReadStarted(std::uint64_t const count)
}
void
BackendCounters::registerReadFinished(std::chrono::steady_clock::time_point const startTime, std::uint64_t const count)
BackendCounters::registerReadFinished(
std::chrono::steady_clock::time_point const startTime,
std::uint64_t const count
)
{
asyncReadCounters_.registerFinished(count);
auto const duration = durationInMillisecondsSince(startTime);
@@ -238,7 +244,8 @@ void
BackendCounters::AsyncOperationCounters::registerError(std::uint64_t count)
{
ASSERT(
pendingCounter_.get().value() >= static_cast<std::int64_t>(count), "Error operations can't be more than pending"
pendingCounter_.get().value() >= static_cast<std::int64_t>(count),
"Error operations can't be more than pending"
);
pendingCounter_.get() -= count;
errorCounter_.get() += count;

4
src/data/BackendCounters.hpp
View File

@@ -46,7 +46,9 @@ concept SomeBackendCounters = requires(T a) {
{ a.registerWriteFinished(std::chrono::steady_clock::time_point{}) } -> std::same_as<void>;
{ a.registerWriteRetry() } -> std::same_as<void>;
{ a.registerReadStarted(std::uint64_t{}) } -> std::same_as<void>;
{ a.registerReadFinished(std::chrono::steady_clock::time_point{}, std::uint64_t{}) } -> std::same_as<void>;
{
a.registerReadFinished(std::chrono::steady_clock::time_point{}, std::uint64_t{})
} -> std::same_as<void>;
{ a.registerReadRetry(std::uint64_t{}) } -> std::same_as<void>;
{ a.registerReadError(std::uint64_t{}) } -> std::same_as<void>;
{ a.report() } -> std::same_as<boost::json::object>;

33
src/data/BackendInterface.cpp
View File

@@ -128,7 +128,8 @@ BackendInterface::fetchLedgerObjects(
misses.push_back(keys[i]);
}
}
LOG(log_.trace()) << "Cache hits = " << keys.size() - misses.size() << " - cache misses = " << misses.size();
LOG(log_.trace()) << "Cache hits = " << keys.size() - misses.size()
<< " - cache misses = " << misses.size();
if (!misses.empty()) {
auto objs = doFetchLedgerObjects(misses, sequence, yield);
@@ -192,7 +193,9 @@ BackendInterface::fetchBookOffers(
ripple::uint256 const bookEnd = ripple::getQualityNext(book);
ripple::uint256 uTipIndex = book;
std::vector<ripple::uint256> keys;
auto getMillis = [](auto diff) { return std::chrono::duration_cast<std::chrono::milliseconds>(diff).count(); };
auto getMillis = [](auto diff) {
return std::chrono::duration_cast<std::chrono::milliseconds>(diff).count();
};
auto begin = std::chrono::system_clock::now();
std::uint32_t numSucc = 0;
std::uint32_t numPages = 0;
@@ -233,20 +236,23 @@ BackendInterface::fetchBookOffers(
auto mid = std::chrono::system_clock::now();
auto objs = fetchLedgerObjects(keys, ledgerSequence, yield);
for (size_t i = 0; i < keys.size() && i < limit; ++i) {
LOG(log_.trace()) << "Key = " << ripple::strHex(keys[i]) << " blob = " << ripple::strHex(objs[i])
LOG(log_.trace()) << "Key = " << ripple::strHex(keys[i])
<< " blob = " << ripple::strHex(objs[i])
<< " ledgerSequence = " << ledgerSequence;
ASSERT(!objs[i].empty(), "Ledger object can't be empty");
page.offers.push_back({keys[i], objs[i]});
}
auto end = std::chrono::system_clock::now();
LOG(log_.debug()) << "Fetching " << std::to_string(keys.size()) << " offers took "
<< std::to_string(getMillis(mid - begin)) << " milliseconds. Fetching next dir took "
<< std::to_string(succMillis) << " milliseconds. Fetched next dir " << std::to_string(numSucc)
<< " times"
<< " Fetching next page of dir took " << std::to_string(pageMillis) << " milliseconds"
<< ". num pages = " << std::to_string(numPages) << ". Fetching all objects took "
<< std::to_string(getMillis(end - mid))
<< " milliseconds. total time = " << std::to_string(getMillis(end - begin)) << " milliseconds"
<< std::to_string(getMillis(mid - begin))
<< " milliseconds. Fetching next dir took " << std::to_string(succMillis)
<< " milliseconds. Fetched next dir " << std::to_string(numSucc) << " times"
<< " Fetching next page of dir took " << std::to_string(pageMillis)
<< " milliseconds"
<< ". num pages = " << std::to_string(numPages)
<< ". Fetching all objects took " << std::to_string(getMillis(end - mid))
<< " milliseconds. total time = " << std::to_string(getMillis(end - begin))
<< " milliseconds"
<< " book = " << ripple::strHex(book);
return page;
@@ -273,7 +279,8 @@ BackendInterface::updateRange(uint32_t newMax)
if (range_.has_value() and newMax < range_->maxSequence) {
ASSERT(
false,
"Range shouldn't exist yet or newMax should be at least range->maxSequence. newMax = {}, "
"Range shouldn't exist yet or newMax should be at least range->maxSequence. newMax = "
"{}, "
"range->maxSequence = {}",
newMax,
range_->maxSequence
@@ -339,8 +346,8 @@ BackendInterface::fetchLedgerPage(
if (!objects[i].empty()) {
page.objects.push_back({keys[i], std::move(objects[i])});
} else if (!outOfOrder) {
LOG(log_.error()) << "Deleted or non-existent object in successor table. key = " << ripple::strHex(keys[i])
<< " - seq = " << ledgerSequence;
LOG(log_.error()) << "Deleted or non-existent object in successor table. key = "
<< ripple::strHex(keys[i]) << " - seq = " << ledgerSequence;
std::stringstream msg;
for (size_t j = 0; j < objects.size(); ++j) {
msg << " - " << ripple::strHex(keys[j]);

96
src/data/BackendInterface.hpp
View File

@@ -109,18 +109,23 @@ synchronous(FnType&& func)
using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
if constexpr (!std::is_same_v<R, void>) {
R res;
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) { res = func(yield); });
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func, &res](auto yield) {
res = func(yield);
});
ctx.run();
return res;
} else {
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) { func(yield); });
util::spawn(ctx, [_ = boost::asio::make_work_guard(ctx), &func](auto yield) {
func(yield);
});
ctx.run();
}
}
/**
* @brief Synchronously execute the given function object and retry until no DatabaseTimeout is thrown.
* @brief Synchronously execute the given function object and retry until no DatabaseTimeout is
* thrown.
*
* @tparam FnType The type of function object to execute
* @param func The function object to execute
@@ -225,7 +230,8 @@ public:
fetchLedgerRange() const;
/**
* @brief Fetch the specified number of account root object indexes by page, the accounts need to exist for seq.
* @brief Fetch the specified number of account root object indexes by page, the accounts need
* to exist for seq.
*
* @param number The number of accounts to fetch
* @param pageSize The maximum number of accounts per page
@@ -296,7 +302,10 @@ public:
* @return A vector of TransactionAndMetadata matching the given hashes
*/
virtual std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const = 0;
fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches all transactions for a specific account.
@@ -325,7 +334,10 @@ public:
* @return Results as a vector of TransactionAndMetadata
*/
virtual std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
fetchAllTransactionsInLedger(
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches all transaction hashes from a specific ledger.
@@ -335,7 +347,10 @@ public:
* @return Hashes as ripple::uint256 in a vector
*/
virtual std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
fetchAllTransactionHashesInLedger(
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches a specific NFT.
@@ -346,7 +361,11 @@ public:
* @return NFT object on success; nullopt otherwise
*/
virtual std::optional<NFT>
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
fetchNFT(
ripple::uint256 const& tokenID,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches all transactions for a specific NFT.
@@ -376,7 +395,8 @@ public:
* @param limit Paging limit.
* @param cursorIn Optional cursor to allow us to pick up from where we last left off.
* @param yield Currently executing coroutine.
* @return NFTs issued by this account, or this issuer/taxon combination if taxon is passed and an optional marker
* @return NFTs issued by this account, or this issuer/taxon combination if taxon is passed and
* an optional marker
*/
virtual NFTsAndCursor
fetchNFTsByIssuer(
@@ -410,8 +430,8 @@ public:
/**
* @brief Fetches a specific ledger object.
*
* Currently the real fetch happens in doFetchLedgerObject and fetchLedgerObject attempts to fetch from Cache first
* and only calls out to the real DB if a cache miss occurred.
* Currently the real fetch happens in doFetchLedgerObject and fetchLedgerObject attempts to
* fetch from Cache first and only calls out to the real DB if a cache miss occurred.
*
* @param key The key of the object
* @param sequence The ledger sequence to fetch for
@@ -419,7 +439,11 @@ public:
* @return The object as a Blob on success; nullopt otherwise
*/
std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
fetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t sequence,
boost::asio::yield_context yield
) const;
/**
* @brief Fetches a specific ledger object sequence.
@@ -432,13 +456,18 @@ public:
* @return The sequence in unit32_t on success; nullopt otherwise
*/
std::optional<std::uint32_t>
fetchLedgerObjectSeq(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const;
fetchLedgerObjectSeq(
ripple::uint256 const& key,
std::uint32_t sequence,
boost::asio::yield_context yield
) const;
/**
* @brief Fetches all ledger objects by their keys.
*
* Currently the real fetch happens in doFetchLedgerObjects and fetchLedgerObjects attempts to fetch from Cache
* first and only calls out to the real DB for each of the keys that was not found in the cache.
* Currently the real fetch happens in doFetchLedgerObjects and fetchLedgerObjects attempts to
* fetch from Cache first and only calls out to the real DB for each of the keys that was not
* found in the cache.
*
* @param keys A vector with the keys of the objects to fetch
* @param sequence The ledger sequence to fetch for
@@ -461,7 +490,11 @@ public:
* @return The object as a Blob on success; nullopt otherwise
*/
virtual std::optional<Blob>
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t sequence, boost::asio::yield_context yield) const = 0;
doFetchLedgerObject(
ripple::uint256 const& key,
std::uint32_t sequence,
boost::asio::yield_context yield
) const = 0;
/**
* @brief The database-specific implementation for fetching a ledger object sequence.
@@ -531,13 +564,18 @@ public:
* @return The successor on success; nullopt otherwise
*/
std::optional<LedgerObject>
fetchSuccessorObject(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const;
fetchSuccessorObject(
ripple::uint256 key,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const;
/**
* @brief Fetches the successor key.
*
* Thea real fetch happens in doFetchSuccessorKey. This function will attempt to lookup the successor in the cache
* first and only if it's not found in the cache will it fetch from the actual DB.
* Thea real fetch happens in doFetchSuccessorKey. This function will attempt to lookup the
* successor in the cache first and only if it's not found in the cache will it fetch from the
* actual DB.
*
* @param key The key to fetch for
* @param ledgerSequence The ledger sequence to fetch for
@@ -545,7 +583,11 @@ public:
* @return The successor key on success; nullopt otherwise
*/
std::optional<ripple::uint256>
fetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const;
fetchSuccessorKey(
ripple::uint256 key,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const;
/**
* @brief Database-specific implementation of fetching the successor key
@@ -556,7 +598,11 @@ public:
* @return The successor on success; nullopt otherwise
*/
virtual std::optional<ripple::uint256>
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t ledgerSequence, boost::asio::yield_context yield) const = 0;
doFetchSuccessorKey(
ripple::uint256 key,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield
) const = 0;
/**
* @brief Fetches book offers.
@@ -583,7 +629,10 @@ public:
* @return The status of the migrator if found; nullopt otherwise
*/
virtual std::optional<std::string>
fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
fetchMigratorStatus(
std::string const& migratorName,
boost::asio::yield_context yield
) const = 0;
/** @brief Return type for fetchClioNodesData() method */
using ClioNodesDataFetchResult =
@@ -601,7 +650,8 @@ public:
/**
* @brief Synchronously fetches the ledger range from DB.
*
* This function just wraps hardFetchLedgerRange(boost::asio::yield_context) using synchronous(FnType&&).
* This function just wraps hardFetchLedgerRange(boost::asio::yield_context) using
* synchronous(FnType&&).
*
* @return The ledger range if available; nullopt otherwise
*/

33
src/data/CassandraBackend.hpp
View File

@@ -102,10 +102,14 @@ public:
this->waitForWritesToFinish();
if (!range_) {
executor_.writeSync(schema_->updateLedgerRange, ledgerSequence_, false, ledgerSequence_);
executor_.writeSync(
schema_->updateLedgerRange, ledgerSequence_, false, ledgerSequence_
);
}
if (not this->executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1))) {
if (not this->executeSyncUpdate(
schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1)
)) {
LOG(log_.warn()) << "Update failed for ledger " << ledgerSequence_;
return false;
}
@@ -139,7 +143,8 @@ public:
r.bindAt(
1,
std::make_tuple(
cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0,
cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn))
: 0,
cursorIn.value_or(ripple::uint256(0))
)
);
@@ -170,9 +175,10 @@ public:
selectNFTStatements.reserve(nftIDs.size());
std::transform(
std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTStatements), [&](auto const& nftID) {
return schema_->selectNFT.bind(nftID, ledgerSequence);
}
std::cbegin(nftIDs),
std::cend(nftIDs),
std::back_inserter(selectNFTStatements),
[&](auto const& nftID) { return schema_->selectNFT.bind(nftID, ledgerSequence); }
);
auto const nftInfos = executor_.readEach(yield, selectNFTStatements);
@@ -181,9 +187,10 @@ public:
selectNFTURIStatements.reserve(nftIDs.size());
std::transform(
std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTURIStatements), [&](auto const& nftID) {
return schema_->selectNFTURI.bind(nftID, ledgerSequence);
}
std::cbegin(nftIDs),
std::cend(nftIDs),
std::back_inserter(selectNFTURIStatements),
[&](auto const& nftID) { return schema_->selectNFTURI.bind(nftID, ledgerSequence); }
);
auto const nftUris = executor_.readEach(yield, selectNFTURIStatements);
@@ -193,7 +200,8 @@ public:
maybeRow.has_value()) {
auto [seq, owner, isBurned] = *maybeRow;
NFT nft(nftIDs[i], seq, owner, isBurned);
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri.has_value())
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>();
maybeUri.has_value())
nft.uri = *maybeUri;
ret.nfts.push_back(nft);
}
@@ -213,8 +221,9 @@ public:
std::optional<ripple::AccountID> lastItem;
while (liveAccounts.size() < number) {
Statement const statement = lastItem ? schema_->selectAccountFromToken.bind(*lastItem, Limit{pageSize})
: schema_->selectAccountFromBeginning.bind(Limit{pageSize});
Statement const statement = lastItem
? schema_->selectAccountFromToken.bind(*lastItem, Limit{pageSize})
: schema_->selectAccountFromBeginning.bind(Limit{pageSize});
auto const res = executor_.read(yield, statement);
if (res) {

44
src/data/DBHelpers.hpp
View File

@@ -83,8 +83,15 @@ struct NFTTransactionsData {
* @param meta The transaction metadata
* @param txHash The transaction hash
*/
NFTTransactionsData(ripple::uint256 const& tokenID, ripple::TxMeta const& meta, ripple::uint256 const& txHash)
: tokenID(tokenID), ledgerSequence(meta.getLgrSeq()), transactionIndex(meta.getIndex()), txHash(txHash)
NFTTransactionsData(
ripple::uint256 const& tokenID,
ripple::TxMeta const& meta,
ripple::uint256 const& txHash
)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
, txHash(txHash)
{
}
};
@@ -94,11 +101,13 @@ struct NFTTransactionsData {
*
* Gets written to nf_tokens table and the like.
*
* The transaction index is only stored because we want to store only the final state of an NFT per ledger.
* Since we pull this from transactions we keep track of which tx index created this so we can de-duplicate, as it is
* possible for one ledger to have multiple txs that change the state of the same NFT.
* The transaction index is only stored because we want to store only the final state of an NFT per
* ledger. Since we pull this from transactions we keep track of which tx index created this so we
* can de-duplicate, as it is possible for one ledger to have multiple txs that change the state of
* the same NFT.
*
* We only set the uri if this is a mint tx, or if we are loading initial state from NFTokenPage objects.
* We only set the uri if this is a mint tx, or if we are loading initial state from NFTokenPage
* objects.
*/
struct NFTsData {
ripple::uint256 tokenID;
@@ -113,8 +122,9 @@ struct NFTsData {
* @brief Construct a new NFTsData object
*
* @note This constructor is used when parsing an NFTokenMint tx
* Unfortunately because of the extreme edge case of being able to re-mint an NFT with the same ID, we must
* explicitly record a null URI. For this reason, we _always_ write this field as a result of this tx.
* Unfortunately because of the extreme edge case of being able to re-mint an NFT with the same
* ID, we must explicitly record a null URI. For this reason, we _always_ write this field as a
* result of this tx.
*
* @param tokenID The token ID
* @param owner The owner
@@ -127,7 +137,11 @@ struct NFTsData {
ripple::Blob const& uri,
ripple::TxMeta const& meta
)
: tokenID(tokenID), ledgerSequence(meta.getLgrSeq()), transactionIndex(meta.getIndex()), owner(owner), uri(uri)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
, owner(owner)
, uri(uri)
{
}
@@ -141,7 +155,12 @@ struct NFTsData {
* @param meta The transaction metadata
* @param isBurned Whether the NFT is burned
*/
NFTsData(ripple::uint256 const& tokenID, ripple::AccountID const& owner, ripple::TxMeta const& meta, bool isBurned)
NFTsData(
ripple::uint256 const& tokenID,
ripple::AccountID const& owner,
ripple::TxMeta const& meta,
bool isBurned
)
: tokenID(tokenID)
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())
@@ -154,8 +173,9 @@ struct NFTsData {
* @brief Construct a new NFTsData object
*
* @note This constructor is used when parsing an NFTokenPage directly from ledger state.
* Unfortunately because of the extreme edge case of being able to re-mint an NFT with the same ID, we must
* explicitly record a null URI. For this reason, we _always_ write this field as a result of this tx.
* Unfortunately because of the extreme edge case of being able to re-mint an NFT with the same
* ID, we must explicitly record a null URI. For this reason, we _always_ write this field as a
* result of this tx.
*
* @param tokenID The token ID
* @param ledgerSequence The ledger sequence

46
src/data/KeyspaceBackend.hpp
View File

@@ -102,11 +102,17 @@ public:
// This would be the first write to the table.
// In this case, insert both min_sequence/max_sequence range into the table.
if (not range_.has_value()) {
executor_.writeSync(schema_->insertLedgerRange, /* isLatestLedger =*/false, ledgerSequence_);
executor_.writeSync(schema_->insertLedgerRange, /* isLatestLedger =*/true, ledgerSequence_);
executor_.writeSync(
schema_->insertLedgerRange, /* isLatestLedger =*/false, ledgerSequence_
);
executor_.writeSync(
schema_->insertLedgerRange, /* isLatestLedger =*/true, ledgerSequence_
);
}
if (not this->executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1))) {
if (not this->executeSyncUpdate(
schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1)
)) {
log_.warn() << "Update failed for ledger " << ledgerSequence_;
return false;
}
@@ -131,7 +137,8 @@ public:
nftIDs = fetchNFTIDsByTaxon(issuer, *taxon, limit, cursorIn, yield);
} else {
// Amazon Keyspaces Workflow for non-taxon queries
auto const startTaxon = cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
auto const startTaxon =
cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
auto const startTokenID = cursorIn.value_or(ripple::uint256(0));
Statement const firstQuery = schema_->selectNFTIDsByIssuerTaxon.bind(issuer);
@@ -163,10 +170,10 @@ public:
/**
* @brief (Unsupported in Keyspaces) Fetches account root object indexes by page.
* @note Loading the cache by enumerating all accounts is currently unsupported by the AWS Keyspaces backend.
* This function's logic relies on "PER PARTITION LIMIT 1", which Keyspaces does not support, and there is
* no efficient alternative. This is acceptable as the cache is primarily loaded via diffs. Calling this
* function will throw an exception.
* @note Loading the cache by enumerating all accounts is currently unsupported by the AWS
* Keyspaces backend. This function's logic relies on "PER PARTITION LIMIT 1", which Keyspaces
* does not support, and there is no efficient alternative. This is acceptable as the cache is
* primarily loaded via diffs. Calling this function will throw an exception.
*
* @param number The total number of accounts to fetch.
* @param pageSize The maximum number of accounts per page.
@@ -220,7 +227,8 @@ private:
{
std::vector<ripple::uint256> nftIDs;
auto const startTaxon = cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
auto const startTaxon =
cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
auto const startTokenID = cursorIn.value_or(ripple::uint256(0));
Statement firstQuery = schema_->selectNFTIDsByIssuerTaxon.bind(issuer);
@@ -250,7 +258,8 @@ private:
}
/**
* @brief Takes a list of NFT IDs, fetches their full data, and assembles the final result with a cursor.
* @brief Takes a list of NFT IDs, fetches their full data, and assembles the final result with
* a cursor.
*/
NFTsAndCursor
populateNFTsAndCreateCursor(
@@ -273,17 +282,19 @@ private:
std::vector<Statement> selectNFTStatements;
selectNFTStatements.reserve(nftIDs.size());
std::transform(
std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTStatements), [&](auto const& nftID) {
return schema_->selectNFT.bind(nftID, ledgerSequence);
}
std::cbegin(nftIDs),
std::cend(nftIDs),
std::back_inserter(selectNFTStatements),
[&](auto const& nftID) { return schema_->selectNFT.bind(nftID, ledgerSequence); }
);
std::vector<Statement> selectNFTURIStatements;
selectNFTURIStatements.reserve(nftIDs.size());
std::transform(
std::cbegin(nftIDs), std::cend(nftIDs), std::back_inserter(selectNFTURIStatements), [&](auto const& nftID) {
return schema_->selectNFTURI.bind(nftID, ledgerSequence);
}
std::cbegin(nftIDs),
std::cend(nftIDs),
std::back_inserter(selectNFTURIStatements),
[&](auto const& nftID) { return schema_->selectNFTURI.bind(nftID, ledgerSequence); }
);
auto const nftInfos = executor_.readEach(yield, selectNFTStatements);
@@ -295,7 +306,8 @@ private:
maybeRow.has_value()) {
auto [seq, owner, isBurned] = *maybeRow;
NFT nft(nftIDs[i], seq, owner, isBurned);
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri.has_value())
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>();
maybeUri.has_value())
nft.uri = *maybeUri;
ret.nfts.push_back(nft);
}

7
src/data/LedgerCache.cpp
View File

@@ -254,7 +254,8 @@ LedgerCache::getSuccessorHitRate() const
{
if (successorReqCounter_.get().value() == 0u)
return 1;
return static_cast<float>(successorHitCounter_.get().value()) / successorReqCounter_.get().value();
return static_cast<float>(successorHitCounter_.get().value()) /
successorReqCounter_.get().value();
}
std::expected<void, std::string>
@@ -266,7 +267,9 @@ LedgerCache::saveToFile(std::string const& path) const
impl::LedgerCacheFile file{path};
std::shared_lock const lock{mtx_};
impl::LedgerCacheFile::DataView const data{.latestSeq = latestSeq_, .map = map_, .deleted = deleted_};
impl::LedgerCacheFile::DataView const data{
.latestSeq = latestSeq_, .map = map_, .deleted = deleted_
};
return file.write(data);
}

47
src/data/LedgerCache.hpp
View File

@@ -58,26 +58,34 @@ public:
private:
// counters for fetchLedgerObject(s) hit rate
std::reference_wrapper<util::prometheus::CounterInt> objectReqCounter_{PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "request"}, {"fetch", "ledger_objects"}}),
"LedgerCache statistics"
)};
std::reference_wrapper<util::prometheus::CounterInt> objectHitCounter_{PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "cache_hit"}, {"fetch", "ledger_objects"}})
)};
std::reference_wrapper<util::prometheus::CounterInt> objectReqCounter_{
PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "request"}, {"fetch", "ledger_objects"}}),
"LedgerCache statistics"
)
};
std::reference_wrapper<util::prometheus::CounterInt> objectHitCounter_{
PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "cache_hit"}, {"fetch", "ledger_objects"}})
)
};
// counters for fetchSuccessorKey hit rate
std::reference_wrapper<util::prometheus::CounterInt> successorReqCounter_{PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "request"}, {"fetch", "successor_key"}}),
"ledgerCache"
)};
std::reference_wrapper<util::prometheus::CounterInt> successorHitCounter_{PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "cache_hit"}, {"fetch", "successor_key"}})
)};
std::reference_wrapper<util::prometheus::CounterInt> successorReqCounter_{
PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "request"}, {"fetch", "successor_key"}}),
"ledgerCache"
)
};
std::reference_wrapper<util::prometheus::CounterInt> successorHitCounter_{
PrometheusService::counterInt(
"ledger_cache_counter_total_number",
util::prometheus::Labels({{"type", "cache_hit"}, {"fetch", "successor_key"}})
)
};
CacheMap map_;
CacheMap deleted_;
@@ -96,7 +104,8 @@ private:
"Whether ledger cache is disabled or not"
)};
// temporary set to prevent background thread from writing already deleted data. not used when cache is full
// temporary set to prevent background thread from writing already deleted data. not used when
// cache is full
std::unordered_set<ripple::uint256, ripple::hardened_hash<>> deletes_;
public:

12
src/data/LedgerCacheInterface.hpp
View File

@@ -126,9 +126,9 @@ public:
/**
* @brief Sets the full flag to true.
*
* This is used when cache loaded in its entirety at startup of the application. This can be either loaded from DB,
* populated together with initial ledger download (on first run) or downloaded from a peer node (specified in
* config).
* This is used when cache loaded in its entirety at startup of the application. This can be
* either loaded from DB, populated together with initial ledger download (on first run) or
* downloaded from a peer node (specified in config).
*/
virtual void
setFull() = 0;
@@ -152,13 +152,15 @@ public:
size() const = 0;
/**
* @return A number representing the success rate of hitting an object in the cache versus missing it.
* @return A number representing the success rate of hitting an object in the cache versus
* missing it.
*/
virtual float
getObjectHitRate() const = 0;
/**
* @return A number representing the success rate of hitting a successor in the cache versus missing it.
* @return A number representing the success rate of hitting a successor in the cache versus
* missing it.
*/
virtual float
getSuccessorHitRate() const = 0;

14
src/data/LedgerCacheSaver.cpp
View File

@@ -29,7 +29,10 @@
namespace data {
LedgerCacheSaver::LedgerCacheSaver(util::config::ClioConfigDefinition const& config, LedgerCacheInterface const& cache)
LedgerCacheSaver::LedgerCacheSaver(
util::config::ClioConfigDefinition const& config,
LedgerCacheInterface const& cache
)
: cacheFilePath_(config.maybeValue<std::string>("cache.file.path"))
, cache_(cache)
, isAsync_(config.get<bool>("cache.file.async_save"))
@@ -51,11 +54,14 @@ LedgerCacheSaver::save()
}
LOG(util::LogService::info()) << "Saving ledger cache to " << *cacheFilePath_;
if (auto const [success, durationMs] = util::timed([&]() { return cache_.get().saveToFile(*cacheFilePath_); });
if (auto const [success, durationMs] =
util::timed([&]() { return cache_.get().saveToFile(*cacheFilePath_); });
success.has_value()) {
LOG(util::LogService::info()) << "Successfully saved ledger cache in " << durationMs << " ms";
LOG(util::LogService::info())
<< "Successfully saved ledger cache in " << durationMs << " ms";
} else {
LOG(util::LogService::error()) << "Error saving LedgerCache to file: " << success.error();
LOG(util::LogService::error())
<< "Error saving LedgerCache to file: " << success.error();
}
});
if (not isAsync_) {

5
src/data/LedgerCacheSaver.hpp
View File

@@ -62,7 +62,10 @@ public:
* @param config The configuration object containing the cache file path setting
* @param cache Reference to the ledger cache interface to be saved
*/
LedgerCacheSaver(util::config::ClioConfigDefinition const& config, LedgerCacheInterface const& cache);
LedgerCacheSaver(
util::config::ClioConfigDefinition const& config,
LedgerCacheInterface const& cache
);
/**
* @brief Destructor that ensures the saving thread is properly joined.

42
src/data/Types.hpp
View File

@@ -81,8 +81,16 @@ struct TransactionAndMetadata {
* @param ledgerSequence The ledger sequence
* @param date The date
*/
TransactionAndMetadata(Blob transaction, Blob metadata, std::uint32_t ledgerSequence, std::uint32_t date)
: transaction{std::move(transaction)}, metadata{std::move(metadata)}, ledgerSequence{ledgerSequence}, date{date}
TransactionAndMetadata(
Blob transaction,
Blob metadata,
std::uint32_t ledgerSequence,
std::uint32_t date
)
: transaction{std::move(transaction)}
, metadata{std::move(metadata)}
, ledgerSequence{ledgerSequence}
, date{date}
{
}
@@ -192,7 +200,11 @@ struct NFT {
ripple::AccountID const& owner,
Blob uri,
bool isBurned)
: tokenID{tokenID}, ledgerSequence{ledgerSequence}, owner{owner}, uri{std::move(uri)}, isBurned{isBurned}
: tokenID{tokenID}
, ledgerSequence{ledgerSequence}
, owner{owner}
, uri{std::move(uri)}
, isBurned{isBurned}
{
}
@@ -204,7 +216,10 @@ struct NFT {
* @param owner The owner
* @param isBurned Whether the token is burned
*/
NFT(ripple::uint256 const& tokenID, std::uint32_t ledgerSequence, ripple::AccountID const& owner, bool isBurned)
NFT(ripple::uint256 const& tokenID,
std::uint32_t ledgerSequence,
ripple::AccountID const& owner,
bool isBurned)
: NFT(tokenID, ledgerSequence, owner, {}, isBurned)
{
}
@@ -212,8 +227,8 @@ struct NFT {
/**
* @brief Check if the NFT is the same as another
*
* Clearly two tokens are the same if they have the same ID, but this struct stores the state of a given
* token at a given ledger sequence, so we also need to compare with ledgerSequence.
* Clearly two tokens are the same if they have the same ID, but this struct stores the state of
* a given token at a given ledger sequence, so we also need to compare with ledgerSequence.
*
* @param other The other NFT
* @return true if they are the same; false otherwise
@@ -293,7 +308,8 @@ struct AmendmentKey {
* @brief Construct a new AmendmentKey
* @param val Anything convertible to a string
*/
AmendmentKey(std::convertible_to<std::string> auto&& val) : name{std::forward<decltype(val)>(val)}
AmendmentKey(std::convertible_to<std::string> auto&& val)
: name{std::forward<decltype(val)>(val)}
{
}
@@ -315,8 +331,14 @@ struct AmendmentKey {
operator<=>(AmendmentKey const& other) const = default;
};
constexpr ripple::uint256 kFIRST_KEY{"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 kLAST_KEY{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
constexpr ripple::uint256 kHI192{"0000000000000000000000000000000000000000000000001111111111111111"};
constexpr ripple::uint256 kFIRST_KEY{
"0000000000000000000000000000000000000000000000000000000000000000"
};
constexpr ripple::uint256 kLAST_KEY{
"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"
};
constexpr ripple::uint256 kHI192{
"0000000000000000000000000000000000000000000000001111111111111111"
};
} // namespace data

232
src/data/cassandra/CassandraBackendFamily.hpp
View File

@@ -104,7 +104,11 @@ public:
* @param cache The ledger cache
* @param readOnly Whether the database should be in readonly mode
*/
CassandraBackendFamily(SettingsProviderType settingsProvider, data::LedgerCacheInterface& cache, bool readOnly)
CassandraBackendFamily(
SettingsProviderType settingsProvider,
data::LedgerCacheInterface& cache,
bool readOnly
)
: BackendInterface(cache)
, settingsProvider_{std::move(settingsProvider)}
, schema_{settingsProvider_}
@@ -116,8 +120,8 @@ public:
if (not readOnly) {
if (auto const res = handle_.execute(schema_.createKeyspace); not res.has_value()) {
// on datastax, creation of keyspaces can be configured to only be done thru the admin
// interface. this does not mean that the keyspace does not already exist tho.
// on datastax, creation of keyspaces can be configured to only be done thru the
// admin interface. this does not mean that the keyspace does not already exist tho.
if (res.error().code() != CASS_ERROR_SERVER_UNAUTHORIZED)
throw std::runtime_error("Could not create keyspace: " + res.error());
}
@@ -130,7 +134,8 @@ public:
schema_.prepareStatements(handle_);
} catch (std::runtime_error const& ex) {
auto const error = fmt::format(
"Failed to prepare the statements: {}; readOnly: {}. ReadOnly should be turned off or another Clio "
"Failed to prepare the statements: {}; readOnly: {}. ReadOnly should be turned off "
"or another Clio "
"node with write access to DB should be started first.",
ex.what(),
readOnly
@@ -169,8 +174,8 @@ public:
auto cursor = txnCursor;
if (cursor) {
statement.bindAt(1, cursor->asTuple());
LOG(log_.debug()) << "account = " << ripple::strHex(account) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
LOG(log_.debug()) << "account = " << ripple::strHex(account)
<< " tuple = " << cursor->ledgerSequence << cursor->transactionIndex;
} else {
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder = forward ? 0u : std::numeric_limits<std::uint32_t>::max();
@@ -195,7 +200,8 @@ public:
auto numRows = results.numRows();
LOG(log_.info()) << "num_rows = " << numRows;
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
for (auto [hash, data] :
extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
hashes.push_back(hash);
if (--numRows == 0) {
LOG(log_.debug()) << "Setting cursor";
@@ -251,7 +257,10 @@ public:
}
std::optional<ripple::LedgerHeader>
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const override
fetchLedgerBySequence(
std::uint32_t const sequence,
boost::asio::yield_context yield
) const override
{
if (auto const lock = ledgerCache_.get(); lock.has_value() && lock->seq == sequence)
return lock->ledger;
@@ -259,7 +268,8 @@ public:
auto const res = executor_.read(yield, schema_->selectLedgerBySeq, sequence);
if (res) {
if (auto const& result = res.value(); result) {
if (auto const maybeValue = result.template get<std::vector<unsigned char>>(); maybeValue) {
if (auto const maybeValue = result.template get<std::vector<unsigned char>>();
maybeValue) {
auto const header = util::deserializeHeader(ripple::makeSlice(*maybeValue));
ledgerCache_.put(FetchLedgerCache::CacheEntry{header, sequence});
return header;
@@ -336,7 +346,10 @@ public:
}
std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
fetchAllTransactionsInLedger(
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield
) const override
{
auto hashes = fetchAllTransactionHashesInLedger(ledgerSequence, yield);
return fetchTransactions(hashes, yield);
@@ -349,7 +362,8 @@ public:
) const override
{
auto start = std::chrono::system_clock::now();
auto const res = executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
auto const res =
executor_.read(yield, schema_->selectAllTransactionHashesInLedger, ledgerSequence);
if (not res) {
LOG(log_.error()) << "Could not fetch all transaction hashes: " << res.error();
@@ -368,9 +382,12 @@ public:
hashes.push_back(std::move(hash));
auto end = std::chrono::system_clock::now();
LOG(log_.debug()) << "Fetched " << hashes.size() << " transaction hashes from database in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
<< " milliseconds";
LOG(
log_.debug()
) << "Fetched "
<< hashes.size() << " transaction hashes from database in "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count()
<< " milliseconds";
return hashes;
}
@@ -386,7 +403,8 @@ public:
if (not res)
return std::nullopt;
if (auto const maybeRow = res->template get<uint32_t, ripple::AccountID, bool>(); maybeRow) {
if (auto const maybeRow = res->template get<uint32_t, ripple::AccountID, bool>();
maybeRow) {
auto [seq, owner, isBurned] = *maybeRow;
auto result = std::make_optional<NFT>(tokenID, seq, owner, isBurned);
@@ -437,8 +455,8 @@ public:
auto cursor = cursorIn;
if (cursor) {
statement.bindAt(1, cursor->asTuple());
LOG(log_.debug()) << "token_id = " << ripple::strHex(tokenID) << " tuple = " << cursor->ledgerSequence
<< cursor->transactionIndex;
LOG(log_.debug()) << "token_id = " << ripple::strHex(tokenID)
<< " tuple = " << cursor->ledgerSequence << cursor->transactionIndex;
} else {
auto const seq = forward ? rng->minSequence : rng->maxSequence;
auto const placeHolder = forward ? 0 : std::numeric_limits<std::uint32_t>::max();
@@ -461,7 +479,8 @@ public:
auto numRows = results.numRows();
LOG(log_.info()) << "num_rows = " << numRows;
for (auto [hash, data] : extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
for (auto [hash, data] :
extract<ripple::uint256, std::tuple<uint32_t, uint32_t>>(results)) {
hashes.push_back(hash);
if (--numRows == 0) {
LOG(log_.debug()) << "Setting cursor";
@@ -495,7 +514,11 @@ public:
) const override
{
auto const holderEntries = executor_.read(
yield, schema_->selectMPTHolders, mptID, cursorIn.value_or(ripple::AccountID(0)), Limit{limit}
yield,
schema_->selectMPTHolders,
mptID,
cursorIn.value_or(ripple::AccountID(0)),
Limit{limit}
);
auto const& holderResults = holderEntries.value();
@@ -513,7 +536,9 @@ public:
auto mptObjects = doFetchLedgerObjects(mptKeys, ledgerSequence, yield);
auto it = std::remove_if(mptObjects.begin(), mptObjects.end(), [](Blob const& mpt) { return mpt.empty(); });
auto it = std::remove_if(mptObjects.begin(), mptObjects.end(), [](Blob const& mpt) {
return mpt.empty();
});
mptObjects.erase(it, mptObjects.end());
@@ -531,7 +556,8 @@ public:
boost::asio::yield_context yield
) const override
{
LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
LOG(log_.debug()) << "Fetching ledger object for seq " << sequence
<< ", key = " << ripple::to_string(key);
if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
if (auto const result = res->template get<Blob>(); result) {
if (result->size())
@@ -553,7 +579,8 @@ public:
boost::asio::yield_context yield
) const override
{
LOG(log_.debug()) << "Fetching ledger object for seq " << sequence << ", key = " << ripple::to_string(key);
LOG(log_.debug()) << "Fetching ledger object for seq " << sequence
<< ", key = " << ripple::to_string(key);
if (auto const res = executor_.read(yield, schema_->selectObject, key, sequence); res) {
if (auto const result = res->template get<Blob, std::uint32_t>(); result) {
auto [_, seq] = result.value();
@@ -571,7 +598,8 @@ public:
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const override
{
if (auto const res = executor_.read(yield, schema_->selectTransaction, hash); res) {
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>(); maybeValue) {
if (auto const maybeValue = res->template get<Blob, Blob, uint32_t, uint32_t>();
maybeValue) {
auto [transaction, meta, seq, date] = *maybeValue;
return std::make_optional<TransactionAndMetadata>(transaction, meta, seq, date);
}
@@ -591,7 +619,8 @@ public:
boost::asio::yield_context yield
) const override
{
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res) {
if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence);
res) {
if (auto const result = res->template get<ripple::uint256>(); result) {
if (*result == kLAST_KEY)
return std::nullopt;
@@ -607,7 +636,10 @@ public:
}
std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const override
fetchTransactions(
std::vector<ripple::uint256> const& hashes,
boost::asio::yield_context yield
) const override
{
if (hashes.empty())
return {};
@@ -622,9 +654,10 @@ public:
auto const timeDiff = util::timed([this, yield, &results, &hashes, &statements]() {
// TODO: seems like a job for "hash IN (list of hashes)" instead?
std::transform(
std::cbegin(hashes), std::cend(hashes), std::back_inserter(statements), [this](auto const& hash) {
return schema_->selectTransaction.bind(hash);
}
std::cbegin(hashes),
std::cend(hashes),
std::back_inserter(statements),
[this](auto const& hash) { return schema_->selectTransaction.bind(hash); }
);
auto const entries = executor_.readEach(yield, statements);
@@ -633,7 +666,8 @@ public:
std::cend(entries),
std::back_inserter(results),
[](auto const& res) -> TransactionAndMetadata {
if (auto const maybeRow = res.template get<Blob, Blob, uint32_t, uint32_t>(); maybeRow)
if (auto const maybeRow = res.template get<Blob, Blob, uint32_t, uint32_t>();
maybeRow)
return *maybeRow;
return {};
@@ -642,8 +676,8 @@ public:
});
ASSERT(numHashes == results.size(), "Number of hashes and results must match");
LOG(log_.debug()) << "Fetched " << numHashes << " transactions from database in " << timeDiff
<< " milliseconds";
LOG(log_.debug()) << "Fetched " << numHashes << " transactions from database in "
<< timeDiff << " milliseconds";
return results;
}
@@ -668,14 +702,18 @@ public:
// TODO: seems like a job for "key IN (list of keys)" instead?
std::transform(
std::cbegin(keys), std::cend(keys), std::back_inserter(statements), [this, &sequence](auto const& key) {
return schema_->selectObject.bind(key, sequence);
}
std::cbegin(keys),
std::cend(keys),
std::back_inserter(statements),
[this, &sequence](auto const& key) { return schema_->selectObject.bind(key, sequence); }
);
auto const entries = executor_.readEach(yield, statements);
std::transform(
std::cbegin(entries), std::cend(entries), std::back_inserter(results), [](auto const& res) -> Blob {
std::cbegin(entries),
std::cend(entries),
std::back_inserter(results),
[](auto const& res) -> Blob {
if (auto const maybeValue = res.template get<Blob>(); maybeValue)
return *maybeValue;
@@ -688,34 +726,40 @@ public:
}
std::vector<LedgerObject>
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const override
fetchLedgerDiff(
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield
) const override
{
auto const [keys, timeDiff] = util::timed([this, &ledgerSequence, yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res) {
LOG(log_.error()) << "Could not fetch ledger diff: " << res.error() << "; ledger = " << ledgerSequence;
return {};
}
auto const [keys, timeDiff] =
util::timed([this, &ledgerSequence, yield]() -> std::vector<ripple::uint256> {
auto const res = executor_.read(yield, schema_->selectDiff, ledgerSequence);
if (not res) {
LOG(log_.error()) << "Could not fetch ledger diff: " << res.error()
<< "; ledger = " << ledgerSequence;
return {};
}
auto const& results = res.value();
if (not results) {
LOG(log_.error()) << "Could not fetch ledger diff - no rows; ledger = " << ledgerSequence;
return {};
}
auto const& results = res.value();
if (not results) {
LOG(log_.error())
<< "Could not fetch ledger diff - no rows; ledger = " << ledgerSequence;
return {};
}
std::vector<ripple::uint256> resultKeys;
for (auto [key] : extract<ripple::uint256>(results))
resultKeys.push_back(key);
std::vector<ripple::uint256> resultKeys;
for (auto [key] : extract<ripple::uint256>(results))
resultKeys.push_back(key);
return resultKeys;
});
return resultKeys;
});
// one of the above errors must have happened
if (keys.empty())
return {};
LOG(log_.debug()) << "Fetched " << keys.size() << " diff hashes from database in " << timeDiff
<< " milliseconds";
LOG(log_.debug()) << "Fetched " << keys.size() << " diff hashes from database in "
<< timeDiff << " milliseconds";
auto const objs = fetchLedgerObjects(keys, ledgerSequence, yield);
std::vector<LedgerObject> results;
@@ -733,7 +777,10 @@ public:
}
std::optional<std::string>
fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override
fetchMigratorStatus(
std::string const& migratorName,
boost::asio::yield_context yield
) const override
{
auto const res = executor_.read(yield, schema_->selectMigratorStatus, Text(migratorName));
if (not res) {
@@ -771,7 +818,8 @@ public:
void
doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
{
LOG(log_.trace()) << " Writing ledger object " << key.size() << ":" << seq << " [" << blob.size() << " bytes]";
LOG(log_.trace()) << " Writing ledger object " << key.size() << ":" << seq << " ["
<< blob.size() << " bytes]";
if (range_)
executor_.write(schema_->insertDiff, seq, key);
@@ -783,7 +831,8 @@ public:
writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) override
{
LOG(log_.trace()) << "Writing successor. key = " << key.size() << " bytes. "
<< " seq = " << std::to_string(seq) << " successor = " << successor.size() << " bytes.";
<< " seq = " << std::to_string(seq) << " successor = " << successor.size()
<< " bytes.";
ASSERT(!key.empty(), "Key must not be empty");
ASSERT(!successor.empty(), "Successor must not be empty");
@@ -797,13 +846,15 @@ public:
statements.reserve(data.size() * 10); // assume 10 transactions avg
for (auto& record : data) {
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
std::ranges::transform(
record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
}
);
}
executor_.write(std::move(statements));
@@ -815,13 +866,15 @@ public:
std::vector<Statement> statements;
statements.reserve(record.accounts.size());
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
std::ranges::transform(
record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
}
);
executor_.write(std::move(statements));
}
@@ -834,7 +887,9 @@ public:
std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
record.tokenID,
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
@@ -854,7 +909,12 @@ public:
executor_.write(schema_->insertLedgerTransaction, seq, hash);
executor_.write(
schema_->insertTransaction, std::move(hash), seq, date, std::move(transaction), std::move(metadata)
schema_->insertTransaction,
std::move(hash),
seq,
date,
std::move(transaction),
std::move(metadata)
);
}
@@ -866,9 +926,9 @@ public:
for (NFTsData const& record : data) {
if (!record.onlyUriChanged) {
statements.push_back(
schema_->insertNFT.bind(record.tokenID, record.ledgerSequence, record.owner, record.isBurned)
);
statements.push_back(schema_->insertNFT.bind(
record.tokenID, record.ledgerSequence, record.owner, record.isBurned
));
// If `uri` is set (and it can be set to an empty uri), we know this
// is a net-new NFT. That is, this NFT has not been seen before by
@@ -881,15 +941,15 @@ public:
static_cast<uint32_t>(ripple::nft::getTaxon(record.tokenID)),
record.tokenID
));
statements.push_back(
schema_->insertNFTURI.bind(record.tokenID, record.ledgerSequence, record.uri.value())
);
statements.push_back(schema_->insertNFTURI.bind(
record.tokenID, record.ledgerSequence, record.uri.value()
));
}
} else {
// only uri changed, we update the uri table only
statements.push_back(
schema_->insertNFTURI.bind(record.tokenID, record.ledgerSequence, record.uri.value())
);
statements.push_back(schema_->insertNFTURI.bind(
record.tokenID, record.ledgerSequence, record.uri.value()
));
}
}
@@ -918,14 +978,18 @@ public:
writeMigratorStatus(std::string const& migratorName, std::string const& status) override
{
executor_.writeSync(
schema_->insertMigratorStatus, data::cassandra::Text{migratorName}, data::cassandra::Text(status)
schema_->insertMigratorStatus,
data::cassandra::Text{migratorName},
data::cassandra::Text(status)
);
}
void
writeNodeMessage(boost::uuids::uuid const& uuid, std::string message) override
{
executor_.writeSync(schema_->updateClioNodeMessage, data::cassandra::Text{std::move(message)}, uuid);
executor_.writeSync(
schema_->updateClioNodeMessage, data::cassandra::Text{std::move(message)}, uuid
);
}
bool

15
src/data/cassandra/Concepts.hpp
View File

@@ -78,12 +78,13 @@ concept SomeExecutionStrategy = requires(
* @brief The requirements of a retry policy.
*/
template <typename T>
concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) {
{ T(ioc) };
{ a.shouldRetry(err) } -> std::same_as<bool>;
{
a.retry([]() {})
} -> std::same_as<void>;
};
concept SomeRetryPolicy =
requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) {
{ T(ioc) };
{ a.shouldRetry(err) } -> std::same_as<bool>;
{
a.retry([]() {})
} -> std::same_as<void>;
};
} // namespace data::cassandra

6
src/data/cassandra/Error.hpp
View File

@@ -105,9 +105,9 @@ public:
bool
isTimeout() const
{
return code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or
code_ == CASS_ERROR_SERVER_UNAVAILABLE or code_ == CASS_ERROR_SERVER_OVERLOADED or
code_ == CASS_ERROR_SERVER_READ_TIMEOUT;
return code_ == CASS_ERROR_LIB_NO_HOSTS_AVAILABLE or
code_ == CASS_ERROR_LIB_REQUEST_TIMED_OUT or code_ == CASS_ERROR_SERVER_UNAVAILABLE or
code_ == CASS_ERROR_SERVER_OVERLOADED or code_ == CASS_ERROR_SERVER_READ_TIMEOUT;
}
/**

21
src/data/cassandra/Handle.cpp
View File

@@ -36,7 +36,8 @@ Handle::Handle(Settings clusterSettings) : cluster_{clusterSettings}
{
}
Handle::Handle(std::string_view contactPoints) : Handle{Settings::defaultSettings().withContactPoints(contactPoints)}
Handle::Handle(std::string_view contactPoints)
: Handle{Settings::defaultSettings().withContactPoints(contactPoints)}
{
}
@@ -85,7 +86,9 @@ Handle::FutureType
Handle::asyncReconnect(std::string_view keyspace) const
{
if (auto rc = asyncDisconnect().await(); not rc) // sync
throw std::logic_error("Reconnect to keyspace '" + std::string{keyspace} + "' failed: " + rc.error());
throw std::logic_error(
"Reconnect to keyspace '" + std::string{keyspace} + "' failed: " + rc.error()
);
return asyncConnect(keyspace);
}
@@ -123,7 +126,10 @@ Handle::asyncExecute(StatementType const& statement) const
}
Handle::FutureWithCallbackType
Handle::asyncExecute(StatementType const& statement, std::function<void(ResultOrErrorType)>&& cb) const
Handle::asyncExecute(
StatementType const& statement,
std::function<void(ResultOrErrorType)>&& cb
) const
{
return Handle::FutureWithCallbackType{cass_session_execute(session_, statement), std::move(cb)};
}
@@ -147,9 +153,14 @@ Handle::execute(std::vector<StatementType> const& statements) const
}
Handle::FutureWithCallbackType
Handle::asyncExecute(std::vector<StatementType> const& statements, std::function<void(ResultOrErrorType)>&& cb) const
Handle::asyncExecute(
std::vector<StatementType> const& statements,
std::function<void(ResultOrErrorType)>&& cb
) const
{
return Handle::FutureWithCallbackType{cass_session_execute_batch(session_, Batch{statements}), std::move(cb)};
return Handle::FutureWithCallbackType{
cass_session_execute_batch(session_, Batch{statements}), std::move(cb)
};
}
Handle::PreparedStatementType

12
src/data/cassandra/Handle.hpp
View File

@@ -293,14 +293,18 @@ public:
execute(std::vector<StatementType> const& statements) const;
/**
* @brief Execute a batch of (bound or simple) statements asynchronously with a completion callback.
* @brief Execute a batch of (bound or simple) statements asynchronously with a completion
* callback.
*
* @param statements The statements to execute
* @param cb The callback to execute when data is ready
* @return A future that holds onto the callback provided
*/
[[nodiscard]] FutureWithCallbackType
asyncExecute(std::vector<StatementType> const& statements, std::function<void(ResultOrErrorType)>&& cb) const;
asyncExecute(
std::vector<StatementType> const& statements,
std::function<void(ResultOrErrorType)>&& cb
) const;
/**
* @brief Prepare a statement.
@@ -314,8 +318,8 @@ public:
};
/**
* @brief Extracts the results into series of std::tuple<Types...> by creating a simple wrapper with an STL input
* iterator inside.
* @brief Extracts the results into series of std::tuple<Types...> by creating a simple wrapper with
* an STL input iterator inside.
*
* You can call .begin() and .end() in order to iterate as usual.
* This also means that you can use it in a range-based for or with some algorithms.

12
src/data/cassandra/Schema.hpp
View File

@@ -43,9 +43,14 @@ namespace data::cassandra {
* @return The qualified table name
*/
template <SomeSettingsProvider SettingsProviderType>
[[nodiscard]] std::string inline qualifiedTableName(SettingsProviderType const& provider, std::string_view name)
[[nodiscard]] std::string inline qualifiedTableName(
SettingsProviderType const& provider,
std::string_view name
)
{
return fmt::format("{}.{}{}", provider.getKeyspace(), provider.getTablePrefix().value_or(""), name);
return fmt::format(
"{}.{}{}", provider.getKeyspace(), provider.getTablePrefix().value_or(""), name
);
}
/**
@@ -65,7 +70,8 @@ public:
*
* @param settingsProvider The settings provider
*/
explicit Schema(SettingsProviderType const& settingsProvider) : settingsProvider_{std::cref(settingsProvider)}
explicit Schema(SettingsProviderType const& settingsProvider)
: settingsProvider_{std::cref(settingsProvider)}
{
}

21
src/data/cassandra/SettingsProvider.cpp
View File

@@ -61,12 +61,18 @@ SettingsProvider::parseOptionalCertificate() const
auto const path = std::filesystem::path(certPath.asString());
std::ifstream fileStream(path.string(), std::ios::in);
if (!fileStream) {
throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string());
throw std::system_error(
errno, std::generic_category(), "Opening certificate " + path.string()
);
}
std::string contents(std::istreambuf_iterator<char>{fileStream}, std::istreambuf_iterator<char>{});
std::string contents(
std::istreambuf_iterator<char>{fileStream}, std::istreambuf_iterator<char>{}
);
if (fileStream.bad()) {
throw std::system_error(errno, std::generic_category(), "Reading certificate " + path.string());
throw std::system_error(
errno, std::generic_category(), "Reading certificate " + path.string()
);
}
return contents;
@@ -82,7 +88,8 @@ SettingsProvider::parseSettings() const
// all config values used in settings is under "database.cassandra" prefix
if (config_.getValueView("secure_connect_bundle").hasValue()) {
auto const bundle = Settings::SecureConnectionBundle{(config_.get<std::string>("secure_connect_bundle"))};
auto const bundle =
Settings::SecureConnectionBundle{(config_.get<std::string>("secure_connect_bundle"))};
settings.connectionInfo = bundle;
} else {
Settings::ContactPoints out;
@@ -101,12 +108,14 @@ SettingsProvider::parseSettings() const
if (config_.getValueView("connect_timeout").hasValue()) {
auto const connectTimeoutSecond = config_.get<uint32_t>("connect_timeout");
settings.connectionTimeout = std::chrono::milliseconds{connectTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
settings.connectionTimeout =
std::chrono::milliseconds{connectTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
}
if (config_.getValueView("request_timeout").hasValue()) {
auto const requestTimeoutSecond = config_.get<uint32_t>("request_timeout");
settings.requestTimeout = std::chrono::milliseconds{requestTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
settings.requestTimeout =
std::chrono::milliseconds{requestTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
}
settings.certificate = parseOptionalCertificate();

12
src/data/cassandra/impl/AsyncExecutor.hpp
View File

@@ -52,7 +52,8 @@ template <
typename StatementType,
typename HandleType = Handle,
SomeRetryPolicy RetryPolicyType = ExponentialBackoffRetryPolicy>
class AsyncExecutor : public std::enable_shared_from_this<AsyncExecutor<StatementType, HandleType, RetryPolicyType>> {
class AsyncExecutor : public std::enable_shared_from_this<
AsyncExecutor<StatementType, HandleType, RetryPolicyType>> {
using FutureWithCallbackType = typename HandleType::FutureWithCallbackType;
using CallbackType = std::function<void(typename HandleType::ResultOrErrorType)>;
using RetryCallbackType = std::function<void()>;
@@ -92,7 +93,9 @@ public:
}
};
auto ptr = std::make_shared<EnableMakeShared>(ioc, std::move(data), std::move(onComplete), std::move(onRetry));
auto ptr = std::make_shared<EnableMakeShared>(
ioc, std::move(data), std::move(onComplete), std::move(onRetry)
);
ptr->execute(handle);
}
@@ -103,7 +106,10 @@ private:
CallbackType&& onComplete,
RetryCallbackType&& onRetry
)
: data_{std::move(data)}, retryPolicy_{ioc}, onComplete_{std::move(onComplete)}, onRetry_{std::move(onRetry)}
: data_{std::move(data)}
, retryPolicy_{ioc}
, onComplete_{std::move(onComplete)}
, onRetry_{std::move(onRetry)}
{
}

3
src/data/cassandra/impl/Batch.cpp
View File

@@ -44,7 +44,8 @@ namespace data::cassandra::impl {
* UNLOGGED: For performance. Sends many separate updates in one network trip to be fast.
* Use this for bulk-loading unrelated data, but know there's NO all-or-nothing guarantee.
*
* More info here: https://docs.datastax.com/en/developer/cpp-driver-dse/1.10/features/basics/batches/index.html
* More info here:
* https://docs.datastax.com/en/developer/cpp-driver-dse/1.10/features/basics/batches/index.html
*/
Batch::Batch(std::vector<Statement> const& statements)
: ManagedObject{cass_batch_new(CASS_BATCH_TYPE_UNLOGGED), kBATCH_DELETER}

55
src/data/cassandra/impl/Cluster.cpp
View File

@@ -44,7 +44,8 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), k
using std::to_string;
cass_cluster_set_token_aware_routing(*this, cass_true);
if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4); rc != CASS_OK) {
if (auto const rc = cass_cluster_set_protocol_version(*this, CASS_PROTOCOL_VERSION_V4);
rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Error setting cassandra protocol version to v4: {}", cass_error_desc(rc))
);
@@ -52,7 +53,11 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), k
if (auto const rc = cass_cluster_set_num_threads_io(*this, settings.threads); rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Error setting cassandra io threads to {}: {}", settings.threads, cass_error_desc(rc))
fmt::format(
"Error setting cassandra io threads to {}: {}",
settings.threads,
cass_error_desc(rc)
)
);
}
@@ -62,24 +67,36 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), k
// TODO: AWS keyspace reads should be local_one to save cost
if (settings.provider == cassandra::impl::Provider::Keyspace) {
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_LOCAL_QUORUM); rc != CASS_OK) {
throw std::runtime_error(fmt::format("Error setting keyspace consistency: {}", cass_error_desc(rc)));
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_LOCAL_QUORUM);
rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Error setting keyspace consistency: {}", cass_error_desc(rc))
);
}
} else {
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_QUORUM); rc != CASS_OK) {
throw std::runtime_error(fmt::format("Error setting cassandra consistency: {}", cass_error_desc(rc)));
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_QUORUM);
rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Error setting cassandra consistency: {}", cass_error_desc(rc))
);
}
}
if (auto const rc = cass_cluster_set_core_connections_per_host(*this, settings.coreConnectionsPerHost);
if (auto const rc =
cass_cluster_set_core_connections_per_host(*this, settings.coreConnectionsPerHost);
rc != CASS_OK) {
throw std::runtime_error(fmt::format("Could not set core connections per host: {}", cass_error_desc(rc)));
throw std::runtime_error(
fmt::format("Could not set core connections per host: {}", cass_error_desc(rc))
);
}
auto const queueSize =
settings.queueSizeIO.value_or(settings.maxWriteRequestsOutstanding + settings.maxReadRequestsOutstanding);
auto const queueSize = settings.queueSizeIO.value_or(
settings.maxWriteRequestsOutstanding + settings.maxReadRequestsOutstanding
);
if (auto const rc = cass_cluster_set_queue_size_io(*this, queueSize); rc != CASS_OK) {
throw std::runtime_error(fmt::format("Could not set queue size for IO per host: {}", cass_error_desc(rc)));
throw std::runtime_error(
fmt::format("Could not set queue size for IO per host: {}", cass_error_desc(rc))
);
}
setupConnection(settings);
@@ -111,7 +128,9 @@ Cluster::setupContactPoints(Settings::ContactPoints const& points)
auto throwErrorIfNeeded = [](CassError rc, std::string const& label, std::string const& value) {
if (rc != CASS_OK) {
throw std::runtime_error(
fmt::format("Cassandra: Error setting {} [{}]: {}", label, value, cass_error_desc(rc))
fmt::format(
"Cassandra: Error setting {} [{}]: {}", label, value, cass_error_desc(rc)
)
);
}
};
@@ -132,8 +151,12 @@ void
Cluster::setupSecureBundle(Settings::SecureConnectionBundle const& bundle)
{
LOG(log_.debug()) << "Attempt connection using secure bundle";
if (auto const rc = cass_cluster_set_cloud_secure_connection_bundle(*this, bundle.bundle.data()); rc != CASS_OK) {
throw std::runtime_error("Failed to connect using secure connection bundle " + bundle.bundle);
if (auto const rc =
cass_cluster_set_cloud_secure_connection_bundle(*this, bundle.bundle.data());
rc != CASS_OK) {
throw std::runtime_error(
"Failed to connect using secure connection bundle " + bundle.bundle
);
}
}
@@ -155,7 +178,9 @@ Cluster::setupCredentials(Settings const& settings)
return;
LOG(log_.debug()) << "Set credentials; username: " << settings.username.value();
cass_cluster_set_credentials(*this, settings.username.value().c_str(), settings.password.value().c_str());
cass_cluster_set_credentials(
*this, settings.username.value().c_str(), settings.password.value().c_str()
);
}
} // namespace data::cassandra::impl

21
src/data/cassandra/impl/Cluster.hpp
View File

@@ -79,7 +79,8 @@ struct Settings {
bool enableLog = false;
/** @brief Connect timeout specified in milliseconds */
std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{kDEFAULT_CONNECTION_TIMEOUT};
std::chrono::milliseconds connectionTimeout =
std::chrono::milliseconds{kDEFAULT_CONNECTION_TIMEOUT};
/** @brief Request timeout specified in milliseconds */
std::chrono::milliseconds requestTimeout = std::chrono::milliseconds{0}; // no timeout at all
@@ -106,25 +107,31 @@ struct Settings {
Provider provider = kDEFAULT_PROVIDER;
/** @brief Size of the IO queue */
std::optional<uint32_t> queueSizeIO = std::nullopt; // NOLINT(readability-redundant-member-init)
std::optional<uint32_t> queueSizeIO =
std::nullopt; // NOLINT(readability-redundant-member-init)
/** @brief SSL certificate */
std::optional<std::string> certificate = std::nullopt; // NOLINT(readability-redundant-member-init)
std::optional<std::string> certificate =
std::nullopt; // NOLINT(readability-redundant-member-init)
/** @brief Username/login */
std::optional<std::string> username = std::nullopt; // NOLINT(readability-redundant-member-init)
std::optional<std::string> username =
std::nullopt; // NOLINT(readability-redundant-member-init)
/** @brief Password to match the `username` */
std::optional<std::string> password = std::nullopt; // NOLINT(readability-redundant-member-init)
std::optional<std::string> password =
std::nullopt; // NOLINT(readability-redundant-member-init)
/**
* @brief Creates a new Settings object as a copy of the current one with overridden contact points.
* @brief Creates a new Settings object as a copy of the current one with overridden contact
* points.
*/
Settings
withContactPoints(std::string_view contactPoints)
{
auto tmp = *this;
tmp.connectionInfo = ContactPoints{.contactPoints = std::string{contactPoints}, .port = std::nullopt};
tmp.connectionInfo =
ContactPoints{.contactPoints = std::string{contactPoints}, .port = std::nullopt};
return tmp;
}

49
src/data/cassandra/impl/ExecutionStrategy.hpp
View File

@@ -267,8 +267,8 @@ public:
}
/**
* @brief Non-blocking query execution used for writing data. Contrast with write, this method does not execute
* the statements in a batch.
* @brief Non-blocking query execution used for writing data. Contrast with write, this method
* does not execute the statements in a batch.
*
* Retries forever with retry policy specified by @ref AsyncExecutor.
*
@@ -278,7 +278,9 @@ public:
void
writeEach(std::vector<StatementType>&& statements)
{
std::ranges::for_each(std::move(statements), [this](auto& statement) { this->write(std::move(statement)); });
std::ranges::for_each(std::move(statements), [this](auto& statement) {
this->write(std::move(statement));
});
}
/**
@@ -328,7 +330,9 @@ public:
future.emplace(handle_.get().asyncExecute(statements, [sself](auto&& res) mutable {
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); }
[sself, res = std::forward<decltype(res)>(res)]() mutable {
sself->complete(std::move(res));
}
);
}));
};
@@ -381,7 +385,9 @@ public:
future.emplace(handle_.get().asyncExecute(statement, [sself](auto&& res) mutable {
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself, res = std::forward<decltype(res)>(res)]() mutable { sself->complete(std::move(res)); }
[sself, res = std::forward<decltype(res)>(res)]() mutable {
sself->complete(std::move(res));
}
);
}));
};
@@ -431,19 +437,23 @@ public:
futures.reserve(numOutstanding);
counters_->registerReadStarted(statements.size());
auto init = [this, &statements, &futures, &errorsCount, &numOutstanding]<typename Self>(Self& self) {
auto init = [this, &statements, &futures, &errorsCount, &numOutstanding]<typename Self>(
Self& self
) {
auto sself = std::make_shared<Self>(std::move(self));
auto executionHandler = [&errorsCount, &numOutstanding, sself](auto const& res) mutable {
if (not res)
++errorsCount;
auto executionHandler =
[&errorsCount, &numOutstanding, sself](auto const& res) mutable {
if (not res)
++errorsCount;
// when all async operations complete unblock the result
if (--numOutstanding == 0) {
boost::asio::post(boost::asio::get_associated_executor(*sself), [sself]() mutable {
sself->complete();
});
}
};
// when all async operations complete unblock the result
if (--numOutstanding == 0) {
boost::asio::post(
boost::asio::get_associated_executor(*sself),
[sself]() mutable { sself->complete(); }
);
}
};
std::transform(
std::cbegin(statements),
@@ -461,7 +471,9 @@ public:
numReadRequestsOutstanding_ -= statements.size();
if (errorsCount > 0) {
ASSERT(errorsCount <= statements.size(), "Errors number cannot exceed statements number");
ASSERT(
errorsCount <= statements.size(), "Errors number cannot exceed statements number"
);
counters_->registerReadError(errorsCount);
counters_->registerReadFinished(startTime, statements.size() - errorsCount);
throw DatabaseTimeout{};
@@ -471,7 +483,8 @@ public:
std::vector<ResultType> results;
results.reserve(futures.size());
// it's safe to call blocking get on futures here as we already waited for the coroutine to resume above.
// it's safe to call blocking get on futures here as we already waited for the coroutine to
// resume above.
std::transform(
std::make_move_iterator(std::begin(futures)),
std::make_move_iterator(std::end(futures)),

4
src/data/cassandra/impl/Future.cpp
View File

@@ -76,8 +76,8 @@ void
invokeHelper(CassFuture* ptr, void* cbPtr)
{
// Note: can't use Future{ptr}.get() because double free will occur :/
// Note2: we are moving/copying it locally as a workaround for an issue we are seeing from asio recently.
// stackoverflow.com/questions/77004137/boost-asio-async-compose-gets-stuck-under-load
// Note2: we are moving/copying it locally as a workaround for an issue we are seeing from asio
// recently. stackoverflow.com/questions/77004137/boost-asio-async-compose-gets-stuck-under-load
auto* cb = static_cast<FutureWithCallback::FnType*>(cbPtr);
auto local = std::make_unique<FutureWithCallback::FnType>(std::move(*cb));
if (auto const rc = cass_future_error_code(ptr); rc) {

4
src/data/cassandra/impl/Result.hpp
View File

@@ -139,7 +139,9 @@ struct Result : public ManagedObject<CassResult const> {
std::size_t idx = 0;
auto advanceId = [&idx]() { return idx++; };
return std::make_optional<std::tuple<RowTypes...>>({extractColumn<RowTypes>(row, advanceId())...});
return std::make_optional<std::tuple<RowTypes...>>(
{extractColumn<RowTypes>(row, advanceId())...}
);
}
template <typename RowType>

8
src/data/cassandra/impl/RetryPolicy.hpp
View File

@@ -63,9 +63,11 @@ public:
[[nodiscard]] bool
shouldRetry([[maybe_unused]] CassandraError err)
{
auto const delayMs = std::chrono::duration_cast<std::chrono::milliseconds>(retry_.delayValue()).count();
LOG(log_.error()) << "Cassandra write error: " << err << ", current retries " << retry_.attemptNumber()
<< ", retrying in " << delayMs << " milliseconds";
auto const delayMs =
std::chrono::duration_cast<std::chrono::milliseconds>(retry_.delayValue()).count();
LOG(log_.error()) << "Cassandra write error: " << err << ", current retries "
<< retry_.attemptNumber() << ", retrying in " << delayMs
<< " milliseconds";
return true; // keep retrying forever
}

7
src/data/cassandra/impl/SslContext.cpp
View File

@@ -32,11 +32,14 @@ constexpr auto kCONTEXT_DELETER = [](CassSsl* ptr) { cass_ssl_free(ptr); };
namespace data::cassandra::impl {
SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), kCONTEXT_DELETER}
SslContext::SslContext(std::string const& certificate)
: ManagedObject{cass_ssl_new(), kCONTEXT_DELETER}
{
cass_ssl_set_verify_flags(*this, CASS_SSL_VERIFY_NONE);
if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str()); rc != CASS_OK) {
throw std::runtime_error(std::string{"Error setting Cassandra SSL Context: "} + cass_error_desc(rc));
throw std::runtime_error(
std::string{"Error setting Cassandra SSL Context: "} + cass_error_desc(rc)
);
}
}

20
src/data/cassandra/impl/Statement.hpp
View File

@@ -97,11 +97,15 @@ public:
using std::to_string;
auto throwErrorIfNeeded = [idx](CassError rc, std::string_view label) {
if (rc != CASS_OK)
throw std::logic_error(fmt::format("[{}] at idx {}: {}", label, idx, cass_error_desc(rc)));
throw std::logic_error(
fmt::format("[{}] at idx {}: {}", label, idx, cass_error_desc(rc))
);
};
auto bindBytes = [this, idx](auto const* data, size_t size) {
return cass_statement_bind_bytes(*this, idx, static_cast<cass_byte_t const*>(data), size);
return cass_statement_bind_bytes(
*this, idx, static_cast<cass_byte_t const*>(data), size
);
};
using DecayedType = std::decay_t<Type>;
@@ -110,7 +114,8 @@ public:
using UintByteTupleType = std::tuple<uint32_t, ripple::uint256>;
using ByteVectorType = std::vector<ripple::uint256>;
if constexpr (std::is_same_v<DecayedType, ripple::uint256> || std::is_same_v<DecayedType, ripple::uint192>) {
if constexpr (std::is_same_v<DecayedType, ripple::uint256> ||
std::is_same_v<DecayedType, ripple::uint192>) {
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind ripple::base_uint");
} else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
@@ -121,17 +126,20 @@ public:
throwErrorIfNeeded(rc, "Bind vector<unsigned char>");
} else if constexpr (std::is_convertible_v<DecayedType, std::string>) {
// reinterpret_cast is needed here :'(
auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
auto const rc =
bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
throwErrorIfNeeded(rc, "Bind string (as bytes)");
} else if constexpr (std::is_convertible_v<DecayedType, Text>) {
auto const rc = cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
auto const rc =
cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
throwErrorIfNeeded(rc, "Bind string (as TEXT)");
} else if constexpr (std::is_same_v<DecayedType, UintTupleType> ||
std::is_same_v<DecayedType, UintByteTupleType>) {
auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});
throwErrorIfNeeded(rc, "Bind tuple<uint32, uint32> or <uint32_t, ripple::uint256>");
} else if constexpr (std::is_same_v<DecayedType, ByteVectorType>) {
auto const rc = cass_statement_bind_collection(*this, idx, Collection{std::forward<Type>(value)});
auto const rc =
cass_statement_bind_collection(*this, idx, Collection{std::forward<Type>(value)});
throwErrorIfNeeded(rc, "Bind collection");
} else if constexpr (std::is_same_v<DecayedType, bool>) {
auto const rc = cass_statement_bind_bool(*this, idx, value ? cass_true : cass_false);

3
src/data/cassandra/impl/Tuple.cpp
View File

@@ -34,7 +34,8 @@ namespace data::cassandra::impl {
{
}
/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr) : ManagedObject{ptr, kTUPLE_ITERATOR_DELETER}
/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr)
: ManagedObject{ptr, kTUPLE_ITERATOR_DELETER}
{
}

4
src/data/cassandra/impl/Tuple.hpp
View File

@@ -65,7 +65,9 @@ public:
auto throwErrorIfNeeded = [idx](CassError rc, std::string_view label) {
if (rc != CASS_OK) {
auto const tag = '[' + std::string{label} + ']';
throw std::logic_error(tag + " at idx " + to_string(idx) + ": " + cass_error_desc(rc));
throw std::logic_error(
tag + " at idx " + to_string(idx) + ": " + cass_error_desc(rc)
);
}
};

35
src/data/impl/LedgerCacheFile.cpp
View File

@@ -92,7 +92,9 @@ LedgerCacheFile::write(DataView dataView)
}
Header const header{
.latestSeq = dataView.latestSeq, .mapSize = dataView.map.size(), .deletedSize = dataView.deleted.size()
.latestSeq = dataView.latestSeq,
.mapSize = dataView.map.size(),
.deletedSize = dataView.deleted.size()
};
file.write(header);
file.write(kSEPARATOR);
@@ -123,7 +125,9 @@ LedgerCacheFile::write(DataView dataView)
try {
std::filesystem::rename(newFilePath, path_);
} catch (std::exception const& e) {
return std::unexpected{fmt::format("Error moving cache file from {} to {}: {}", newFilePath, path_, e.what())};
return std::unexpected{
fmt::format("Error moving cache file from {} to {}: {}", newFilePath, path_, e.what())
};
}
return {};
@@ -145,12 +149,14 @@ LedgerCacheFile::read(uint32_t minLatestSequence)
return std::unexpected{"Error reading cache header"};
}
if (header.version != kVERSION) {
return std::unexpected{
fmt::format("Cache has wrong version: expected {} found {}", kVERSION, header.version)
};
return std::unexpected{fmt::format(
"Cache has wrong version: expected {} found {}", kVERSION, header.version
)};
}
if (header.latestSeq < minLatestSequence) {
return std::unexpected{fmt::format("Latest sequence ({}) in the cache file is too low.", header.latestSeq)};
return std::unexpected{
fmt::format("Latest sequence ({}) in the cache file is too low.", header.latestSeq)
};
}
result.latestSeq = header.latestSeq;
@@ -158,7 +164,8 @@ LedgerCacheFile::read(uint32_t minLatestSequence)
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading cache header"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
if (auto verificationResult = verifySeparator(separator);
not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
@@ -167,15 +174,16 @@ LedgerCacheFile::read(uint32_t minLatestSequence)
if (not cacheEntryExpected.has_value()) {
return std::unexpected{std::move(cacheEntryExpected).error()};
}
// Using insert with hint here to decrease insert operation complexity to the amortized constant instead of
// logN
// Using insert with hint here to decrease insert operation complexity to the amortized
// constant instead of logN
result.map.insert(result.map.end(), std::move(cacheEntryExpected).value());
}
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading separator"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
if (auto verificationResult = verifySeparator(separator);
not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
@@ -190,13 +198,16 @@ LedgerCacheFile::read(uint32_t minLatestSequence)
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading separator"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
if (auto verificationResult = verifySeparator(separator);
not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
auto const dataHash = file.hash();
ripple::uint256 hashFromFile{};
if (not file.readRaw(reinterpret_cast<char*>(hashFromFile.data()), decltype(hashFromFile)::bytes)) {
if (not file.readRaw(
reinterpret_cast<char*>(hashFromFile.data()), decltype(hashFromFile)::bytes
)) {
return std::unexpected{"Error reading hash"};
}

15
src/etl/CacheLoader.hpp
View File

@@ -110,7 +110,9 @@ public:
if (settings_.numCacheCursorsFromDiff != 0) {
LOG(log_.info()) << "Loading cache with cursor from num_cursors_from_diff="
<< settings_.numCacheCursorsFromDiff;
provider = std::make_shared<impl::CursorFromDiffProvider>(backend_, settings_.numCacheCursorsFromDiff);
provider = std::make_shared<impl::CursorFromDiffProvider>(
backend_, settings_.numCacheCursorsFromDiff
);
} else if (settings_.numCacheCursorsFromAccount != 0) {
LOG(log_.info()) << "Loading cache with cursor from num_cursors_from_account="
<< settings_.numCacheCursorsFromAccount;
@@ -118,8 +120,11 @@ public:
backend_, settings_.numCacheCursorsFromAccount, settings_.cachePageFetchSize
);
} else {
LOG(log_.info()) << "Loading cache with cursor from num_diffs=" << settings_.numCacheDiffs;
provider = std::make_shared<impl::CursorFromFixDiffNumProvider>(backend_, settings_.numCacheDiffs);
LOG(log_.info()) << "Loading cache with cursor from num_diffs="
<< settings_.numCacheDiffs;
provider = std::make_shared<impl::CursorFromFixDiffNumProvider>(
backend_, settings_.numCacheDiffs
);
}
loader_ = std::make_unique<CacheLoaderType>(
@@ -169,7 +174,9 @@ private:
auto const minLatestSequence =
backend_->fetchLedgerRange()
.transform([this](data::LedgerRange const& range) {
return std::max(range.maxSequence - settings_.cacheFileSettings->maxAge, range.minSequence);
return std::max(
range.maxSequence - settings_.cacheFileSettings->maxAge, range.minSequence
);
})
.value_or(0);

3
src/etl/CacheLoaderSettings.cpp
View File

@@ -66,7 +66,8 @@ makeCacheLoaderSettings(util::config::ClioConfigDefinition const& config)
if (auto filePath = cache.maybeValue<std::string>("file.path"); filePath.has_value()) {
settings.cacheFileSettings = CacheLoaderSettings::CacheFileSettings{
.path = std::move(filePath).value(), .maxAge = cache.get<uint32_t>("file.max_sequence_age")
.path = std::move(filePath).value(),
.maxAge = cache.get<uint32_t>("file.max_sequence_age")
};
}

17
src/etl/CacheLoaderSettings.hpp
View File

@@ -37,22 +37,25 @@ struct CacheLoaderSettings {
/** @brief Settings for cache file operations */
struct CacheFileSettings {
std::string path; /**< path to the file to load cache from on start and save cache to on shutdown */
std::string
path; /**< path to the file to load cache from on start and save cache to on shutdown */
uint32_t maxAge = 5000; /**< max difference between latest sequence in cache file and DB */
auto
operator<=>(CacheFileSettings const&) const = default;
};
size_t numCacheDiffs = 32; /**< number of diffs to use to generate cursors */
size_t numCacheMarkers = 48; /**< number of markers to use at one time to traverse the ledger */
size_t cachePageFetchSize = 512; /**< number of ledger objects to fetch concurrently per marker */
size_t numThreads = 2; /**< number of threads to use for loading cache */
size_t numCacheDiffs = 32; /**< number of diffs to use to generate cursors */
size_t numCacheMarkers = 48; /**< number of markers to use at one time to traverse the ledger */
size_t cachePageFetchSize =
512; /**< number of ledger objects to fetch concurrently per marker */
size_t numThreads = 2; /**< number of threads to use for loading cache */
size_t numCacheCursorsFromDiff = 0; /**< number of cursors to fetch from diff */
size_t numCacheCursorsFromAccount = 0; /**< number of cursors to fetch from account_tx */
LoadStyle loadStyle = LoadStyle::ASYNC; /**< how to load the cache */
std::optional<CacheFileSettings> cacheFileSettings; /**< optional settings for cache file operations */
LoadStyle loadStyle = LoadStyle::ASYNC; /**< how to load the cache */
std::optional<CacheFileSettings>
cacheFileSettings; /**< optional settings for cache file operations */
auto
operator<=>(CacheLoaderSettings const&) const = default;

4
src/etl/CorruptionDetector.hpp
View File

@@ -59,7 +59,9 @@ public:
if (not state_.get().isCorruptionDetected) {
state_.get().isCorruptionDetected = true;
LOG(log_.error()) << "Disabling the cache to avoid corrupting the DB further. Please investigate.";
LOG(
log_.error()
) << "Disabling the cache to avoid corrupting the DB further. Please investigate.";
cache_.get().setDisabled();
}
}

11
src/etl/ETLHelpers.hpp
View File

@@ -33,12 +33,13 @@
namespace etl {
// TODO: does the note make sense? lockfree queues provide the same blocking behaviour just without mutex, don't they?
// TODO: does the note make sense? lockfree queues provide the same blocking behaviour just without
// mutex, don't they?
/**
* @brief Generic thread-safe queue with a max capacity.
*
* @note (original note) We can't use a lockfree queue here, since we need the ability to wait for an element to be
* added or removed from the queue. These waits are blocking calls.
* @note (original note) We can't use a lockfree queue here, since we need the ability to wait for
* an element to be added or removed from the queue. These waits are blocking calls.
*/
template <typename T>
class ThreadSafeQueue {
@@ -52,8 +53,8 @@ public:
/**
* @brief Create an instance of the queue.
*
* @param maxSize maximum size of the queue. Calls that would cause the queue to exceed this size will block until
* free space is available.
* @param maxSize maximum size of the queue. Calls that would cause the queue to exceed this
* size will block until free space is available.
*/
ThreadSafeQueue(uint32_t maxSize) : maxSize_(maxSize)
{

31
src/etl/ETLService.cpp
View File

@@ -112,7 +112,8 @@ ETLService::makeETLService(
state
);
auto taskManagerProvider = std::make_shared<impl::TaskManagerProvider>(*ledgers, extractor, loader);
auto taskManagerProvider =
std::make_shared<impl::TaskManagerProvider>(*ledgers, extractor, loader);
ret = std::make_shared<ETLService>(
ctx,
@@ -131,7 +132,8 @@ ETLService::makeETLService(
state
);
// inject networkID into subscriptions, as transaction feed require it to inject CTID in response
// inject networkID into subscriptions, as transaction feed require it to inject CTID in
// response
if (auto const etlState = ret->getETLState(); etlState)
subscriptions->setNetworkID(etlState->networkID);
@@ -181,7 +183,8 @@ ETLService::ETLService(
if (finishSequence_.has_value())
LOG(log_.info()) << "Finish sequence: " << *finishSequence_;
LOG(log_.info()) << "Starting in " << (state_->isStrictReadonly ? "STRICT READONLY MODE" : "WRITE MODE");
LOG(log_.info()) << "Starting in "
<< (state_->isStrictReadonly ? "STRICT READONLY MODE" : "WRITE MODE");
}
ETLService::~ETLService()
@@ -213,7 +216,8 @@ ETLService::run()
}
auto const nextSequence = syncCacheWithDb();
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = "
<< nextSequence;
startMonitor(nextSequence);
@@ -290,7 +294,8 @@ ETLService::loadInitialLedgerIfNeeded()
if (not rng.has_value()) {
ASSERT(
not state_->isStrictReadonly,
"Database is empty but this node is in strict readonly mode. Can't write initial ledger."
"Database is empty but this node is in strict readonly mode. Can't write initial "
"ledger."
);
LOG(log_.info()) << "Database is empty. Will download a ledger from the network.";
@@ -309,9 +314,11 @@ ETLService::loadInitialLedgerIfNeeded()
auto [ledger, timeDiff] = ::util::timed<std::chrono::duration<double>>([this, seq]() {
return extractor_->extractLedgerOnly(seq).and_then(
[this, seq](auto&& data) -> std::optional<ripple::LedgerHeader> {
// TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or similar
// TODO: loadInitialLedger in balancer should be called fetchEdgeKeys or
// similar
auto res = balancer_->loadInitialLedger(seq, *initialLoadObserver_);
if (not res.has_value() and res.error() == InitialLedgerLoadError::Cancelled) {
if (not res.has_value() and
res.error() == InitialLedgerLoadError::Cancelled) {
LOG(log_.debug()) << "Initial ledger load got cancelled";
return std::nullopt;
}
@@ -330,7 +337,8 @@ ETLService::loadInitialLedgerIfNeeded()
}
LOG(log_.debug()) << "Time to download and store ledger = " << timeDiff;
LOG(log_.info()) << "Finished loadInitialLedger. cache size = " << backend_->cache().size();
LOG(log_.info()) << "Finished loadInitialLedger. cache size = "
<< backend_->cache().size();
return backend_->hardFetchLedgerRangeNoThrow();
}
@@ -353,7 +361,8 @@ ETLService::syncCacheWithDb()
{
auto rng = backend_->hardFetchLedgerRangeNoThrow();
while (not backend_->cache().isDisabled() and rng->maxSequence > backend_->cache().latestLedgerSequence()) {
while (not backend_->cache().isDisabled() and
rng->maxSequence > backend_->cache().latestLedgerSequence()) {
LOG(log_.info()) << "Syncing cache with DB. DB latest seq: " << rng->maxSequence
<< ". Cache latest seq: " << backend_->cache().latestLedgerSequence();
for (auto seq = backend_->cache().latestLedgerSequence(); seq <= rng->maxSequence; ++seq) {
@@ -443,8 +452,8 @@ ETLService::attemptTakeoverWriter()
if (backend_->cache().latestLedgerSequence() != rng->maxSequence) {
LOG(log_.info()) << "Wanted to take over the ETL writer seat but LedgerCache is outdated";
// Give ETL time to update LedgerCache. This method will be called because ClusterCommunication will likely to
// continue sending StartWriting signal every 1 second
// Give ETL time to update LedgerCache. This method will be called because
// ClusterCommunication will likely to continue sending StartWriting signal every 1 second
return;
}

20
src/etl/ETLService.hpp
View File

@@ -80,17 +80,19 @@
namespace etl {
/**
* @brief This class is responsible for continuously extracting data from a p2p node, and writing that data to the
* databases.
* @brief This class is responsible for continuously extracting data from a p2p node, and writing
* that data to the databases.
*
* Usually, multiple different processes share access to the same network accessible databases, in which case only one
* such process is performing ETL and writing to the database. The other processes simply monitor the database for new
* ledgers, and publish those ledgers to the various subscription streams. If a monitoring process determines that the
* ETL writer has failed (no new ledgers written for some time), the process will attempt to become the ETL writer.
* Usually, multiple different processes share access to the same network accessible databases, in
* which case only one such process is performing ETL and writing to the database. The other
* processes simply monitor the database for new ledgers, and publish those ledgers to the various
* subscription streams. If a monitoring process determines that the ETL writer has failed (no new
* ledgers written for some time), the process will attempt to become the ETL writer.
*
* If there are multiple monitoring processes that try to become the ETL writer at the same time, one will win out, and
* the others will fall back to monitoring/publishing. In this sense, this class dynamically transitions from monitoring
* to writing and from writing to monitoring, based on the activity of other processes running on different machines.
* If there are multiple monitoring processes that try to become the ETL writer at the same time,
* one will win out, and the others will fall back to monitoring/publishing. In this sense, this
* class dynamically transitions from monitoring to writing and from writing to monitoring, based on
* the activity of other processes running on different machines.
*/
class ETLService : public ETLServiceInterface {
util::Logger log_{"ETL"};

4
src/etl/ETLServiceInterface.hpp
View File

@@ -30,8 +30,8 @@ namespace etl {
/**
* @brief This is a base class for any ETL service implementations.
* @note A ETL service is responsible for continuously extracting data from a p2p node, and writing that data to the
* databases.
* @note A ETL service is responsible for continuously extracting data from a p2p node, and writing
* that data to the databases.
*/
struct ETLServiceInterface {
virtual ~ETLServiceInterface() = default;

3
src/etl/ETLState.cpp
View File

@@ -36,7 +36,8 @@ tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv)
ETLState state;
auto const& jsonObject = jv.as_object();
if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) {
if (jsonObject.contains(JS(result)) &&
jsonObject.at(JS(result)).as_object().contains(JS(info))) {
auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object();
if (rippledInfo.contains(JS(network_id)))
state.networkID = boost::json::value_to<int64_t>(rippledInfo.at(JS(network_id)));

22
src/etl/ETLState.hpp
View File

@@ -35,13 +35,14 @@
namespace etl {
/**
* @brief This class is responsible for fetching and storing the state of the ETL information, such as the network id
* @brief This class is responsible for fetching and storing the state of the ETL information, such
* as the network id
*/
struct ETLState {
/*
* NOTE: Rippled NetworkID: Mainnet = 0; Testnet = 1; Devnet = 2
* However, if rippled is running on neither of these (ie. standalone mode) rippled will default to 0, but
* is not included in the stateOpt response. Must manually add it here.
* However, if rippled is running on neither of these (ie. standalone mode) rippled will default
* to 0, but is not included in the stateOpt response. Must manually add it here.
*/
uint32_t networkID{0};
@@ -54,12 +55,15 @@ struct ETLState {
static std::optional<ETLState>
fetchETLStateFromSource(Forward& source) noexcept
{
auto const serverInfoRippled = data::synchronous([&source](auto yield) -> std::optional<boost::json::object> {
if (auto result = source.forwardToRippled({{"command", "server_info"}}, std::nullopt, {}, yield)) {
return std::move(result).value();
}
return std::nullopt;
});
auto const serverInfoRippled =
data::synchronous([&source](auto yield) -> std::optional<boost::json::object> {
if (auto result = source.forwardToRippled(
{{"command", "server_info"}}, std::nullopt, {}, yield
)) {
return std::move(result).value();
}
return std::nullopt;
});
if (serverInfoRippled && not serverInfoRippled->contains(JS(error))) {
return boost::json::value_to<ETLState>(boost::json::value(*serverInfoRippled));

14
src/etl/LedgerFetcherInterface.hpp
View File

@@ -40,11 +40,12 @@ struct LedgerFetcherInterface {
/**
* @brief Extract data for a particular ledger from an ETL source
*
* This function continuously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
* This function continuously tries to extract the specified ledger (using all available ETL
* sources) until the extraction succeeds, or the server shuts down.
*
* @param seq sequence of the ledger to extract
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is
* shutting down
*/
[[nodiscard]] virtual OptionalGetLedgerResponseType
fetchData(uint32_t seq) = 0;
@@ -52,11 +53,12 @@ struct LedgerFetcherInterface {
/**
* @brief Extract diff data for a particular ledger from an ETL source.
*
* This function continuously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
* This function continuously tries to extract the specified ledger (using all available ETL
* sources) until the extraction succeeds, or the server shuts down.
*
* @param seq sequence of the ledger to extract
* @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
* @return Ledger data diff between sequance and parent; Empty optional if the server is
* shutting down
*/
[[nodiscard]] virtual OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t seq) = 0;

48
src/etl/LoadBalancer.cpp
View File

@@ -111,7 +111,8 @@ LoadBalancer::LoadBalancer(
.retries = PrometheusService::counterInt(
"forwarding_retries_counter",
Labels(),
"The number of retries before a forwarded request was successful. Initial attempt excluded"
"The number of retries before a forwarded request was successful. Initial attempt "
"excluded"
),
.cacheHit = PrometheusService::counterInt(
"forwarding_cache_hit_counter",
@@ -147,7 +148,9 @@ LoadBalancer::LoadBalancer(
LOG(log_.warn()) << log;
if (!allowNoEtl) {
LOG(log_.error()) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources.";
LOG(
log_.error()
) << "Set allow_no_etl as true in config to allow clio run without valid ETL sources.";
throw std::logic_error("ETL configuration error.");
}
};
@@ -185,7 +188,8 @@ LoadBalancer::LoadBalancer(
} else if (etlState_ && etlState_->networkID != stateOpt->networkID) {
checkOnETLFailure(
fmt::format(
"ETL sources must be on the same network. Source network id = {} does not match others network id "
"ETL sources must be on the same network. Source network id = {} does not "
"match others network id "
"= {}",
stateOpt->networkID,
etlState_->networkID
@@ -200,7 +204,9 @@ LoadBalancer::LoadBalancer(
}
if (!etlState_)
checkOnETLFailure("Failed to fetch ETL state from any source. Please check the configuration and network");
checkOnETLFailure(
"Failed to fetch ETL state from any source. Please check the configuration and network"
);
if (sources_.empty())
checkOnETLFailure("No ETL sources configured. Please check the configuration");
@@ -227,7 +233,8 @@ LoadBalancer::loadInitialLedger(
if (not res.has_value() and res.error() == InitialLedgerLoadError::Errored) {
LOG(log_.error()) << "Failed to download initial ledger."
<< " Sequence = " << sequence << " source = " << source->toString();
<< " Sequence = " << sequence
<< " source = " << source->toString();
return false; // should retry on error
}
@@ -252,7 +259,8 @@ LoadBalancer::fetchLedger(
GetLedgerResponseType response;
execute(
[&response, ledgerSequence, getObjects, getObjectNeighbors, log = log_](auto& source) {
auto [status, data] = source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
auto [status, data] =
source->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
response = std::move(data);
if (status.ok() && response.validated()) {
LOG(log.info()) << "Successfully fetched ledger = " << ledgerSequence
@@ -260,8 +268,10 @@ LoadBalancer::fetchLedger(
return true;
}
LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence << ", Reply: " << response.DebugString()
<< ", error_code: " << status.error_code() << ", error_msg: " << status.error_message()
LOG(log.warn()) << "Could not fetch ledger " << ledgerSequence
<< ", Reply: " << response.DebugString()
<< ", error_code: " << status.error_code()
<< ", error_msg: " << status.error_message()
<< ", source = " << source->toString();
return false;
},
@@ -301,8 +311,9 @@ LoadBalancer::forwardToRippled(
std::optional<boost::json::object> response;
rpc::ClioError error = rpc::ClioError::EtlConnectionError;
while (numAttempts < sources_.size()) {
auto [res, duration] =
util::timed([&]() { return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield); });
auto [res, duration] = util::timed([&]() {
return sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
});
if (res) {
forwardingCounters_.successDuration.get() += duration;
response = std::move(res).value();
@@ -337,7 +348,11 @@ LoadBalancer::toJson() const
template <typename Func>
void
LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter)
LoadBalancer::execute(
Func f,
uint32_t ledgerSequence,
std::chrono::steady_clock::duration retryAfter
)
{
ASSERT(not sources_.empty(), "ETL sources must be configured to execute functions.");
size_t sourceIdx = randomGenerator_->uniform(0ul, sources_.size() - 1);
@@ -370,8 +385,11 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock
sourceIdx = (sourceIdx + 1) % sources_.size();
numAttempts++;
if (numAttempts % sources_.size() == 0) {
LOG(log_.info()) << "Ledger sequence " << ledgerSequence
<< " is not yet available from any configured sources. Sleeping and trying again";
LOG(
log_.info()
) << "Ledger sequence "
<< ledgerSequence
<< " is not yet available from any configured sources. Sleeping and trying again";
std::this_thread::sleep_for(retryAfter);
}
}
@@ -392,7 +410,9 @@ LoadBalancer::stop(boost::asio::yield_context yield)
{
util::CoroutineGroup group{yield};
std::ranges::for_each(sources_, [&group, yield](auto& source) {
group.spawn(yield, [&source](boost::asio::yield_context innerYield) { source->stop(innerYield); });
group.spawn(yield, [&source](boost::asio::yield_context innerYield) {
source->stop(innerYield);
});
});
group.asyncWait(yield);
}

41
src/etl/LoadBalancer.hpp
View File

@@ -70,9 +70,10 @@ concept SomeLoadBalancer = std::derived_from<T, LoadBalancerTag>;
/**
* @brief This class is used to manage connections to transaction processing processes.
*
* This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of
* which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
* allows requests for ledger data to be load balanced across all possible ETL sources.
* This class spawns a listener for each etl source, which listens to messages on the ledgers stream
* (to keep track of which ledgers have been validated by the network, and the range of ledgers each
* etl source has). This class also allows requests for ledger data to be load balanced across all
* possible ETL sources.
*/
class LoadBalancer : public LoadBalancerInterface, LoadBalancerTag {
public:
@@ -84,7 +85,8 @@ private:
static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16;
util::Logger log_{"ETL"};
// Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
// Forwarding cache must be destroyed after sources because sources have a callback to
// invalidate cache
std::optional<util::ResponseExpirationCache> forwardingCache_;
std::optional<std::string> forwardingXUserValue_;
@@ -92,8 +94,8 @@ private:
std::vector<SourcePtr> sources_;
std::optional<ETLState> etlState_;
std::uint32_t downloadRanges_ =
kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
std::uint32_t downloadRanges_ = kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when
downloading initial ledger */
struct ForwardingCounters {
std::reference_wrapper<util::prometheus::CounterInt> successDuration;
@@ -104,7 +106,8 @@ private:
} forwardingCounters_;
// Using mutex instead of atomic_bool because choosing a new source to
// forward messages should be done with a mutual exclusion otherwise there will be a race condition
// forward messages should be done with a mutual exclusion otherwise there will be a race
// condition
util::Mutex<bool> hasForwardingSource_{false};
public:
@@ -164,12 +167,14 @@ public:
/**
* @brief Load the initial ledger, writing data to the queue.
* @note This function will retry indefinitely until the ledger is downloaded or the download is cancelled.
* @note This function will retry indefinitely until the ledger is downloaded or the download is
* cancelled.
*
* @param sequence Sequence of ledger to download
* @param observer The observer to notify of progress
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::expected with ledger edge keys on success, or InitialLedgerLoadError on failure
* @return A std::expected with ledger edge keys on success, or InitialLedgerLoadError on
* failure
*/
InitialLedgerLoadResult
loadInitialLedger(
@@ -181,8 +186,8 @@ public:
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
* This function will continuously try to fetch data for the specified ledger until the fetch
* succeeds, the ledger is found in the database, or the server is shutting down.
*
* @param ledgerSequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one
@@ -245,17 +250,23 @@ private:
* @brief Execute a function on a randomly selected source.
*
* @note f is a function that takes an Source as an argument and returns a bool.
* Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another
* randomly chosen Source is used. The process repeats until f returns true.
* Attempt to execute f for one randomly chosen Source that has the specified ledger. If f
* returns false, another randomly chosen Source is used. The process repeats until f returns
* true.
*
* @param f Function to execute. This function takes the ETL source as an argument, and returns a bool
* @param f Function to execute. This function takes the ETL source as an argument, and returns
* a bool
* @param ledgerSequence f is executed for each Source that has this ledger
* @param retryAfter Time to wait between retries (2 seconds by default)
* server is shutting down
*/
template <typename Func>
void
execute(Func f, uint32_t ledgerSequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2});
execute(
Func f,
uint32_t ledgerSequence,
std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
);
/**
* @brief Choose a new source to forward requests

10
src/etl/LoadBalancerInterface.hpp
View File

@@ -66,12 +66,14 @@ public:
/**
* @brief Load the initial ledger, writing data to the queue.
* @note This function will retry indefinitely until the ledger is downloaded or the download is cancelled.
* @note This function will retry indefinitely until the ledger is downloaded or the download is
* cancelled.
*
* @param sequence Sequence of ledger to download
* @param loader InitialLoadObserverInterface implementation
* @param retryAfter Time to wait between retries (2 seconds by default)
* @return A std::expected with ledger edge keys on success, or InitialLedgerLoadError on failure
* @return A std::expected with ledger edge keys on success, or InitialLedgerLoadError on
* failure
*/
[[nodiscard]] virtual InitialLedgerLoadResult
loadInitialLedger(
@@ -83,8 +85,8 @@ public:
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
* This function will continuously try to fetch data for the specified ledger until the fetch
* succeeds, the ledger is found in the database, or the server is shutting down.
*
* @param ledgerSequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one

16
src/etl/MPTHelpers.cpp
View File

@@ -53,7 +53,8 @@ getMPTokenAuthorize(ripple::TxMeta const& txMeta)
if (node.getFName() == ripple::sfCreatedNode) {
auto const& newMPT = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
return MPTHolderData{
.mptID = newMPT[ripple::sfMPTokenIssuanceID], .holder = newMPT.getAccountID(ripple::sfAccount)
.mptID = newMPT[ripple::sfMPTokenIssuanceID],
.holder = newMPT.getAccountID(ripple::sfAccount)
};
}
}
@@ -63,7 +64,8 @@ getMPTokenAuthorize(ripple::TxMeta const& txMeta)
std::optional<MPTHolderData>
getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
if (txMeta.getResultTER() != ripple::tesSUCCESS || sttx.getTxnType() != ripple::TxType::ttMPTOKEN_AUTHORIZE)
if (txMeta.getResultTER() != ripple::tesSUCCESS ||
sttx.getTxnType() != ripple::TxType::ttMPTOKEN_AUTHORIZE)
return {};
return getMPTokenAuthorize(txMeta);
@@ -73,10 +75,14 @@ std::optional<MPTHolderData>
getMPTHolderFromObj(std::string const& key, std::string const& blob)
{
// https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0033-multi-purpose-tokens#2121-mptoken-ledger-identifier
ASSERT(key.size() == ripple::uint256::size(), "The size of the key is expected to fit uint256 exactly");
ASSERT(
key.size() == ripple::uint256::size(),
"The size of the key is expected to fit uint256 exactly"
);
ripple::STLedgerEntry const sle =
ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
ripple::STLedgerEntry const sle = ripple::STLedgerEntry(
ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data())
);
if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
return {};

4
src/etl/Models.hpp
View File

@@ -43,8 +43,8 @@ namespace etl::model {
/**
* @brief A specification for the Registry.
*
* This specification simply defines the transaction types that are to be filtered out from the incoming transactions by
* the Registry for its `onTransaction` and `onInitialTransaction` hooks.
* This specification simply defines the transaction types that are to be filtered out from the
* incoming transactions by the Registry for its `onTransaction` and `onInitialTransaction` hooks.
* It's a compilation error to list the same transaction type more than once.
*/
template <ripple::TxType... Types>

7
src/etl/MonitorInterface.hpp
View File

@@ -30,8 +30,8 @@ namespace etl {
/**
* @brief An interface for the monitor service
* An implementation of this service is responsible for periodically checking various datasources to detect newly
* ingested ledgers.
* An implementation of this service is responsible for periodically checking various datasources to
* detect newly ingested ledgers.
*/
class MonitorInterface {
public:
@@ -65,7 +65,8 @@ public:
subscribeToNewSequence(NewSequenceSignalType::slot_type const& subscriber) = 0;
/**
* @brief Allows clients to get notified when no database update is detected for a configured period.
* @brief Allows clients to get notified when no database update is detected for a configured
* period.
*
* @param subscriber The slot to connect
* @return A connection object that automatically disconnects the subscription once destroyed

104
src/etl/NFTHelpers.cpp
View File

@@ -54,7 +54,9 @@ getNftokenModifyData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
auto const tokenID = sttx.getFieldH256(ripple::sfNFTokenID);
// note: sfURI is optional, if it is absent, we will update the uri as empty string
return {
{NFTTransactionsData(sttx.getFieldH256(ripple::sfNFTokenID), txMeta, sttx.getTransactionID())},
{NFTTransactionsData(
sttx.getFieldH256(ripple::sfNFTokenID), txMeta, sttx.getTransactionID()
)},
NFTsData(tokenID, txMeta, sttx.getFieldVL(ripple::sfURI))
};
}
@@ -83,8 +85,9 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
owner = ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data());
if (node.getFName() == ripple::sfCreatedNode) {
ripple::STArray const& toAddNFTs =
node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
ripple::STArray const& toAddNFTs = node.peekAtField(ripple::sfNewFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
std::ranges::transform(
toAddNFTs,
@@ -117,8 +120,9 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
[](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
);
ripple::STArray const& toAddFinalNFTs =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
ripple::STArray const& toAddFinalNFTs = node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
std::ranges::transform(
toAddFinalNFTs,
@@ -134,7 +138,8 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
// Find the first NFT ID that doesn't match. We're looking for an
// added NFT, so the one we want will be the mismatch in finalIDs.
// NOLINTNEXTLINE(modernize-use-ranges)
auto const diff = std::mismatch(finalIDs.begin(), finalIDs.end(), prevIDs.begin(), prevIDs.end());
auto const diff =
std::mismatch(finalIDs.begin(), finalIDs.end(), prevIDs.begin(), prevIDs.end());
// There should always be a difference so the returned finalIDs
// iterator should never be end(). But better safe than sorry.
@@ -154,7 +159,9 @@ std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
ripple::uint256 const tokenID = sttx.getFieldH256(ripple::sfNFTokenID);
std::vector<NFTTransactionsData> const txs = {NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())};
std::vector<NFTTransactionsData> const txs = {
NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())
};
// Determine who owned the token when it was burned by finding an
// NFTokenPage that was deleted or modified that contains this
@@ -180,22 +187,27 @@ getNFTokenBurnData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
if (previousFields.isFieldPresent(ripple::sfNFTokens))
prevNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
} else if (node.getFName() == ripple::sfDeletedNode) {
prevNFTs =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
prevNFTs = node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldArray(ripple::sfNFTokens);
}
if (!prevNFTs)
continue;
auto const nft =
std::find_if(prevNFTs->begin(), prevNFTs->end(), [&tokenID](ripple::STObject const& candidate) {
auto const nft = std::find_if(
prevNFTs->begin(), prevNFTs->end(), [&tokenID](ripple::STObject const& candidate) {
return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
});
}
);
if (nft != prevNFTs->end()) {
return std::make_pair(
txs,
NFTsData(
tokenID, ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data()), txMeta, true
tokenID,
ripple::AccountID::fromVoid(node.getFieldH256(ripple::sfLedgerIndex).data()),
txMeta,
true
)
);
}
@@ -213,10 +225,14 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
// more easily by just looking at the owner of the accepted NFTokenOffer
// object.
if (sttx.isFieldPresent(ripple::sfNFTokenBuyOffer)) {
auto const affectedBuyOffer =
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenBuyOffer);
});
auto const affectedBuyOffer = std::find_if(
txMeta.getNodes().begin(),
txMeta.getNodes().end(),
[&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) ==
sttx.getFieldH256(ripple::sfNFTokenBuyOffer);
}
);
if (affectedBuyOffer == txMeta.getNodes().end()) {
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
@@ -231,15 +247,18 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
.downcast<ripple::STObject>()
.getAccountID(ripple::sfOwner);
return {
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())}, NFTsData(tokenID, owner, txMeta, false)
{NFTTransactionsData(tokenID, txMeta, sttx.getTransactionID())},
NFTsData(tokenID, owner, txMeta, false)
};
}
// Otherwise we have to infer the new owner from the affected nodes.
auto const affectedSellOffer =
std::find_if(txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) == sttx.getFieldH256(ripple::sfNFTokenSellOffer);
});
auto const affectedSellOffer = std::find_if(
txMeta.getNodes().begin(), txMeta.getNodes().end(), [&sttx](ripple::STObject const& node) {
return node.getFieldH256(ripple::sfLedgerIndex) ==
sttx.getFieldH256(ripple::sfNFTokenSellOffer);
}
);
if (affectedSellOffer == txMeta.getNodes().end()) {
std::stringstream msg;
msg << " - unexpected NFTokenAcceptOffer data in tx " << sttx.getTransactionID();
@@ -303,8 +322,9 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_OFFER)
continue;
ripple::uint256 const tokenID =
node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldH256(ripple::sfNFTokenID);
ripple::uint256 const tokenID = node.peekAtField(ripple::sfFinalFields)
.downcast<ripple::STObject>()
.getFieldH256(ripple::sfNFTokenID);
txs.emplace_back(tokenID, txMeta, sttx.getTransactionID());
}
@@ -312,9 +332,10 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
std::ranges::sort(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID < b.tokenID;
});
auto [last, end] = std::ranges::unique(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID == b.tokenID;
});
auto [last, end] =
std::ranges::unique(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
return a.tokenID == b.tokenID;
});
txs.erase(last, end);
return {txs, {}};
}
@@ -324,7 +345,12 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTokenCreateOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
{
return {{NFTTransactionsData(sttx.getFieldH256(ripple::sfNFTokenID), txMeta, sttx.getTransactionID())}, {}};
return {
{NFTTransactionsData(
sttx.getFieldH256(ripple::sfNFTokenID), txMeta, sttx.getTransactionID()
)},
{}
};
}
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
@@ -361,10 +387,14 @@ std::vector<NFTsData>
getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob)
{
// https://github.com/XRPLF/XRPL-Standards/tree/master/XLS-0020-non-fungible-tokens#tokenpage-id-format
ASSERT(key.size() == ripple::uint256::size(), "The size of the key (token) is expected to fit uint256 exactly");
ASSERT(
key.size() == ripple::uint256::size(),
"The size of the key (token) is expected to fit uint256 exactly"
);
auto const sle =
ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
auto const sle = ripple::STLedgerEntry(
ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data())
);
if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltNFTOKEN_PAGE)
return {};
@@ -373,7 +403,9 @@ getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string c
std::vector<NFTsData> nfts;
for (ripple::STObject const& node : sle.getFieldArray(ripple::sfNFTokens))
nfts.emplace_back(node.getFieldH256(ripple::sfNFTokenID), seq, owner, node.getFieldVL(ripple::sfURI));
nfts.emplace_back(
node.getFieldH256(ripple::sfNFTokenID), seq, owner, node.getFieldVL(ripple::sfURI)
);
return nfts;
}
@@ -384,11 +416,13 @@ getUniqueNFTsDatas(std::vector<NFTsData> const& nfts)
std::vector<NFTsData> results = nfts;
std::ranges::sort(results, [](NFTsData const& a, NFTsData const& b) {
return a.tokenID == b.tokenID ? a.transactionIndex > b.transactionIndex : a.tokenID > b.tokenID;
return a.tokenID == b.tokenID ? a.transactionIndex > b.transactionIndex
: a.tokenID > b.tokenID;
});
auto const [last, end] =
std::ranges::unique(results, [](NFTsData const& a, NFTsData const& b) { return a.tokenID == b.tokenID; });
auto const [last, end] = std::ranges::unique(results, [](NFTsData const& a, NFTsData const& b) {
return a.tokenID == b.tokenID;
});
results.erase(last, end);
return results;
}

3
src/etl/NFTHelpers.hpp
View File

@@ -115,7 +115,8 @@ std::vector<NFTsData>
getNFTDataFromObj(std::uint32_t seq, std::string const& key, std::string const& blob);
/**
* @brief Get the unique NFTs data from a vector of NFTsData happening in the same ledger. For example, if a NFT has
* @brief Get the unique NFTs data from a vector of NFTsData happening in the same ledger. For
example, if a NFT has
* both accept offer and burn happening in the same ledger,we only keep the final state of the NFT.
* @param nfts The NFTs data to filter, happening in the same ledger

5
src/etl/NetworkValidatedLedgers.cpp
View File

@@ -54,7 +54,10 @@ NetworkValidatedLedgers::getMostRecent()
}
bool
NetworkValidatedLedgers::waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs)
NetworkValidatedLedgers::waitUntilValidatedByNetwork(
uint32_t sequence,
std::optional<uint32_t> maxWaitMs
)
{
std::unique_lock lck(mtx_);
auto pred = [sequence, this]() -> bool { return (latest_ && sequence <= *latest_); };

24
src/etl/NetworkValidatedLedgers.hpp
View File

@@ -34,12 +34,13 @@
namespace etl {
/**
* @brief This datastructure is used to keep track of the sequence of the most recent ledger validated by the network.
* @brief This datastructure is used to keep track of the sequence of the most recent ledger
* validated by the network.
*
* There are two methods that will wait until certain conditions are met. This datastructure is able to be "stopped".
* When the datastructure is stopped, any threads currently waiting are unblocked.
* Any later calls to methods of this datastructure will not wait. Once the datastructure is stopped, the datastructure
* remains stopped for the rest of its lifetime.
* There are two methods that will wait until certain conditions are met. This datastructure is able
* to be "stopped". When the datastructure is stopped, any threads currently waiting are unblocked.
* Any later calls to methods of this datastructure will not wait. Once the datastructure is
* stopped, the datastructure remains stopped for the rest of its lifetime.
*/
class NetworkValidatedLedgers : public NetworkValidatedLedgersInterface {
std::optional<uint32_t> latest_; // currently known latest sequence validated by network
@@ -69,9 +70,11 @@ public:
/**
* @brief Get most recently validated sequence.
*
* If no ledgers are known to have been validated, this function waits until the next ledger is validated
* If no ledgers are known to have been validated, this function waits until the next ledger is
* validated
*
* @return Sequence of most recently validated ledger. empty optional if the datastructure has been stopped
* @return Sequence of most recently validated ledger. empty optional if the datastructure has
* been stopped
*/
std::optional<uint32_t>
getMostRecent() final;
@@ -80,9 +83,10 @@ public:
* @brief Waits for the sequence to be validated by the network.
*
* @param sequence The sequence to wait for
* @param maxWaitMs Maximum time to wait for the sequence to be validated. If empty, wait indefinitely
* @return true if sequence was validated, false otherwise a return value of false means the datastructure has been
* stopped
* @param maxWaitMs Maximum time to wait for the sequence to be validated. If empty, wait
* indefinitely
* @return true if sequence was validated, false otherwise a return value of false means the
* datastructure has been stopped
*/
bool
waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs = {}) final;

13
src/etl/NetworkValidatedLedgersInterface.hpp
View File

@@ -49,9 +49,11 @@ public:
/**
* @brief Get most recently validated sequence.
*
* If no ledgers are known to have been validated, this function waits until the next ledger is validated
* If no ledgers are known to have been validated, this function waits until the next ledger is
* validated
*
* @return Sequence of most recently validated ledger. empty optional if the datastructure has been stopped
* @return Sequence of most recently validated ledger. empty optional if the datastructure has
* been stopped
*/
[[nodiscard]] virtual std::optional<uint32_t>
getMostRecent() = 0;
@@ -60,9 +62,10 @@ public:
* @brief Waits for the sequence to be validated by the network.
*
* @param sequence The sequence to wait for
* @param maxWaitMs Maximum time to wait for the sequence to be validated. If empty, wait indefinitely
* @return true if sequence was validated, false otherwise a return value of false means the datastructure has been
* stopped
* @param maxWaitMs Maximum time to wait for the sequence to be validated. If empty, wait
* indefinitely
* @return true if sequence was validated, false otherwise a return value of false means the
* datastructure has been stopped
*/
virtual bool
waitUntilValidatedByNetwork(uint32_t sequence, std::optional<uint32_t> maxWaitMs = {}) = 0;

19
src/etl/RegistryInterface.hpp
View File

@@ -46,13 +46,14 @@ namespace etl {
* - void onInitialObjects(uint32_t, std::vector<etl::model::Object> const&, std::string)
* - void onInitialObject(uint32_t, etl::model::Object const&)
*
* When the registry dispatches (initial)data or objects, each of the above hooks will be called in order on each
* registered extension.
* This means that the order of execution is from left to right (hooks) and top to bottom (registered extensions).
* When the registry dispatches (initial)data or objects, each of the above hooks will be called in
* order on each registered extension. This means that the order of execution is from left to right
* (hooks) and top to bottom (registered extensions).
*
* If either `onTransaction` or `onInitialTransaction` are defined, the extension will have to additionally define a
* Specification. The specification lists transaction types to filter from the incoming data such that `onTransaction`
* and `onInitialTransaction` are only called for the transactions that are of interest for the given extension.
* If either `onTransaction` or `onInitialTransaction` are defined, the extension will have to
* additionally define a Specification. The specification lists transaction types to filter from the
* incoming data such that `onTransaction` and `onInitialTransaction` are only called for the
* transactions that are of interest for the given extension.
*
* The specification is setup like so:
* @code{.cpp}
@@ -82,7 +83,11 @@ struct RegistryInterface {
* @param lastKey The predcessor of the first object in data if known; an empty string otherwise
*/
virtual void
dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) = 0;
dispatchInitialObjects(
uint32_t seq,
std::vector<model::Object> const& data,
std::string lastKey
) = 0;
/**
* @brief Dispatch initial ledger data.

7
src/etl/Source.cpp
View File

@@ -66,7 +66,12 @@ makeSource(
);
return std::make_unique<impl::SourceImpl<>>(
ip, wsPort, grpcPort, std::move(grpcSource), std::move(subscriptionSource), std::move(forwardingSource)
ip,
wsPort,
grpcPort,
std::move(grpcSource),
std::move(subscriptionSource),
std::move(forwardingSource)
);
}

17
src/etl/Source.hpp
View File

@@ -112,11 +112,12 @@ public:
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
* This function will continuously try to fetch data for the specified ledger until the fetch
* succeeds, the ledger is found in the database, or the server is shutting down.
*
* @param sequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
* @param getObjects Whether to get the account state diff between this ledger and the prior
* one; defaults to true
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
* @return A std::pair of the response status and the response itself
*/
@@ -132,7 +133,11 @@ public:
* @return A std::pair of the data and a bool indicating whether the download was successful
*/
virtual InitialLedgerLoadResult
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, InitialLoadObserverInterface& loader) = 0;
loadInitialLedger(
uint32_t sequence,
std::uint32_t numMarkers,
InitialLoadObserverInterface& loader
) = 0;
/**
* @brief Forward a request to rippled.
@@ -175,8 +180,8 @@ using SourceFactory = std::function<SourcePtr(
* @param forwardingTimeout The timeout for forwarding to rippled
* @param onConnect The hook to call on connect
* @param onDisconnect The hook to call on disconnect
* @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed and the source is set
* as forwarding.
* @param onLedgerClosed The hook to call on ledger closed. This is called when a ledger is closed
* and the source is set as forwarding.
* @return The created source
*/
[[nodiscard]] SourcePtr

17
src/etl/SystemState.hpp
View File

@@ -58,8 +58,8 @@ struct SystemState {
/**
* @brief Whether the process is in strict read-only mode.
*
* In strict read-only mode, the process will never attempt to become the ETL writer, and will only publish ledgers
* as they are written to the database.
* In strict read-only mode, the process will never attempt to become the ETL writer, and will
* only publish ledgers as they are written to the database.
*/
util::prometheus::Bool isStrictReadonly = PrometheusService::boolMetric(
"read_only",
@@ -84,7 +84,8 @@ struct SystemState {
/**
* @brief Commands for controlling the ETL writer state.
*
* These commands are emitted via writeCommandSignal to coordinate writer state transitions across components.
* These commands are emitted via writeCommandSignal to coordinate writer state transitions
* across components.
*/
enum class WriteCommand {
StartWriting, /**< Request to attempt taking over as the ETL writer */
@@ -103,9 +104,9 @@ struct SystemState {
/**
* @brief Whether clio detected an amendment block.
*
* Being amendment blocked means that Clio was compiled with libxrpl that does not yet support some field that
* arrived from rippled and therefore can't extract the ledger diff. When this happens, Clio can't proceed with ETL
* and should log this error and only handle RPC requests.
* Being amendment blocked means that Clio was compiled with libxrpl that does not yet support
* some field that arrived from rippled and therefore can't extract the ledger diff. When this
* happens, Clio can't proceed with ETL and should log this error and only handle RPC requests.
*/
util::prometheus::Bool isAmendmentBlocked = PrometheusService::boolMetric(
"etl_amendment_blocked",
@@ -116,8 +117,8 @@ struct SystemState {
/**
* @brief Whether clio detected a corruption that needs manual attention.
*
* When corruption is detected, Clio should disable cache and stop the cache loading process in order to prevent
* further corruption.
* When corruption is detected, Clio should disable cache and stop the cache loading process in
* order to prevent further corruption.
*/
util::prometheus::Bool isCorruptionDetected = PrometheusService::boolMetric(
"etl_corruption_detected",

3
src/etl/WriterState.hpp
View File

@@ -117,7 +117,8 @@ public:
*/
class WriterState : public WriterStateInterface {
private:
std::shared_ptr<SystemState> systemState_; /**< @brief Shared system state for ETL coordination */
std::shared_ptr<SystemState>
systemState_; /**< @brief Shared system state for ETL coordination */
public:
/**

14
src/etl/impl/AmendmentBlockHandler.cpp
View File

@@ -30,11 +30,15 @@
namespace etl::impl {
AmendmentBlockHandler::ActionType const AmendmentBlockHandler::kDEFAULT_AMENDMENT_BLOCK_ACTION = []() {
static util::Logger const log{"ETL"}; // NOLINT(readability-identifier-naming)
LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
<< "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
};
AmendmentBlockHandler::ActionType const AmendmentBlockHandler::kDEFAULT_AMENDMENT_BLOCK_ACTION =
[]() {
static util::Logger const log{"ETL"}; // NOLINT(readability-identifier-naming)
LOG(
log.fatal()
) << "Can't process new ledgers: The current ETL source is not compatible with the version "
"of "
<< "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
};
AmendmentBlockHandler::AmendmentBlockHandler(
util::async::AnyExecutionContext ctx,

12
src/etl/impl/AsyncGrpcCall.cpp
View File

@@ -138,14 +138,17 @@ AsyncGrpcCall::process(
if (not data.empty())
loader.onInitialLoadGotMoreObjects(request_.ledger().sequence(), data, predecessorKey_);
predecessorKey_ = lastKey_; // but for ongoing onInitialObjects calls we need to pass along the key we left
// off at so that we can link the two lists correctly
predecessorKey_ = lastKey_; // but for ongoing onInitialObjects calls we need to pass along the
// key we left off at so that we can link the two lists correctly
return more ? CallStatus::More : CallStatus::Done;
}
void
AsyncGrpcCall::call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
AsyncGrpcCall::call(
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub,
grpc::CompletionQueue& cq
)
{
context_ = std::make_unique<grpc::ClientContext>();
auto rpc = stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq);
@@ -157,7 +160,8 @@ AsyncGrpcCall::call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stu
std::string
AsyncGrpcCall::getMarkerPrefix()
{
return next_->marker().empty() ? std::string{} : ripple::strHex(std::string{next_->marker().data()[0]});
return next_->marker().empty() ? std::string{}
: ripple::strHex(std::string{next_->marker().data()[0]});
}
// this is used to generate edgeKeys - keys that were the last one in the onInitialObjects list

11
src/etl/impl/AsyncGrpcCall.hpp
View File

@@ -59,7 +59,11 @@ private:
std::optional<std::string> predecessorKey_;
public:
AsyncGrpcCall(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker);
AsyncGrpcCall(
uint32_t seq,
ripple::uint256 const& marker,
std::optional<ripple::uint256> const& nextMarker
);
static std::vector<AsyncGrpcCall>
makeAsyncCalls(uint32_t const sequence, uint32_t const numMarkers);
@@ -73,7 +77,10 @@ public:
);
void
call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq);
call(
std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub,
grpc::CompletionQueue& cq
);
std::string
getMarkerPrefix();

24
src/etl/impl/CacheLoader.hpp
View File

@@ -70,7 +70,11 @@ public:
std::size_t const cachePageFetchSize,
std::vector<CursorPair> const& cursors
)
: ctx_{ctx}, backend_{backend}, cache_{std::ref(cache)}, queue_{cursors.size()}, remaining_{cursors.size()}
: ctx_{ctx}
, backend_{backend}
, cache_{std::ref(cache)}
, queue_{cursors.size()}
, remaining_{cursors.size()}
{
std::ranges::for_each(cursors, [this](auto const& cursor) { queue_.push(cursor); });
load(seq, numCacheMarkers, cachePageFetchSize);
@@ -123,19 +127,25 @@ private:
LOG(log_.debug()) << "Starting a cursor: " << ripple::strHex(start);
while (not token.isStopRequested() and not cache_.get().isDisabled()) {
auto res = data::retryOnTimeout([this, seq, cachePageFetchSize, &start, token]() {
return backend_->fetchLedgerPage(start, seq, cachePageFetchSize, false, token);
});
auto res =
data::retryOnTimeout([this, seq, cachePageFetchSize, &start, token]() {
return backend_->fetchLedgerPage(
start, seq, cachePageFetchSize, false, token
);
});
cache_.get().update(res.objects, seq, true);
if (not res.cursor or res.cursor > end) {
if (--remaining_ <= 0) {
auto endTime = std::chrono::steady_clock::now();
auto duration = std::chrono::duration_cast<std::chrono::seconds>(endTime - startTime_);
auto duration = std::chrono::duration_cast<std::chrono::seconds>(
endTime - startTime_
);
LOG(log_.info()) << "Finished loading cache. Cache size = " << cache_.get().size()
<< ". Took " << duration.count() << " seconds";
LOG(log_.info())
<< "Finished loading cache. Cache size = " << cache_.get().size()
<< ". Took " << duration.count() << " seconds";
cache_.get().setFull();
} else {

6
src/etl/impl/CursorFromAccountProvider.hpp
View File

@@ -41,7 +41,11 @@ class CursorFromAccountProvider : public BaseCursorProvider {
size_t pageSize_;
public:
CursorFromAccountProvider(std::shared_ptr<BackendInterface> const& backend, size_t numCursors, size_t pageSize)
CursorFromAccountProvider(
std::shared_ptr<BackendInterface> const& backend,
size_t numCursors,
size_t pageSize
)
: backend_{backend}, numCursors_{numCursors}, pageSize_{pageSize}
{
}

5
src/etl/impl/CursorFromDiffProvider.hpp
View File

@@ -86,7 +86,10 @@ public:
}
std::vector<ripple::uint256> cursors{data::kFIRST_KEY};
rg::copy(liveCursors | vs::take(std::min(liveCursors.size(), numCursors_)), std::back_inserter(cursors));
rg::copy(
liveCursors | vs::take(std::min(liveCursors.size(), numCursors_)),
std::back_inserter(cursors)
);
rg::sort(cursors);
cursors.push_back(data::kLAST_KEY);

4
src/etl/impl/CursorFromFixDiffNumProvider.hpp
View File

@@ -58,7 +58,9 @@ public:
auto diffs = std::vector<data::LedgerObject>{};
auto const append = [](auto&& a, auto&& b) { a.insert(std::end(a), std::begin(b), std::end(b)); };
auto const append = [](auto&& a, auto&& b) {
a.insert(std::end(a), std::begin(b), std::end(b));
};
auto const fetchDiff = [this, seq](uint32_t offset) {
return data::synchronousAndRetryOnTimeout([this, seq, offset](auto yield) {
return backend_->fetchLedgerDiff(seq - offset, yield);

30
src/etl/impl/Extraction.cpp
View File

@@ -62,7 +62,11 @@ extractModType(PBModType type)
case PBObjType::DELETED:
return model::Object::ModType::Deleted;
default: // some gRPC system values that we don't care about
ASSERT(false, "Tried to extract bogus mod type '{}'", PBObjType::ModificationType_Name(type));
ASSERT(
false,
"Tried to extract bogus mod type '{}'",
PBObjType::ModificationType_Name(type)
);
}
std::unreachable();
@@ -97,7 +101,10 @@ extractTxs(PBTxListType transactions, uint32_t seq)
std::vector<model::Transaction> output;
output.reserve(transactions.size());
rg::move(transactions | vs::transform([seq](auto&& tx) { return extractTx(tx, seq); }), std::back_inserter(output));
rg::move(
transactions | vs::transform([seq](auto&& tx) { return extractTx(tx, seq); }),
std::back_inserter(output)
);
return output;
}
@@ -134,7 +141,10 @@ extractObjs(PBObjListType objects)
std::vector<model::Object> output;
output.reserve(objects.size());
rg::move(objects | vs::transform([](auto&& obj) { return extractObj(obj); }), std::back_inserter(output));
rg::move(
objects | vs::transform([](auto&& obj) { return extractObj(obj); }),
std::back_inserter(output)
);
return output;
}
@@ -174,8 +184,9 @@ Extractor::unpack()
auto header = ::util::deserializeHeader(ripple::makeSlice(data.ledger_header()));
return std::make_optional<model::LedgerData>({
.transactions =
extractTxs(std::move(*data.mutable_transactions_list()->mutable_transactions()), header.seq),
.transactions = extractTxs(
std::move(*data.mutable_transactions_list()->mutable_transactions()), header.seq
),
.objects = extractObjs(std::move(*data.mutable_ledger_objects()->mutable_objects())),
.successors = maybeExtractSuccessors(data),
.edgeKeys = std::nullopt,
@@ -197,7 +208,8 @@ Extractor::extractLedgerWithDiff(uint32_t seq)
LOG(log_.debug()) << "Extracted and Transformed diff for " << seq << " in " << time << "ms";
// can be nullopt. this means that either the server is stopping or another node took over ETL writing.
// can be nullopt. this means that either the server is stopping or another node took over ETL
// writing.
return batch;
}
@@ -210,9 +222,11 @@ Extractor::extractLedgerOnly(uint32_t seq)
return fetcher_->fetchData(seq).and_then(unpack());
});
LOG(log_.debug()) << "Extracted and Transformed full ledger for " << seq << " in " << time << "ms";
LOG(log_.debug()) << "Extracted and Transformed full ledger for " << seq << " in " << time
<< "ms";
// can be nullopt. this means that either the server is stopping or another node took over ETL writing.
// can be nullopt. this means that either the server is stopping or another node took over ETL
// writing.
return batch;
}

6
src/etl/impl/ForwardingSource.cpp
View File

@@ -52,7 +52,8 @@ ForwardingSource::ForwardingSource(
{
connectionBuilder_.setConnectionTimeout(connTimeout)
.addHeader(
{boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)}
{boost::beast::http::field::user_agent,
fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)}
);
}
@@ -103,7 +104,8 @@ ForwardingSource::forwardToRippled(
if (not parsedResponse.is_object())
throw std::runtime_error("response is not an object");
} catch (std::exception const& e) {
LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
LOG(log_.debug()) << "Error parsing response from rippled: " << e.what()
<< ". Response: " << *response;
return std::unexpected{rpc::ClioError::EtlInvalidResponse};
}

22
src/etl/impl/GrpcSource.cpp
View File

@@ -65,7 +65,11 @@ resolve(std::string const& ip, std::string const& port)
namespace etl::impl {
GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::chrono::system_clock::duration deadline)
GrpcSource::GrpcSource(
std::string const& ip,
std::string const& grpcPort,
std::chrono::system_clock::duration deadline
)
: log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort))
, initialLoadShouldStop_(std::make_unique<std::atomic_bool>(false))
, deadline_{deadline}
@@ -75,11 +79,16 @@ GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::
chArgs.SetMaxReceiveMessageSize(-1);
chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIME_MS, kKEEPALIVE_PING_INTERVAL_MS);
chArgs.SetInt(GRPC_ARG_KEEPALIVE_TIMEOUT_MS, kKEEPALIVE_TIMEOUT_MS);
chArgs.SetInt(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS, static_cast<int>(kKEEPALIVE_PERMIT_WITHOUT_CALLS));
chArgs.SetInt(
GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS,
static_cast<int>(kKEEPALIVE_PERMIT_WITHOUT_CALLS)
);
chArgs.SetInt(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA, kMAX_PINGS_WITHOUT_DATA);
stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
grpc::CreateCustomChannel(resolve(ip, grpcPort), grpc::InsecureChannelCredentials(), chArgs)
grpc::CreateCustomChannel(
resolve(ip, grpcPort), grpc::InsecureChannelCredentials(), chArgs
)
);
LOG(log_.debug()) << "Made stub for remote.";
@@ -98,7 +107,9 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb
org::xrpl::rpc::v1::GetLedgerRequest request;
grpc::ClientContext context;
context.set_deadline(std::chrono::system_clock::now() + deadline_); // Prevent indefinite blocking
context.set_deadline(
std::chrono::system_clock::now() + deadline_
); // Prevent indefinite blocking
request.mutable_ledger()->set_sequence(sequence);
request.set_transactions(true);
@@ -110,7 +121,8 @@ GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighb
grpc::Status const status = stub_->GetLedger(&context, request, &response);
if (status.ok() and not response.is_unlimited()) {
log_.warn() << "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. Status = "
log_.warn() << "is_unlimited is false. Make sure secure_gateway is set correctly on the "
"ETL source. Status = "
<< status.error_message();
}

18
src/etl/impl/GrpcSource.hpp
View File

@@ -45,8 +45,9 @@ class GrpcSource {
static constexpr auto kKEEPALIVE_PING_INTERVAL_MS = 10000;
static constexpr auto kKEEPALIVE_TIMEOUT_MS = 5000;
static constexpr auto kKEEPALIVE_PERMIT_WITHOUT_CALLS = true; // Allow keepalive pings when no calls
static constexpr auto kMAX_PINGS_WITHOUT_DATA = 0; // No limit
static constexpr auto kKEEPALIVE_PERMIT_WITHOUT_CALLS =
true; // Allow keepalive pings when no calls
static constexpr auto kMAX_PINGS_WITHOUT_DATA = 0; // No limit
static constexpr auto kDEADLINE = std::chrono::seconds(30);
public:
@@ -59,11 +60,12 @@ public:
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
* This function will continuously try to fetch data for the specified ledger until the fetch
* succeeds, the ledger is found in the database, or the server is shutting down.
*
* @param sequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
* @param getObjects Whether to get the account state diff between this ledger and the prior
* one; defaults to true
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
* @return A std::pair of the response status and the response itself
*/
@@ -79,7 +81,11 @@ public:
* @return Downloaded data or an indication of error or cancellation
*/
InitialLedgerLoadResult
loadInitialLedger(uint32_t sequence, uint32_t numMarkers, InitialLoadObserverInterface& observer);
loadInitialLedger(
uint32_t sequence,
uint32_t numMarkers,
InitialLoadObserverInterface& observer
);
/**
* @brief Stop any ongoing operations

22
src/etl/impl/LedgerFetcher.hpp
View File

@@ -47,7 +47,10 @@ public:
/**
* @brief Create an instance of the fetcher
*/
LedgerFetcher(std::shared_ptr<BackendInterface> backend, std::shared_ptr<LoadBalancerInterface> balancer)
LedgerFetcher(
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<LoadBalancerInterface> balancer
)
: backend_(std::move(backend)), loadBalancer_(std::move(balancer))
{
}
@@ -55,11 +58,12 @@ public:
/**
* @brief Extract data for a particular ledger from an ETL source
*
* This function continuously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
* This function continuously tries to extract the specified ledger (using all available ETL
* sources) until the extraction succeeds, or the server shuts down.
*
* @param sequence sequence of the ledger to extract
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
* @return Ledger header and transaction+metadata blobs; Empty optional if the server is
* shutting down
*/
[[nodiscard]] OptionalGetLedgerResponseType
fetchData(uint32_t sequence) override
@@ -75,11 +79,12 @@ public:
/**
* @brief Extract diff data for a particular ledger from an ETL source.
*
* This function continuously tries to extract the specified ledger (using all available ETL sources) until the
* extraction succeeds, or the server shuts down.
* This function continuously tries to extract the specified ledger (using all available ETL
* sources) until the extraction succeeds, or the server shuts down.
*
* @param sequence sequence of the ledger to extract
* @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
* @return Ledger data diff between sequance and parent; Empty optional if the server is
* shutting down
*/
[[nodiscard]] OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t sequence) override
@@ -89,7 +94,8 @@ public:
auto const isCacheFull = backend_->cache().isFull();
auto const isLedgerCached = backend_->cache().latestLedgerSequence() >= sequence;
if (isLedgerCached) {
LOG(log_.info()) << sequence << " is already cached, the current latest seq in cache is "
LOG(log_.info()) << sequence
<< " is already cached, the current latest seq in cache is "
<< backend_->cache().latestLedgerSequence() << " and the cache is "
<< (isCacheFull ? "full" : "not full");
}

76
src/etl/impl/LedgerPublisher.hpp
View File

@@ -64,13 +64,14 @@ namespace etl::impl {
/**
* @brief Publishes ledgers in a synchronized fashion.
*
* If ETL is started far behind the network, ledgers will be written and published very rapidly. Monitoring processes
* will publish ledgers as they are written. However, to publish a ledger, the monitoring process needs to read all of
* the transactions for that ledger from the database. Reading the transactions from the database requires network
* calls, which can be slow. It is imperative however that the monitoring processes keep up with the writer, else the
* monitoring processes will not be able to detect if the writer failed. Therefore, publishing each ledger (which
* includes reading all of the transactions from the database) is done from the application wide asio io_service, and a
* strand is used to ensure ledgers are published in order.
* If ETL is started far behind the network, ledgers will be written and published very rapidly.
* Monitoring processes will publish ledgers as they are written. However, to publish a ledger, the
* monitoring process needs to read all of the transactions for that ledger from the database.
* Reading the transactions from the database requires network calls, which can be slow. It is
* imperative however that the monitoring processes keep up with the writer, else the monitoring
* processes will not be able to detect if the writer failed. Therefore, publishing each ledger
* (which includes reading all of the transactions from the database) is done from the application
* wide asio io_service, and a strand is used to ensure ledgers are published in order.
*/
class LedgerPublisher : public LedgerPublisherInterface {
util::Logger log_{"ETL"};
@@ -85,11 +86,12 @@ class LedgerPublisher : public LedgerPublisherInterface {
util::Mutex<std::chrono::time_point<ripple::NetClock>, std::shared_mutex> lastCloseTime_;
std::reference_wrapper<util::prometheus::CounterInt> lastPublishSeconds_ = PrometheusService::counterInt(
"etl_last_publish_seconds",
{},
"Seconds since epoch of the last published ledger"
);
std::reference_wrapper<util::prometheus::CounterInt> lastPublishSeconds_ =
PrometheusService::counterInt(
"etl_last_publish_seconds",
{},
"Seconds since epoch of the last published ledger"
);
util::Mutex<std::optional<uint32_t>, std::shared_mutex> lastPublishedSequence_;
@@ -111,8 +113,8 @@ public:
}
/**
* @brief Attempt to read the specified ledger from the database, and then publish that ledger to the ledgers
* stream.
* @brief Attempt to read the specified ledger from the database, and then publish that ledger
* to the ledgers stream.
*
* @param ledgerSequence the sequence of the ledger to publish
* @param maxAttempts the number of times to attempt to read the ledger from the database
@@ -133,11 +135,14 @@ public:
if (!range || range->maxSequence < ledgerSequence) {
++numAttempts;
LOG(log_.debug()) << "Trying to publish. Could not find ledger with sequence = " << ledgerSequence;
LOG(log_.debug()) << "Trying to publish. Could not find ledger with sequence = "
<< ledgerSequence;
// We try maxAttempts times to publish the ledger, waiting one second in between each attempt.
// We try maxAttempts times to publish the ledger, waiting one second in between
// each attempt.
if (maxAttempts && numAttempts >= maxAttempts) {
LOG(log_.debug()) << "Failed to publish ledger after " << numAttempts << " attempts.";
LOG(log_.debug())
<< "Failed to publish ledger after " << numAttempts << " attempts.";
return false;
}
std::this_thread::sleep_for(attemptsDelay);
@@ -148,7 +153,11 @@ public:
return backend_->fetchLedgerBySequence(ledgerSequence, yield);
});
ASSERT(lgr.has_value(), "Ledger must exist in database. Ledger sequence = {}", ledgerSequence);
ASSERT(
lgr.has_value(),
"Ledger must exist in database. Ledger sequence = {}",
ledgerSequence
);
publish(*lgr);
return true;
@@ -159,7 +168,8 @@ public:
/**
* @brief Publish the passed ledger asynchronously.
*
* All ledgers are published thru publishStrand_ which ensures that all publishes are performed in a serial fashion.
* All ledgers are published thru publishStrand_ which ensures that all publishes are performed
* in a serial fashion.
*
* @param lgrInfo the ledger to publish
*/
@@ -172,12 +182,14 @@ public:
setLastClose(lgrInfo.closeTime);
auto age = lastCloseAgeSeconds();
// if the ledger closed over MAX_LEDGER_AGE_SECONDS ago, assume we are still catching up and don't publish
// if the ledger closed over MAX_LEDGER_AGE_SECONDS ago, assume we are still catching up
// and don't publish
static constexpr std::uint32_t kMAX_LEDGER_AGE_SECONDS = 600;
if (age < kMAX_LEDGER_AGE_SECONDS) {
std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchFees(lgrInfo.seq, yield);
});
std::optional<ripple::Fees> fees =
data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchFees(lgrInfo.seq, yield);
});
ASSERT(fees.has_value(), "Fees must exist for ledger {}", lgrInfo.seq);
auto transactions = data::synchronousAndRetryOnTimeout([&](auto yield) {
@@ -187,7 +199,8 @@ public:
auto const ledgerRange = backend_->fetchLedgerRange();
ASSERT(ledgerRange.has_value(), "Ledger range must exist");
auto const range = fmt::format("{}-{}", ledgerRange->minSequence, ledgerRange->maxSequence);
auto const range =
fmt::format("{}-{}", ledgerRange->minSequence, ledgerRange->maxSequence);
subscriptions_->pubLedger(lgrInfo, *fees, range, transactions.size());
// order with transaction index
@@ -222,7 +235,9 @@ public:
std::uint32_t
lastPublishAgeSeconds() const override
{
return std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now() - getLastPublish())
return std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now() - getLastPublish()
)
.count();
}
@@ -244,7 +259,9 @@ public:
lastCloseAgeSeconds() const override
{
auto closeTime = lastCloseTime_.lock()->time_since_epoch().count();
auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
auto now = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()
)
.count();
if (now < (kRIPPLE_EPOCH_START + closeTime))
return 0;
@@ -252,8 +269,8 @@ public:
}
/**
* @brief Get the sequence of the last schueduled ledger to publish, Be aware that the ledger may not have been
* published to network
* @brief Get the sequence of the last schueduled ledger to publish, Be aware that the ledger
* may not have been published to network
*/
std::optional<uint32_t>
getLastPublishedSequence() const
@@ -285,7 +302,8 @@ private:
setLastPublishTime()
{
using namespace std::chrono;
auto const nowSeconds = duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
auto const nowSeconds =
duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
lastPublishSeconds_.get().set(nowSeconds);
}

39
src/etl/impl/Loading.cpp
View File

@@ -69,17 +69,20 @@ Loader::load(model::LedgerData const& data)
// Only a writer should attempt to commit to DB
// This is also where conflicts with other writer nodes will be detected
if (state_->isWriting) {
auto [success, duration] =
::util::timed<std::chrono::milliseconds>([&]() { return backend_->finishWrites(data.seq); });
LOG(log_.info()) << "Finished writes to DB for " << data.seq << ": " << (success ? "YES" : "NO")
<< "; took " << duration << "ms";
auto [success, duration] = ::util::timed<std::chrono::milliseconds>([&]() {
return backend_->finishWrites(data.seq);
});
LOG(log_.info()) << "Finished writes to DB for " << data.seq << ": "
<< (success ? "YES" : "NO") << "; took " << duration << "ms";
if (not success) {
// Write conflict detected - another node wrote to the database
// This triggers the fallback mechanism and stops this node from writing
state_->writeCommandSignal(SystemState::WriteCommand::StopWriting);
state_->isWriterDecidingFallback = true;
LOG(log_.warn()) << "Another node wrote a ledger into the DB - we have a write conflict";
LOG(
log_.warn()
) << "Another node wrote a ledger into the DB - we have a write conflict";
return std::unexpected(LoaderError::WriteConflict);
}
}
@@ -103,11 +106,14 @@ Loader::onInitialLoadGotMoreObjects(
static auto kINITIAL_LOAD_START_TIME = std::chrono::steady_clock::now();
try {
LOG(log_.trace()) << "On initial load: got more objects for seq " << seq << ". size = " << data.size();
LOG(log_.trace()) << "On initial load: got more objects for seq " << seq
<< ". size = " << data.size();
registry_->dispatchInitialObjects(
seq,
data,
std::move(lastKey).value_or(std::string{}) // TODO: perhaps use optional all the way to extensions?
std::move(lastKey).value_or(
std::string{}
) // TODO: perhaps use optional all the way to extensions?
);
initialLoadWrittenObjects_ += data.size();
@@ -116,13 +122,15 @@ Loader::onInitialLoadGotMoreObjects(
auto elapsedSinceStart = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::steady_clock::now() - kINITIAL_LOAD_START_TIME
);
auto elapsedSeconds = elapsedSinceStart.count() / static_cast<double>(util::kMILLISECONDS_PER_SECOND);
auto objectsPerSecond =
elapsedSeconds > 0.0 ? static_cast<double>(initialLoadWrittenObjects_) / elapsedSeconds : 0.0;
auto elapsedSeconds =
elapsedSinceStart.count() / static_cast<double>(util::kMILLISECONDS_PER_SECOND);
auto objectsPerSecond = elapsedSeconds > 0.0
? static_cast<double>(initialLoadWrittenObjects_) / elapsedSeconds
: 0.0;
LOG(log_.info()) << "Wrote " << initialLoadWrittenObjects_
<< " initial ledger objects so far with average rate of " << objectsPerSecond
<< " objects per second";
<< " initial ledger objects so far with average rate of "
<< objectsPerSecond << " objects per second";
}
} catch (std::runtime_error const& e) {
@@ -142,8 +150,11 @@ Loader::loadInitialLedger(model::LedgerData const& data)
LOG(log_.debug()) << "Deserialized ledger header. " << ::util::toString(data.header);
auto seconds = ::util::timed<std::chrono::seconds>([this, &data]() { registry_->dispatchInitialData(data); });
LOG(log_.info()) << "Dispatching initial data and submitting all writes took " << seconds << " seconds.";
auto seconds = ::util::timed<std::chrono::seconds>([this, &data]() {
registry_->dispatchInitialData(data);
});
LOG(log_.info()) << "Dispatching initial data and submitting all writes took " << seconds
<< " seconds.";
backend_->finishWrites(data.seq);
LOG(log_.debug()) << "Loaded initial ledger";

37
src/etl/impl/Monitor.cpp
View File

@@ -77,7 +77,8 @@ void
Monitor::notifyWriteConflict(uint32_t seq)
{
LOG(log_.warn()) << "Loader notified Monitor about write conflict at " << seq;
nextSequence_ = seq + 1; // we already loaded the cache for seq just before we detected conflict
nextSequence_ =
seq + 1; // we already loaded the cache for seq just before we detected conflict
LOG(log_.warn()) << "Resume monitoring from " << nextSequence_;
}
@@ -87,13 +88,17 @@ Monitor::run(std::chrono::steady_clock::duration repeatInterval)
ASSERT(not repeatedTask_.has_value(), "Monitor attempted to run more than once");
{
auto lck = updateData_.lock();
LOG(log_.debug()) << "Starting monitor with repeat interval: "
<< std::chrono::duration_cast<std::chrono::seconds>(repeatInterval).count()
<< "s and dbStalledReportDelay: "
<< std::chrono::duration_cast<std::chrono::seconds>(lck->dbStalledReportDelay).count() << "s";
LOG(
log_.debug()
) << "Starting monitor with repeat interval: "
<< std::chrono::duration_cast<std::chrono::seconds>(repeatInterval).count()
<< "s and dbStalledReportDelay: "
<< std::chrono::duration_cast<std::chrono::seconds>(lck->dbStalledReportDelay).count()
<< "s";
}
repeatedTask_ = strand_.executeRepeatedly(repeatInterval, std::bind_front(&Monitor::doWork, this));
repeatedTask_ =
strand_.executeRepeatedly(repeatInterval, std::bind_front(&Monitor::doWork, this));
subscription_ = validatedLedgers_->subscribe(std::bind_front(&Monitor::onNextSequence, this));
}
@@ -143,23 +148,27 @@ Monitor::doWork()
}
while (lck->lastSeenMaxSeqInDb >= nextSequence_) {
LOG(log_.trace()) << "Publishing from Monitor::doWork. nextSequence_ = " << nextSequence_
LOG(log_.trace()) << "Publishing from Monitor::doWork. nextSequence_ = "
<< nextSequence_
<< ", lastSeenMaxSeqInDb_ = " << lck->lastSeenMaxSeqInDb;
notificationChannel_(nextSequence_++);
dbProgressedThisCycle = true;
}
} else {
LOG(log_.trace()) << "DB range is not available or empty. lastSeenMaxSeqInDb_ = " << lck->lastSeenMaxSeqInDb
<< ", nextSequence_ = " << nextSequence_;
LOG(log_.trace()) << "DB range is not available or empty. lastSeenMaxSeqInDb_ = "
<< lck->lastSeenMaxSeqInDb << ", nextSequence_ = " << nextSequence_;
}
if (dbProgressedThisCycle) {
lck->lastDbCheckTime = std::chrono::steady_clock::now();
} else if (std::chrono::steady_clock::now() - lck->lastDbCheckTime > lck->dbStalledReportDelay) {
LOG(log_.info()) << "No DB update detected for "
<< std::chrono::duration_cast<std::chrono::seconds>(lck->dbStalledReportDelay).count()
<< " seconds. Firing dbStalledChannel. Last seen max seq in DB: " << lck->lastSeenMaxSeqInDb
<< ". Expecting next: " << nextSequence_;
} else if (std::chrono::steady_clock::now() - lck->lastDbCheckTime >
lck->dbStalledReportDelay) {
LOG(
log_.info()
) << "No DB update detected for "
<< std::chrono::duration_cast<std::chrono::seconds>(lck->dbStalledReportDelay).count()
<< " seconds. Firing dbStalledChannel. Last seen max seq in DB: "
<< lck->lastSeenMaxSeqInDb << ". Expecting next: " << nextSequence_;
dbStalledChannel_();
lck->lastDbCheckTime = std::chrono::steady_clock::now();
}

3
src/etl/impl/Monitor.hpp
View File

@@ -47,7 +47,8 @@ class Monitor : public MonitorInterface {
std::atomic_uint32_t nextSequence_;
std::optional<util::async::AnyOperation<void>> repeatedTask_;
std::optional<boost::signals2::scoped_connection> subscription_; // network validated ledgers subscription
std::optional<boost::signals2::scoped_connection>
subscription_; // network validated ledgers subscription
NewSequenceSignalType notificationChannel_;
DbStalledSignalType dbStalledChannel_;

6
src/etl/impl/MonitorProvider.hpp
View File

@@ -45,7 +45,11 @@ public:
) override
{
return std::make_unique<Monitor>(
std::move(ctx), std::move(backend), std::move(validatedLedgers), startSequence, dbStalledReportDelay
std::move(ctx),
std::move(backend),
std::move(validatedLedgers),
startSequence,
dbStalledReportDelay
);
}
};

36
src/etl/impl/Registry.hpp
View File

@@ -58,12 +58,16 @@ concept HasObjectHook = requires(T p) {
template <typename T>
concept HasInitialTransactionHook = requires(T p) {
{ p.onInitialTransaction(uint32_t{}, std::declval<model::Transaction>()) } -> std::same_as<void>;
{
p.onInitialTransaction(uint32_t{}, std::declval<model::Transaction>())
} -> std::same_as<void>;
};
template <typename T>
concept HasInitialObjectsHook = requires(T p) {
{ p.onInitialObjects(uint32_t{}, std::declval<std::vector<model::Object>>(), std::string{}) } -> std::same_as<void>;
{
p.onInitialObjects(uint32_t{}, std::declval<std::vector<model::Object>>(), std::string{})
} -> std::same_as<void>;
};
template <typename T>
@@ -75,9 +79,10 @@ template <typename T>
concept ContainsSpec = std::decay_t<T>::spec::kSPEC_TAG;
template <typename T>
concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or
(HasTransactionHook<T> and ContainsSpec<T>) or (HasInitialTransactionHook<T> and ContainsSpec<T>) or
HasObjectHook<T> or HasInitialObjectsHook<T> or HasInitialObjectHook<T>;
concept ContainsValidHook =
HasLedgerDataHook<T> or HasInitialDataHook<T> or (HasTransactionHook<T> and ContainsSpec<T>) or
(HasInitialTransactionHook<T> and ContainsSpec<T>) or HasObjectHook<T> or
HasInitialObjectsHook<T> or HasInitialObjectHook<T>;
template <typename T>
concept NoTwoOfKind = not(HasLedgerDataHook<T> and HasTransactionHook<T>) and
@@ -98,7 +103,8 @@ class Registry : public RegistryInterface {
);
static_assert(
(((not HasInitialTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and ...),
(((not HasInitialTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and
...),
"Spec must be specified when 'onInitialTransaction' function exists."
);
@@ -158,13 +164,19 @@ public:
}
constexpr void
dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) override
dispatchInitialObjects(
uint32_t seq,
std::vector<model::Object> const& data,
std::string lastKey
) override
{
// send entire vector path
{
auto const expand = [&](auto&& p) {
if constexpr (requires { p.onInitialObjects(seq, data, lastKey); })
executeIfAllowed(p, [seq, &data, &lastKey](auto& p) { p.onInitialObjects(seq, data, lastKey); });
executeIfAllowed(p, [seq, &data, &lastKey](auto& p) {
p.onInitialObjects(seq, data, lastKey);
});
};
std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
@@ -201,7 +213,9 @@ public:
auto const expand = [&]<typename P>(P&& p, model::Transaction const& tx) {
if constexpr (requires { p.onInitialTransaction(data.seq, tx); }) {
if (std::decay_t<P>::spec::wants(tx.type))
executeIfAllowed(p, [&data, &tx](auto& p) { p.onInitialTransaction(data.seq, tx); });
executeIfAllowed(p, [&data, &tx](auto& p) {
p.onInitialTransaction(data.seq, tx);
});
}
};
@@ -228,7 +242,9 @@ private:
static auto
makeRegistry(SystemState const& state, auto&&... exts)
{
return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(state, std::forward<decltype(exts)>(exts)...);
return std::make_unique<Registry<std::decay_t<decltype(exts)>...>>(
state, std::forward<decltype(exts)>(exts)...
);
}
} // namespace etl::impl

Some files were not shown because too many files have changed in this diff Show More