mirror of
https://github.com/XRPLF/clio.git
synced 2025-11-20 19:56:00 +00:00
Merge branch 'develop' into release/2.6.0
This commit is contained in:
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
@@ -51,9 +51,9 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
overwrite_release: false
|
overwrite_release: false
|
||||||
prerelease: ${{ contains(github.ref_name, '-') }}
|
prerelease: ${{ contains(github.ref_name, '-') }}
|
||||||
title: "${{ github.ref_name}}"
|
title: "${{ github.ref_name }}"
|
||||||
version: "${{ github.ref_name }}"
|
version: "${{ github.ref_name }}"
|
||||||
header: >
|
header: >
|
||||||
${{ contains(github.ref_name, '-') && '> **Note:** Please remember that this is a release candidate and it is not recommended for production use.' || '' }}
|
${{ contains(github.ref_name, '-') && '> **Note:** Please remember that this is a release candidate and it is not recommended for production use.' || '' }}
|
||||||
generate_changelog: ${{ !contains(github.ref_name, '-') }}
|
generate_changelog: ${{ !contains(github.ref_name, '-') }}
|
||||||
draft: true
|
draft: ${{ !contains(github.ref_name, '-') }}
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ namespace data {
|
|||||||
inline std::shared_ptr<BackendInterface>
|
inline std::shared_ptr<BackendInterface>
|
||||||
makeBackend(util::config::ClioConfigDefinition const& config, data::LedgerCacheInterface& cache)
|
makeBackend(util::config::ClioConfigDefinition const& config, data::LedgerCacheInterface& cache)
|
||||||
{
|
{
|
||||||
|
using namespace cassandra::impl;
|
||||||
static util::Logger const log{"Backend"}; // NOLINT(readability-identifier-naming)
|
static util::Logger const log{"Backend"}; // NOLINT(readability-identifier-naming)
|
||||||
LOG(log.info()) << "Constructing BackendInterface";
|
LOG(log.info()) << "Constructing BackendInterface";
|
||||||
|
|
||||||
@@ -56,7 +57,7 @@ makeBackend(util::config::ClioConfigDefinition const& config, data::LedgerCacheI
|
|||||||
|
|
||||||
if (boost::iequals(type, "cassandra")) {
|
if (boost::iequals(type, "cassandra")) {
|
||||||
auto const cfg = config.getObject("database." + type);
|
auto const cfg = config.getObject("database." + type);
|
||||||
if (cfg.getValueView("provider").asString() == toString(cassandra::impl::Provider::Keyspace)) {
|
if (providerFromString(cfg.getValueView("provider").asString()) == Provider::Keyspace) {
|
||||||
backend = std::make_shared<data::cassandra::KeyspaceBackend>(
|
backend = std::make_shared<data::cassandra::KeyspaceBackend>(
|
||||||
data::cassandra::SettingsProvider{cfg}, cache, readOnly
|
data::cassandra::SettingsProvider{cfg}, cache, readOnly
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -189,10 +189,11 @@ public:
|
|||||||
auto const nftUris = executor_.readEach(yield, selectNFTURIStatements);
|
auto const nftUris = executor_.readEach(yield, selectNFTURIStatements);
|
||||||
|
|
||||||
for (auto i = 0u; i < nftIDs.size(); i++) {
|
for (auto i = 0u; i < nftIDs.size(); i++) {
|
||||||
if (auto const maybeRow = nftInfos[i].template get<uint32_t, ripple::AccountID, bool>(); maybeRow) {
|
if (auto const maybeRow = nftInfos[i].template get<uint32_t, ripple::AccountID, bool>();
|
||||||
|
maybeRow.has_value()) {
|
||||||
auto [seq, owner, isBurned] = *maybeRow;
|
auto [seq, owner, isBurned] = *maybeRow;
|
||||||
NFT nft(nftIDs[i], seq, owner, isBurned);
|
NFT nft(nftIDs[i], seq, owner, isBurned);
|
||||||
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri)
|
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri.has_value())
|
||||||
nft.uri = *maybeUri;
|
nft.uri = *maybeUri;
|
||||||
ret.nfts.push_back(nft);
|
ret.nfts.push_back(nft);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -57,9 +57,9 @@ namespace data::cassandra {
|
|||||||
/**
|
/**
|
||||||
* @brief Implements @ref CassandraBackendFamily for Keyspace
|
* @brief Implements @ref CassandraBackendFamily for Keyspace
|
||||||
*
|
*
|
||||||
* @tparam SettingsProviderType The settings provider type to use
|
* @tparam SettingsProviderType The settings provider type
|
||||||
* @tparam ExecutionStrategyType The execution strategy type to use
|
* @tparam ExecutionStrategyType The execution strategy type
|
||||||
* @tparam FetchLedgerCacheType The ledger header cache type to use
|
* @tparam FetchLedgerCacheType The ledger header cache type
|
||||||
*/
|
*/
|
||||||
template <
|
template <
|
||||||
SomeSettingsProvider SettingsProviderType,
|
SomeSettingsProvider SettingsProviderType,
|
||||||
@@ -101,9 +101,9 @@ public:
|
|||||||
// !range_.has_value() means the table 'ledger_range' is not populated;
|
// !range_.has_value() means the table 'ledger_range' is not populated;
|
||||||
// This would be the first write to the table.
|
// This would be the first write to the table.
|
||||||
// In this case, insert both min_sequence/max_sequence range into the table.
|
// In this case, insert both min_sequence/max_sequence range into the table.
|
||||||
if (not(range_.has_value())) {
|
if (not range_.has_value()) {
|
||||||
executor_.writeSync(schema_->insertLedgerRange, false, ledgerSequence_);
|
executor_.writeSync(schema_->insertLedgerRange, /* isLatestLedger =*/false, ledgerSequence_);
|
||||||
executor_.writeSync(schema_->insertLedgerRange, true, ledgerSequence_);
|
executor_.writeSync(schema_->insertLedgerRange, /* isLatestLedger =*/true, ledgerSequence_);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (not this->executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1))) {
|
if (not this->executeSyncUpdate(schema_->updateLedgerRange.bind(ledgerSequence_, true, ledgerSequence_ - 1))) {
|
||||||
@@ -130,7 +130,7 @@ public:
|
|||||||
// Keyspace and ScyllaDB uses the same logic for taxon-filtered queries
|
// Keyspace and ScyllaDB uses the same logic for taxon-filtered queries
|
||||||
nftIDs = fetchNFTIDsByTaxon(issuer, *taxon, limit, cursorIn, yield);
|
nftIDs = fetchNFTIDsByTaxon(issuer, *taxon, limit, cursorIn, yield);
|
||||||
} else {
|
} else {
|
||||||
// --- Amazon Keyspaces Workflow for non-taxon queries ---
|
// Amazon Keyspaces Workflow for non-taxon queries
|
||||||
auto const startTaxon = cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
|
auto const startTaxon = cursorIn.has_value() ? ripple::nft::toUInt32(ripple::nft::getTaxon(*cursorIn)) : 0;
|
||||||
auto const startTokenID = cursorIn.value_or(ripple::uint256(0));
|
auto const startTokenID = cursorIn.value_or(ripple::uint256(0));
|
||||||
|
|
||||||
@@ -140,8 +140,8 @@ public:
|
|||||||
firstQuery.bindAt(3, Limit{limit});
|
firstQuery.bindAt(3, Limit{limit});
|
||||||
|
|
||||||
auto const firstRes = executor_.read(yield, firstQuery);
|
auto const firstRes = executor_.read(yield, firstQuery);
|
||||||
if (firstRes) {
|
if (firstRes.has_value()) {
|
||||||
for (auto const [nftID] : extract<ripple::uint256>(firstRes.value()))
|
for (auto const [nftID] : extract<ripple::uint256>(*firstRes))
|
||||||
nftIDs.push_back(nftID);
|
nftIDs.push_back(nftID);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -152,8 +152,8 @@ public:
|
|||||||
secondQuery.bindAt(2, Limit{remainingLimit});
|
secondQuery.bindAt(2, Limit{remainingLimit});
|
||||||
|
|
||||||
auto const secondRes = executor_.read(yield, secondQuery);
|
auto const secondRes = executor_.read(yield, secondQuery);
|
||||||
if (secondRes) {
|
if (secondRes.has_value()) {
|
||||||
for (auto const [nftID] : extract<ripple::uint256>(secondRes.value()))
|
for (auto const [nftID] : extract<ripple::uint256>(*secondRes))
|
||||||
nftIDs.push_back(nftID);
|
nftIDs.push_back(nftID);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -163,7 +163,7 @@ public:
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief (Unsupported in Keyspaces) Fetches account root object indexes by page.
|
* @brief (Unsupported in Keyspaces) Fetches account root object indexes by page.
|
||||||
* * @note Loading the cache by enumerating all accounts is currently unsupported by the AWS Keyspaces backend.
|
* @note Loading the cache by enumerating all accounts is currently unsupported by the AWS Keyspaces backend.
|
||||||
* This function's logic relies on "PER PARTITION LIMIT 1", which Keyspaces does not support, and there is
|
* This function's logic relies on "PER PARTITION LIMIT 1", which Keyspaces does not support, and there is
|
||||||
* no efficient alternative. This is acceptable as the cache is primarily loaded via diffs. Calling this
|
* no efficient alternative. This is acceptable as the cache is primarily loaded via diffs. Calling this
|
||||||
* function will throw an exception.
|
* function will throw an exception.
|
||||||
@@ -203,8 +203,8 @@ private:
|
|||||||
statement.bindAt(3, Limit{limit});
|
statement.bindAt(3, Limit{limit});
|
||||||
|
|
||||||
auto const res = executor_.read(yield, statement);
|
auto const res = executor_.read(yield, statement);
|
||||||
if (res && res.value().hasRows()) {
|
if (res.has_value() && res->hasRows()) {
|
||||||
for (auto const [nftID] : extract<ripple::uint256>(res.value()))
|
for (auto const [nftID] : extract<ripple::uint256>(*res))
|
||||||
nftIDs.push_back(nftID);
|
nftIDs.push_back(nftID);
|
||||||
}
|
}
|
||||||
return nftIDs;
|
return nftIDs;
|
||||||
@@ -229,8 +229,8 @@ private:
|
|||||||
firstQuery.bindAt(3, Limit{limit});
|
firstQuery.bindAt(3, Limit{limit});
|
||||||
|
|
||||||
auto const firstRes = executor_.read(yield, firstQuery);
|
auto const firstRes = executor_.read(yield, firstQuery);
|
||||||
if (firstRes) {
|
if (firstRes.has_value()) {
|
||||||
for (auto const [nftID] : extract<ripple::uint256>(firstRes.value()))
|
for (auto const [nftID] : extract<ripple::uint256>(*firstRes))
|
||||||
nftIDs.push_back(nftID);
|
nftIDs.push_back(nftID);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,8 +241,8 @@ private:
|
|||||||
secondQuery.bindAt(2, Limit{remainingLimit});
|
secondQuery.bindAt(2, Limit{remainingLimit});
|
||||||
|
|
||||||
auto const secondRes = executor_.read(yield, secondQuery);
|
auto const secondRes = executor_.read(yield, secondQuery);
|
||||||
if (secondRes) {
|
if (secondRes.has_value()) {
|
||||||
for (auto const [nftID] : extract<ripple::uint256>(secondRes.value()))
|
for (auto const [nftID] : extract<ripple::uint256>(*secondRes))
|
||||||
nftIDs.push_back(nftID);
|
nftIDs.push_back(nftID);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -291,10 +291,11 @@ private:
|
|||||||
|
|
||||||
// Combine the results into final NFT objects.
|
// Combine the results into final NFT objects.
|
||||||
for (auto i = 0u; i < nftIDs.size(); ++i) {
|
for (auto i = 0u; i < nftIDs.size(); ++i) {
|
||||||
if (auto const maybeRow = nftInfos[i].template get<uint32_t, ripple::AccountID, bool>(); maybeRow) {
|
if (auto const maybeRow = nftInfos[i].template get<uint32_t, ripple::AccountID, bool>();
|
||||||
|
maybeRow.has_value()) {
|
||||||
auto [seq, owner, isBurned] = *maybeRow;
|
auto [seq, owner, isBurned] = *maybeRow;
|
||||||
NFT nft(nftIDs[i], seq, owner, isBurned);
|
NFT nft(nftIDs[i], seq, owner, isBurned);
|
||||||
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri)
|
if (auto const maybeUri = nftUris[i].template get<ripple::Blob>(); maybeUri.has_value())
|
||||||
nft.uri = *maybeUri;
|
nft.uri = *maybeUri;
|
||||||
ret.nfts.push_back(nft);
|
ret.nfts.push_back(nft);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -70,10 +70,10 @@ namespace data::cassandra {
|
|||||||
*
|
*
|
||||||
* Note: This is a safer and more correct rewrite of the original implementation of the backend.
|
* Note: This is a safer and more correct rewrite of the original implementation of the backend.
|
||||||
*
|
*
|
||||||
* @tparam SettingsProviderType The settings provider type to use
|
* @tparam SettingsProviderType The settings provider type
|
||||||
* @tparam ExecutionStrategyType The execution strategy type to use
|
* @tparam ExecutionStrategyType The execution strategy type
|
||||||
* @tparam SchemaType The Schema type to use
|
* @tparam SchemaType The Schema type
|
||||||
* @tparam FetchLedgerCacheType The ledger header cache type to use
|
* @tparam FetchLedgerCacheType The ledger header cache type
|
||||||
*/
|
*/
|
||||||
template <
|
template <
|
||||||
SomeSettingsProvider SettingsProviderType,
|
SomeSettingsProvider SettingsProviderType,
|
||||||
@@ -100,8 +100,8 @@ public:
|
|||||||
/**
|
/**
|
||||||
* @brief Create a new cassandra/scylla backend instance.
|
* @brief Create a new cassandra/scylla backend instance.
|
||||||
*
|
*
|
||||||
* @param settingsProvider The settings provider to use
|
* @param settingsProvider The settings provider
|
||||||
* @param cache The ledger cache to use
|
* @param cache The ledger cache
|
||||||
* @param readOnly Whether the database should be in readonly mode
|
* @param readOnly Whether the database should be in readonly mode
|
||||||
*/
|
*/
|
||||||
CassandraBackendFamily(SettingsProviderType settingsProvider, data::LedgerCacheInterface& cache, bool readOnly)
|
CassandraBackendFamily(SettingsProviderType settingsProvider, data::LedgerCacheInterface& cache, bool readOnly)
|
||||||
@@ -111,18 +111,18 @@ public:
|
|||||||
, handle_{settingsProvider_.getSettings()}
|
, handle_{settingsProvider_.getSettings()}
|
||||||
, executor_{settingsProvider_.getSettings(), handle_}
|
, executor_{settingsProvider_.getSettings(), handle_}
|
||||||
{
|
{
|
||||||
if (auto const res = handle_.connect(); not res)
|
if (auto const res = handle_.connect(); not res.has_value())
|
||||||
throw std::runtime_error("Could not connect to database: " + res.error());
|
throw std::runtime_error("Could not connect to database: " + res.error());
|
||||||
|
|
||||||
if (not readOnly) {
|
if (not readOnly) {
|
||||||
if (auto const res = handle_.execute(schema_.createKeyspace); not res) {
|
if (auto const res = handle_.execute(schema_.createKeyspace); not res.has_value()) {
|
||||||
// on datastax, creation of keyspaces can be configured to only be done thru the admin
|
// on datastax, creation of keyspaces can be configured to only be done thru the admin
|
||||||
// interface. this does not mean that the keyspace does not already exist tho.
|
// interface. this does not mean that the keyspace does not already exist tho.
|
||||||
if (res.error().code() != CASS_ERROR_SERVER_UNAUTHORIZED)
|
if (res.error().code() != CASS_ERROR_SERVER_UNAUTHORIZED)
|
||||||
throw std::runtime_error("Could not create keyspace: " + res.error());
|
throw std::runtime_error("Could not create keyspace: " + res.error());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (auto const res = handle_.executeEach(schema_.createSchema); not res)
|
if (auto const res = handle_.executeEach(schema_.createSchema); not res.has_value())
|
||||||
throw std::runtime_error("Could not create schema: " + res.error());
|
throw std::runtime_error("Could not create schema: " + res.error());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -233,10 +233,10 @@ public:
|
|||||||
std::optional<std::uint32_t>
|
std::optional<std::uint32_t>
|
||||||
fetchLatestLedgerSequence(boost::asio::yield_context yield) const override
|
fetchLatestLedgerSequence(boost::asio::yield_context yield) const override
|
||||||
{
|
{
|
||||||
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res) {
|
if (auto const res = executor_.read(yield, schema_->selectLatestLedger); res.has_value()) {
|
||||||
if (auto const& result = res.value(); result) {
|
if (auto const& rows = *res; rows) {
|
||||||
if (auto const maybeValue = result.template get<uint32_t>(); maybeValue)
|
if (auto const maybeRow = rows.template get<uint32_t>(); maybeRow.has_value())
|
||||||
return maybeValue;
|
return maybeRow;
|
||||||
|
|
||||||
LOG(log_.error()) << "Could not fetch latest ledger - no rows";
|
LOG(log_.error()) << "Could not fetch latest ledger - no rows";
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ SettingsProvider::parseSettings() const
|
|||||||
settings.coreConnectionsPerHost = config_.get<uint32_t>("core_connections_per_host");
|
settings.coreConnectionsPerHost = config_.get<uint32_t>("core_connections_per_host");
|
||||||
settings.queueSizeIO = config_.maybeValue<uint32_t>("queue_size_io");
|
settings.queueSizeIO = config_.maybeValue<uint32_t>("queue_size_io");
|
||||||
settings.writeBatchSize = config_.get<std::size_t>("write_batch_size");
|
settings.writeBatchSize = config_.get<std::size_t>("write_batch_size");
|
||||||
settings.provider = config_.get<std::string>("provider");
|
settings.provider = impl::providerFromString(config_.get<std::string>("provider"));
|
||||||
|
|
||||||
if (config_.getValueView("connect_timeout").hasValue()) {
|
if (config_.getValueView("connect_timeout").hasValue()) {
|
||||||
auto const connectTimeoutSecond = config_.get<uint32_t>("connect_timeout");
|
auto const connectTimeoutSecond = config_.get<uint32_t>("connect_timeout");
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), k
|
|||||||
cass_cluster_set_request_timeout(*this, settings.requestTimeout.count());
|
cass_cluster_set_request_timeout(*this, settings.requestTimeout.count());
|
||||||
|
|
||||||
// TODO: AWS keyspace reads should be local_one to save cost
|
// TODO: AWS keyspace reads should be local_one to save cost
|
||||||
if (settings.provider == toString(cassandra::impl::Provider::Keyspace)) {
|
if (settings.provider == cassandra::impl::Provider::Keyspace) {
|
||||||
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_LOCAL_QUORUM); rc != CASS_OK) {
|
if (auto const rc = cass_cluster_set_consistency(*this, CASS_CONSISTENCY_LOCAL_QUORUM); rc != CASS_OK) {
|
||||||
throw std::runtime_error(fmt::format("Error setting keyspace consistency: {}", cass_error_desc(rc)));
|
throw std::runtime_error(fmt::format("Error setting keyspace consistency: {}", cass_error_desc(rc)));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "data/cassandra/impl/ManagedObject.hpp"
|
#include "data/cassandra/impl/ManagedObject.hpp"
|
||||||
|
#include "util/Assert.hpp"
|
||||||
#include "util/log/Logger.hpp"
|
#include "util/log/Logger.hpp"
|
||||||
|
|
||||||
#include <cassandra.h>
|
#include <cassandra.h>
|
||||||
@@ -31,29 +32,22 @@
|
|||||||
#include <string>
|
#include <string>
|
||||||
#include <string_view>
|
#include <string_view>
|
||||||
#include <thread>
|
#include <thread>
|
||||||
#include <utility>
|
|
||||||
#include <variant>
|
#include <variant>
|
||||||
|
|
||||||
namespace data::cassandra::impl {
|
namespace data::cassandra::impl {
|
||||||
|
|
||||||
namespace {
|
|
||||||
|
|
||||||
enum class Provider { Cassandra, Keyspace };
|
enum class Provider { Cassandra, Keyspace };
|
||||||
|
|
||||||
inline std::string
|
inline Provider
|
||||||
toString(Provider provider)
|
providerFromString(std::string const& provider)
|
||||||
{
|
{
|
||||||
switch (provider) {
|
ASSERT(
|
||||||
case Provider::Cassandra:
|
provider == "cassandra" || provider == "aws_keyspace",
|
||||||
return "cassandra";
|
"Provider type must be one of 'cassandra' or 'aws_keyspace'"
|
||||||
case Provider::Keyspace:
|
);
|
||||||
return "aws_keyspace";
|
return provider == "cassandra" ? Provider::Cassandra : Provider::Keyspace;
|
||||||
}
|
|
||||||
std::unreachable();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
} // namespace
|
|
||||||
|
|
||||||
// TODO: move Settings to public interface, not impl
|
// TODO: move Settings to public interface, not impl
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -109,7 +103,7 @@ struct Settings {
|
|||||||
std::size_t writeBatchSize = kDEFAULT_BATCH_SIZE;
|
std::size_t writeBatchSize = kDEFAULT_BATCH_SIZE;
|
||||||
|
|
||||||
/** @brief Provider to know if we are using scylladb or keyspace */
|
/** @brief Provider to know if we are using scylladb or keyspace */
|
||||||
std::string provider = toString(kDEFAULT_PROVIDER);
|
Provider provider = kDEFAULT_PROVIDER;
|
||||||
|
|
||||||
/** @brief Size of the IO queue */
|
/** @brief Size of the IO queue */
|
||||||
std::optional<uint32_t> queueSizeIO = std::nullopt; // NOLINT(readability-redundant-member-init)
|
std::optional<uint32_t> queueSizeIO = std::nullopt; // NOLINT(readability-redundant-member-init)
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ class ConfigValue;
|
|||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for logger levels in config.
|
* @brief specific values that are accepted for logger levels in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 6> kLOG_LEVELS = {
|
static constexpr std::array<std::string_view, 6> kLOG_LEVELS = {
|
||||||
"trace",
|
"trace",
|
||||||
"debug",
|
"debug",
|
||||||
"info",
|
"info",
|
||||||
@@ -57,7 +57,7 @@ static constexpr std::array<char const*, 6> kLOG_LEVELS = {
|
|||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for logger tag style in config.
|
* @brief specific values that are accepted for logger tag style in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 5> kLOG_TAGS = {
|
static constexpr std::array<std::string_view, 5> kLOG_TAGS = {
|
||||||
"int",
|
"int",
|
||||||
"uint",
|
"uint",
|
||||||
"null",
|
"null",
|
||||||
@@ -68,7 +68,7 @@ static constexpr std::array<char const*, 5> kLOG_TAGS = {
|
|||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for cache loading in config.
|
* @brief specific values that are accepted for cache loading in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 3> kLOAD_CACHE_MODE = {
|
static constexpr std::array<std::string_view, 3> kLOAD_CACHE_MODE = {
|
||||||
"sync",
|
"sync",
|
||||||
"async",
|
"async",
|
||||||
"none",
|
"none",
|
||||||
@@ -77,17 +77,17 @@ static constexpr std::array<char const*, 3> kLOAD_CACHE_MODE = {
|
|||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for database type in config.
|
* @brief specific values that are accepted for database type in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 1> kDATABASE_TYPE = {"cassandra"};
|
static constexpr std::array<std::string_view, 1> kDATABASE_TYPE = {"cassandra"};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for server's processing_policy in config.
|
* @brief specific values that are accepted for server's processing_policy in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 2> kPROCESSING_POLICY = {"parallel", "sequent"};
|
static constexpr std::array<std::string_view, 2> kPROCESSING_POLICY = {"parallel", "sequent"};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief specific values that are accepted for database provider in config.
|
* @brief specific values that are accepted for database provider in config.
|
||||||
*/
|
*/
|
||||||
static constexpr std::array<char const*, 2> kPROVIDER = {"cassandra", "aws_keyspace"};
|
static constexpr std::array<std::string_view, 2> kPROVIDER = {"cassandra", "aws_keyspace"};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @brief An interface to enforce constraints on certain values within ClioConfigDefinition.
|
* @brief An interface to enforce constraints on certain values within ClioConfigDefinition.
|
||||||
@@ -123,7 +123,7 @@ protected:
|
|||||||
*/
|
*/
|
||||||
template <std::size_t ArrSize>
|
template <std::size_t ArrSize>
|
||||||
constexpr std::string
|
constexpr std::string
|
||||||
makeErrorMsg(std::string_view key, Value const& value, std::array<char const*, ArrSize> arr) const
|
makeErrorMsg(std::string_view key, Value const& value, std::array<std::string_view, ArrSize> arr) const
|
||||||
{
|
{
|
||||||
// Extract the value from the variant
|
// Extract the value from the variant
|
||||||
auto const valueStr = std::visit([](auto const& v) { return fmt::format("{}", v); }, value);
|
auto const valueStr = std::visit([](auto const& v) { return fmt::format("{}", v); }, value);
|
||||||
@@ -271,7 +271,7 @@ public:
|
|||||||
* @param key The key of the ConfigValue that has this constraint
|
* @param key The key of the ConfigValue that has this constraint
|
||||||
* @param arr The value that has this constraint must be of the values in arr
|
* @param arr The value that has this constraint must be of the values in arr
|
||||||
*/
|
*/
|
||||||
constexpr OneOf(std::string_view key, std::array<char const*, ArrSize> arr) : key_{key}, arr_{arr}
|
constexpr OneOf(std::string_view key, std::array<std::string_view, ArrSize> arr) : key_{key}, arr_{arr}
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -318,7 +318,7 @@ private:
|
|||||||
print(std::ostream& stream) const override
|
print(std::ostream& stream) const override
|
||||||
{
|
{
|
||||||
std::string valuesStream;
|
std::string valuesStream;
|
||||||
std::ranges::for_each(arr_, [&valuesStream](std::string const& elem) {
|
std::ranges::for_each(arr_, [&valuesStream](std::string_view elem) {
|
||||||
valuesStream += fmt::format(" `{}`,", elem);
|
valuesStream += fmt::format(" `{}`,", elem);
|
||||||
});
|
});
|
||||||
// replace the last "," with "."
|
// replace the last "," with "."
|
||||||
@@ -327,7 +327,7 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::string_view key_;
|
std::string_view key_;
|
||||||
std::array<char const*, ArrSize> arr_;
|
std::array<std::string_view, ArrSize> arr_;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -220,10 +220,10 @@ LogService::createFileSink(FileLoggingParams const& params, std::string const& f
|
|||||||
* @param defaultSeverity The default severity level to use if not overridden.
|
* @param defaultSeverity The default severity level to use if not overridden.
|
||||||
* @return A map of channel names to their minimum severity levels, or an error message if parsing fails.
|
* @return A map of channel names to their minimum severity levels, or an error message if parsing fails.
|
||||||
*/
|
*/
|
||||||
static std::expected<std::unordered_map<std::string, Severity>, std::string>
|
static std::expected<std::unordered_map<std::string_view, Severity>, std::string>
|
||||||
getMinSeverity(config::ClioConfigDefinition const& config, Severity defaultSeverity)
|
getMinSeverity(config::ClioConfigDefinition const& config, Severity defaultSeverity)
|
||||||
{
|
{
|
||||||
std::unordered_map<std::string, Severity> minSeverity;
|
std::unordered_map<std::string_view, Severity> minSeverity;
|
||||||
for (auto const& channel : Logger::kCHANNELS)
|
for (auto const& channel : Logger::kCHANNELS)
|
||||||
minSeverity[channel] = defaultSeverity;
|
minSeverity[channel] = defaultSeverity;
|
||||||
|
|
||||||
@@ -284,13 +284,15 @@ LogServiceState::reset()
|
|||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<spdlog::logger>
|
std::shared_ptr<spdlog::logger>
|
||||||
LogServiceState::registerLogger(std::string const& channel, std::optional<Severity> severity)
|
LogServiceState::registerLogger(std::string_view channel, std::optional<Severity> severity)
|
||||||
{
|
{
|
||||||
if (not initialized_) {
|
if (not initialized_) {
|
||||||
throw std::logic_error("LogService is not initialized");
|
throw std::logic_error("LogService is not initialized");
|
||||||
}
|
}
|
||||||
|
|
||||||
std::shared_ptr<spdlog::logger> existingLogger = spdlog::get(channel);
|
std::string const channelStr{channel};
|
||||||
|
|
||||||
|
std::shared_ptr<spdlog::logger> existingLogger = spdlog::get(channelStr);
|
||||||
if (existingLogger != nullptr) {
|
if (existingLogger != nullptr) {
|
||||||
if (severity.has_value())
|
if (severity.has_value())
|
||||||
existingLogger->set_level(toSpdlogLevel(*severity));
|
existingLogger->set_level(toSpdlogLevel(*severity));
|
||||||
@@ -300,10 +302,10 @@ LogServiceState::registerLogger(std::string const& channel, std::optional<Severi
|
|||||||
std::shared_ptr<spdlog::logger> logger;
|
std::shared_ptr<spdlog::logger> logger;
|
||||||
if (isAsync_) {
|
if (isAsync_) {
|
||||||
logger = std::make_shared<spdlog::async_logger>(
|
logger = std::make_shared<spdlog::async_logger>(
|
||||||
channel, sinks_.begin(), sinks_.end(), spdlog::thread_pool(), spdlog::async_overflow_policy::block
|
channelStr, sinks_.begin(), sinks_.end(), spdlog::thread_pool(), spdlog::async_overflow_policy::block
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
logger = std::make_shared<spdlog::logger>(channel, sinks_.begin(), sinks_.end());
|
logger = std::make_shared<spdlog::logger>(channelStr, sinks_.begin(), sinks_.end());
|
||||||
}
|
}
|
||||||
|
|
||||||
logger->set_level(toSpdlogLevel(severity.value_or(defaultSeverity_)));
|
logger->set_level(toSpdlogLevel(severity.value_or(defaultSeverity_)));
|
||||||
@@ -427,10 +429,25 @@ LogServiceState::replaceSinks(std::vector<std::shared_ptr<spdlog::sinks::sink>>
|
|||||||
spdlog::apply_all([](std::shared_ptr<spdlog::logger> logger) { logger->sinks() = sinks_; });
|
spdlog::apply_all([](std::shared_ptr<spdlog::logger> logger) { logger->sinks() = sinks_; });
|
||||||
}
|
}
|
||||||
|
|
||||||
Logger::Logger(std::string channel) : logger_(LogServiceState::registerLogger(channel))
|
Logger::Logger(std::string_view const channel) : logger_(LogServiceState::registerLogger(channel))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Logger::~Logger()
|
||||||
|
{
|
||||||
|
// One reference is held by logger_ and the other by spdlog registry
|
||||||
|
static constexpr size_t kLAST_LOGGER_REF_COUNT = 2;
|
||||||
|
|
||||||
|
if (logger_ == nullptr) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool const isDynamic = !std::ranges::contains(kCHANNELS, logger_->name());
|
||||||
|
if (isDynamic && logger_.use_count() == kLAST_LOGGER_REF_COUNT) {
|
||||||
|
spdlog::drop(logger_->name());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Logger::Pump::Pump(std::shared_ptr<spdlog::logger> logger, Severity sev, SourceLocationType const& loc)
|
Logger::Pump::Pump(std::shared_ptr<spdlog::logger> logger, Severity sev, SourceLocationType const& loc)
|
||||||
: logger_(std::move(logger))
|
: logger_(std::move(logger))
|
||||||
, severity_(sev)
|
, severity_(sev)
|
||||||
|
|||||||
@@ -29,6 +29,7 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
#include <sstream>
|
#include <sstream>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
// We forward declare spdlog::logger and spdlog::sinks::sink
|
// We forward declare spdlog::logger and spdlog::sinks::sink
|
||||||
@@ -91,7 +92,7 @@ enum class Severity {
|
|||||||
* otherwise. See @ref LogService::init() for setup of the logging core and
|
* otherwise. See @ref LogService::init() for setup of the logging core and
|
||||||
* severity levels for each channel.
|
* severity levels for each channel.
|
||||||
*/
|
*/
|
||||||
class Logger final {
|
class Logger {
|
||||||
std::shared_ptr<spdlog::logger> logger_;
|
std::shared_ptr<spdlog::logger> logger_;
|
||||||
|
|
||||||
friend class LogService; // to expose the Pump interface
|
friend class LogService; // to expose the Pump interface
|
||||||
@@ -145,7 +146,7 @@ class Logger final {
|
|||||||
};
|
};
|
||||||
|
|
||||||
public:
|
public:
|
||||||
static constexpr std::array<char const*, 8> kCHANNELS = {
|
static constexpr std::array<std::string_view, 8> kCHANNELS = {
|
||||||
"General",
|
"General",
|
||||||
"WebServer",
|
"WebServer",
|
||||||
"Backend",
|
"Backend",
|
||||||
@@ -165,10 +166,10 @@ public:
|
|||||||
*
|
*
|
||||||
* @param channel The channel this logger will report into.
|
* @param channel The channel this logger will report into.
|
||||||
*/
|
*/
|
||||||
Logger(std::string channel);
|
Logger(std::string_view const channel);
|
||||||
|
|
||||||
Logger(Logger const&) = default;
|
Logger(Logger const&) = default;
|
||||||
~Logger() = default;
|
~Logger();
|
||||||
|
|
||||||
Logger(Logger&&) = default;
|
Logger(Logger&&) = default;
|
||||||
Logger&
|
Logger&
|
||||||
@@ -291,7 +292,7 @@ protected:
|
|||||||
* @return Shared pointer to the registered spdlog logger
|
* @return Shared pointer to the registered spdlog logger
|
||||||
*/
|
*/
|
||||||
static std::shared_ptr<spdlog::logger>
|
static std::shared_ptr<spdlog::logger>
|
||||||
registerLogger(std::string const& channel, std::optional<Severity> severity = std::nullopt);
|
registerLogger(std::string_view channel, std::optional<Severity> severity = std::nullopt);
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
static bool isAsync_; // NOLINT(readability-identifier-naming)
|
static bool isAsync_; // NOLINT(readability-identifier-naming)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ LoggerFixture::init()
|
|||||||
{
|
{
|
||||||
util::LogServiceState::init(false, util::Severity::FTL, {});
|
util::LogServiceState::init(false, util::Severity::FTL, {});
|
||||||
|
|
||||||
std::ranges::for_each(util::Logger::kCHANNELS, [](char const* channel) {
|
std::ranges::for_each(util::Logger::kCHANNELS, [](std::string_view const channel) {
|
||||||
util::LogService::registerLogger(channel);
|
util::LogService::registerLogger(channel);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -85,15 +85,17 @@ using namespace data::cassandra;
|
|||||||
|
|
||||||
class BackendCassandraTestBase : public SyncAsioContextTest, public WithPrometheus {
|
class BackendCassandraTestBase : public SyncAsioContextTest, public WithPrometheus {
|
||||||
protected:
|
protected:
|
||||||
|
static constexpr auto kCASSANDRA = "cassandra";
|
||||||
|
|
||||||
ClioConfigDefinition cfg_{
|
ClioConfigDefinition cfg_{
|
||||||
{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
|
{"database.type", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)},
|
||||||
{"database.cassandra.contact_points",
|
{"database.cassandra.contact_points",
|
||||||
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
|
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
|
||||||
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
|
{"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
|
||||||
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
|
{"database.cassandra.port", ConfigValue{ConfigType::Integer}.optional()},
|
||||||
{"database.cassandra.keyspace",
|
{"database.cassandra.keyspace",
|
||||||
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
|
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
|
||||||
{"database.cassandra.provider", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
|
{"database.cassandra.provider", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)},
|
||||||
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
||||||
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
|
{"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
|
||||||
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
|
{"database.cassandra.max_write_requests_outstanding", ConfigValue{ConfigType::Integer}.defaultValue(10'000)},
|
||||||
|
|||||||
@@ -95,14 +95,15 @@ class MigrationCassandraSimpleTest : public WithPrometheus {
|
|||||||
}
|
}
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
ClioConfigDefinition cfg_{
|
static constexpr auto kCASSANDRA = "cassandra";
|
||||||
|
|
||||||
{{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
|
ClioConfigDefinition cfg_{
|
||||||
|
{{"database.type", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)},
|
||||||
{"database.cassandra.contact_points",
|
{"database.cassandra.contact_points",
|
||||||
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
|
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
|
||||||
{"database.cassandra.keyspace",
|
{"database.cassandra.keyspace",
|
||||||
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
|
ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
|
||||||
{"database.cassandra.provider", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
|
{"database.cassandra.provider", ConfigValue{ConfigType::String}.defaultValue(kCASSANDRA)},
|
||||||
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
||||||
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
{"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
|
||||||
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)},
|
{"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)},
|
||||||
|
|||||||
@@ -31,6 +31,7 @@
|
|||||||
#include <optional>
|
#include <optional>
|
||||||
#include <ostream>
|
#include <ostream>
|
||||||
#include <string>
|
#include <string>
|
||||||
|
#include <string_view>
|
||||||
|
|
||||||
using namespace util::config;
|
using namespace util::config;
|
||||||
|
|
||||||
@@ -164,7 +165,7 @@ TEST_F(ConstraintTest, SetValuesOnPortConstraint)
|
|||||||
|
|
||||||
TEST_F(ConstraintTest, OneOfConstraintOneValue)
|
TEST_F(ConstraintTest, OneOfConstraintOneValue)
|
||||||
{
|
{
|
||||||
std::array<char const*, 1> const arr = {"tracer"};
|
std::array<std::string_view, 1> const arr = {"tracer"};
|
||||||
auto const databaseConstraint{OneOf{"database.type", arr}};
|
auto const databaseConstraint{OneOf{"database.type", arr}};
|
||||||
EXPECT_FALSE(databaseConstraint.checkConstraint("tracer").has_value());
|
EXPECT_FALSE(databaseConstraint.checkConstraint("tracer").has_value());
|
||||||
|
|
||||||
@@ -180,7 +181,7 @@ TEST_F(ConstraintTest, OneOfConstraintOneValue)
|
|||||||
|
|
||||||
TEST_F(ConstraintTest, OneOfConstraint)
|
TEST_F(ConstraintTest, OneOfConstraint)
|
||||||
{
|
{
|
||||||
std::array<char const*, 3> const arr = {"123", "trace", "haha"};
|
std::array<std::string_view, 3> const arr = {"123", "trace", "haha"};
|
||||||
auto const oneOfCons{OneOf{"log.level", arr}};
|
auto const oneOfCons{OneOf{"log.level", arr}};
|
||||||
|
|
||||||
EXPECT_FALSE(oneOfCons.checkConstraint("trace").has_value());
|
EXPECT_FALSE(oneOfCons.checkConstraint("trace").has_value());
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ TEST_F(LogServiceInitTests, DefaultLogLevel)
|
|||||||
EXPECT_TRUE(LogService::init(config_));
|
EXPECT_TRUE(LogService::init(config_));
|
||||||
|
|
||||||
std::string const logString = "some log";
|
std::string const logString = "some log";
|
||||||
for (auto const& channel : Logger::kCHANNELS) {
|
for (std::string_view const channel : Logger::kCHANNELS) {
|
||||||
Logger const log{channel};
|
Logger const log{channel};
|
||||||
log.trace() << logString;
|
log.trace() << logString;
|
||||||
auto loggerStr = getLoggerString();
|
auto loggerStr = getLoggerString();
|
||||||
|
|||||||
@@ -21,11 +21,23 @@
|
|||||||
#include "util/log/Logger.hpp"
|
#include "util/log/Logger.hpp"
|
||||||
|
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
#include <spdlog/logger.h>
|
||||||
|
#include <spdlog/spdlog.h>
|
||||||
|
|
||||||
#include <cstddef>
|
#include <cstddef>
|
||||||
#include <string>
|
#include <string>
|
||||||
using namespace util;
|
using namespace util;
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
size_t
|
||||||
|
loggersNum()
|
||||||
|
{
|
||||||
|
size_t counter = 0;
|
||||||
|
spdlog::apply_all([&counter](std::shared_ptr<spdlog::logger>) { ++counter; });
|
||||||
|
return counter;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
// Used as a fixture for tests with enabled logging
|
// Used as a fixture for tests with enabled logging
|
||||||
class LoggerTest : public LoggerFixture {};
|
class LoggerTest : public LoggerFixture {};
|
||||||
|
|
||||||
@@ -71,3 +83,24 @@ TEST_F(LoggerTest, LOGMacro)
|
|||||||
EXPECT_TRUE(computeCalled);
|
EXPECT_TRUE(computeCalled);
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
TEST_F(LoggerTest, ManyDynamicLoggers)
|
||||||
|
{
|
||||||
|
static constexpr size_t kNUM_LOGGERS = 10'000;
|
||||||
|
|
||||||
|
auto initialLoggers = loggersNum();
|
||||||
|
|
||||||
|
for (size_t i = 0; i < kNUM_LOGGERS; ++i) {
|
||||||
|
std::string const loggerName = "DynamicLogger" + std::to_string(i);
|
||||||
|
|
||||||
|
Logger log{loggerName};
|
||||||
|
log.info() << "Logger number " << i;
|
||||||
|
ASSERT_EQ(getLoggerString(), "inf:" + loggerName + " - Logger number " + std::to_string(i) + "\n");
|
||||||
|
|
||||||
|
Logger copy = log;
|
||||||
|
copy.info() << "Copy of logger number " << i;
|
||||||
|
ASSERT_EQ(getLoggerString(), "inf:" + loggerName + " - Copy of logger number " + std::to_string(i) + "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
ASSERT_EQ(loggersNum(), initialLoggers);
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user