From 3c4903a339d9d79b14ce13ec01285542fa92d4a0 Mon Sep 17 00:00:00 2001 From: Peter Chen <34582813+PeterChen13579@users.noreply.github.com> Date: Mon, 16 Dec 2024 15:33:32 -0800 Subject: [PATCH] refactor: Replace all old instances of Config with New Config (#1627) Fixes #1184 Previous PR's found [here](https://github.com/XRPLF/clio/pull/1593) and [here](https://github.com/XRPLF/clio/pull/1544) --- .../examples/config/cloud-example-config.json | 43 ----- src/app/ClioApplication.cpp | 13 +- src/app/ClioApplication.hpp | 7 +- src/data/BackendFactory.hpp | 10 +- src/data/CassandraBackend.hpp | 2 +- src/data/cassandra/SettingsProvider.cpp | 80 ++++----- src/data/cassandra/SettingsProvider.hpp | 9 +- src/etl/CacheLoader.hpp | 6 +- src/etl/CacheLoaderSettings.cpp | 43 +++-- src/etl/CacheLoaderSettings.hpp | 4 +- src/etl/ETLService.cpp | 10 +- src/etl/ETLService.hpp | 4 +- src/etl/LoadBalancer.cpp | 31 ++-- src/etl/LoadBalancer.hpp | 6 +- src/etl/Source.cpp | 11 +- src/etl/Source.hpp | 7 +- src/feed/SubscriptionManager.hpp | 9 +- src/main/Main.cpp | 22 ++- src/rpc/RPCEngine.hpp | 9 +- src/rpc/WorkQueue.cpp | 10 +- src/rpc/WorkQueue.hpp | 5 +- src/rpc/common/impl/APIVersionParser.cpp | 22 ++- src/rpc/common/impl/APIVersionParser.hpp | 33 +--- src/rpc/common/impl/HandlerProvider.cpp | 4 +- src/rpc/common/impl/HandlerProvider.hpp | 4 +- src/rpc/handlers/VersionHandler.hpp | 10 +- src/util/SignalsHandler.cpp | 8 +- src/util/SignalsHandler.hpp | 8 +- src/util/Taggable.hpp | 46 +++--- src/util/log/Logger.cpp | 52 +++--- src/util/log/Logger.hpp | 16 +- src/util/newconfig/ConfigConstraints.cpp | 4 +- src/util/newconfig/ConfigConstraints.hpp | 9 + src/util/newconfig/ConfigDefinition.cpp | 98 ++--------- src/util/newconfig/ConfigDefinition.hpp | 156 +++++++++++++++++- src/util/newconfig/ConfigDescription.hpp | 56 +++++-- src/util/newconfig/ConfigFileJson.cpp | 6 +- src/util/newconfig/ConfigValue.hpp | 4 +- src/util/newconfig/ObjectView.cpp | 4 +- src/util/newconfig/ObjectView.hpp | 31 +++- src/util/newconfig/ValueView.cpp | 22 ++- src/util/newconfig/ValueView.hpp | 52 +++++- src/util/prometheus/Prometheus.cpp | 8 +- src/util/prometheus/Prometheus.hpp | 5 +- src/web/AdminVerificationStrategy.cpp | 4 +- src/web/AdminVerificationStrategy.hpp | 4 +- src/web/RPCServerHandler.hpp | 6 +- src/web/Server.hpp | 13 +- src/web/dosguard/DOSGuard.cpp | 31 ++-- src/web/dosguard/DOSGuard.hpp | 10 +- src/web/dosguard/IntervalSweepHandler.cpp | 7 +- src/web/dosguard/IntervalSweepHandler.hpp | 4 +- src/web/dosguard/WhitelistHandler.hpp | 23 +-- src/web/ng/RPCServerHandler.hpp | 5 +- src/web/ng/Server.cpp | 26 ++- src/web/ng/Server.hpp | 4 +- src/web/ng/impl/ServerSslContext.cpp | 12 +- src/web/ng/impl/ServerSslContext.hpp | 4 +- tests/common/util/MockBackend.hpp | 4 +- tests/common/util/MockBackendTestFixture.hpp | 9 +- tests/common/util/MockPrometheus.hpp | 18 +- tests/common/util/MockSource.hpp | 7 +- tests/common/util/MockWsBase.hpp | 8 +- .../integration/data/BackendFactoryTests.cpp | 133 +++++---------- .../data/cassandra/BackendTests.cpp | 26 +-- tests/unit/CMakeLists.txt | 2 +- tests/unit/ConfigTests.cpp | 2 +- tests/unit/app/WebHandlersTests.cpp | 9 +- .../data/cassandra/SettingsProviderTests.cpp | 88 +++++++--- tests/unit/etl/CacheLoaderSettingsTests.cpp | 49 ++++-- tests/unit/etl/CacheLoaderTests.cpp | 38 ++++- tests/unit/etl/GrpcSourceTests.cpp | 5 +- tests/unit/etl/LedgerPublisherTests.cpp | 6 +- tests/unit/etl/LoadBalancerTests.cpp | 82 +++++++-- tests/unit/rpc/APIVersionTests.cpp | 22 ++- tests/unit/rpc/ForwardingProxyTests.cpp | 7 +- tests/unit/rpc/RPCEngineTests.cpp | 63 +++++-- tests/unit/rpc/WorkQueueTests.cpp | 22 +-- .../unit/rpc/handlers/VersionHandlerTests.cpp | 25 ++- tests/unit/util/SignalsHandlerTests.cpp | 15 +- tests/unit/util/newconfig/ArrayViewTests.cpp | 17 +- .../newconfig/ClioConfigDefinitionTests.cpp | 84 ++++++---- .../unit/util/newconfig/ConfigValueTests.cpp | 4 +- tests/unit/util/newconfig/JsonFileTests.cpp | 99 +++++++++++ tests/unit/util/newconfig/ObjectViewTests.cpp | 50 ++++-- tests/unit/util/newconfig/ValueViewTests.cpp | 50 +++++- tests/unit/util/prometheus/HttpTests.cpp | 26 ++- tests/unit/web/AdminVerificationTests.cpp | 23 ++- tests/unit/web/RPCServerHandlerTests.cpp | 12 +- tests/unit/web/ServerTests.cpp | 130 +++++++++++---- tests/unit/web/SubscriptionContextTests.cpp | 8 +- tests/unit/web/dosguard/DOSGuardTests.cpp | 14 +- .../dosguard/IntervalSweepHandlerTests.cpp | 7 +- .../web/dosguard/WhitelistHandlerTests.cpp | 26 ++- tests/unit/web/impl/ErrorHandlingTests.cpp | 9 +- tests/unit/web/ng/RPCServerHandlerTests.cpp | 14 +- tests/unit/web/ng/ResponseTests.cpp | 12 +- tests/unit/web/ng/ServerTests.cpp | 73 +++++--- .../unit/web/ng/SubscriptionContextTests.cpp | 11 +- .../web/ng/impl/ConnectionHandlerTests.cpp | 13 +- .../unit/web/ng/impl/HttpConnectionTests.cpp | 9 +- .../web/ng/impl/ServerSslContextTests.cpp | 29 +++- tests/unit/web/ng/impl/WsConnectionTests.cpp | 10 +- 103 files changed, 1624 insertions(+), 898 deletions(-) delete mode 100644 docs/examples/config/cloud-example-config.json create mode 100644 tests/unit/util/newconfig/JsonFileTests.cpp diff --git a/docs/examples/config/cloud-example-config.json b/docs/examples/config/cloud-example-config.json deleted file mode 100644 index 2de07975..00000000 --- a/docs/examples/config/cloud-example-config.json +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This is an example configuration file. Please do not use without modifying to suit your needs. - */ -{ - "database": { - "type": "cassandra", - "cassandra": { - // This option can be used to setup a secure connect bundle connection - "secure_connect_bundle": "[path/to/zip. ignore if using contact_points]", - // The following options are used only if using contact_points - "contact_points": "[ip. ignore if using secure_connect_bundle]", - "port": "[port. ignore if using_secure_connect_bundle]", - // Authentication settings - "username": "[username, if any]", - "password": "[password, if any]", - // Other common settings - "keyspace": "clio", - "max_write_requests_outstanding": 25000, - "max_read_requests_outstanding": 30000, - "threads": 8 - } - }, - "etl_sources": [ - { - "ip": "[rippled ip]", - "ws_port": "6006", - "grpc_port": "50051" - } - ], - "dos_guard": { - "whitelist": [ - "127.0.0.1" - ] - }, - "server": { - "ip": "0.0.0.0", - "port": 8080 - }, - "log_level": "debug", - "log_file": "./clio.log", - "extractor_threads": 8, - "read_only": false -} diff --git a/src/app/ClioApplication.cpp b/src/app/ClioApplication.cpp index b2f54e1a..2a05235a 100644 --- a/src/app/ClioApplication.cpp +++ b/src/app/ClioApplication.cpp @@ -31,8 +31,8 @@ #include "rpc/WorkQueue.hpp" #include "rpc/common/impl/HandlerProvider.hpp" #include "util/build/Build.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Prometheus.hpp" #include "web/AdminVerificationStrategy.hpp" #include "web/RPCServerHandler.hpp" @@ -78,7 +78,8 @@ start(boost::asio::io_context& ioc, std::uint32_t numThreads) } // namespace -ClioApplication::ClioApplication(util::Config const& config) : config_(config), signalsHandler_{config_} +ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& config) + : config_(config), signalsHandler_{config_} { LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString(); PrometheusService::init(config); @@ -87,11 +88,7 @@ ClioApplication::ClioApplication(util::Config const& config) : config_(config), int ClioApplication::run(bool const useNgWebServer) { - auto const threads = config_.valueOr("io_threads", 2); - if (threads <= 0) { - LOG(util::LogService::fatal()) << "io_threads is less than 1"; - return EXIT_FAILURE; - } + auto const threads = config_.get("io_threads"); LOG(util::LogService::info()) << "Number of io threads = " << threads; // IO context to handle all incoming requests, as well as other things. @@ -132,7 +129,7 @@ ClioApplication::run(bool const useNgWebServer) auto const rpcEngine = RPCEngineType::make_RPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider); - if (useNgWebServer or config_.valueOr("server.__ng_web_server", false)) { + if (useNgWebServer or config_.get("server.__ng_web_server")) { web::ng::RPCServerHandler handler{config_, backend, rpcEngine, etl}; auto expectedAdminVerifier = web::make_AdminVerificationStrategy(config_); diff --git a/src/app/ClioApplication.hpp b/src/app/ClioApplication.hpp index 30fbaf8c..95263f8b 100644 --- a/src/app/ClioApplication.hpp +++ b/src/app/ClioApplication.hpp @@ -18,9 +18,8 @@ //============================================================================== #pragma once - #include "util/SignalsHandler.hpp" -#include "util/config//Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" namespace app { @@ -28,7 +27,7 @@ namespace app { * @brief The main application class */ class ClioApplication { - util::Config const& config_; + util::config::ClioConfigDefinition const& config_; util::SignalsHandler signalsHandler_; public: @@ -37,7 +36,7 @@ public: * * @param config The configuration of the application */ - ClioApplication(util::Config const& config); + ClioApplication(util::config::ClioConfigDefinition const& config); /** * @brief Run the application diff --git a/src/data/BackendFactory.hpp b/src/data/BackendFactory.hpp index 7e2c1b63..d3990faf 100644 --- a/src/data/BackendFactory.hpp +++ b/src/data/BackendFactory.hpp @@ -22,8 +22,8 @@ #include "data/BackendInterface.hpp" #include "data/CassandraBackend.hpp" #include "data/cassandra/SettingsProvider.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -41,18 +41,18 @@ namespace data { * @return A shared_ptr with the selected implementation */ inline std::shared_ptr -make_Backend(util::Config const& config) +make_Backend(util::config::ClioConfigDefinition const& config) { static util::Logger const log{"Backend"}; LOG(log.info()) << "Constructing BackendInterface"; - auto const readOnly = config.valueOr("read_only", false); + auto const readOnly = config.get("read_only"); - auto const type = config.value("database.type"); + auto const type = config.get("database.type"); std::shared_ptr backend = nullptr; if (boost::iequals(type, "cassandra")) { - auto cfg = config.section("database." + type); + auto const cfg = config.getObject("database." + type); backend = std::make_shared(data::cassandra::SettingsProvider{cfg}, readOnly); } diff --git a/src/data/CassandraBackend.hpp b/src/data/CassandraBackend.hpp index fd304aef..f74ff7e9 100644 --- a/src/data/CassandraBackend.hpp +++ b/src/data/CassandraBackend.hpp @@ -93,7 +93,7 @@ public: , executor_{settingsProvider_.getSettings(), handle_} { if (auto const res = handle_.connect(); not res) - throw std::runtime_error("Could not connect to databse: " + res.error()); + throw std::runtime_error("Could not connect to database: " + res.error()); if (not readOnly) { if (auto const res = handle_.execute(schema_.createKeyspace); not res) { diff --git a/src/data/cassandra/SettingsProvider.cpp b/src/data/cassandra/SettingsProvider.cpp index 38cca8cd..56c51e72 100644 --- a/src/data/cassandra/SettingsProvider.cpp +++ b/src/data/cassandra/SettingsProvider.cpp @@ -22,7 +22,7 @@ #include "data/cassandra/Types.hpp" #include "data/cassandra/impl/Cluster.hpp" #include "util/Constants.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -36,43 +36,17 @@ #include #include #include -#include #include +#include #include namespace data::cassandra { -namespace impl { -inline static Settings::ContactPoints -tag_invoke(boost::json::value_to_tag, boost::json::value const& value) -{ - if (not value.is_object()) { - throw std::runtime_error("Feed entire Cassandra section to parse Settings::ContactPoints instead"); - } - - util::Config const obj{value}; - Settings::ContactPoints out; - - out.contactPoints = obj.valueOrThrow("contact_points", "`contact_points` must be a string"); - out.port = obj.maybeValue("port"); - - return out; -} - -inline static Settings::SecureConnectionBundle -tag_invoke(boost::json::value_to_tag, boost::json::value const& value) -{ - if (not value.is_string()) - throw std::runtime_error("`secure_connect_bundle` must be a string"); - return Settings::SecureConnectionBundle{value.as_string().data()}; -} -} // namespace impl - -SettingsProvider::SettingsProvider(util::Config const& cfg) +SettingsProvider::SettingsProvider(util::config::ObjectView const& cfg) : config_{cfg} - , keyspace_{cfg.valueOr("keyspace", "clio")} + , keyspace_{cfg.get("keyspace")} , tablePrefix_{cfg.maybeValue("table_prefix")} - , replicationFactor_{cfg.valueOr("replication_factor", 3)} + , replicationFactor_{cfg.get("replication_factor")} , settings_{parseSettings()} { } @@ -86,8 +60,8 @@ SettingsProvider::getSettings() const std::optional SettingsProvider::parseOptionalCertificate() const { - if (auto const certPath = config_.maybeValue("certfile"); certPath) { - auto const path = std::filesystem::path(*certPath); + if (auto const certPath = config_.getValueView("certfile"); certPath.hasValue()) { + auto const path = std::filesystem::path(certPath.asString()); std::ifstream fileStream(path.string(), std::ios::in); if (!fileStream) { throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string()); @@ -108,30 +82,34 @@ Settings SettingsProvider::parseSettings() const { auto settings = Settings::defaultSettings(); - if (auto const bundle = config_.maybeValue("secure_connect_bundle"); bundle) { - settings.connectionInfo = *bundle; + + // all config values used in settings is under "database.cassandra" prefix + if (config_.getValueView("secure_connect_bundle").hasValue()) { + auto const bundle = Settings::SecureConnectionBundle{(config_.get("secure_connect_bundle"))}; + settings.connectionInfo = bundle; } else { - settings.connectionInfo = - config_.valueOrThrow("Missing contact_points in Cassandra config"); + Settings::ContactPoints out; + out.contactPoints = config_.get("contact_points"); + out.port = config_.maybeValue("port"); + settings.connectionInfo = out; } - settings.threads = config_.valueOr("threads", settings.threads); - settings.maxWriteRequestsOutstanding = - config_.valueOr("max_write_requests_outstanding", settings.maxWriteRequestsOutstanding); - settings.maxReadRequestsOutstanding = - config_.valueOr("max_read_requests_outstanding", settings.maxReadRequestsOutstanding); - settings.coreConnectionsPerHost = - config_.valueOr("core_connections_per_host", settings.coreConnectionsPerHost); + settings.threads = config_.get("threads"); + settings.maxWriteRequestsOutstanding = config_.get("max_write_requests_outstanding"); + settings.maxReadRequestsOutstanding = config_.get("max_read_requests_outstanding"); + settings.coreConnectionsPerHost = config_.get("core_connections_per_host"); settings.queueSizeIO = config_.maybeValue("queue_size_io"); - settings.writeBatchSize = config_.valueOr("write_batch_size", settings.writeBatchSize); + settings.writeBatchSize = config_.get("write_batch_size"); - auto const connectTimeoutSecond = config_.maybeValue("connect_timeout"); - if (connectTimeoutSecond) - settings.connectionTimeout = std::chrono::milliseconds{*connectTimeoutSecond * util::MILLISECONDS_PER_SECOND}; + if (config_.getValueView("connect_timeout").hasValue()) { + auto const connectTimeoutSecond = config_.get("connect_timeout"); + settings.connectionTimeout = std::chrono::milliseconds{connectTimeoutSecond * util::MILLISECONDS_PER_SECOND}; + } - auto const requestTimeoutSecond = config_.maybeValue("request_timeout"); - if (requestTimeoutSecond) - settings.requestTimeout = std::chrono::milliseconds{*requestTimeoutSecond * util::MILLISECONDS_PER_SECOND}; + if (config_.getValueView("request_timeout").hasValue()) { + auto const requestTimeoutSecond = config_.get("request_timeout"); + settings.requestTimeout = std::chrono::milliseconds{requestTimeoutSecond * util::MILLISECONDS_PER_SECOND}; + } settings.certificate = parseOptionalCertificate(); settings.username = config_.maybeValue("username"); diff --git a/src/data/cassandra/SettingsProvider.hpp b/src/data/cassandra/SettingsProvider.hpp index 517498a7..73ceb2fd 100644 --- a/src/data/cassandra/SettingsProvider.hpp +++ b/src/data/cassandra/SettingsProvider.hpp @@ -19,10 +19,9 @@ #pragma once -#include "data/cassandra/Handle.hpp" #include "data/cassandra/Types.hpp" -#include "util/config/Config.hpp" -#include "util/log/Logger.hpp" +#include "data/cassandra/impl/Cluster.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -34,7 +33,7 @@ namespace data::cassandra { * @brief Provides settings for @ref BasicCassandraBackend. */ class SettingsProvider { - util::Config config_; + util::config::ObjectView config_; std::string keyspace_; std::optional tablePrefix_; @@ -47,7 +46,7 @@ public: * * @param cfg The config of Clio to use */ - explicit SettingsProvider(util::Config const& cfg); + explicit SettingsProvider(util::config::ObjectView const& cfg); /** * @return The cluster settings diff --git a/src/etl/CacheLoader.hpp b/src/etl/CacheLoader.hpp index ddee6220..739d94e8 100644 --- a/src/etl/CacheLoader.hpp +++ b/src/etl/CacheLoader.hpp @@ -64,7 +64,11 @@ public: * @param backend The backend to use * @param cache The cache to load into */ - CacheLoader(util::Config const& config, std::shared_ptr const& backend, CacheType& cache) + CacheLoader( + util::config::ClioConfigDefinition const& config, + std::shared_ptr const& backend, + CacheType& cache + ) : backend_{backend}, cache_{cache}, settings_{make_CacheLoaderSettings(config)}, ctx_{settings_.numThreads} { } diff --git a/src/etl/CacheLoaderSettings.cpp b/src/etl/CacheLoaderSettings.cpp index 9b69d67b..08f43ed8 100644 --- a/src/etl/CacheLoaderSettings.cpp +++ b/src/etl/CacheLoaderSettings.cpp @@ -19,11 +19,12 @@ #include "etl/CacheLoaderSettings.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include +#include #include namespace etl { @@ -47,31 +48,29 @@ CacheLoaderSettings::isDisabled() const } [[nodiscard]] CacheLoaderSettings -make_CacheLoaderSettings(util::Config const& config) +make_CacheLoaderSettings(util::config::ClioConfigDefinition const& config) { CacheLoaderSettings settings; - settings.numThreads = config.valueOr("io_threads", settings.numThreads); - if (config.contains("cache")) { - auto const cache = config.section("cache"); - // Given diff number to generate cursors - settings.numCacheDiffs = cache.valueOr("num_diffs", settings.numCacheDiffs); - // Given cursors number fetching from diff - settings.numCacheCursorsFromDiff = cache.valueOr("num_cursors_from_diff", 0); - // Given cursors number fetching from account - settings.numCacheCursorsFromAccount = cache.valueOr("num_cursors_from_account", 0); + settings.numThreads = config.get("io_threads"); + auto const cache = config.getObject("cache"); + // Given diff number to generate cursors + settings.numCacheDiffs = cache.get("num_diffs"); + // Given cursors number fetching from diff + settings.numCacheCursorsFromDiff = cache.get("num_cursors_from_diff"); + // Given cursors number fetching from account + settings.numCacheCursorsFromAccount = cache.get("num_cursors_from_account"); - settings.numCacheMarkers = cache.valueOr("num_markers", settings.numCacheMarkers); - settings.cachePageFetchSize = cache.valueOr("page_fetch_size", settings.cachePageFetchSize); + settings.numCacheMarkers = cache.get("num_markers"); + settings.cachePageFetchSize = cache.get("page_fetch_size"); + + auto const entry = cache.get("load"); + if (boost::iequals(entry, "sync")) + settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC; + if (boost::iequals(entry, "async")) + settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC; + if (boost::iequals(entry, "none") or boost::iequals(entry, "no")) + settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE; - if (auto entry = cache.maybeValue("load"); entry) { - if (boost::iequals(*entry, "sync")) - settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC; - if (boost::iequals(*entry, "async")) - settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC; - if (boost::iequals(*entry, "none") or boost::iequals(*entry, "no")) - settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE; - } - } return settings; } diff --git a/src/etl/CacheLoaderSettings.hpp b/src/etl/CacheLoaderSettings.hpp index b6e2bca7..367d915b 100644 --- a/src/etl/CacheLoaderSettings.hpp +++ b/src/etl/CacheLoaderSettings.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include @@ -64,6 +64,6 @@ struct CacheLoaderSettings { * @returns The CacheLoaderSettings object */ [[nodiscard]] CacheLoaderSettings -make_CacheLoaderSettings(util::Config const& config); +make_CacheLoaderSettings(util::config::ClioConfigDefinition const& config); } // namespace etl diff --git a/src/etl/ETLService.cpp b/src/etl/ETLService.cpp index 3017e417..6065cd0a 100644 --- a/src/etl/ETLService.cpp +++ b/src/etl/ETLService.cpp @@ -26,8 +26,8 @@ #include "feed/SubscriptionManagerInterface.hpp" #include "util/Assert.hpp" #include "util/Constants.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -262,7 +262,7 @@ ETLService::doWork() } ETLService::ETLService( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -280,9 +280,9 @@ ETLService::ETLService( { startSequence_ = config.maybeValue("start_sequence"); finishSequence_ = config.maybeValue("finish_sequence"); - state_.isReadOnly = config.valueOr("read_only", static_cast(state_.isReadOnly)); - extractorThreads_ = config.valueOr("extractor_threads", extractorThreads_); - txnThreshold_ = config.valueOr("txn_threshold", txnThreshold_); + state_.isReadOnly = config.get("read_only"); + extractorThreads_ = config.get("extractor_threads"); + txnThreshold_ = config.get("txn_threshold"); // This should probably be done in the backend factory but we don't have state available until here backend_->setCorruptionDetector(CorruptionDetector{state_, backend->cache()}); diff --git a/src/etl/ETLService.hpp b/src/etl/ETLService.hpp index 7502699d..3583bb89 100644 --- a/src/etl/ETLService.hpp +++ b/src/etl/ETLService.hpp @@ -119,7 +119,7 @@ public: * @param ledgers The network validated ledgers datastructure */ ETLService( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -142,7 +142,7 @@ public: */ static std::shared_ptr make_ETLService( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, diff --git a/src/etl/LoadBalancer.cpp b/src/etl/LoadBalancer.cpp index 4edda00a..f10a7bc2 100644 --- a/src/etl/LoadBalancer.cpp +++ b/src/etl/LoadBalancer.cpp @@ -29,6 +29,9 @@ #include "util/Random.hpp" #include "util/ResponseExpirationCache.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -51,13 +54,13 @@ #include #include -using namespace util; +using namespace util::config; namespace etl { std::shared_ptr LoadBalancer::make_LoadBalancer( - Config const& config, + ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -71,7 +74,7 @@ LoadBalancer::make_LoadBalancer( } LoadBalancer::LoadBalancer( - Config const& config, + ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -79,23 +82,23 @@ LoadBalancer::LoadBalancer( SourceFactory sourceFactory ) { - auto const forwardingCacheTimeout = config.valueOr("forwarding.cache_timeout", 0.f); + auto const forwardingCacheTimeout = config.get("forwarding.cache_timeout"); if (forwardingCacheTimeout > 0.f) { forwardingCache_ = util::ResponseExpirationCache{ - Config::toMilliseconds(forwardingCacheTimeout), + util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout), {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"} }; } - static constexpr std::uint32_t MAX_DOWNLOAD = 256; - if (auto value = config.maybeValue("num_markers"); value) { - ASSERT(*value > 0 and *value <= MAX_DOWNLOAD, "'num_markers' value in config must be in range 1-256"); - downloadRanges_ = *value; + auto const numMarkers = config.getValueView("num_markers"); + if (numMarkers.hasValue()) { + auto const value = numMarkers.asIntType(); + downloadRanges_ = value; } else if (backend->fetchLedgerRange()) { downloadRanges_ = 4; } - auto const allowNoEtl = config.valueOr("allow_no_etl", false); + auto const allowNoEtl = config.get("allow_no_etl"); auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) { LOG(log_.warn()) << log; @@ -106,10 +109,12 @@ LoadBalancer::LoadBalancer( } }; - auto const forwardingTimeout = Config::toMilliseconds(config.valueOr("forwarding.request_timeout", 10.)); - for (auto const& entry : config.array("etl_sources")) { + auto const forwardingTimeout = + ClioConfigDefinition::toMilliseconds(config.get("forwarding.request_timeout")); + auto const etlArray = config.getArray("etl_sources"); + for (auto it = etlArray.begin(); it != etlArray.end(); ++it) { auto source = sourceFactory( - entry, + *it, ioc, backend, subscriptions, diff --git a/src/etl/LoadBalancer.hpp b/src/etl/LoadBalancer.hpp index aa7e70c2..9d19aeed 100644 --- a/src/etl/LoadBalancer.hpp +++ b/src/etl/LoadBalancer.hpp @@ -27,8 +27,8 @@ #include "rpc/Errors.hpp" #include "util/Mutex.hpp" #include "util/ResponseExpirationCache.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -103,7 +103,7 @@ public: * @param sourceFactory A factory function to create a source */ LoadBalancer( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -124,7 +124,7 @@ public: */ static std::shared_ptr make_LoadBalancer( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, diff --git a/src/etl/Source.cpp b/src/etl/Source.cpp index 534cc89b..a678bbf0 100644 --- a/src/etl/Source.cpp +++ b/src/etl/Source.cpp @@ -26,7 +26,8 @@ #include "etl/impl/SourceImpl.hpp" #include "etl/impl/SubscriptionSource.hpp" #include "feed/SubscriptionManagerInterface.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" #include @@ -39,7 +40,7 @@ namespace etl { SourcePtr make_Source( - util::Config const& config, + util::config::ObjectView const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, @@ -50,9 +51,9 @@ make_Source( SourceBase::OnLedgerClosedHook onLedgerClosed ) { - auto const ip = config.valueOr("ip", {}); - auto const wsPort = config.valueOr("ws_port", {}); - auto const grpcPort = config.valueOr("grpc_port", {}); + auto const ip = config.get("ip"); + auto const wsPort = config.get("ws_port"); + auto const grpcPort = config.get("grpc_port"); impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout}; impl::GrpcSource grpcSource{ip, grpcPort, std::move(backend)}; diff --git a/src/etl/Source.hpp b/src/etl/Source.hpp index 01fea998..6adda85d 100644 --- a/src/etl/Source.hpp +++ b/src/etl/Source.hpp @@ -23,8 +23,9 @@ #include "etl/NetworkValidatedLedgersInterface.hpp" #include "feed/SubscriptionManagerInterface.hpp" #include "rpc/Errors.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -147,7 +148,7 @@ public: using SourcePtr = std::unique_ptr; using SourceFactory = std::function backend, std::shared_ptr subscriptions, @@ -175,7 +176,7 @@ using SourceFactory = std::function backend, std::shared_ptr subscriptions, diff --git a/src/feed/SubscriptionManager.hpp b/src/feed/SubscriptionManager.hpp index 1a3a3f66..e07fa34a 100644 --- a/src/feed/SubscriptionManager.hpp +++ b/src/feed/SubscriptionManager.hpp @@ -30,8 +30,8 @@ #include "feed/impl/TransactionFeed.hpp" #include "util/async/AnyExecutionContext.hpp" #include "util/async/context/BasicExecutionContext.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -77,9 +77,12 @@ public: * @return A shared pointer to a new instance of SubscriptionManager */ static std::shared_ptr - make_SubscriptionManager(util::Config const& config, std::shared_ptr const& backend) + make_SubscriptionManager( + util::config::ClioConfigDefinition const& config, + std::shared_ptr const& backend + ) { - auto const workersNum = config.valueOr("subscription_workers", 1); + auto const workersNum = config.get("subscription_workers"); util::Logger const logger{"Subscriptions"}; LOG(logger.info()) << "Starting subscription manager with " << workersNum << " workers"; diff --git a/src/main/Main.cpp b/src/main/Main.cpp index ffd4dbd9..1c1d6650 100644 --- a/src/main/Main.cpp +++ b/src/main/Main.cpp @@ -21,13 +21,16 @@ #include "app/ClioApplication.hpp" #include "rpc/common/impl/HandlerProvider.hpp" #include "util/TerminationHandler.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ConfigFileJson.hpp" #include #include #include +using namespace util::config; + int main(int argc, char const* argv[]) try { @@ -37,14 +40,19 @@ try { return action.apply( [](app::CliArgs::Action::Exit const& exit) { return exit.exitCode; }, [](app::CliArgs::Action::Run const& run) { - auto const config = util::ConfigReader::open(run.configPath); - if (!config) { - std::cerr << "Couldnt parse config '" << run.configPath << "'." << std::endl; + auto const json = ConfigFileJson::make_ConfigFileJson(run.configPath); + if (!json.has_value()) { + std::cerr << json.error().error << std::endl; return EXIT_FAILURE; } - util::LogService::init(config); - app::ClioApplication clio{config}; - + auto const errors = ClioConfig.parse(json.value()); + if (errors.has_value()) { + for (auto const& err : errors.value()) + std::cerr << err.error << std::endl; + return EXIT_FAILURE; + } + util::LogService::init(ClioConfig); + app::ClioApplication clio{ClioConfig}; return clio.run(run.useNgWebServer); } ); diff --git a/src/rpc/RPCEngine.hpp b/src/rpc/RPCEngine.hpp index 2194b22f..bb140bbd 100644 --- a/src/rpc/RPCEngine.hpp +++ b/src/rpc/RPCEngine.hpp @@ -84,7 +84,7 @@ public: * @param handlerProvider The handler provider to use */ RPCEngine( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, @@ -100,13 +100,14 @@ public: , forwardingProxy_{balancer, counters, handlerProvider} { // Let main thread catch the exception if config type is wrong - auto const cacheTimeout = config.valueOr("rpc.cache_timeout", 0.f); + auto const cacheTimeout = config.get("rpc.cache_timeout"); if (cacheTimeout > 0.f) { LOG(log_.info()) << fmt::format("Init RPC Cache, timeout: {} seconds", cacheTimeout); responseCache_.emplace( - util::Config::toMilliseconds(cacheTimeout), std::unordered_set{"server_info"} + util::config::ClioConfigDefinition::toMilliseconds(cacheTimeout), + std::unordered_set{"server_info"} ); } } @@ -125,7 +126,7 @@ public: */ static std::shared_ptr make_RPCEngine( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& balancer, web::dosguard::DOSGuardInterface const& dosGuard, diff --git a/src/rpc/WorkQueue.cpp b/src/rpc/WorkQueue.cpp index 305c35b6..93a526b9 100644 --- a/src/rpc/WorkQueue.cpp +++ b/src/rpc/WorkQueue.cpp @@ -19,7 +19,6 @@ #include "rpc/WorkQueue.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" #include "util/prometheus/Label.hpp" #include "util/prometheus/Prometheus.hpp" @@ -29,7 +28,6 @@ #include #include #include -#include #include namespace rpc { @@ -92,12 +90,12 @@ WorkQueue::stop(std::function onQueueEmpty) } WorkQueue -WorkQueue::make_WorkQueue(util::Config const& config) +WorkQueue::make_WorkQueue(util::config::ClioConfigDefinition const& config) { static util::Logger const log{"RPC"}; - auto const serverConfig = config.section("server"); - auto const numThreads = config.valueOr("workers", std::thread::hardware_concurrency()); - auto const maxQueueSize = serverConfig.valueOr("max_queue_size", 0); // 0 is no limit + auto const serverConfig = config.getObject("server"); + auto const numThreads = config.get("workers"); + auto const maxQueueSize = serverConfig.get("max_queue_size"); LOG(log.info()) << "Number of workers = " << numThreads << ". Max queue size = " << maxQueueSize; return WorkQueue{numThreads, maxQueueSize}; diff --git a/src/rpc/WorkQueue.hpp b/src/rpc/WorkQueue.hpp index 1f540cba..16997cf8 100644 --- a/src/rpc/WorkQueue.hpp +++ b/src/rpc/WorkQueue.hpp @@ -19,10 +19,9 @@ #pragma once -#include "util/Assert.hpp" #include "util/Mutex.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" @@ -97,7 +96,7 @@ public: * @return The work queue */ static WorkQueue - make_WorkQueue(util::Config const& config); + make_WorkQueue(util::config::ClioConfigDefinition const& config); /** * @brief Submit a job to the work queue. diff --git a/src/rpc/common/impl/APIVersionParser.cpp b/src/rpc/common/impl/APIVersionParser.cpp index 2d177853..3a89b28e 100644 --- a/src/rpc/common/impl/APIVersionParser.cpp +++ b/src/rpc/common/impl/APIVersionParser.cpp @@ -19,8 +19,7 @@ #include "rpc/common/impl/APIVersionParser.hpp" -#include "rpc/common/APIVersion.hpp" -#include "util/log/Logger.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -33,11 +32,22 @@ using namespace std; namespace rpc::impl { -ProductionAPIVersionParser::ProductionAPIVersionParser(util::Config const& config) +ProductionAPIVersionParser::ProductionAPIVersionParser( + uint32_t defaultVersion, + uint32_t minVersion, + uint32_t maxVersion +) + : defaultVersion_{defaultVersion}, minVersion_{minVersion}, maxVersion_{maxVersion} +{ + LOG(log_.info()) << "API version settings: [min = " << minVersion_ << "; max = " << maxVersion_ + << "; default = " << defaultVersion_ << "]"; +} + +ProductionAPIVersionParser::ProductionAPIVersionParser(util::config::ObjectView const& config) : ProductionAPIVersionParser( - config.valueOr("default", API_VERSION_DEFAULT), - config.valueOr("min", API_VERSION_MIN), - config.valueOr("max", API_VERSION_MAX) + config.get("default"), + config.get("min"), + config.get("max") ) { } diff --git a/src/rpc/common/impl/APIVersionParser.hpp b/src/rpc/common/impl/APIVersionParser.hpp index af0f91c0..b4f8d454 100644 --- a/src/rpc/common/impl/APIVersionParser.hpp +++ b/src/rpc/common/impl/APIVersionParser.hpp @@ -20,12 +20,11 @@ #pragma once #include "rpc/common/APIVersion.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ObjectView.hpp" #include -#include #include #include #include @@ -40,39 +39,13 @@ class ProductionAPIVersionParser : public APIVersionParser { uint32_t maxVersion_; public: - // Note: this constructor must remain in the header because of UNITTEST_BUILD definition below ProductionAPIVersionParser( uint32_t defaultVersion = API_VERSION_DEFAULT, uint32_t minVersion = API_VERSION_MIN, uint32_t maxVersion = API_VERSION_MAX - ) - : defaultVersion_{defaultVersion}, minVersion_{minVersion}, maxVersion_{maxVersion} - { -#ifndef UNITTEST_BUILD - // in production, we don't want the ability to misconfigure clio with bogus versions - // that are not actually supported by the code itself. for testing it is desired however. - auto checkRange = [this](uint32_t version, std::string label) { - if (std::clamp(version, API_VERSION_MIN, API_VERSION_MAX) != version) { - LOG(log_.error()) << "API version settings issue detected: " << label << " version with value " - << version << " is outside of supported range " << API_VERSION_MIN << "-" - << API_VERSION_MAX << "; Falling back to hardcoded values."; + ); - defaultVersion_ = API_VERSION_DEFAULT; - minVersion_ = API_VERSION_MIN; - maxVersion_ = API_VERSION_MAX; - } - }; - - checkRange(defaultVersion, "default"); - checkRange(minVersion, "minimum"); - checkRange(maxVersion, "maximum"); -#endif - - LOG(log_.info()) << "API version settings: [min = " << minVersion_ << "; max = " << maxVersion_ - << "; default = " << defaultVersion_ << "]"; - } - - ProductionAPIVersionParser(util::Config const& config); + ProductionAPIVersionParser(util::config::ObjectView const& config); std::expected parse(boost::json::object const& request) const override; diff --git a/src/rpc/common/impl/HandlerProvider.cpp b/src/rpc/common/impl/HandlerProvider.cpp index 1b905e56..90ec3f7b 100644 --- a/src/rpc/common/impl/HandlerProvider.cpp +++ b/src/rpc/common/impl/HandlerProvider.cpp @@ -60,7 +60,7 @@ #include "rpc/handlers/Tx.hpp" #include "rpc/handlers/Unsubscribe.hpp" #include "rpc/handlers/VersionHandler.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -69,7 +69,7 @@ namespace rpc::impl { ProductionHandlerProvider::ProductionHandlerProvider( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, std::shared_ptr const& balancer, diff --git a/src/rpc/common/impl/HandlerProvider.hpp b/src/rpc/common/impl/HandlerProvider.hpp index 906f88b1..89ea5661 100644 --- a/src/rpc/common/impl/HandlerProvider.hpp +++ b/src/rpc/common/impl/HandlerProvider.hpp @@ -25,7 +25,7 @@ #include "rpc/common/AnyHandler.hpp" #include "rpc/common/HandlerProvider.hpp" #include "rpc/common/Types.hpp" -#include "util/config/Config.hpp" +#include "util/log/Logger.hpp" #include #include @@ -52,7 +52,7 @@ class ProductionHandlerProvider final : public HandlerProvider { public: ProductionHandlerProvider( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& subscriptionManager, std::shared_ptr const& balancer, diff --git a/src/rpc/handlers/VersionHandler.hpp b/src/rpc/handlers/VersionHandler.hpp index a84feaa8..515377b1 100644 --- a/src/rpc/handlers/VersionHandler.hpp +++ b/src/rpc/handlers/VersionHandler.hpp @@ -22,7 +22,7 @@ #include "rpc/common/APIVersion.hpp" #include "rpc/common/Types.hpp" #include "rpc/common/impl/APIVersionParser.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -53,11 +53,11 @@ public: * * @param config The configuration to use */ - explicit VersionHandler(util::Config const& config) + explicit VersionHandler(util::config::ClioConfigDefinition const& config) : apiVersionParser_( - config.valueOr("default", API_VERSION_DEFAULT), - config.valueOr("min", API_VERSION_MIN), - config.valueOr("max", API_VERSION_MAX) + config.get("api_version.default"), + config.get("api_version.min"), + config.get("api_version.max") ) { } diff --git a/src/util/SignalsHandler.cpp b/src/util/SignalsHandler.cpp index c4f212f5..427a7b93 100644 --- a/src/util/SignalsHandler.cpp +++ b/src/util/SignalsHandler.cpp @@ -20,8 +20,8 @@ #include "util/SignalsHandler.hpp" #include "util/Assert.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -70,7 +70,7 @@ SignalsHandler* SignalsHandlerStatic::handler_ = nullptr; } // namespace impl -SignalsHandler::SignalsHandler(Config const& config, std::function forceExitHandler) +SignalsHandler::SignalsHandler(config::ClioConfigDefinition const& config, std::function forceExitHandler) : gracefulPeriod_(0) , context_(1) , stopHandler_([this, forceExitHandler](int) mutable { @@ -99,9 +99,7 @@ SignalsHandler::SignalsHandler(Config const& config, std::function force { impl::SignalsHandlerStatic::registerHandler(*this); - gracefulPeriod_ = Config::toMilliseconds(config.valueOr("graceful_period", 10.f)); - ASSERT(gracefulPeriod_.count() >= 0, "Graceful period must be non-negative"); - + gracefulPeriod_ = util::config::ClioConfigDefinition::toMilliseconds(config.get("graceful_period")); setHandler(impl::SignalsHandlerStatic::handleSignal); } diff --git a/src/util/SignalsHandler.hpp b/src/util/SignalsHandler.hpp index 49f5b4f2..9c51d8a5 100644 --- a/src/util/SignalsHandler.hpp +++ b/src/util/SignalsHandler.hpp @@ -20,8 +20,8 @@ #pragma once #include "util/async/context/BasicExecutionContext.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -35,7 +35,6 @@ #include #include #include -#include namespace util { @@ -70,7 +69,10 @@ public: * @param config The configuration. * @param forceExitHandler The handler for forced exit. */ - SignalsHandler(Config const& config, std::function forceExitHandler = defaultForceExitHandler_); + SignalsHandler( + util::config::ClioConfigDefinition const& config, + std::function forceExitHandler = defaultForceExitHandler_ + ); SignalsHandler(SignalsHandler const&) = delete; SignalsHandler(SignalsHandler&&) = delete; diff --git a/src/util/Taggable.hpp b/src/util/Taggable.hpp index 71ce59ba..9092c6a1 100644 --- a/src/util/Taggable.hpp +++ b/src/util/Taggable.hpp @@ -19,7 +19,8 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/Assert.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -33,7 +34,9 @@ #include #include #include -#include +#include +#include +#include namespace util { namespace impl { @@ -178,6 +181,22 @@ class TagDecoratorFactory final { Type type_; /*< The type of TagDecorator this factory produces */ ParentType parent_ = std::nullopt; /*< The parent tag decorator to bind */ + static Type + getLogTagType(std::string_view style) + { + if (boost::iequals(style, "int") || boost::iequals(style, "uint")) + return TagDecoratorFactory::Type::UINT; + + if (boost::iequals(style, "null") || boost::iequals(style, "none")) + return TagDecoratorFactory::Type::NONE; + + if (boost::iequals(style, "uuid")) + return TagDecoratorFactory::Type::UUID; + + ASSERT(false, "log_tag_style does not have valid value"); + std::unreachable(); + } + public: ~TagDecoratorFactory() = default; @@ -186,7 +205,8 @@ public: * * @param config The configuration as a json object */ - explicit TagDecoratorFactory(util::Config const& config) : type_{config.valueOr("log_tag_style", Type::NONE)} + explicit TagDecoratorFactory(util::config::ClioConfigDefinition const& config) + : type_{getLogTagType(config.get("log_tag_style"))} { } @@ -212,26 +232,6 @@ public: */ TagDecoratorFactory with(ParentType parent) const noexcept; - -private: - friend Type - tag_invoke(boost::json::value_to_tag, boost::json::value const& value) - { - if (not value.is_string()) - throw std::runtime_error("`log_tag_style` must be a string"); - auto const& style = value.as_string(); - - if (boost::iequals(style, "int") || boost::iequals(style, "uint")) - return TagDecoratorFactory::Type::UINT; - - if (boost::iequals(style, "null") || boost::iequals(style, "none")) - return TagDecoratorFactory::Type::NONE; - - if (boost::iequals(style, "uuid")) - return TagDecoratorFactory::Type::UUID; - - throw std::runtime_error("Could not parse `log_tag_style`: expected `uint`, `uuid` or `null`"); - } }; /** diff --git a/src/util/log/Logger.cpp b/src/util/log/Logger.cpp index 06a7782c..b7b4cc94 100644 --- a/src/util/log/Logger.cpp +++ b/src/util/log/Logger.cpp @@ -19,8 +19,11 @@ #include "util/log/Logger.hpp" +#include "util/Assert.hpp" #include "util/SourceLocation.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" #include #include @@ -53,9 +56,11 @@ #include #include #include +#include #include #include #include +#include #include #include @@ -80,13 +85,15 @@ operator<<(std::ostream& stream, Severity sev) return stream << labels.at(static_cast(sev)); } +/** + * @brief converts the loglevel to string to a corresponding Severity enum value. + * + * @param logLevel A string representing the log level + * @return Severity The corresponding Severity enum value. + */ Severity -tag_invoke(boost::json::value_to_tag, boost::json::value const& value) +getSeverityLevel(std::string_view logLevel) { - if (not value.is_string()) - throw std::runtime_error("`log_level` must be a string"); - auto const& logLevel = value.as_string(); - if (boost::iequals(logLevel, "trace")) return Severity::TRC; if (boost::iequals(logLevel, "debug")) @@ -100,23 +107,22 @@ tag_invoke(boost::json::value_to_tag, boost::json::value const& value) if (boost::iequals(logLevel, "fatal")) return Severity::FTL; - throw std::runtime_error( - "Could not parse `log_level`: expected `trace`, `debug`, `info`, `warning`, `error` or `fatal`" - ); + // already checked during parsing of config that value must be valid + ASSERT(false, "Parsing of log_level is incorrect"); + std::unreachable(); } void -LogService::init(util::Config const& config) +LogService::init(config::ClioConfigDefinition const& config) { namespace keywords = boost::log::keywords; namespace sinks = boost::log::sinks; boost::log::add_common_attributes(); boost::log::register_simple_formatter_factory("Severity"); - auto const defaultFormat = "%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%"; - std::string format = config.valueOr("log_format", defaultFormat); + std::string format = config.get("log_format"); - if (config.valueOr("log_to_console", false)) { + if (config.get("log_to_console")) { boost::log::add_console_log( std::cout, keywords::format = format, keywords::filter = log_severity < Severity::FTL ); @@ -125,13 +131,14 @@ LogService::init(util::Config const& config) // Always print fatal logs to cerr boost::log::add_console_log(std::cerr, keywords::format = format, keywords::filter = log_severity >= Severity::FTL); - if (auto logDir = config.maybeValue("log_directory"); logDir) { + auto const logDir = config.maybeValue("log_directory"); + if (logDir) { boost::filesystem::path dirPath{logDir.value()}; if (!boost::filesystem::exists(dirPath)) boost::filesystem::create_directories(dirPath); - auto const rotationSize = config.valueOr("log_rotation_size", 2048u) * 1024u * 1024u; - auto const rotationPeriod = config.valueOr("log_rotation_hour_interval", 12u); - auto const dirSize = config.valueOr("log_directory_max_size", 50u * 1024u) * 1024u * 1024u; + auto const rotationSize = config.get("log_rotation_size"); + auto const rotationPeriod = config.get("log_rotation_hour_interval"); + auto const dirSize = config.get("log_directory_max_size"); auto fileSink = boost::log::add_file_log( keywords::file_name = dirPath / "clio.log", keywords::target_file_name = dirPath / "clio_%Y-%m-%d_%H-%M-%S.log", @@ -149,19 +156,22 @@ LogService::init(util::Config const& config) } // get default severity, can be overridden per channel using the `log_channels` array - auto defaultSeverity = config.valueOr("log_level", Severity::NFO); + auto const defaultSeverity = getSeverityLevel(config.get("log_level")); std::unordered_map min_severity; for (auto const& channel : Logger::CHANNELS) min_severity[channel] = defaultSeverity; min_severity["Alert"] = Severity::WRN; // Channel for alerts, always warning severity - for (auto const overrides = config.arrayOr("log_channels", {}); auto const& cfg : overrides) { - auto name = cfg.valueOrThrow("channel", "Channel name is required"); + auto const overrides = config.getArray("log_channels"); + + for (auto it = overrides.begin(); it != overrides.end(); ++it) { + auto const& cfg = *it; + auto name = cfg.get("channel"); if (std::count(std::begin(Logger::CHANNELS), std::end(Logger::CHANNELS), name) == 0) throw std::runtime_error("Can't override settings for log channel " + name + ": invalid channel"); - min_severity[name] = cfg.valueOr("log_level", defaultSeverity); + min_severity[name] = getSeverityLevel(config.get("log_level")); } auto log_filter = [min_severity = std::move(min_severity), diff --git a/src/util/log/Logger.hpp b/src/util/log/Logger.hpp index 46ecd476..a891b90b 100644 --- a/src/util/log/Logger.hpp +++ b/src/util/log/Logger.hpp @@ -52,7 +52,9 @@ namespace util { -class Config; +namespace config { +class ClioConfigDefinition; +} // namespace config /** * @brief Skips evaluation of expensive argument lists if the given logger is disabled for the required severity level. @@ -164,16 +166,6 @@ class Logger final { private: [[nodiscard]] static std::string pretty_path(SourceLocationType const& loc, size_t max_depth = 3); - - /** - * @brief Custom JSON parser for @ref Severity. - * - * @param value The JSON string to parse - * @return The parsed severity - * @throws std::runtime_error Thrown if severity is not in the right format - */ - friend Severity - tag_invoke(boost::json::value_to_tag, boost::json::value const& value); }; public: @@ -285,7 +277,7 @@ public: * @param config The configuration to use */ static void - init(Config const& config); + init(config::ClioConfigDefinition const& config); /** * @brief Globally accesible General logger at Severity::TRC severity diff --git a/src/util/newconfig/ConfigConstraints.cpp b/src/util/newconfig/ConfigConstraints.cpp index a45fc818..3c1507b0 100644 --- a/src/util/newconfig/ConfigConstraints.cpp +++ b/src/util/newconfig/ConfigConstraints.cpp @@ -95,9 +95,11 @@ PositiveDouble::checkTypeImpl(Value const& num) const std::optional PositiveDouble::checkValueImpl(Value const& num) const { + if (std::holds_alternative(num) && std::get(num) >= 0) + return std::nullopt; if (std::get(num) >= 0) return std::nullopt; - return Error{"Double number must be greater than 0"}; + return Error{"Double number must be greater than or equal to 0"}; } } // namespace util::config diff --git a/src/util/newconfig/ConfigConstraints.hpp b/src/util/newconfig/ConfigConstraints.hpp index 79027d55..a7adc1f6 100644 --- a/src/util/newconfig/ConfigConstraints.hpp +++ b/src/util/newconfig/ConfigConstraints.hpp @@ -79,6 +79,11 @@ static constexpr std::array LOAD_CACHE_MODE = { */ static constexpr std::array DATABASE_TYPE = {"cassandra"}; +/** + * @brief specific values that are accepted for server's processing_policy in config. + */ +static constexpr std::array PROCESSING_POLICY = {"parallel", "sequent"}; + /** * @brief An interface to enforce constraints on certain values within ClioConfigDefinition. */ @@ -346,9 +351,13 @@ static constinit OneOf validateLogLevelName{"log_level", LOG_LEVELS}; static constinit OneOf validateCassandraName{"database.type", DATABASE_TYPE}; static constinit OneOf validateLoadMode{"cache.load", LOAD_CACHE_MODE}; static constinit OneOf validateLogTag{"log_tag_style", LOG_TAGS}; +static constinit OneOf validateProcessingPolicy{"server.processing_policy", PROCESSING_POLICY}; static constinit PositiveDouble validatePositiveDouble{}; +static constinit NumberValueConstraint validateNumMarkers{1, 256}; +static constinit NumberValueConstraint validateIOThreads{1, std::numeric_limits::max()}; + static constinit NumberValueConstraint validateUint16{ std::numeric_limits::min(), std::numeric_limits::max() diff --git a/src/util/newconfig/ConfigDefinition.cpp b/src/util/newconfig/ConfigDefinition.cpp index 330c0f68..921a6185 100644 --- a/src/util/newconfig/ConfigDefinition.cpp +++ b/src/util/newconfig/ConfigDefinition.cpp @@ -20,6 +20,7 @@ #include "util/newconfig/ConfigDefinition.hpp" #include "util/Assert.hpp" +#include "util/Constants.hpp" #include "util/OverloadSet.hpp" #include "util/newconfig/Array.hpp" #include "util/newconfig/ArrayView.hpp" @@ -34,100 +35,18 @@ #include #include +#include +#include #include -#include #include #include #include #include -#include #include #include #include namespace util::config { -/** - * @brief Full Clio Configuration definition. - * - * Specifies which keys are valid in Clio Config and provides default values if user's do not specify one. Those - * without default values must be present in the user's config file. - */ -static ClioConfigDefinition ClioConfig = ClioConfigDefinition{ - {{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra").withConstraint(validateCassandraName)}, - {"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("localhost")}, - {"database.cassandra.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)}, - {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue("clio")}, - {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(3u)}, - {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.defaultValue("table_prefix")}, - {"database.cassandra.max_write_requests_outstanding", - ConfigValue{ConfigType::Integer}.defaultValue(10'000).withConstraint(validateUint32)}, - {"database.cassandra.max_read_requests_outstanding", - ConfigValue{ConfigType::Integer}.defaultValue(100'000).withConstraint(validateUint32)}, - {"database.cassandra.threads", - ConfigValue{ConfigType::Integer} - .defaultValue(static_cast(std::thread::hardware_concurrency())) - .withConstraint(validateUint32)}, - {"database.cassandra.core_connections_per_host", - ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint16)}, - {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint16)}, - {"database.cassandra.write_batch_size", - ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint16)}, - {"etl_source.[].ip", Array{ConfigValue{ConfigType::String}.withConstraint(validateIP)}}, - {"etl_source.[].ws_port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}}, - {"etl_source.[].grpc_port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}}, - {"forwarding.cache_timeout", - ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)}, - {"forwarding.request_timeout", - ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)}, - {"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}}, - {"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000).withConstraint(validateUint32)}, - {"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint32)}, - {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint32)}, - {"dos_guard.sweep_interval", - ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)}, - {"cache.peers.[].ip", Array{ConfigValue{ConfigType::String}.withConstraint(validateIP)}}, - {"cache.peers.[].port", Array{ConfigValue{ConfigType::String}.withConstraint(validatePort)}}, - {"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)}, - {"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)}, - {"server.workers", ConfigValue{ConfigType::Integer}.withConstraint(validateUint32)}, - {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)}, - {"server.local_admin", ConfigValue{ConfigType::Boolean}.optional()}, - {"server.admin_password", ConfigValue{ConfigType::String}.optional()}, - {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, - {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, - {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateUint16)}, - {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)}, - {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)}, - {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, - {"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16) - }, - {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)}, - {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)}, - {"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}}, - {"log_channels.[].log_level", - Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}}, - {"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)}, - {"log_format", - ConfigValue{ConfigType::String}.defaultValue( - R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)" - )}, - {"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - {"log_directory", ConfigValue{ConfigType::String}.optional()}, - {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048u).withConstraint(validateUint32)}, - {"log_directory_max_size", - ConfigValue{ConfigType::Integer}.defaultValue(50u * 1024u).withConstraint(validateUint32)}, - {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)}, - {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("uint").withConstraint(validateLogTag)}, - {"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(2u).withConstraint(validateUint32)}, - {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, - {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, - {"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, - {"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, - {"ssl_cert_file", ConfigValue{ConfigType::String}.optional()}, - {"ssl_key_file", ConfigValue{ConfigType::String}.optional()}, - {"api_version.min", ConfigValue{ConfigType::Integer}}, - {"api_version.max", ConfigValue{ConfigType::Integer}}} -}; ClioConfigDefinition::ClioConfigDefinition(std::initializer_list pair) { @@ -154,7 +73,7 @@ ClioConfigDefinition::getObject(std::string_view prefix, std::optional(map_.at(fullKey))) { @@ -198,6 +117,13 @@ ClioConfigDefinition::getValue(std::string_view fullKey) const std::unreachable(); } +std::chrono::milliseconds +ClioConfigDefinition::toMilliseconds(float value) +{ + ASSERT(value >= 0.0f, "Floating point value of seconds must be non-negative, got: {}", value); + return std::chrono::milliseconds{std::lroundf(value * static_cast(util::MILLISECONDS_PER_SECOND))}; +} + ValueView ClioConfigDefinition::getValueInArray(std::string_view fullKey, std::size_t index) const { diff --git a/src/util/newconfig/ConfigDefinition.hpp b/src/util/newconfig/ConfigDefinition.hpp index 01019339..85c08601 100644 --- a/src/util/newconfig/ConfigDefinition.hpp +++ b/src/util/newconfig/ConfigDefinition.hpp @@ -19,25 +19,33 @@ #pragma once +#include "rpc/common/APIVersion.hpp" #include "util/Assert.hpp" #include "util/newconfig/Array.hpp" +#include "util/newconfig/ConfigConstraints.hpp" #include "util/newconfig/ConfigDescription.hpp" #include "util/newconfig/ConfigFileInterface.hpp" #include "util/newconfig/ConfigValue.hpp" #include "util/newconfig/Error.hpp" #include "util/newconfig/ObjectView.hpp" +#include "util/newconfig/Types.hpp" #include "util/newconfig/ValueView.hpp" +#include +#include #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -113,7 +121,26 @@ public: * @return ValueView associated with the given key */ [[nodiscard]] ValueView - getValue(std::string_view fullKey) const; + getValueView(std::string_view fullKey) const; + + /** + * @brief Returns the specified value of given string if value exists + * + * @tparam T The type T to return + * @param fullKey The config key to search for + * @return Value of key of type T + */ + template + T + get(std::string_view fullKey) const + { + ASSERT(map_.contains(fullKey), "key {} does not exist in config", fullKey); + auto const val = map_.at(fullKey); + if (std::holds_alternative(val)) { + return ValueView{std::get(val)}.getValueImpl(); + } + std::unreachable(); + } /** * @brief Returns the specified ValueView object in an array with a given index @@ -170,6 +197,29 @@ public: [[nodiscard]] std::size_t arraySize(std::string_view prefix) const; + /** + * @brief Method to convert a float seconds value to milliseconds. + * + * @param value The value to convert + * @return The value in milliseconds + */ + static std::chrono::milliseconds + toMilliseconds(float value); + + /** + * @brief Returns the specified value of given string of type T if type and value exists + * + * @tparam T The type T to return + * @param fullKey The config key to search for + * @return The value of type T if it exists, std::nullopt otherwise. + */ + template + std::optional + maybeValue(std::string_view fullKey) const + { + return getValueView(fullKey).asOptional(); + } + /** * @brief Returns an iterator to the beginning of the configuration map. * @@ -229,4 +279,108 @@ private: std::unordered_map> map_; }; +/** + * @brief Full Clio Configuration definition. + * + * Specifies which keys are valid in Clio Config and provides default values if user's do not specify one. Those + * without default values must be present in the user's config file. + */ +static ClioConfigDefinition ClioConfig = ClioConfigDefinition{ + {{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra").withConstraint(validateCassandraName)}, + {"database.cassandra.contact_points", ConfigValue{ConfigType::String}.defaultValue("localhost")}, + {"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort).optional()}, + {"database.cassandra.keyspace", ConfigValue{ConfigType::String}.defaultValue("clio")}, + {"database.cassandra.replication_factor", + ConfigValue{ConfigType::Integer}.defaultValue(3u).withConstraint(validateUint16)}, + {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.max_write_requests_outstanding", + ConfigValue{ConfigType::Integer}.defaultValue(10'000).withConstraint(validateUint32)}, + {"database.cassandra.max_read_requests_outstanding", + ConfigValue{ConfigType::Integer}.defaultValue(100'000).withConstraint(validateUint32)}, + {"database.cassandra.threads", + ConfigValue{ConfigType::Integer} + .defaultValue(static_cast(std::thread::hardware_concurrency())) + .withConstraint(validateUint32)}, + {"database.cassandra.core_connections_per_host", + ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint16)}, + {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint16)}, + {"database.cassandra.write_batch_size", + ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint16)}, + {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"database.cassandra.request_timeout", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()}, + {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()}, + {"allow_no_etl", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"etl_sources.[].ip", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateIP)}}, + {"etl_sources.[].ws_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}}, + {"etl_sources.[].grpc_port", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validatePort)}}, + {"forwarding.cache_timeout", + ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)}, + {"forwarding.request_timeout", + ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)}, + {"rpc.cache_timeout", ConfigValue{ConfigType::Double}.defaultValue(0.0).withConstraint(validatePositiveDouble)}, + {"num_markers", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateNumMarkers)}, + {"dos_guard.whitelist.[]", Array{ConfigValue{ConfigType::String}}}, + {"dos_guard.max_fetches", ConfigValue{ConfigType::Integer}.defaultValue(1000'000u).withConstraint(validateUint32)}, + {"dos_guard.max_connections", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)}, + {"dos_guard.max_requests", ConfigValue{ConfigType::Integer}.defaultValue(20u).withConstraint(validateUint32)}, + {"dos_guard.sweep_interval", + ConfigValue{ConfigType::Double}.defaultValue(1.0).withConstraint(validatePositiveDouble)}, + {"workers", + ConfigValue{ConfigType::Integer}.defaultValue(std::thread::hardware_concurrency()).withConstraint(validateUint32) + }, + {"server.ip", ConfigValue{ConfigType::String}.withConstraint(validateIP)}, + {"server.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort)}, + {"server.max_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint32)}, + {"server.local_admin", ConfigValue{ConfigType::Boolean}.optional()}, + {"server.admin_password", ConfigValue{ConfigType::String}.optional()}, + {"server.processing_policy", + ConfigValue{ConfigType::String}.defaultValue("parallel").withConstraint(validateProcessingPolicy)}, + {"server.parallel_requests_limit", ConfigValue{ConfigType::Integer}.optional()}, + {"server.ws_max_sending_queue_size", ConfigValue{ConfigType::Integer}.defaultValue(1500)}, + {"server.__ng_web_server", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"prometheus.enabled", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, + {"prometheus.compress_reply", ConfigValue{ConfigType::Boolean}.defaultValue(true)}, + {"io_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateIOThreads)}, + {"subscription_workers", ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint32)}, + {"graceful_period", ConfigValue{ConfigType::Double}.defaultValue(10.0).withConstraint(validatePositiveDouble)}, + {"cache.num_diffs", ConfigValue{ConfigType::Integer}.defaultValue(32).withConstraint(validateUint16)}, + {"cache.num_markers", ConfigValue{ConfigType::Integer}.defaultValue(48).withConstraint(validateUint16)}, + {"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, + {"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16) + }, + {"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(validateUint16)}, + {"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(validateLoadMode)}, + {"log_channels.[].channel", Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateChannelName)}}, + {"log_channels.[].log_level", + Array{ConfigValue{ConfigType::String}.optional().withConstraint(validateLogLevelName)}}, + {"log_level", ConfigValue{ConfigType::String}.defaultValue("info").withConstraint(validateLogLevelName)}, + {"log_format", + ConfigValue{ConfigType::String}.defaultValue( + R"(%TimeStamp% (%SourceLocation%) [%ThreadID%] %Channel%:%Severity% %Message%)" + )}, + {"log_to_console", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"log_directory", ConfigValue{ConfigType::String}.optional()}, + {"log_rotation_size", ConfigValue{ConfigType::Integer}.defaultValue(2048).withConstraint(validateUint32)}, + {"log_directory_max_size", ConfigValue{ConfigType::Integer}.defaultValue(50 * 1024).withConstraint(validateUint32) + }, + {"log_rotation_hour_interval", ConfigValue{ConfigType::Integer}.defaultValue(12).withConstraint(validateUint32)}, + {"log_tag_style", ConfigValue{ConfigType::String}.defaultValue("none").withConstraint(validateLogTag)}, + {"extractor_threads", ConfigValue{ConfigType::Integer}.defaultValue(1u).withConstraint(validateUint32)}, + {"read_only", ConfigValue{ConfigType::Boolean}.defaultValue(false)}, + {"txn_threshold", ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(validateUint16)}, + {"start_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"finish_sequence", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)}, + {"ssl_cert_file", ConfigValue{ConfigType::String}.optional()}, + {"ssl_key_file", ConfigValue{ConfigType::String}.optional()}, + {"api_version.default", + ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_DEFAULT).withConstraint(validateApiVersion)}, + {"api_version.min", + ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MIN).withConstraint(validateApiVersion)}, + {"api_version.max", + ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MAX).withConstraint(validateApiVersion)}} +}; + } // namespace util::config diff --git a/src/util/newconfig/ConfigDescription.hpp b/src/util/newconfig/ConfigDescription.hpp index 094a8850..a9c0dfd3 100644 --- a/src/util/newconfig/ConfigDescription.hpp +++ b/src/util/newconfig/ConfigDescription.hpp @@ -63,9 +63,15 @@ public: private: static constexpr auto configDescription = std::array{ - KV{.key = "database.type", .value = "Type of database to use."}, + KV{.key = "database.type", .value = "Type of database to use. Default is Scylladb."}, KV{.key = "database.cassandra.contact_points", - .value = "Comma-separated list of contact points for Cassandra nodes."}, + .value = + "A list of IP addresses or hostnames of the initial nodes (Cassandra/Scylladb cluster nodes) that the " + "client will connect to when establishing a connection with the database."}, + KV{.key = "database.cassandra.secure_connect_bundle", + .value = "Configuration file that contains the necessary security credentials and connection details for " + "securely " + "connecting to a Cassandra database cluster."}, KV{.key = "database.cassandra.port", .value = "Port number to connect to Cassandra."}, KV{.key = "database.cassandra.keyspace", .value = "Keyspace to use in Cassandra."}, KV{.key = "database.cassandra.replication_factor", .value = "Number of replicated nodes for Scylladb."}, @@ -79,30 +85,59 @@ private: .value = "Number of core connections per host for Cassandra."}, KV{.key = "database.cassandra.queue_size_io", .value = "Queue size for I/O operations in Cassandra."}, KV{.key = "database.cassandra.write_batch_size", .value = "Batch size for write operations in Cassandra."}, - KV{.key = "etl_source.[].ip", .value = "IP address of the ETL source."}, - KV{.key = "etl_source.[].ws_port", .value = "WebSocket port of the ETL source."}, - KV{.key = "etl_source.[].grpc_port", .value = "gRPC port of the ETL source."}, + KV{.key = "database.cassandra.connect_timeout", + .value = "The maximum amount of time in seconds the system will wait for a connection to be successfully " + "established " + "with the database."}, + KV{.key = "database.cassandra.request_timeout", + .value = + "The maximum amount of time in seconds the system will wait for a request to be fetched from database."}, + KV{.key = "database.cassandra.username", .value = "The username used for authenticating with the database."}, + KV{.key = "database.cassandra.password", .value = "The password used for authenticating with the database."}, + KV{.key = "database.cassandra.certfile", + .value = "The path to the SSL/TLS certificate file used to establish a secure connection between the client " + "and the " + "Cassandra database."}, + KV{.key = "allow_no_etl", .value = "If True, no ETL nodes will run with Clio."}, + KV{.key = "etl_sources.[].ip", .value = "IP address of the ETL source."}, + KV{.key = "etl_sources.[].ws_port", .value = "WebSocket port of the ETL source."}, + KV{.key = "etl_sources.[].grpc_port", .value = "gRPC port of the ETL source."}, KV{.key = "forwarding.cache_timeout", .value = "Timeout duration for the forwarding cache used in Rippled communication."}, KV{.key = "forwarding.request_timeout", .value = "Timeout duration for the forwarding request used in Rippled communication."}, + KV{.key = "rpc.cache_timeout", .value = "Timeout duration for the rpc request."}, + KV{.key = "num_markers", + .value = "The number of markers is the number of coroutines to load the cache concurrently."}, KV{.key = "dos_guard.[].whitelist", .value = "List of IP addresses to whitelist for DOS protection."}, KV{.key = "dos_guard.max_fetches", .value = "Maximum number of fetch operations allowed by DOS guard."}, KV{.key = "dos_guard.max_connections", .value = "Maximum number of concurrent connections allowed by DOS guard." }, KV{.key = "dos_guard.max_requests", .value = "Maximum number of requests allowed by DOS guard."}, KV{.key = "dos_guard.sweep_interval", .value = "Interval in seconds for DOS guard to sweep/clear its state."}, - KV{.key = "cache.peers.[].ip", .value = "IP address of peer nodes to cache."}, - KV{.key = "cache.peers.[].port", .value = "Port number of peer nodes to cache."}, + KV{.key = "workers", .value = "Number of threads to process RPC requests."}, KV{.key = "server.ip", .value = "IP address of the Clio HTTP server."}, KV{.key = "server.port", .value = "Port number of the Clio HTTP server."}, - KV{.key = "server.max_queue_size", .value = "Maximum size of the server's request queue."}, - KV{.key = "server.workers", .value = "Maximum number of threads for server to run with."}, + KV{.key = "server.max_queue_size", + .value = "Maximum size of the server's request queue. Value of 0 is no limit."}, KV{.key = "server.local_admin", .value = "Indicates if the server should run with admin privileges."}, KV{.key = "server.admin_password", .value = "Password for Clio admin-only APIs."}, + KV{.key = "server.processing_policy", + .value = R"(Could be "sequent" or "parallel". For the sequent policy, requests from a single client + connection are processed one by one, with the next request read only after the previous one is processed. For the parallel policy, Clio will accept + all requests and process them in parallel, sending a reply for each request as soon as it is ready.)"}, + KV{.key = "server.parallel_requests_limit", + .value = R"(Optional parameter, used only if "processing_strategy" is + "parallel". It limits the number of requests for a single client connection that are processed in parallel. If not specified, the limit is infinite.)" + }, + KV{.key = "server.ws_max_sending_queue_size", .value = "Maximum size of the websocket sending queue."}, KV{.key = "prometheus.enabled", .value = "Enable or disable Prometheus metrics."}, KV{.key = "prometheus.compress_reply", .value = "Enable or disable compression of Prometheus responses."}, - KV{.key = "io_threads", .value = "Number of I/O threads."}, + KV{.key = "io_threads", .value = "Number of I/O threads. Value must be greater than 1"}, + KV{.key = "subscription_workers", + .value = "The number of worker threads or processes that are responsible for managing and processing " + "subscription-based tasks."}, + KV{.key = "graceful_period", .value = "Number of milliseconds server will wait to shutdown gracefully."}, KV{.key = "cache.num_diffs", .value = "Number of diffs to cache."}, KV{.key = "cache.num_markers", .value = "Number of markers to cache."}, KV{.key = "cache.num_cursors_from_diff", .value = "Num of cursors that are different."}, @@ -126,6 +161,7 @@ private: KV{.key = "finish_sequence", .value = "Ending ledger index."}, KV{.key = "ssl_cert_file", .value = "Path to the SSL certificate file."}, KV{.key = "ssl_key_file", .value = "Path to the SSL key file."}, + KV{.key = "api_version.default", .value = "Default API version Clio will run on."}, KV{.key = "api_version.min", .value = "Minimum API version."}, KV{.key = "api_version.max", .value = "Maximum API version."} }; diff --git a/src/util/newconfig/ConfigFileJson.cpp b/src/util/newconfig/ConfigFileJson.cpp index d59671d3..4bd5c6ef 100644 --- a/src/util/newconfig/ConfigFileJson.cpp +++ b/src/util/newconfig/ConfigFileJson.cpp @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -58,6 +59,9 @@ extractJsonValue(boost::json::value const& jsonValue) if (jsonValue.is_int64()) { return jsonValue.as_int64(); } + if (jsonValue.is_uint64()) { + return static_cast(jsonValue.as_uint64()); + } if (jsonValue.is_string()) { return jsonValue.as_string().c_str(); } @@ -67,7 +71,7 @@ extractJsonValue(boost::json::value const& jsonValue) if (jsonValue.is_double()) { return jsonValue.as_double(); } - ASSERT(false, "Json is not of type int, string, bool or double"); + ASSERT(false, "Json is not of type int, uint, string, bool or double"); std::unreachable(); } } // namespace diff --git a/src/util/newconfig/ConfigValue.hpp b/src/util/newconfig/ConfigValue.hpp index eb26b69f..8b80db47 100644 --- a/src/util/newconfig/ConfigValue.hpp +++ b/src/util/newconfig/ConfigValue.hpp @@ -219,7 +219,9 @@ private: if (type == ConfigType::Boolean && !std::holds_alternative(value)) { return Error{"value does not match type boolean"}; } - if (type == ConfigType::Double && !std::holds_alternative(value)) { + if (type == ConfigType::Double && (!std::holds_alternative(value))) { + if (std::holds_alternative(value)) + return std::nullopt; return Error{"value does not match type double"}; } if (type == ConfigType::Integer && !std::holds_alternative(value)) { diff --git a/src/util/newconfig/ObjectView.cpp b/src/util/newconfig/ObjectView.cpp index 75ef617f..c8624333 100644 --- a/src/util/newconfig/ObjectView.cpp +++ b/src/util/newconfig/ObjectView.cpp @@ -51,13 +51,13 @@ ObjectView::containsKey(std::string_view key) const } ValueView -ObjectView::getValue(std::string_view key) const +ObjectView::getValueView(std::string_view key) const { auto const fullKey = getFullKey(key); if (arrayIndex_.has_value()) { return clioConfig_.get().getArray(fullKey).valueAt(arrayIndex_.value()); } - return clioConfig_.get().getValue(fullKey); + return clioConfig_.get().getValueView(fullKey); } ObjectView diff --git a/src/util/newconfig/ObjectView.hpp b/src/util/newconfig/ObjectView.hpp index fd4fba58..42977d17 100644 --- a/src/util/newconfig/ObjectView.hpp +++ b/src/util/newconfig/ObjectView.hpp @@ -19,7 +19,6 @@ #pragma once -#include "util/newconfig/ConfigValue.hpp" #include "util/newconfig/ValueView.hpp" #include @@ -73,7 +72,35 @@ public: * @return A ValueView object representing the value associated with the key */ [[nodiscard]] ValueView - getValue(std::string_view key) const; + getValueView(std::string_view key) const; + + /** + * @brief Returns the specified value of given string if value exists + * + * @tparam T The type T to return + * @param key The config key to add to prefix and then search for + * @return Value of key of type T + */ + template + T + get(std::string_view key) const + { + return getValueView(key).getValueImpl(); + } + + /** + * @brief Returns the specified value of given string of type T if type and value exists + * + * @tparam T The type T to return + * @param key The config key to add to prefix and then search for + * @return The value of type T if it exists, std::nullopt otherwise. + */ + template + std::optional + maybeValue(std::string_view key) const + { + return getValueView(key).asOptional(); + } /** * @brief Retrieves an ObjectView in ClioConfigDefinition with key that starts with prefix_.key. The view must be of diff --git a/src/util/newconfig/ValueView.cpp b/src/util/newconfig/ValueView.cpp index a8b16a8b..63b7eba6 100644 --- a/src/util/newconfig/ValueView.cpp +++ b/src/util/newconfig/ValueView.cpp @@ -25,8 +25,8 @@ #include #include -#include #include +#include namespace util::config { @@ -34,7 +34,7 @@ ValueView::ValueView(ConfigValue const& configVal) : configVal_{configVal} { } -std::string_view +std::string ValueView::asString() const { if (this->type() == ConfigType::String && configVal_.get().hasValue()) @@ -56,9 +56,14 @@ double ValueView::asDouble() const { if (configVal_.get().hasValue()) { - if (type() == ConfigType::Double) - return std::get(configVal_.get().getValue()); + auto const& val = configVal_.get().getValue(); + if (type() == ConfigType::Double) { + if (std::holds_alternative(val)) { + return static_cast(std::get(val)); + } + return static_cast(std::get(val)); + } if (type() == ConfigType::Integer) return static_cast(std::get(configVal_.get().getValue())); } @@ -70,11 +75,16 @@ float ValueView::asFloat() const { if (configVal_.get().hasValue()) { + auto const& val = configVal_.get().getValue(); + if (type() == ConfigType::Double) { - return static_cast(std::get(configVal_.get().getValue())); + if (std::holds_alternative(val)) { + return static_cast(std::get(val)); + } + return static_cast(std::get(val)); } if (type() == ConfigType::Integer) - return static_cast(std::get(configVal_.get().getValue())); + return static_cast(std::get(val)); } ASSERT(false, "Value view is not of Float type"); std::unreachable(); diff --git a/src/util/newconfig/ValueView.hpp b/src/util/newconfig/ValueView.hpp index 5fd0d40a..534b03f0 100644 --- a/src/util/newconfig/ValueView.hpp +++ b/src/util/newconfig/ValueView.hpp @@ -29,7 +29,10 @@ #include #include #include -#include +#include +#include +#include +#include namespace util::config { @@ -53,7 +56,7 @@ public: * @return The value as a string * @throws std::bad_variant_access if the value is not a string */ - [[nodiscard]] std::string_view + [[nodiscard]] std::string asString() const; /** @@ -140,6 +143,51 @@ public: return configVal_.get().isOptional(); } + /** + * @brief Retrieves the stored value as the specified type T + * + * @tparam T The type to cast the stored value to + * @return The value cast to the specified type T + */ + template + T + getValueImpl() const + { + ASSERT(configVal_.get().hasValue(), "ConfigValue does not have a value"); + if constexpr (std::is_same_v) { + ASSERT(type() == ConfigType::Boolean, "Value type is not a bool"); + return asBool(); + } else if constexpr (std::is_integral_v) { + ASSERT(type() == ConfigType::Integer, "Value type is not an int"); + return asIntType(); + } else if constexpr (std::is_same_v) { + ASSERT(type() == ConfigType::String, "Value type is not a string"); + return asString(); + } else if constexpr (std::is_floating_point_v) { + ASSERT(type() == ConfigType::Double || type() == ConfigType::Integer, "Value type is not a floating point"); + return asDouble(); + } + + std::unreachable(); + } + + /** + * @brief Returns an optional value of the specified type T if valid + * + * @tparam T The type of value to retrieve (must be compatible with internal type) + * @return Returns the value as an optional value exists, or std::nullopt if not + */ + template + std::optional + asOptional() const + { + ASSERT(isOptional(), "A Config Value is not an optional value"); + if (!hasValue()) + return std::nullopt; + + return std::make_optional(getValueImpl()); + } + private: std::reference_wrapper configVal_; }; diff --git a/src/util/prometheus/Prometheus.cpp b/src/util/prometheus/Prometheus.cpp index 24ad51f8..8898f9a7 100644 --- a/src/util/prometheus/Prometheus.cpp +++ b/src/util/prometheus/Prometheus.cpp @@ -20,7 +20,7 @@ #include "util/prometheus/Prometheus.hpp" #include "util/Assert.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" @@ -176,10 +176,10 @@ PrometheusImpl::getMetric( } // namespace util::prometheus void -PrometheusService::init(util::Config const& config) +PrometheusService::init(util::config::ClioConfigDefinition const& config) { - bool const enabled = config.valueOr("prometheus.enabled", true); - bool const compressReply = config.valueOr("prometheus.compress_reply", true); + bool const enabled = config.get("prometheus.enabled"); + bool const compressReply = config.get("prometheus.compress_reply"); instance_ = std::make_unique(enabled, compressReply); } diff --git a/src/util/prometheus/Prometheus.hpp b/src/util/prometheus/Prometheus.hpp index fe9629bd..c1bc1d8a 100644 --- a/src/util/prometheus/Prometheus.hpp +++ b/src/util/prometheus/Prometheus.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/log/Logger.hpp" #include "util/prometheus/Bool.hpp" #include "util/prometheus/Counter.hpp" #include "util/prometheus/Gauge.hpp" @@ -28,6 +28,7 @@ #include "util/prometheus/MetricBase.hpp" #include "util/prometheus/MetricsFamily.hpp" +#include #include #include #include @@ -256,7 +257,7 @@ public: * * @param config The configuration to use */ - void static init(util::Config const& config = util::Config{}); + void static init(util::config::ClioConfigDefinition const& config); /** * @brief Get a bool based metric. It will be created if it doesn't exist diff --git a/src/web/AdminVerificationStrategy.cpp b/src/web/AdminVerificationStrategy.cpp index 520d6640..84265eb9 100644 --- a/src/web/AdminVerificationStrategy.cpp +++ b/src/web/AdminVerificationStrategy.cpp @@ -20,7 +20,7 @@ #include "web/AdminVerificationStrategy.hpp" #include "util/JsonUtils.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -81,7 +81,7 @@ make_AdminVerificationStrategy(std::optional password) } std::expected, std::string> -make_AdminVerificationStrategy(util::Config const& config) +make_AdminVerificationStrategy(util::config::ClioConfigDefinition const& config) { auto adminPassword = config.maybeValue("server.admin_password"); auto const localAdmin = config.maybeValue("server.local_admin"); diff --git a/src/web/AdminVerificationStrategy.hpp b/src/web/AdminVerificationStrategy.hpp index 4bc17ca4..6a6b1abc 100644 --- a/src/web/AdminVerificationStrategy.hpp +++ b/src/web/AdminVerificationStrategy.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -116,6 +116,6 @@ make_AdminVerificationStrategy(std::optional password); * @return Admin verification strategy according to the config or an error message. */ std::expected, std::string> -make_AdminVerificationStrategy(util::Config const& serverConfig); +make_AdminVerificationStrategy(util::config::ClioConfigDefinition const& serverConfig); } // namespace web diff --git a/src/web/RPCServerHandler.hpp b/src/web/RPCServerHandler.hpp index 46ee718b..963643a3 100644 --- a/src/web/RPCServerHandler.hpp +++ b/src/web/RPCServerHandler.hpp @@ -28,8 +28,8 @@ #include "util/JsonUtils.hpp" #include "util/Profiler.hpp" #include "util/Taggable.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "web/impl/ErrorHandling.hpp" #include "web/interface/ConnectionBase.hpp" @@ -79,7 +79,7 @@ public: * @param etl The ETL to use */ RPCServerHandler( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, std::shared_ptr const& etl @@ -88,7 +88,7 @@ public: , rpcEngine_(rpcEngine) , etl_(etl) , tagFactory_(config) - , apiVersionParser_(config.sectionOr("api_version", {})) + , apiVersionParser_(config.getObject("api_version")) { } diff --git a/src/web/Server.hpp b/src/web/Server.hpp index c01d7f3e..efdb3675 100644 --- a/src/web/Server.hpp +++ b/src/web/Server.hpp @@ -336,7 +336,7 @@ using HttpServer = Server; template static std::shared_ptr> make_HttpServer( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ioc, dosguard::DOSGuardInterface& dosGuard, std::shared_ptr const& handler @@ -350,12 +350,9 @@ make_HttpServer( return nullptr; } - if (!config.contains("server")) - return nullptr; - - auto const serverConfig = config.section("server"); - auto const address = boost::asio::ip::make_address(serverConfig.value("ip")); - auto const port = serverConfig.value("port"); + auto const serverConfig = config.getObject("server"); + auto const address = boost::asio::ip::make_address(serverConfig.get("ip")); + auto const port = serverConfig.get("port"); auto expectedAdminVerification = make_AdminVerificationStrategy(config); if (not expectedAdminVerification.has_value()) { @@ -365,7 +362,7 @@ make_HttpServer( // If the transactions number is 200 per ledger, A client which subscribes everything will send 400+ feeds for // each ledger. we allow user delay 3 ledgers by default - auto const maxWsSendingQueueSize = serverConfig.valueOr("ws_max_sending_queue_size", 1500); + auto const maxWsSendingQueueSize = serverConfig.get("ws_max_sending_queue_size"); auto server = std::make_shared>( ioc, diff --git a/src/web/dosguard/DOSGuard.cpp b/src/web/dosguard/DOSGuard.cpp index f3b52961..7896c307 100644 --- a/src/web/dosguard/DOSGuard.cpp +++ b/src/web/dosguard/DOSGuard.cpp @@ -20,8 +20,10 @@ #include "web/dosguard/DOSGuard.hpp" #include "util/Assert.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ValueView.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" #include @@ -33,13 +35,15 @@ #include #include +using namespace util::config; + namespace web::dosguard { -DOSGuard::DOSGuard(util::Config const& config, WhitelistHandlerInterface const& whitelistHandler) +DOSGuard::DOSGuard(ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler) : whitelistHandler_{std::cref(whitelistHandler)} - , maxFetches_{config.valueOr("dos_guard.max_fetches", DEFAULT_MAX_FETCHES)} - , maxConnCount_{config.valueOr("dos_guard.max_connections", DEFAULT_MAX_CONNECTIONS)} - , maxRequestCount_{config.valueOr("dos_guard.max_requests", DEFAULT_MAX_REQUESTS)} + , maxFetches_{config.get("dos_guard.max_fetches")} + , maxConnCount_{config.get("dos_guard.max_connections")} + , maxRequestCount_{config.get("dos_guard.max_requests")} { } @@ -134,15 +138,16 @@ DOSGuard::clear() noexcept } [[nodiscard]] std::unordered_set -DOSGuard::getWhitelist(util::Config const& config) +DOSGuard::getWhitelist(ClioConfigDefinition const& config) { - using T = std::unordered_set const; - auto whitelist = config.arrayOr("dos_guard.whitelist", {}); - auto const transform = [](auto const& elem) { return elem.template value(); }; - return T{ - boost::transform_iterator(std::begin(whitelist), transform), - boost::transform_iterator(std::end(whitelist), transform) - }; + std::unordered_set ips; + auto const whitelist = config.getArray("dos_guard.whitelist"); + + for (auto it = whitelist.begin(); it != whitelist.end(); ++it) { + ips.insert((*it).asString()); + } + + return ips; } } // namespace web::dosguard diff --git a/src/web/dosguard/DOSGuard.hpp b/src/web/dosguard/DOSGuard.hpp index 79798f05..6fa645dd 100644 --- a/src/web/dosguard/DOSGuard.hpp +++ b/src/web/dosguard/DOSGuard.hpp @@ -19,8 +19,8 @@ #pragma once -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "web/dosguard/DOSGuardInterface.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" @@ -63,17 +63,13 @@ class DOSGuard : public DOSGuardInterface { util::Logger log_{"RPC"}; public: - static constexpr std::uint32_t DEFAULT_MAX_FETCHES = 1000'000u; /**< Default maximum fetches per sweep */ - static constexpr std::uint32_t DEFAULT_MAX_CONNECTIONS = 20u; /**< Default maximum concurrent connections */ - static constexpr std::uint32_t DEFAULT_MAX_REQUESTS = 20u; /**< Default maximum requests per sweep */ - /** * @brief Constructs a new DOS guard. * * @param config Clio config * @param whitelistHandler Whitelist handler that checks whitelist for IP addresses */ - DOSGuard(util::Config const& config, WhitelistHandlerInterface const& whitelistHandler); + DOSGuard(util::config::ClioConfigDefinition const& config, WhitelistHandlerInterface const& whitelistHandler); /** * @brief Check whether an ip address is in the whitelist or not. @@ -148,7 +144,7 @@ public: private: [[nodiscard]] static std::unordered_set - getWhitelist(util::Config const& config); + getWhitelist(util::config::ClioConfigDefinition const& config); }; } // namespace web::dosguard diff --git a/src/web/dosguard/IntervalSweepHandler.cpp b/src/web/dosguard/IntervalSweepHandler.cpp index 55edf7d4..348fc4dd 100644 --- a/src/web/dosguard/IntervalSweepHandler.cpp +++ b/src/web/dosguard/IntervalSweepHandler.cpp @@ -19,7 +19,7 @@ #include "web/dosguard/IntervalSweepHandler.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "web/dosguard/DOSGuardInterface.hpp" #include @@ -32,14 +32,15 @@ namespace web::dosguard { IntervalSweepHandler::IntervalSweepHandler( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ctx, BaseDOSGuard& dosGuard ) : repeat_{std::ref(ctx)} { auto const sweepInterval{std::max( - std::chrono::milliseconds{1u}, util::Config::toMilliseconds(config.valueOr("dos_guard.sweep_interval", 1.0)) + std::chrono::milliseconds{1u}, + util::config::ClioConfigDefinition::toMilliseconds(config.get("dos_guard.sweep_interval")) )}; repeat_.start(sweepInterval, [&dosGuard] { dosGuard.clear(); }); } diff --git a/src/web/dosguard/IntervalSweepHandler.hpp b/src/web/dosguard/IntervalSweepHandler.hpp index 5162eca4..27cd2da4 100644 --- a/src/web/dosguard/IntervalSweepHandler.hpp +++ b/src/web/dosguard/IntervalSweepHandler.hpp @@ -20,7 +20,7 @@ #pragma once #include "util/Repeat.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include @@ -43,7 +43,7 @@ public: * @param dosGuard The DOS guard to use */ IntervalSweepHandler( - util::Config const& config, + util::config::ClioConfigDefinition const& config, boost::asio::io_context& ctx, web::dosguard::BaseDOSGuard& dosGuard ); diff --git a/src/web/dosguard/WhitelistHandler.hpp b/src/web/dosguard/WhitelistHandler.hpp index e4d72b9b..f7abbfd0 100644 --- a/src/web/dosguard/WhitelistHandler.hpp +++ b/src/web/dosguard/WhitelistHandler.hpp @@ -19,7 +19,9 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/newconfig/ArrayView.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ValueView.hpp" #include "web/Resolver.hpp" #include "web/dosguard/WhitelistHandlerInterface.hpp" @@ -99,7 +101,7 @@ public: * @param resolver The resolver to use for hostname resolution */ template - WhitelistHandler(util::Config const& config, HostnameResolverType&& resolver = {}) + WhitelistHandler(util::config::ClioConfigDefinition const& config, HostnameResolverType&& resolver = {}) { std::unordered_set const arr = getWhitelist(config, std::forward(resolver)); for (auto const& net : arr) @@ -121,24 +123,23 @@ public: private: template [[nodiscard]] static std::unordered_set - getWhitelist(util::Config const& config, HostnameResolverType&& resolver) + getWhitelist(util::config::ClioConfigDefinition const& config, HostnameResolverType&& resolver) { - auto whitelist = config.arrayOr("dos_guard.whitelist", {}); - auto const transform = [](auto const& elem) { return elem.template value(); }; - - std::unordered_set const hostnames{ - boost::transform_iterator(std::begin(whitelist), transform), - boost::transform_iterator(std::end(whitelist), transform) - }; - + auto const whitelist = config.getArray("dos_guard.whitelist"); + std::unordered_set hostnames{}; // resolve hostnames to ips std::unordered_set ips; + + for (auto it = whitelist.begin(); it != whitelist.end(); ++it) + hostnames.insert((*it).asString()); + for (auto const& hostname : hostnames) { auto resolvedIps = resolver.resolve(hostname); for (auto& ip : resolvedIps) { ips.insert(std::move(ip)); } }; + return ips; } }; diff --git a/src/web/ng/RPCServerHandler.hpp b/src/web/ng/RPCServerHandler.hpp index 2774fa93..bc076b5a 100644 --- a/src/web/ng/RPCServerHandler.hpp +++ b/src/web/ng/RPCServerHandler.hpp @@ -30,7 +30,6 @@ #include "util/JsonUtils.hpp" #include "util/Profiler.hpp" #include "util/Taggable.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" #include "web/SubscriptionContextInterface.hpp" #include "web/ng/Connection.hpp" @@ -87,7 +86,7 @@ public: * @param etl The ETL to use */ RPCServerHandler( - util::Config const& config, + util::config::ClioConfigDefinition const& config, std::shared_ptr const& backend, std::shared_ptr const& rpcEngine, std::shared_ptr const& etl @@ -96,7 +95,7 @@ public: , rpcEngine_(rpcEngine) , etl_(etl) , tagFactory_(config) - , apiVersionParser_(config.sectionOr("api_version", {})) + , apiVersionParser_(config.getObject("api_version")) { } diff --git a/src/web/ng/Server.cpp b/src/web/ng/Server.cpp index 8d905e25..987bfa6f 100644 --- a/src/web/ng/Server.cpp +++ b/src/web/ng/Server.cpp @@ -21,8 +21,9 @@ #include "util/Assert.hpp" #include "util/Taggable.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" +#include "util/newconfig/ObjectView.hpp" #include "web/ng/Connection.hpp" #include "web/ng/MessageHandler.hpp" #include "web/ng/ProcessingPolicy.hpp" @@ -56,22 +57,17 @@ namespace web::ng { namespace { std::expected -makeEndpoint(util::Config const& serverConfig) +makeEndpoint(util::config::ObjectView const& serverConfig) { - auto const ip = serverConfig.maybeValue("ip"); - if (not ip.has_value()) - return std::unexpected{"Missing 'ip` in server config."}; + auto const ip = serverConfig.get("ip"); boost::system::error_code error; - auto const address = boost::asio::ip::make_address(*ip, error); + auto const address = boost::asio::ip::make_address(ip, error); if (error) return std::unexpected{fmt::format("Error parsing provided IP: {}", error.message())}; - auto const port = serverConfig.maybeValue("port"); - if (not port.has_value()) - return std::unexpected{"Missing 'port` in server config."}; - - return boost::asio::ip::tcp::endpoint{address, *port}; + auto const port = serverConfig.get("port"); + return boost::asio::ip::tcp::endpoint{address, port}; } std::expected @@ -305,13 +301,13 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield std::expected make_Server( - util::Config const& config, + util::config::ClioConfigDefinition const& config, Server::OnConnectCheck onConnectCheck, Server::OnDisconnectHook onDisconnectHook, boost::asio::io_context& context ) { - auto const serverConfig = config.section("server"); + auto const serverConfig = config.getObject("server"); auto endpoint = makeEndpoint(serverConfig); if (not endpoint.has_value()) @@ -324,7 +320,7 @@ make_Server( ProcessingPolicy processingPolicy{ProcessingPolicy::Parallel}; std::optional parallelRequestLimit; - auto const processingStrategyStr = serverConfig.valueOr("processing_policy", "parallel"); + auto const processingStrategyStr = serverConfig.get("processing_policy"); if (processingStrategyStr == "sequent") { processingPolicy = ProcessingPolicy::Sequential; } else if (processingStrategyStr == "parallel") { @@ -333,7 +329,7 @@ make_Server( return std::unexpected{fmt::format("Invalid 'server.processing_strategy': {}", processingStrategyStr)}; } - auto const maxSubscriptionSendQueueSize = serverConfig.maybeValue("ws_max_sending_queue_size"); + auto const maxSubscriptionSendQueueSize = serverConfig.get("ws_max_sending_queue_size"); return Server{ context, diff --git a/src/web/ng/Server.hpp b/src/web/ng/Server.hpp index 1df7361a..cbf3ca79 100644 --- a/src/web/ng/Server.hpp +++ b/src/web/ng/Server.hpp @@ -20,8 +20,8 @@ #pragma once #include "util/Taggable.hpp" -#include "util/config/Config.hpp" #include "util/log/Logger.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include "web/ng/Connection.hpp" #include "web/ng/MessageHandler.hpp" #include "web/ng/ProcessingPolicy.hpp" @@ -170,7 +170,7 @@ private: */ std::expected make_Server( - util::Config const& config, + util::config::ClioConfigDefinition const& config, Server::OnConnectCheck onConnectCheck, Server::OnDisconnectHook onDisconnectHook, boost::asio::io_context& context diff --git a/src/web/ng/impl/ServerSslContext.cpp b/src/web/ng/impl/ServerSslContext.cpp index 46e9fb2a..f1f0e3bb 100644 --- a/src/web/ng/impl/ServerSslContext.cpp +++ b/src/web/ng/impl/ServerSslContext.cpp @@ -19,7 +19,7 @@ #include "web/ng/impl/ServerSslContext.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -52,10 +52,10 @@ readFile(std::string const& path) } // namespace std::expected, std::string> -makeServerSslContext(util::Config const& config) +makeServerSslContext(util::config::ClioConfigDefinition const& config) { - bool const configHasCertFile = config.contains("ssl_cert_file"); - bool const configHasKeyFile = config.contains("ssl_key_file"); + bool const configHasCertFile = config.getValueView("ssl_cert_file").hasValue(); + bool const configHasKeyFile = config.getValueView("ssl_key_file").hasValue(); if (configHasCertFile != configHasKeyFile) return std::unexpected{"Config entries 'ssl_cert_file' and 'ssl_key_file' must be set or unset together."}; @@ -63,12 +63,12 @@ makeServerSslContext(util::Config const& config) if (not configHasCertFile) return std::nullopt; - auto const certFilename = config.value("ssl_cert_file"); + auto const certFilename = config.get("ssl_cert_file"); auto const certContent = readFile(certFilename); if (!certContent) return std::unexpected{"Can't read SSL certificate: " + certFilename}; - auto const keyFilename = config.value("ssl_key_file"); + auto const keyFilename = config.get("ssl_key_file"); auto const keyContent = readFile(keyFilename); if (!keyContent) return std::unexpected{"Can't read SSL key: " + keyFilename}; diff --git a/src/web/ng/impl/ServerSslContext.hpp b/src/web/ng/impl/ServerSslContext.hpp index 151f8b06..be272058 100644 --- a/src/web/ng/impl/ServerSslContext.hpp +++ b/src/web/ng/impl/ServerSslContext.hpp @@ -19,7 +19,7 @@ #pragma once -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include @@ -30,7 +30,7 @@ namespace web::ng::impl { std::expected, std::string> -makeServerSslContext(util::Config const& config); +makeServerSslContext(util::config::ClioConfigDefinition const& config); std::expected makeServerSslContext(std::string const& certData, std::string const& keyData); diff --git a/tests/common/util/MockBackend.hpp b/tests/common/util/MockBackend.hpp index d1b4674a..4b09c417 100644 --- a/tests/common/util/MockBackend.hpp +++ b/tests/common/util/MockBackend.hpp @@ -22,7 +22,7 @@ #include "data/BackendInterface.hpp" #include "data/DBHelpers.hpp" #include "data/Types.hpp" -#include "util/config/Config.hpp" +#include "util/newconfig/ConfigDefinition.hpp" #include #include @@ -39,7 +39,7 @@ using namespace data; struct MockBackend : public BackendInterface { - MockBackend(util::Config) + MockBackend(util::config::ClioConfigDefinition) { } diff --git a/tests/common/util/MockBackendTestFixture.hpp b/tests/common/util/MockBackendTestFixture.hpp index eee31145..97f90f9d 100644 --- a/tests/common/util/MockBackendTestFixture.hpp +++ b/tests/common/util/MockBackendTestFixture.hpp @@ -19,13 +19,20 @@ #pragma once +#include "data/BackendInterface.hpp" #include "util/LoggerFixtures.hpp" #include "util/MockBackend.hpp" +#include "util/newconfig/ConfigDefinition.hpp" + +#include + +#include template