ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 11:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement an abstraction for the config (#358)

Browse Source 
Fixes #321
This commit is contained in:
Alex Kremer
2022-11-01 17:59:23 +01:00
committed by GitHub
parent 8bd8ab9b8a
commit ea2837749a
22 changed files with 1159 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@@ -100,13 +100,16 @@ target_sources(clio PRIVATE
src/rpc/handlers/ServerInfo.cpp
# Utility
src/rpc/handlers/Random.cpp
src/util/Taggable.cpp)
src/util/Taggable.cpp
src/config/Config.cpp)
add_executable(clio_server src/main/main.cpp)
target_link_libraries(clio_server PUBLIC clio)
if(BUILD_TESTS)
add_executable(clio_tests unittests/main.cpp)
add_executable(clio_tests
unittests/config.cpp
unittests/main.cpp)
include(CMake/deps/gtest.cmake)
endif()

21
src/backend/BackendFactory.h
View File

@@ -4,30 +4,23 @@
#include <boost/algorithm/string.hpp>
#include <backend/BackendInterface.h>
#include <backend/CassandraBackend.h>
#include <config/Config.h>
namespace Backend {
std::shared_ptr<BackendInterface>
make_Backend(boost::asio::io_context& ioc, boost::json::object const& config)
make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
{
BOOST_LOG_TRIVIAL(info) << __func__ << ": Constructing BackendInterface";
boost::json::object dbConfig = config.at("database").as_object();
bool readOnly = false;
if (config.contains("read_only"))
readOnly = config.at("read_only").as_bool();
auto type = dbConfig.at("type").as_string();
auto readOnly = config.valueOr("read_only", false);
auto type = config.value<std::string>("database.type");
std::shared_ptr<BackendInterface> backend = nullptr;
if (boost::iequals(type, "cassandra"))
{
if (config.contains("online_delete"))
dbConfig.at(type).as_object()["ttl"] =
config.at("online_delete").as_int64() * 4;
backend = std::make_shared<CassandraBackend>(
ioc, dbConfig.at(type).as_object());
auto cfg = config.section("database." + type);
auto ttl = config.valueOr<uint32_t>("online_delete", 0) * 4;
backend = std::make_shared<CassandraBackend>(ioc, cfg, ttl);
}
if (!backend)

4
src/backend/BackendInterface.h
View File

@@ -11,6 +11,8 @@
#include <backend/SimpleCache.h>
#include <backend/Types.h>
#include <config/Config.h>
#include <thread>
#include <type_traits>
@@ -96,7 +98,7 @@ protected:
SimpleCache cache_;
public:
BackendInterface(boost::json::object const& config)
BackendInterface(clio::Config const& config)
{
}
virtual ~BackendInterface()

69
src/backend/CassandraBackend.cpp
View File

@@ -979,19 +979,6 @@ CassandraBackend::isTooBusy() const
void
CassandraBackend::open(bool readOnly)
{
auto getString = [this](std::string const& field) -> std::string {
if (config_.contains(field))
{
auto jsonStr = config_[field].as_string();
return {jsonStr.c_str(), jsonStr.size()};
}
return {""};
};
auto getInt = [this](std::string const& field) -> std::optional<int> {
if (config_.contains(field) && config_.at(field).is_int64())
return config_[field].as_int64();
return {};
};
if (open_)
{
assert(false);
@@ -1005,7 +992,8 @@ CassandraBackend::open(bool readOnly)
if (!cluster)
throw std::runtime_error("nodestore:: Failed to create CassCluster");
std::string secureConnectBundle = getString("secure_connect_bundle");
std::string secureConnectBundle =
config_.valueOr<std::string>("secure_connect_bundle", "");
if (!secureConnectBundle.empty())
{
@@ -1025,12 +1013,9 @@ CassandraBackend::open(bool readOnly)
}
else
{
std::string contact_points = getString("contact_points");
if (contact_points.empty())
{
throw std::runtime_error(
"nodestore: Missing contact_points in Cassandra config");
}
std::string contact_points = config_.valueOrThrow<std::string>(
"contact_points",
"nodestore: Missing contact_points in Cassandra config");
CassError rc =
cass_cluster_set_contact_points(cluster, contact_points.c_str());
if (rc != CASS_OK)
@@ -1043,7 +1028,7 @@ CassandraBackend::open(bool readOnly)
throw std::runtime_error(ss.str());
}
auto port = getInt("port");
auto port = config_.maybeValue<int>("port");
if (port)
{
rc = cass_cluster_set_port(cluster, *port);
@@ -1069,15 +1054,16 @@ CassandraBackend::open(bool readOnly)
throw std::runtime_error(ss.str());
}
std::string username = getString("username");
if (username.size())
auto username = config_.maybeValue<std::string>("username");
if (username)
{
BOOST_LOG_TRIVIAL(debug) << "user = " << username.c_str();
BOOST_LOG_TRIVIAL(debug) << "user = " << *username;
auto password = config_.value<std::string>("password");
cass_cluster_set_credentials(
cluster, username.c_str(), getString("password").c_str());
cluster, username->c_str(), password.c_str());
}
int threads =
getInt("threads").value_or(std::thread::hardware_concurrency());
auto threads =
config_.valueOr<int>("threads", std::thread::hardware_concurrency());
rc = cass_cluster_set_num_threads_io(cluster, threads);
if (rc != CASS_OK)
@@ -1087,14 +1073,13 @@ CassandraBackend::open(bool readOnly)
<< ", result: " << rc << ", " << cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
if (getInt("max_write_requests_outstanding"))
maxWriteRequestsOutstanding = *getInt("max_write_requests_outstanding");
if (getInt("max_read_requests_outstanding"))
maxReadRequestsOutstanding = *getInt("max_read_requests_outstanding");
maxWriteRequestsOutstanding = config_.valueOr<int>(
"max_write_requests_outstanding", maxWriteRequestsOutstanding);
maxReadRequestsOutstanding = config_.valueOr<int>(
"max_read_requests_outstanding", maxReadRequestsOutstanding);
syncInterval_ = config_.valueOr<int>("sync_interval", syncInterval_);
if (getInt("sync_interval"))
syncInterval_ = *getInt("sync_interval");
BOOST_LOG_TRIVIAL(info)
<< __func__ << " sync interval is " << syncInterval_
<< ". max write requests outstanding is " << maxWriteRequestsOutstanding
@@ -1117,15 +1102,14 @@ CassandraBackend::open(bool readOnly)
throw std::runtime_error(ss.str());
}
std::string certfile = getString("certfile");
if (certfile.size())
if (auto certfile = config_.maybeValue<std::string>("certfile"); certfile)
{
std::ifstream fileStream(
boost::filesystem::path(certfile).string(), std::ios::in);
boost::filesystem::path(*certfile).string(), std::ios::in);
if (!fileStream)
{
std::stringstream ss;
ss << "opening config file " << certfile;
ss << "opening config file " << *certfile;
throw std::system_error(errno, std::generic_category(), ss.str());
}
std::string cert(
@@ -1134,7 +1118,7 @@ CassandraBackend::open(bool readOnly)
if (fileStream.bad())
{
std::stringstream ss;
ss << "reading config file " << certfile;
ss << "reading config file " << *certfile;
throw std::system_error(errno, std::generic_category(), ss.str());
}
@@ -1153,7 +1137,7 @@ CassandraBackend::open(bool readOnly)
cass_ssl_free(context);
}
std::string keyspace = getString("keyspace");
auto keyspace = config_.valueOr<std::string>("keyspace", "");
if (keyspace.empty())
{
BOOST_LOG_TRIVIAL(warning)
@@ -1161,9 +1145,8 @@ CassandraBackend::open(bool readOnly)
keyspace = "clio";
}
int rf = getInt("replication_factor") ? *getInt("replication_factor") : 3;
std::string tablePrefix = getString("table_prefix");
auto rf = config_.valueOr<int>("replication_factor", 3);
auto tablePrefix = config_.valueOr<std::string>("table_prefix", "");
if (tablePrefix.empty())
{
BOOST_LOG_TRIVIAL(warning) << "Table prefix is empty";
@@ -1171,7 +1154,7 @@ CassandraBackend::open(bool readOnly)
cass_cluster_set_connect_timeout(cluster, 10000);
int ttl = getInt("ttl") ? *getInt("ttl") * 2 : 0;
auto ttl = ttl_ * 2;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " setting ttl to " << std::to_string(ttl);

10
src/backend/CassandraBackend.h
View File

@@ -18,6 +18,8 @@
#include <mutex>
#include <thread>
#include <config/Config.h>
namespace Backend {
class CassandraPreparedStatement
@@ -670,15 +672,17 @@ private:
std::optional<boost::asio::io_context::work> work_;
std::thread ioThread_;
boost::json::object config_;
clio::Config config_;
uint32_t ttl_ = 0;
mutable std::uint32_t ledgerSequence_ = 0;
public:
CassandraBackend(
boost::asio::io_context& ioc,
boost::json::object const& config)
: BackendInterface(config), config_(config)
clio::Config const& config,
uint32_t ttl)
: BackendInterface(config), config_(config), ttl_(ttl)
{
work_.emplace(ioContext_);
ioThread_ = std::thread([this]() { ioContext_.run(); });

172
src/config/Config.cpp Normal file
View File

@@ -0,0 +1,172 @@
#include <config/Config.h>
#include <boost/log/trivial.hpp>
#include <fstream>
namespace clio {
// Note: `store_(store)` MUST use `()` instead of `{}` otherwise gcc
// picks `initializer_list` constructor and anything passed becomes an
// array :-D
Config::Config(boost::json::value store) : store_(std::move(store))
{
}
Config::operator bool() const noexcept
{
return not store_.is_null();
}
bool
Config::contains(key_type key) const
{
return lookup(key).has_value();
}
std::optional<boost::json::value>
Config::lookup(key_type key) const
{
if (store_.is_null())
return std::nullopt;
std::reference_wrapper<boost::json::value const> cur = std::cref(store_);
auto hasBrokenPath = false;
auto tokenized = detail::Tokenizer<key_type, Separator>{key};
std::string subkey{};
auto maybeSection = tokenized.next();
while (maybeSection.has_value())
{
auto section = maybeSection.value();
subkey += section;
if (not hasBrokenPath)
{
if (not cur.get().is_object())
throw detail::StoreException(
"Not an object at '" + subkey + "'");
if (not cur.get().as_object().contains(section))
hasBrokenPath = true;
else
cur = std::cref(cur.get().as_object().at(section));
}
subkey += Separator;
maybeSection = tokenized.next();
}
if (hasBrokenPath)
return std::nullopt;
return std::make_optional(cur);
}
std::optional<Config::array_type>
Config::maybeArray(key_type key) const
{
try
{
auto maybe_arr = lookup(key);
if (maybe_arr && maybe_arr->is_array())
{
auto& arr = maybe_arr->as_array();
array_type out;
out.reserve(arr.size());
std::transform(
std::begin(arr),
std::end(arr),
std::back_inserter(out),
[](auto&& element) { return Config{std::move(element)}; });
return std::make_optional<array_type>(std::move(out));
}
}
catch (detail::StoreException const&)
{
// ignore store error, but rethrow key errors
}
return std::nullopt;
}
Config::array_type
Config::array(key_type key) const
{
if (auto maybe_arr = maybeArray(key); maybe_arr)
return maybe_arr.value();
throw std::logic_error("No array found at '" + key + "'");
}
Config::array_type
Config::arrayOr(key_type key, array_type fallback) const
{
if (auto maybe_arr = maybeArray(key); maybe_arr)
return maybe_arr.value();
return fallback;
}
Config::array_type
Config::arrayOrThrow(key_type key, std::string_view err) const
{
try
{
return maybeArray(key).value();
}
catch (std::exception const&)
{
throw std::runtime_error(err.data());
}
}
Config
Config::section(key_type key) const
{
auto maybe_element = lookup(key);
if (maybe_element && maybe_element->is_object())
return Config{std::move(*maybe_element)};
throw std::logic_error("No section found at '" + key + "'");
}
Config::array_type
Config::array() const
{
if (not store_.is_array())
throw std::logic_error("_self_ is not an array");
array_type out;
auto const& arr = store_.as_array();
out.reserve(arr.size());
std::transform(
std::cbegin(arr),
std::cend(arr),
std::back_inserter(out),
[](auto const& element) { return Config{element}; });
return out;
}
Config
ConfigReader::open(std::filesystem::path path)
{
try
{
std::ifstream in(path, std::ios::in | std::ios::binary);
if (in)
{
std::stringstream contents;
contents << in.rdbuf();
auto opts = boost::json::parse_options{};
opts.allow_comments = true;
return Config{boost::json::parse(contents.str(), {}, opts)};
}
}
catch (std::exception const& e)
{
BOOST_LOG_TRIVIAL(error) << "Could not read configuration file from '"
<< path.string() << "': " << e.what();
}
BOOST_LOG_TRIVIAL(warning) << "Using empty default configuration";
return Config{};
}
} // namespace clio

389
src/config/Config.h Normal file
View File

@@ -0,0 +1,389 @@
#ifndef RIPPLE_APP_CONFIG_H_INCLUDED
#define RIPPLE_APP_CONFIG_H_INCLUDED
#include <config/detail/Helpers.h>
#include <boost/json.hpp>
#include <filesystem>
#include <optional>
#include <string>
#include <string_view>
namespace clio {
/**
* @brief Convenience wrapper to query a JSON configuration file.
*
* Any custom data type can be supported by implementing the right `tag_invoke`
* for `boost::json::value_to`.
*/
class Config final
{
boost::json::value store_;
static constexpr char Separator = '.';
public:
using key_type = std::string; /*! The type of key used */
using array_type = std::vector<Config>; /*! The type of array used */
using write_cursor_type = std::pair<
std::optional<std::reference_wrapper<boost::json::value>>,
key_type>;
/**
* @brief Construct a new Config object.
* @param store boost::json::value that backs this instance
*/
explicit Config(boost::json::value store = {});
//
// Querying the store
//
/**
* @brief Checks whether underlying store is not null.
*
* @return true If the store is null
* @return false If the store is not null
*/
operator bool() const noexcept;
/**
* @brief Checks whether something exists under given key.
*
* @param key The key to check
* @return true If something exists under key
* @return false If nothing exists under key
* @throws std::logic_error If the key is of invalid format
*/
[[nodiscard]] bool
contains(key_type key) const;
//
// Key value access
//
/**
* @brief Interface for fetching values by key that returns std::optional.
*
* Will attempt to fetch the value under the desired key. If the value
* exists and can be represented by the desired type Result then it will be
* returned wrapped in an optional. If the value exists but the conversion
* to Result is not possible - a runtime_error will be thrown. If the value
* does not exist under the specified key - std::nullopt is returned.
*
* @tparam Result The desired return type
* @param key The key to check
* @return std::optional<Result> Optional value of desired type
* @throws std::logic_error Thrown if conversion to Result is not possible
* or key is of invalid format
*/
template <typename Result>
[[nodiscard]] std::optional<Result>
maybeValue(key_type key) const
{
auto maybe_element = lookup(key);
if (maybe_element)
return std::make_optional<Result>(
checkedAs<Result>(key, *maybe_element));
return std::nullopt;
}
/**
* @brief Interface for fetching values by key.
*
* Will attempt to fetch the value under the desired key. If the value
* exists and can be represented by the desired type Result then it will be
* returned. If the value exists but the conversion
* to Result is not possible OR the value does not exist - a logic_error
* will be thrown.
*
* @tparam Result The desired return type
* @param key The key to check
* @return Result Value of desired type
* @throws std::logic_error Thrown if conversion to Result is not
* possible, value does not exist under specified key path or the key is of
* invalid format
*/
template <typename Result>
[[nodiscard]] Result
value(key_type key) const
{
return maybeValue<Result>(key).value();
}
/**
* @brief Interface for fetching values by key with fallback.
*
* Will attempt to fetch the value under the desired key. If the value
* exists and can be represented by the desired type Result then it will be
* returned. If the value exists but the conversion
* to Result is not possible - a logic_error will be thrown. If the value
* does not exist under the specified key - user specified fallback is
* returned.
*
* @tparam Result The desired return type
* @param key The key to check
* @param fallback The fallback value
* @return Result Value of desired type
* @throws std::logic_error Thrown if conversion to Result is not possible
* or the key is of invalid format
*/
template <typename Result>
[[nodiscard]] Result
valueOr(key_type key, Result fallback) const
{
try
{
return maybeValue<Result>(key).value_or(fallback);
}
catch (detail::StoreException const&)
{
return fallback;
}
}
/**
* @brief Interface for fetching values by key with custom error handling.
*
* Will attempt to fetch the value under the desired key. If the value
* exists and can be represented by the desired type Result then it will be
* returned. If the value exists but the conversion
* to Result is not possible OR the value does not exist - a runtime_error
* will be thrown with the user specified message.
*
* @tparam Result The desired return type
* @param key The key to check
* @param err The custom error message
* @return Result Value of desired type
* @throws std::runtime_error Thrown if conversion to Result is not possible
* or value does not exist under key
*/
template <typename Result>
[[nodiscard]] Result
valueOrThrow(key_type key, std::string_view err) const
{
try
{
return maybeValue<Result>(key).value();
}
catch (std::exception const&)
{
throw std::runtime_error(err.data());
}
}
/**
* @brief Interface for fetching an array by key that returns std::optional.
*
* Will attempt to fetch an array under the desired key. If the array
* exists then it will be
* returned wrapped in an optional. If the array does not exist under the
* specified key - std::nullopt is returned.
*
* @param key The key to check
* @return std::optional<array_type> Optional array
* @throws std::logic_error Thrown if the key is of invalid format
*/
[[nodiscard]] std::optional<array_type>
maybeArray(key_type key) const;
/**
* @brief Interface for fetching an array by key.
*
* Will attempt to fetch an array under the desired key. If the array
* exists then it will be
* returned. If the array does not exist under the
* specified key an std::logic_error is thrown.
*
* @param key The key to check
* @return array_type The array
* @throws std::logic_error Thrown if there is no array under the desired
* key or the key is of invalid format
*/
[[nodiscard]] array_type
array(key_type key) const;
/**
* @brief Interface for fetching an array by key with fallback.
*
* Will attempt to fetch an array under the desired key. If the array
* exists then it will be returned.
* If the array does not exist or another type is stored under the desired
* key - user specified fallback is returned.
*
* @param key The key to check
* @param fallback The fallback array
* @return array_type The array
* @throws std::logic_error Thrown if the key is of invalid format
*/
[[nodiscard]] array_type
arrayOr(key_type key, array_type fallback) const;
/**
* @brief Interface for fetching an array by key with custom error handling.
*
* Will attempt to fetch an array under the desired key. If the array
* exists then it will be returned.
* If the array does not exist or another type is stored under the desired
* key - std::runtime_error is thrown with the user specified error message.
*
* @param key The key to check
* @param err The custom error message
* @return array_type The array
* @throws std::runtime_error Thrown if there is no array under the desired
* key
*/
[[nodiscard]] array_type
arrayOrThrow(key_type key, std::string_view err) const;
/**
* @brief Interface for fetching a sub section by key.
*
* Will attempt to fetch an entire section under the desired key and return
* it as a Config instance. If the section does not exist or another type is
* stored under the desired key - std::logic_error is thrown.
*
* @param key The key to check
* @return Config Section represented as a separate instance of Config
* @throws std::logic_error Thrown if there is no section under the
* desired key or the key is of invalid format
*/
[[nodiscard]] Config
section(key_type key) const;
//
// Direct self-value access
//
/**
* @brief Interface for reading the value directly referred to by the
* instance. Wraps as std::optional.
*
* See @ref maybeValue(key_type) const for how this works.
*/
template <typename Result>
[[nodiscard]] std::optional<Result>
maybeValue() const
{
if (store_.is_null())
return std::nullopt;
return std::make_optional<Result>(checkedAs<Result>("_self_", store_));
}
/**
* @brief Interface for reading the value directly referred to by the
* instance.
*
* See @ref value(key_type) const for how this works.
*/
template <typename Result>
[[nodiscard]] Result
value() const
{
return maybeValue<Result>().value();
}
/**
* @brief Interface for reading the value directly referred to by the
* instance with user-specified fallback.
*
* See @ref valueOr(key_type, Result) const for how this works.
*/
template <typename Result>
[[nodiscard]] Result
valueOr(Result fallback) const
{
return maybeValue<Result>().valueOr(fallback);
}
/**
* @brief Interface for reading the value directly referred to by the
* instance with user-specified error message.
*
* See @ref valueOrThrow(key_type, std::string_view) const for how this
* works.
*/
template <typename Result>
[[nodiscard]] Result
valueOrThrow(std::string_view err) const
{
try
{
return maybeValue<Result>().value();
}
catch (std::exception const&)
{
throw std::runtime_error(err.data());
}
}
/**
* @brief Interface for reading the array directly referred to by the
* instance.
*
* See @ref array(key_type) const for how this works.
*/
[[nodiscard]] array_type
array() const;
private:
template <typename Return>
[[nodiscard]] Return
checkedAs(key_type key, boost::json::value const& value) const
{
auto has_error = false;
if constexpr (std::is_same_v<Return, bool>)
{
if (not value.is_bool())
has_error = true;
}
else if constexpr (std::is_same_v<Return, std::string>)
{
if (not value.is_string())
has_error = true;
}
else if constexpr (std::is_same_v<Return, double>)
{
if (not value.is_double())
has_error = true;
}
else if constexpr (
std::is_convertible_v<Return, uint64_t> ||
std::is_convertible_v<Return, int64_t>)
{
if (not value.is_int64() && not value.is_uint64())
has_error = true;
}
if (has_error)
throw std::runtime_error(
"Type for key '" + key + "' is '" +
std::string{to_string(value.kind())} +
"' in JSON but requested '" + detail::typeName<Return>() + "'");
return boost::json::value_to<Return>(value);
}
std::optional<boost::json::value>
lookup(key_type key) const;
write_cursor_type
lookupForWrite(key_type key);
};
/**
* @brief Simple configuration file reader.
*
* Reads the JSON file under specified path and creates a @ref Config object
* from its contents.
*/
class ConfigReader final
{
public:
static Config
open(std::filesystem::path path);
};
} // namespace clio
#endif // RIPPLE_APP_CONFIG_H_INCLUDED

148
src/config/detail/Helpers.h Normal file
View File

@@ -0,0 +1,148 @@
#ifndef RIPPLE_APP_CONFIG_DETAIL_H_INCLUDED
#define RIPPLE_APP_CONFIG_DETAIL_H_INCLUDED
#include <optional>
#include <queue>
#include <stdexcept>
#include <string>
namespace clio::detail {
/**
* @brief Thrown when a KeyPath related error occurs
*/
struct KeyException : public ::std::logic_error
{
KeyException(::std::string msg) : ::std::logic_error{msg}
{
}
};
/**
* @brief Thrown when a Store (config's storage) related error occurs.
*/
struct StoreException : public ::std::logic_error
{
StoreException(::std::string msg) : ::std::logic_error{msg}
{
}
};
/**
* @brief Simple string tokenizer. Used by @ref Config.
*
* @tparam KeyType The type of key to use
* @tparam Separator The separator character
*/
template <typename KeyType, char Separator>
class Tokenizer final
{
using opt_key_t = std::optional<KeyType>;
KeyType key_;
KeyType token_{};
std::queue<KeyType> tokens_{};
public:
explicit Tokenizer(KeyType key) : key_{key}
{
if (key.empty())
throw KeyException("Empty key");
for (auto const& c : key)
{
if (c == Separator)
saveToken();
else
token_ += c;
}
saveToken();
}
[[nodiscard]] opt_key_t
next()
{
if (tokens_.empty())
return std::nullopt;
auto token = tokens_.front();
tokens_.pop();
return std::make_optional(std::move(token));
}
private:
void
saveToken()
{
if (token_.empty())
throw KeyException("Empty token in key '" + key_ + "'.");
tokens_.push(std::move(token_));
token_ = {};
}
};
template <typename T>
static constexpr const char*
typeName()
{
return typeid(T).name();
}
template <>
constexpr const char*
typeName<uint64_t>()
{
return "uint64_t";
}
template <>
constexpr const char*
typeName<int64_t>()
{
return "int64_t";
}
template <>
constexpr const char*
typeName<uint32_t>()
{
return "uint32_t";
}
template <>
constexpr const char*
typeName<int32_t>()
{
return "int32_t";
}
template <>
constexpr const char*
typeName<bool>()
{
return "bool";
}
template <>
constexpr const char*
typeName<std::string>()
{
return "std::string";
}
template <>
constexpr const char*
typeName<const char*>()
{
return "const char*";
}
template <>
constexpr const char*
typeName<double>()
{
return "double";
}
}; // namespace clio::detail
#endif // RIPPLE_APP_CONFIG_DETAIL_H_INCLUDED

18
src/etl/ETLSource.cpp
View File

@@ -913,7 +913,7 @@ ETLSourceImpl<Derived>::fetchLedger(
static std::unique_ptr<ETLSource>
make_ETLSource(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -934,27 +934,21 @@ make_ETLSource(
}
ETLLoadBalancer::ETLLoadBalancer(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl)
{
if (config.contains("num_markers") && config.at("num_markers").is_int64())
{
downloadRanges_ = config.at("num_markers").as_int64();
downloadRanges_ = std::clamp(downloadRanges_, {1}, {256});
}
if (auto value = config.maybeValue<uint32_t>("num_markers"); value)
downloadRanges_ = std::clamp(*value, 1u, 256u);
else if (backend->fetchLedgerRange())
{
downloadRanges_ = 4;
}
for (auto& entry : config.at("etl_sources").as_array())
for (auto const& entry : config.array("etl_sources"))
{
std::unique_ptr<ETLSource> source = make_ETLSource(
entry.as_object(), ioContext, backend, subscriptions, nwvl, *this);
entry, ioContext, backend, subscriptions, nwvl, *this);
sources_.push_back(std::move(source));
BOOST_LOG_TRIVIAL(info) << __func__ << " : added etl source - "

68
src/etl/ETLSource.h
View File

@@ -7,11 +7,13 @@
#include <boost/beast/core/string.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/websocket.hpp>
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <etl/ETLHelpers.h>
#include <subscriptions/SubscriptionManager.h>
#include "org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h"
#include <etl/ETLHelpers.h>
#include <grpcpp/grpcpp.h>
class ETLLoadBalancer;
@@ -44,33 +46,27 @@ class ForwardCache
public:
ForwardCache(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
ETLSource const& source)
: strand_(ioc), timer_(strand_), source_(source)
{
if (config.contains("cache") && !config.at("cache").is_array())
throw std::runtime_error("ETLSource cache must be array");
if (config.contains("cache_duration") &&
!config.at("cache_duration").is_int64())
throw std::runtime_error(
"ETLSource cache_duration must be a number");
duration_ = config.contains("cache_duration")
? config.at("cache_duration").as_int64()
: 10;
auto commands = config.contains("cache") ? config.at("cache").as_array()
: boost::json::array{};
for (auto const& command : commands)
if (config.contains("cache"))
{
if (!command.is_string())
throw std::runtime_error(
"ETLSource forward command must be array of strings");
auto commands =
config.arrayOrThrow("cache", "ETLSource cache must be array");
latestForwarded_[command.as_string().c_str()] = {};
if (config.contains("cache_duration"))
duration_ = config.valueOrThrow<uint32_t>(
"cache_duration",
"ETLSource cache_duration must be a number");
for (auto const& command : commands)
{
auto key = command.valueOrThrow<std::string>(
"ETLSource forward command must be array of strings");
latestForwarded_[key] = {};
}
}
}
@@ -262,7 +258,7 @@ public:
/// Fetch ledger and load initial ledger will fail for this source
/// Primarly used in read-only mode, to monitor when ledgers are validated
ETLSourceImpl(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -279,20 +275,12 @@ public:
, timer_(ioContext)
, hooks_(hooks)
{
if (config.contains("ip"))
ip_ = config.valueOr<std::string>("ip", {});
wsPort_ = config.valueOr<std::string>("ws_port", {});
if (auto value = config.maybeValue<std::string>("grpc_port"); value)
{
auto ipJs = config.at("ip").as_string();
ip_ = {ipJs.c_str(), ipJs.size()};
}
if (config.contains("ws_port"))
{
auto portjs = config.at("ws_port").as_string();
wsPort_ = {portjs.c_str(), portjs.size()};
}
if (config.contains("grpc_port"))
{
auto portjs = config.at("grpc_port").as_string();
grpcPort_ = {portjs.c_str(), portjs.size()};
grpcPort_ = *value;
try
{
boost::asio::ip::tcp::endpoint endpoint{
@@ -498,7 +486,7 @@ class PlainETLSource : public ETLSourceImpl<PlainETLSource>
public:
PlainETLSource(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -547,7 +535,7 @@ class SslETLSource : public ETLSourceImpl<SslETLSource>
public:
SslETLSource(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx,
std::shared_ptr<BackendInterface> backend,
@@ -610,7 +598,7 @@ private:
public:
ETLLoadBalancer(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -618,7 +606,7 @@ public:
static std::shared_ptr<ETLLoadBalancer>
make_ETLLoadBalancer(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

2
src/etl/ProbingETLSource.cpp
View File

@@ -1,7 +1,7 @@
#include <etl/ProbingETLSource.h>
ProbingETLSource::ProbingETLSource(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

7
src/etl/ProbingETLSource.h
View File

@@ -6,9 +6,12 @@
#include <boost/beast/core/string.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/websocket.hpp>
#include <etl/ETLSource.h>
#include <mutex>
#include <config/Config.h>
#include <etl/ETLSource.h>
/// This ETLSource implementation attempts to connect over both secure websocket
/// and plain websocket. First to connect pauses the other and the probing is
/// considered done at this point. If however the connected source loses
@@ -23,7 +26,7 @@ class ProbingETLSource : public ETLSource
public:
ProbingETLSource(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

71
src/etl/ReportingETL.cpp
View File

@@ -1317,7 +1317,7 @@ ReportingETL::doWork()
}
ReportingETL::ReportingETL(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -1330,60 +1330,55 @@ ReportingETL::ReportingETL(
, publishStrand_(ioc)
, networkValidatedLedgers_(ledgers)
{
if (config.contains("start_sequence"))
startSequence_ = config.at("start_sequence").as_int64();
if (config.contains("finish_sequence"))
finishSequence_ = config.at("finish_sequence").as_int64();
if (config.contains("read_only"))
readOnly_ = config.at("read_only").as_bool();
if (config.contains("online_delete"))
startSequence_ = config.maybeValue<uint32_t>("start_sequence");
finishSequence_ = config.maybeValue<uint32_t>("finish_sequence");
readOnly_ = config.valueOr("read_only", readOnly_);
if (auto interval = config.maybeValue<uint32_t>("online_delete"); interval)
{
int64_t interval = config.at("online_delete").as_int64();
uint32_t max = std::numeric_limits<uint32_t>::max();
if (interval > max)
auto const max = std::numeric_limits<uint32_t>::max();
if (*interval > max)
{
std::stringstream msg;
msg << "online_delete cannot be greater than "
<< std::to_string(max);
throw std::runtime_error(msg.str());
}
if (interval > 0)
onlineDeleteInterval_ = static_cast<uint32_t>(interval);
if (*interval > 0)
onlineDeleteInterval_ = *interval;
}
if (config.contains("extractor_threads"))
extractorThreads_ = config.at("extractor_threads").as_int64();
if (config.contains("txn_threshold"))
txnThreshold_ = config.at("txn_threshold").as_int64();
extractorThreads_ =
config.valueOr<uint32_t>("extractor_threads", extractorThreads_);
txnThreshold_ = config.valueOr<size_t>("txn_threshold", txnThreshold_);
if (config.contains("cache"))
{
auto cache = config.at("cache").as_object();
if (cache.contains("load") && cache.at("load").is_string())
auto const cache = config.section("cache");
if (auto entry = cache.maybeValue<std::string>("load"); entry)
{
auto entry = config.at("cache").as_object().at("load").as_string();
boost::algorithm::to_lower(entry);
if (entry == "sync")
if (boost::iequals(*entry, "sync"))
cacheLoadStyle_ = CacheLoadStyle::SYNC;
if (entry == "async")
if (boost::iequals(*entry, "async"))
cacheLoadStyle_ = CacheLoadStyle::ASYNC;
if (entry == "none" || entry == "no")
if (boost::iequals(*entry, "none") or boost::iequals(*entry, "no"))
cacheLoadStyle_ = CacheLoadStyle::NOT_AT_ALL;
}
if (cache.contains("num_diffs") && cache.at("num_diffs").is_int64())
numCacheDiffs_ = cache.at("num_diffs").as_int64();
if (cache.contains("num_markers") && cache.at("num_markers").is_int64())
numCacheMarkers_ = cache.at("num_markers").as_int64();
if (cache.contains("page_fetch_size") &&
cache.at("page_fetch_size").is_int64())
cachePageFetchSize_ = cache.at("page_fetch_size").as_int64();
if (cache.contains("peers") && cache.at("peers").is_array())
numCacheDiffs_ = cache.valueOr<size_t>("num_diffs", numCacheDiffs_);
numCacheMarkers_ =
cache.valueOr<size_t>("num_markers", numCacheMarkers_);
cachePageFetchSize_ =
cache.valueOr<size_t>("page_fetch_size", cachePageFetchSize_);
if (auto peers = cache.maybeArray("peers"); peers)
{
auto const& peers = cache.at("peers").as_array();
for (auto const& peer : peers)
for (auto const& peer : *peers)
{
auto const& clio = peer.as_object();
auto ip = clio.at("ip").as_string().c_str();
auto port = clio.at("port").as_int64();
clioPeers.emplace_back(ip, port);
auto ip = peer.value<std::string>("ip");
auto port = peer.value<uint32_t>("port");
// todo: use emplace_back when clang is ready
clioPeers.push_back({ip, port});
}
unsigned seed =
std::chrono::system_clock::now().time_since_epoch().count();

4
src/etl/ReportingETL.h
View File

@@ -311,7 +311,7 @@ private:
public:
ReportingETL(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -320,7 +320,7 @@ public:
static std::shared_ptr<ReportingETL>
make_ReportingETL(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

90
src/main/main.cpp
View File

@@ -23,6 +23,7 @@
#include <boost/log/utility/setup/file.hpp>
#include <algorithm>
#include <backend/BackendFactory.h>
#include <config/Config.h>
#include <cstdlib>
#include <etl/ReportingETL.h>
#include <fstream>
@@ -36,37 +37,16 @@
#include <vector>
#include <webserver/Listener.h>
std::optional<boost::json::object>
parse_config(const char* filename)
{
try
{
std::ifstream in(filename, std::ios::in | std::ios::binary);
if (in)
{
std::stringstream contents;
contents << in.rdbuf();
in.close();
std::cout << contents.str() << std::endl;
boost::json::value value = boost::json::parse(contents.str());
return value.as_object();
}
}
catch (std::exception const& e)
{
std::cout << e.what() << std::endl;
}
return {};
}
using namespace clio;
std::optional<ssl::context>
parse_certs(boost::json::object const& config)
parse_certs(Config const& config)
{
if (!config.contains("ssl_cert_file") || !config.contains("ssl_key_file"))
return {};
auto certFilename = config.at("ssl_cert_file").as_string().c_str();
auto keyFilename = config.at("ssl_key_file").as_string().c_str();
auto certFilename = config.value<std::string>("ssl_cert_file");
auto keyFilename = config.value<std::string>("ssl_key_file");
std::ifstream readCert(certFilename, std::ios::in | std::ios::binary);
if (!readCert)
@@ -74,7 +54,6 @@ parse_certs(boost::json::object const& config)
std::stringstream contents;
contents << readCert.rdbuf();
readCert.close();
std::string cert = contents.str();
std::ifstream readKey(keyFilename, std::ios::in | std::ios::binary);
@@ -102,7 +81,7 @@ parse_certs(boost::json::object const& config)
}
void
initLogging(boost::json::object const& config)
initLogging(Config const& config)
{
namespace src = boost::log::sources;
namespace keywords = boost::log::keywords;
@@ -110,35 +89,30 @@ initLogging(boost::json::object const& config)
namespace trivial = boost::log::trivial;
boost::log::add_common_attributes();
std::string format = "[%TimeStamp%] [%ThreadID%] [%Severity%] %Message%";
if (!config.contains("log_to_console") ||
config.at("log_to_console").as_bool())
if (config.valueOr<bool>("log_to_console", true))
{
boost::log::add_console_log(std::cout, keywords::format = format);
}
if (config.contains("log_directory"))
if (auto logDir = config.maybeValue<std::string>("log_directory"); logDir)
{
if (!config.at("log_directory").is_string())
throw std::runtime_error("log directory must be a string");
boost::filesystem::path dirPath{
config.at("log_directory").as_string().c_str()};
boost::filesystem::path dirPath{*logDir};
if (!boost::filesystem::exists(dirPath))
boost::filesystem::create_directories(dirPath);
const int64_t rotationSize = config.contains("log_rotation_size")
? config.at("log_rotation_size").as_int64() * 1024 * 1024u
: 2 * 1024 * 1024 * 1024u;
auto const rotationSize =
config.valueOr<int64_t>("log_rotation_size", 2 * 1024) * 1024 *
1024u;
if (rotationSize <= 0)
throw std::runtime_error(
"log rotation size must be greater than 0");
const int64_t rotationPeriod =
config.contains("log_rotation_hour_interval")
? config.at("log_rotation_hour_interval").as_int64()
: 12u;
auto const rotationPeriod =
config.valueOr<int64_t>("log_rotation_hour_interval", 12u);
if (rotationPeriod <= 0)
throw std::runtime_error(
"log rotation time interval must be greater than 0");
const int64_t dirSize = config.contains("log_directory_max_size")
? config.at("log_directory_max_size").as_int64() * 1024 * 1024u
: 50 * 1024 * 1024 * 1024u;
auto const dirSize =
config.valueOr<int64_t>("log_directory_max_size", 50 * 1024) *
1024 * 1024u;
if (dirSize <= 0)
throw std::runtime_error(
"log rotation directory max size must be greater than 0");
@@ -157,9 +131,7 @@ initLogging(boost::json::object const& config)
keywords::target = dirPath, keywords::max_size = dirSize));
fileSink->locked_backend()->scan_for_files();
}
auto const logLevel = config.contains("log_level")
? config.at("log_level").as_string()
: "info";
auto const logLevel = config.valueOr<std::string>("log_level", "info");
if (boost::iequals(logLevel, "trace"))
boost::log::core::get()->set_filter(
trivial::severity >= trivial::trace);
@@ -217,28 +189,24 @@ main(int argc, char* argv[])
return EXIT_SUCCESS;
}
auto const config = parse_config(argv[1]);
auto const config = ConfigReader::open(argv[1]);
if (!config)
{
std::cerr << "Couldnt parse config. Exiting..." << std::endl;
return EXIT_FAILURE;
}
initLogging(*config);
initLogging(config);
// Announce Clio version
BOOST_LOG_TRIVIAL(info)
<< "Clio version: " << Build::getClioFullVersionString();
auto ctx = parse_certs(*config);
auto ctx = parse_certs(config);
auto ctxRef = ctx
? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()}
: std::nullopt;
auto const threads = config->contains("io_threads")
? config->at("io_threads").as_int64()
: 2;
auto const threads = config.valueOr("io_threads", 2);
if (threads <= 0)
{
BOOST_LOG_TRIVIAL(fatal) << "io_threads is less than 0";
@@ -251,15 +219,15 @@ main(int argc, char* argv[])
boost::asio::io_context ioc{threads};
// Rate limiter, to prevent abuse
DOSGuard dosGuard{config.value(), ioc};
DOSGuard dosGuard{config, ioc};
// Interface to the database
std::shared_ptr<BackendInterface> backend{
Backend::make_Backend(ioc, *config)};
Backend::make_Backend(ioc, config)};
// Manages clients subscribed to streams
std::shared_ptr<SubscriptionManager> subscriptions{
SubscriptionManager::make_SubscriptionManager(*config, backend)};
SubscriptionManager::make_SubscriptionManager(config, backend)};
// Tracks which ledgers have been validated by the
// network
@@ -272,16 +240,16 @@ main(int argc, char* argv[])
// The balancer itself publishes to streams (transactions_proposed and
// accounts_proposed)
auto balancer = ETLLoadBalancer::make_ETLLoadBalancer(
*config, ioc, backend, subscriptions, ledgers);
config, ioc, backend, subscriptions, ledgers);
// ETL is responsible for writing and publishing to streams. In read-only
// mode, ETL only publishes
auto etl = ReportingETL::make_ReportingETL(
*config, ioc, backend, subscriptions, balancer, ledgers);
config, ioc, backend, subscriptions, balancer, ledgers);
// The server handles incoming RPCs
auto httpServer = Server::make_HttpServer(
*config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
config, ioc, ctxRef, backend, subscriptions, balancer, etl, dosGuard);
// Blocks until stopped.
// When stopped, shared_ptrs fall out of scope

12
src/subscriptions/SubscriptionManager.h
View File

@@ -2,6 +2,7 @@
#define SUBSCRIPTION_MANAGER_H
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <memory>
#include <subscriptions/Message.h>
@@ -107,17 +108,10 @@ class SubscriptionManager
public:
static std::shared_ptr<SubscriptionManager>
make_SubscriptionManager(
boost::json::object const& config,
clio::Config const& config,
std::shared_ptr<Backend::BackendInterface const> const& b)
{
auto numThreads = 1;
if (config.contains("subscription_workers") &&
config.at("subscription_workers").is_int64())
{
numThreads = config.at("subscription_workers").as_int64();
}
auto numThreads = config.valueOr<uint64_t>("subscription_workers", 1);
return std::make_shared<SubscriptionManager>(numThreads, b);
}

20
src/util/Taggable.cpp
View File

@@ -1,6 +1,5 @@
#include <util/Taggable.h>
#include <boost/algorithm/string/predicate.hpp>
#include <boost/json.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
@@ -49,25 +48,6 @@ TagDecoratorFactory::make() const
}
}
TagDecoratorFactory::Type
TagDecoratorFactory::parseType(boost::json::object const& config)
{
if (!config.contains("log_tag_style"))
return TagDecoratorFactory::Type::NONE;
auto style = config.at("log_tag_style").as_string();
if (boost::iequals(style, "int") || boost::iequals(style, "uint"))
return TagDecoratorFactory::Type::UINT;
else if (boost::iequals(style, "null") || boost::iequals(style, "none"))
return TagDecoratorFactory::Type::NONE;
else if (boost::iequals(style, "uuid"))
return TagDecoratorFactory::Type::UUID;
else
throw std::runtime_error(
"Could not parse `log_tag_style`: expected `uint`, `uuid` or "
"`null`");
}
TagDecoratorFactory
TagDecoratorFactory::with(parent_t parent) const noexcept
{

26
src/util/Taggable.h
View File

@@ -1,6 +1,7 @@
#ifndef RIPPLE_UTIL_TAGDECORATOR_H
#define RIPPLE_UTIL_TAGDECORATOR_H
#include <boost/algorithm/string/predicate.hpp>
#include <boost/json.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_io.hpp>
@@ -10,6 +11,8 @@
#include <ostream>
#include <string>
#include <config/Config.h>
namespace util {
namespace detail {
@@ -170,8 +173,8 @@ public:
* @brief Instantiates a tag decorator factory from `clio` configuration.
* @param config The configuration as a json object
*/
explicit TagDecoratorFactory(boost::json::object const& config)
: type_{TagDecoratorFactory::parseType(config)}
explicit TagDecoratorFactory(clio::Config const& config)
: type_{config.valueOr<Type>("log_tag_style", Type::NONE)}
{
}
@@ -203,8 +206,23 @@ public:
with(parent_t parent) const noexcept;
private:
static Type
parseType(boost::json::object const& config);
friend Type
tag_invoke(boost::json::value_to_tag<Type>, boost::json::value const& value)
{
assert(value.is_string());
auto const& style = value.as_string();
if (boost::iequals(style, "int") || boost::iequals(style, "uint"))
return TagDecoratorFactory::Type::UINT;
else if (boost::iequals(style, "null") || boost::iequals(style, "none"))
return TagDecoratorFactory::Type::NONE;
else if (boost::iequals(style, "uuid"))
return TagDecoratorFactory::Type::UUID;
else
throw std::runtime_error(
"Could not parse `log_tag_style`: expected `uint`, `uuid` or "
"`null`");
}
};
/**

80
src/webserver/DOSGuard.h
View File

@@ -6,6 +6,8 @@
#include <unordered_map>
#include <unordered_set>
#include <config/Config.h>
class DOSGuard
{
boost::asio::io_context& ctx_;
@@ -15,66 +17,12 @@ class DOSGuard
std::uint32_t const maxFetches_;
std::uint32_t const sweepInterval_;
// Load config setting for DOSGuard
std::optional<boost::json::object>
getConfig(boost::json::object const& config) const
{
if (!config.contains("dos_guard"))
return {};
return config.at("dos_guard").as_object();
}
std::uint32_t
get(boost::json::object const& config,
std::string const& key,
std::uint32_t const fallback) const
{
try
{
if (auto const c = getConfig(config))
return c->at(key).as_int64();
}
catch (std::exception const& e)
{
}
return fallback;
}
std::unordered_set<std::string> const
getWhitelist(boost::json::object const& config) const
{
using T = std::unordered_set<std::string> const;
try
{
auto const& c = getConfig(config);
if (!c)
return T();
auto const& w = c->at("whitelist").as_array();
auto const transform = [](auto const& elem) {
return std::string(elem.as_string().c_str());
};
return T(
boost::transform_iterator(w.begin(), transform),
boost::transform_iterator(w.end(), transform));
}
catch (std::exception const& e)
{
return T();
}
}
public:
DOSGuard(boost::json::object const& config, boost::asio::io_context& ctx)
: ctx_(ctx)
, whitelist_(getWhitelist(config))
, maxFetches_(get(config, "max_fetches", 100))
, sweepInterval_(get(config, "sweep_interval", 1))
DOSGuard(clio::Config const& config, boost::asio::io_context& ctx)
: ctx_{ctx}
, whitelist_{getWhitelist(config)}
, maxFetches_{config.valueOr("dos_guard.max_fetches", 100u)}
, sweepInterval_{config.valueOr("dos_guard.sweep_interval", 1u)}
{
createTimer();
}
@@ -137,5 +85,19 @@ public:
std::unique_lock lck(mtx_);
ipFetchCount_.clear();
}
private:
std::unordered_set<std::string> const
getWhitelist(clio::Config const& config) const
{
using T = std::unordered_set<std::string> const;
auto whitelist = config.arrayOr("dos_guard.whitelist", {});
auto const transform = [](auto const& elem) {
return elem.template value<std::string>();
};
return T{
boost::transform_iterator(std::begin(whitelist), transform),
boost::transform_iterator(std::end(whitelist), transform)};
}
};
#endif

22
src/webserver/Listener.h
View File

@@ -327,7 +327,7 @@ using HttpServer = Listener<HttpSession, SslHttpSession>;
static std::shared_ptr<HttpServer>
make_HttpServer(
boost::json::object const& config,
clio::Config const& config,
boost::asio::io_context& ioc,
std::optional<std::reference_wrapper<ssl::context>> sslCtx,
std::shared_ptr<BackendInterface const> backend,
@@ -339,19 +339,15 @@ make_HttpServer(
if (!config.contains("server"))
return nullptr;
auto const& serverConfig = config.at("server").as_object();
auto const serverConfig = config.section("server");
auto const address =
boost::asio::ip::make_address(serverConfig.value<std::string>("ip"));
auto const port = serverConfig.value<unsigned short>("port");
auto const numThreads = config.valueOr<uint32_t>(
"workers", std::thread::hardware_concurrency());
auto const maxQueueSize =
serverConfig.valueOr<uint32_t>("max_queue_size", 0); // 0 is no limit
auto const address = boost::asio::ip::make_address(
serverConfig.at("ip").as_string().c_str());
auto const port =
static_cast<unsigned short>(serverConfig.at("port").as_int64());
uint32_t numThreads = std::thread::hardware_concurrency();
if (config.contains("workers"))
numThreads = config.at("workers").as_int64();
uint32_t maxQueueSize = 0; // no max
if (serverConfig.contains("max_queue_size"))
maxQueueSize = serverConfig.at("max_queue_size").as_int64();
BOOST_LOG_TRIVIAL(info) << __func__ << " Number of workers = " << numThreads
<< ". Max queue size = " << maxQueueSize;

240
unittests/config.cpp Normal file
View File

@@ -0,0 +1,240 @@
#include <config/Config.h>
#include <boost/filesystem.hpp>
#include <boost/json/parse.hpp>
#include <boost/log/core.hpp>
#include <gtest/gtest.h>
#include <cstdio>
#include <exception>
#include <fstream>
#include <iostream>
using namespace clio;
using namespace boost::log;
using namespace std;
namespace json = boost::json;
constexpr static auto JSONData = R"JSON(
{
"arr": [
{ "first": 1234 },
{ "second": true },
{ "inner_section": [{ "inner": "works" }] },
["127.0.0.1", "192.168.0.255"]
],
"section": {
"test": {
"str": "hello",
"int": 9042,
"bool": true
}
},
"top": 420
}
)JSON";
class ConfigTest : public ::testing::Test
{
protected:
Config cfg{json::parse(JSONData)};
void
SetUp() override
{
// disable logging in test
core::get()->set_logging_enabled(false);
}
};
TEST_F(ConfigTest, SanityCheck)
{
// throw on wrong key format etc.:
ASSERT_ANY_THROW((void)cfg.value<bool>(""));
ASSERT_ANY_THROW((void)cfg.value<bool>("a."));
ASSERT_ANY_THROW((void)cfg.value<bool>(".a"));
ASSERT_ANY_THROW((void)cfg.valueOr<bool>("", false));
ASSERT_ANY_THROW((void)cfg.valueOr<bool>("a.", false));
ASSERT_ANY_THROW((void)cfg.valueOr<bool>(".a", false));
ASSERT_ANY_THROW((void)cfg.maybeValue<bool>(""));
ASSERT_ANY_THROW((void)cfg.maybeValue<bool>("a."));
ASSERT_ANY_THROW((void)cfg.maybeValue<bool>(".a"));
ASSERT_ANY_THROW((void)cfg.valueOrThrow<bool>("", "custom"));
ASSERT_ANY_THROW((void)cfg.valueOrThrow<bool>("a.", "custom"));
ASSERT_ANY_THROW((void)cfg.valueOrThrow<bool>(".a", "custom"));
ASSERT_ANY_THROW((void)cfg.contains(""));
ASSERT_ANY_THROW((void)cfg.contains("a."));
ASSERT_ANY_THROW((void)cfg.contains(".a"));
ASSERT_ANY_THROW((void)cfg.section(""));
ASSERT_ANY_THROW((void)cfg.section("a."));
ASSERT_ANY_THROW((void)cfg.section(".a"));
// valid path, value does not exists -> optional functions should not throw
ASSERT_ANY_THROW((void)cfg.value<bool>("b"));
ASSERT_EQ(cfg.valueOr<bool>("b", false), false);
ASSERT_EQ(cfg.maybeValue<bool>("b"), std::nullopt);
ASSERT_ANY_THROW((void)cfg.valueOrThrow<bool>("b", "custom"));
}
TEST_F(ConfigTest, Access)
{
ASSERT_EQ(cfg.value<int64_t>("top"), 420);
ASSERT_EQ(cfg.value<string>("section.test.str"), "hello");
ASSERT_EQ(cfg.value<int64_t>("section.test.int"), 9042);
ASSERT_EQ(cfg.value<bool>("section.test.bool"), true);
ASSERT_ANY_THROW((void)cfg.value<uint64_t>(
"section.test.bool")); // wrong type requested
ASSERT_ANY_THROW((void)cfg.value<bool>("section.doesnotexist"));
ASSERT_EQ(cfg.valueOr<string>("section.test.str", "fallback"), "hello");
ASSERT_EQ(
cfg.valueOr<string>("section.test.nonexistent", "fallback"),
"fallback");
ASSERT_EQ(cfg.valueOr("section.test.bool", false), true);
ASSERT_ANY_THROW(
(void)cfg.valueOr("section.test.bool", 1234)); // wrong type requested
}
TEST_F(ConfigTest, ErrorHandling)
{
try
{
(void)cfg.valueOrThrow<bool>("section.test.int", "msg");
ASSERT_FALSE(true); // should not get here
}
catch (std::runtime_error const& e)
{
ASSERT_STREQ(e.what(), "msg");
}
ASSERT_EQ(cfg.valueOrThrow<bool>("section.test.bool", ""), true);
auto arr = cfg.array("arr");
try
{
(void)arr[3].array()[1].valueOrThrow<int>("msg"); // wrong type
ASSERT_FALSE(true); // should not get here
}
catch (std::runtime_error const& e)
{
ASSERT_STREQ(e.what(), "msg");
}
ASSERT_EQ(arr[3].array()[1].valueOrThrow<string>(""), "192.168.0.255");
try
{
(void)cfg.arrayOrThrow("nonexisting.key", "msg");
ASSERT_FALSE(true); // should not get here
}
catch (std::runtime_error const& e)
{
ASSERT_STREQ(e.what(), "msg");
}
ASSERT_EQ(cfg.arrayOrThrow("arr", "")[0].value<int>("first"), 1234);
}
TEST_F(ConfigTest, Section)
{
auto sub = cfg.section("section.test");
ASSERT_EQ(sub.value<string>("str"), "hello");
ASSERT_EQ(sub.value<int64_t>("int"), 9042);
ASSERT_EQ(sub.value<bool>("bool"), true);
}
TEST_F(ConfigTest, Array)
{
auto arr = cfg.array("arr");
ASSERT_EQ(arr.size(), 4);
ASSERT_EQ(arr[0].value<int64_t>("first"), 1234);
// check twice to verify that previous array(key) access did not destroy the
// store by using move
ASSERT_EQ(arr[2].array("inner_section")[0].value<string>("inner"), "works");
ASSERT_EQ(arr[2].array("inner_section")[0].value<string>("inner"), "works");
ASSERT_EQ(arr[3].array()[1].value<string>(), "192.168.0.255");
vector exp{"192.168.0.255", "127.0.0.1"};
for (auto inner = arr[3].array(); auto const& el : inner)
{
ASSERT_EQ(el.value<string>(), exp.back());
exp.pop_back();
}
ASSERT_TRUE(exp.empty());
}
/**
* @brief Simple custom data type with json parsing support
*/
struct Custom
{
string a;
int b;
bool c;
friend Custom
tag_invoke(json::value_to_tag<Custom>, json::value const& value)
{
assert(value.is_object());
auto const& obj = value.as_object();
return {
obj.at("str").as_string().c_str(),
obj.at("int").as_int64(),
obj.at("bool").as_bool()};
}
};
TEST_F(ConfigTest, Extend)
{
auto custom = cfg.value<Custom>("section.test");
ASSERT_EQ(custom.a, "hello");
ASSERT_EQ(custom.b, 9042);
ASSERT_EQ(custom.c, true);
}
/**
* @brief Simple temporary file util
*/
class TmpFile
{
public:
TmpFile(std::string const& data)
: tmpPath_{boost::filesystem::unique_path().string()}
{
std::ofstream of;
of.open(tmpPath_);
of << data;
of.close();
}
~TmpFile()
{
std::remove(tmpPath_.c_str());
}
std::string
path() const
{
return tmpPath_;
}
private:
std::string tmpPath_;
};
TEST_F(ConfigTest, File)
{
auto tmp = TmpFile(JSONData);
auto conf = ConfigReader::open(tmp.path());
ASSERT_EQ(conf.value<int64_t>("top"), 420);
auto doesntexist = ConfigReader::open("nope");
ASSERT_EQ(doesntexist.valueOr<bool>("found", false), false);
}

5
unittests/main.cpp
View File

@@ -9,6 +9,7 @@
#include <boost/log/trivial.hpp>
#include <backend/BackendFactory.h>
#include <backend/BackendInterface.h>
#include <config/Config.h>
TEST(BackendTest, Basic)
{
@@ -41,7 +42,7 @@ TEST(BackendTest, Basic)
for (auto& config : configs)
{
std::cout << keyspace << std::endl;
auto backend = Backend::make_Backend(ioc, config);
auto backend = Backend::make_Backend(ioc, clio::Config{config});
std::string rawHeader =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335"
@@ -1844,7 +1845,7 @@ TEST(Backend, cacheIntegration)
for (auto& config : configs)
{
std::cout << keyspace << std::endl;
auto backend = Backend::make_Backend(ioc, config);
auto backend = Backend::make_Backend(ioc, clio::Config{config});
backend->cache().setFull();
std::string rawHeader =