ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-18 18:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge develop

Browse Source
This commit is contained in:
Peter Chen
2025-08-21 12:37:19 -04:00
parent 0d9a83fd4d ec40cc93ff
commit 8f6bec2e25
36 changed files with 388 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/actions/generate/action.yml vendored
View File

@@ -37,6 +37,10 @@ inputs:
description: Whether to use mold linker
required: true
default: "false"
package:
description: Whether to generate Debian package
required: true
default: "false"
runs:
using: composite
@@ -55,6 +59,7 @@ runs:
BENCHMARK_OPTION: "${{ inputs.build_benchmark == 'true' && 'True' || 'False' }}"
TIME_TRACE: "${{ inputs.time_trace == 'true' && 'True' || 'False' }}"
USE_MOLD: "${{ inputs.use_mold == 'true' && 'True' || 'False' }}"
PACKAGE: "${{ inputs.package == 'true' && 'True' || 'False' }}"
run: |
cd build
conan \
@@ -70,6 +75,7 @@ runs:
-o "&:coverage=${CODE_COVERAGE}" \
-o "&:time_trace=${TIME_TRACE}" \
-o "&:use_mold=${USE_MOLD}" \
-o "&:package=${PACKAGE}" \
--profile:all "${{ inputs.conan_profile }}"
- name: Run cmake

17
.github/workflows/build.yml vendored
View File

@@ -85,6 +85,23 @@ jobs:
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
package:
name: Build packages
uses: ./.github/workflows/build_impl.yml
with:
runs_on: heavy
container: '{ "image": "ghcr.io/xrplf/clio-ci:8ad111655c4d04bfedb7e7cb3bbfba6d4204852d" }'
conan_profile: gcc
build_type: Release
disable_cache: false
code_coverage: false
static: true
upload_clio_server: false
package: true
targets: package
analyze_build_time: false
check_config:
name: Check Config Description
needs: build-and-test

7
.github/workflows/build_and_test.yml vendored
View File

@@ -63,6 +63,12 @@ on:
type: string
default: ""
package:
description: Whether to generate Debian package
required: false
type: boolean
default: false
jobs:
build:
uses: ./.github/workflows/build_impl.yml
@@ -78,6 +84,7 @@ jobs:
targets: ${{ inputs.targets }}
analyze_build_time: false
expected_version: ${{ inputs.expected_version }}
package: ${{ inputs.package }}
test:
needs: build

17
.github/workflows/build_impl.yml vendored
View File

@@ -59,6 +59,11 @@ on:
type: string
default: ""
package:
description: Whether to generate Debian package
required: false
type: boolean
secrets:
CODECOV_TOKEN:
required: false
@@ -111,6 +116,7 @@ jobs:
static: ${{ inputs.static }}
time_trace: ${{ inputs.analyze_build_time }}
use_mold: ${{ runner.os != 'macOS' }}
package: ${{ inputs.package }}
- name: Build Clio
uses: ./.github/actions/build_clio
@@ -158,19 +164,26 @@ jobs:
path: build/clio_server
- name: Upload clio_tests
if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }}
if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }}
uses: actions/upload-artifact@v4
with:
name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
path: build/clio_tests
- name: Upload clio_integration_tests
if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }}
if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }}
uses: actions/upload-artifact@v4
with:
name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
path: build/clio_integration_tests
- name: Upload Clio Linux package
if: inputs.package
uses: actions/upload-artifact@v4
with:
name: clio_deb_package_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
path: build/*.deb
- name: Save cache
if: ${{ !inputs.disable_cache && github.ref == 'refs/heads/develop' }}
uses: ./.github/actions/save_cache

4
CMakeLists.txt
View File

@@ -94,8 +94,8 @@ if (docs)
endif ()
include(install/install)
if (packaging)
include(cmake/packaging.cmake) # This file exists only in build runner
if (package)
include(ClioPackage)
endif ()
if (snapshot)

3
benchmarks/util/log/LoggerBenchmark.cpp
View File

@@ -45,7 +45,7 @@ struct BenchmarkLoggingInitializer {
static std::shared_ptr<spdlog::sinks::sink>
createFileSink(LogService::FileLoggingParams const& params)
{
return LogService::createFileSink(params);
return LogService::createFileSink(params, kLOG_FORMAT);
}
static Logger
@@ -107,7 +107,6 @@ benchmarkConcurrentFileLogging(benchmark::State& state)
auto logger = std::make_shared<spdlog::async_logger>(
channel, fileSink, spdlog::thread_pool(), spdlog::async_overflow_policy::block
);
logger->set_pattern(kLOG_FORMAT);
spdlog::register_logger(logger);
Logger const threadLogger = BenchmarkLoggingInitializer::getLogger(std::move(logger));

8
cmake/ClioPackage.cmake Normal file
View File

@@ -0,0 +1,8 @@
include("${CMAKE_CURRENT_LIST_DIR}/ClioVersion.cmake")
set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/clio")
set(CPACK_PACKAGE_VERSION "${CLIO_VERSION}")
set(CPACK_STRIP_FILES TRUE)
include(pkg/deb)
include(CPack)

17
cmake/install/clio.service.in
View File

@@ -1,17 +0,0 @@
[Unit]
Description=Clio XRPL API server
Documentation=https://github.com/XRPLF/clio.git
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=@CLIO_INSTALL_DIR@/bin/clio_server @CLIO_INSTALL_DIR@/etc/config.json
Restart=on-failure
User=clio
Group=clio
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target

12
cmake/install/install.cmake
View File

@@ -1,13 +1,13 @@
set(CLIO_INSTALL_DIR "/opt/clio")
set(CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR})
set(CMAKE_INSTALL_PREFIX "${CLIO_INSTALL_DIR}" CACHE PATH "Install prefix" FORCE)
install(TARGETS clio_server DESTINATION bin)
set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
include(GNUInstallDirs)
install(TARGETS clio_server DESTINATION ${CMAKE_INSTALL_BINDIR})
file(READ docs/examples/config/example-config.json config)
string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
configure_file("${CMAKE_SOURCE_DIR}/cmake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)

12
cmake/pkg/deb.cmake Normal file
View File

@@ -0,0 +1,12 @@
set(CPACK_GENERATOR "DEB")
set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/XRPLF/clio")
set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Ripple Labs Inc. <support@ripple.com>")
set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA ${CMAKE_SOURCE_DIR}/cmake/pkg/postinst)
# We must replace "-" with "~" otherwise dpkg will sort "X.Y.Z-b1" as greater than "X.Y.Z"
string(REPLACE "-" "~" git "${CPACK_PACKAGE_VERSION}")

46
cmake/pkg/postinst Executable file
View File

@@ -0,0 +1,46 @@
#!/bin/sh
set -e
USER_NAME=clio
GROUP_NAME="${USER_NAME}"
CLIO_EXECUTABLE="clio_server"
CLIO_PREFIX="/opt/clio"
CLIO_BIN="$CLIO_PREFIX/bin/${CLIO_EXECUTABLE}"
CLIO_CONFIG="$CLIO_PREFIX/etc/config.json"
case "$1" in
configure)
if ! id -u "$USER_NAME" >/dev/null 2>&1; then
# Users who should not have a home directory should have their home directory set to /nonexistent
# https://www.debian.org/doc/debian-policy/ch-opersys.html#non-existent-home-directories
useradd \
--system \
--home-dir /nonexistent \
--no-create-home \
--shell /usr/sbin/nologin \
--comment "system user for ${CLIO_EXECUTABLE}" \
--user-group \
${USER_NAME}
fi
install -d -o "$USER_NAME" -g "$GROUP_NAME" /var/log/clio
if [ -f "$CLIO_CONFIG" ]; then
chown "$USER_NAME:$GROUP_NAME" "$CLIO_CONFIG"
fi
chown -R "$USER_NAME:$GROUP_NAME" "$CLIO_PREFIX"
ln -sf "$CLIO_BIN" "/usr/bin/${CLIO_EXECUTABLE}"
;;
abort-upgrade|abort-remove|abort-deconfigure)
;;
*)
echo "postinst called with unknown argument \`$1'" >&2
exit 1
;;
esac
exit 0

4
conanfile.py
View File

@@ -16,7 +16,7 @@ class ClioConan(ConanFile):
'integration_tests': [True, False], # build integration tests; create `clio_integration_tests` binary
'benchmark': [True, False], # build benchmarks; create `clio_benchmarks` binary
'docs': [True, False], # doxygen API docs; create custom target 'docs'
'packaging': [True, False], # create distribution packages
'package': [True, False], # create distribution packages
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
'lint': [True, False], # run clang-tidy checks during compilation
'snapshot': [True, False], # build export/import snapshot tool
@@ -43,7 +43,7 @@ class ClioConan(ConanFile):
'tests': False,
'integration_tests': False,
'benchmark': False,
'packaging': False,
'package': False,
'coverage': False,
'lint': False,
'docs': False,

22
docs/config-description.md
View File

@@ -3,7 +3,9 @@
This document provides a list of all available Clio configuration properties in detail.
> [!NOTE]
> Dot notation in configuration key names represents nested fields. For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
> Dot notation in configuration key names represents nested fields.
> For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object.
> If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
## Configuration Details
@@ -453,7 +455,23 @@ This document provides a list of all available Clio configuration properties in
- **Type**: string
- **Default value**: `%Y-%m-%d %H:%M:%S.%f %^%3!l:%n%$ - %v`
- **Constraints**: None
- **Description**: The format string for log messages using spdlog format patterns. Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.
- **Description**: The format string for log messages using spdlog format patterns.
Each of the variables expands like so:
- `%Y-%m-%d %H:%M:%S.%f`: The full date and time of the log entry with microsecond precision
- `%^`: Start color range
- `%3!l`: The severity (aka log level) the entry was sent at stripped to 3 characters
- `%n`: The logger name (channel) that this log entry was sent to
- `%$`: End color range
- `%v`: The actual log message
Some additional variables that might be useful:
- `%@`: A partial path to the C++ file and the line number in the said file (`src/file/path:linenumber`)
- `%t`: The ID of the thread the log entry is written from
Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.
### log.is_async

8
docs/configure-clio.md
View File

@@ -88,13 +88,15 @@ Exactly equal password gains admin rights for the request or a websocket connect
Clio can cache requests to ETL sources to reduce the load on the ETL source.
Only following commands are cached: `server_info`, `server_state`, `server_definitions`, `fee`, `ledger_closed`.
By default the forwarding cache is off.
To enable the caching for a source, `forwarding_cache_timeout` value should be added to the configuration file, e.g.:
To enable the caching for a source, `forwarding.cache_timeout` value should be added to the configuration file, e.g.:
```json
"forwarding_cache_timeout": 0.250,
"forwarding": {
"cache_timeout": 0.250,
}
```
`forwarding_cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
`forwarding.cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
Zero value turns off the cache feature.
## Graceful shutdown (not fully implemented yet)

17
src/app/CliArgs.cpp
View File

@@ -23,6 +23,7 @@
#include "util/build/Build.hpp"
#include "util/config/ConfigDescription.hpp"
#include <boost/program_options/errors.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/positional_options.hpp>
@@ -56,12 +57,22 @@ CliArgs::parse(int argc, char const* argv[])
po::positional_options_description positional;
positional.add("conf", 1);
auto const printHelp = [&description]() {
std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
};
po::variables_map parsed;
po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
po::notify(parsed);
try {
po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
po::notify(parsed);
} catch (po::error const& e) {
std::cerr << "Error: " << e.what() << std::endl << std::endl;
printHelp();
return Action{Action::Exit{EXIT_FAILURE}};
}
if (parsed.contains("help")) {
std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
printHelp();
return Action{Action::Exit{EXIT_SUCCESS}};
}

35
src/main/Main.cpp
View File

@@ -34,19 +34,16 @@
using namespace util::config;
[[nodiscard]]
int
main(int argc, char const* argv[])
try {
util::setTerminationHandler();
util::ScopeGuard const loggerShutdownGuard{[]() { util::LogService::shutdown(); }};
runApp(int argc, char const* argv[])
{
auto const action = app::CliArgs::parse(argc, argv);
return action.apply(
[](app::CliArgs::Action::Exit const& exit) { return exit.exitCode; },
[](app::CliArgs::Action::VerifyConfig const& verify) {
if (app::parseConfig(verify.configPath)) {
std::cout << "Config " << verify.configPath << " is correct"
<< "\n";
std::cout << "Config " << verify.configPath << " is correct" << "\n";
return EXIT_SUCCESS;
}
return EXIT_FAILURE;
@@ -76,10 +73,22 @@ try {
return migrator.run();
}
);
} catch (std::exception const& e) {
LOG(util::LogService::fatal()) << "Exit on exception: " << e.what();
return EXIT_FAILURE;
} catch (...) {
LOG(util::LogService::fatal()) << "Exit on exception: unknown";
return EXIT_FAILURE;
}
int
main(int argc, char const* argv[])
{
util::setTerminationHandler();
util::ScopeGuard const loggerShutdownGuard{[] { util::LogService::shutdown(); }};
try {
return runApp(argc, argv);
} catch (std::exception const& e) {
LOG(util::LogService::fatal()) << "Exit on exception: " << e.what();
return EXIT_FAILURE;
} catch (...) {
LOG(util::LogService::fatal()) << "Exit on exception: unknown";
return EXIT_FAILURE;
}
}

36
src/rpc/handlers/AccountInfo.cpp
View File

@@ -43,6 +43,7 @@
#include <algorithm>
#include <iterator>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
@@ -90,6 +91,17 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
auto const isClawbackEnabled = isEnabled(Amendments::Clawback);
auto const isTokenEscrowEnabled = isEnabled(Amendments::TokenEscrow);
Output out{
.ledgerIndex = lgrInfo.seq,
.ledgerHash = ripple::strHex(lgrInfo.hash),
.accountData = sle,
.isDisallowIncomingEnabled = isDisallowIncomingEnabled,
.isClawbackEnabled = isClawbackEnabled,
.isTokenEscrowEnabled = isTokenEscrowEnabled,
.apiVersion = ctx.apiVersion,
.signerLists = std::nullopt
};
// Return SignerList(s) if that is requested.
if (input.signerLists) {
// We put the SignerList in an array because of an anticipated
@@ -99,7 +111,6 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
// This code will need to be revisited if in the future we
// support multiple SignerLists on one account.
auto const signers = sharedPtrBackend_->fetchLedgerObject(signersKey.key, lgrInfo.seq, ctx.yield);
std::vector<ripple::STLedgerEntry> signerList;
if (signers) {
ripple::STLedgerEntry const sleSigners{
@@ -109,30 +120,11 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
if (!signersKey.check(sleSigners))
return Error{Status{RippledError::rpcDB_DESERIALIZATION}};
signerList.push_back(sleSigners);
out.signerLists = std::vector<ripple::STLedgerEntry>{sleSigners};
}
return Output(
lgrInfo.seq,
ripple::strHex(lgrInfo.hash),
sle,
isDisallowIncomingEnabled,
isClawbackEnabled,
isTokenEscrowEnabled,
ctx.apiVersion,
signerList
);
}
return Output(
lgrInfo.seq,
ripple::strHex(lgrInfo.hash),
sle,
isDisallowIncomingEnabled,
isClawbackEnabled,
isTokenEscrowEnabled,
ctx.apiVersion
);
return out;
}
void

33
src/rpc/handlers/AccountInfo.hpp
View File

@@ -66,39 +66,6 @@ public:
std::optional<std::vector<ripple::STLedgerEntry>> signerLists;
// validated should be sent via framework
bool validated = true;
/**
* @brief Construct a new Output object
*
* @param ledgerId The ledger index
* @param ledgerHash The ledger hash
* @param sle The account data
* @param isDisallowIncomingEnabled Whether disallow incoming is enabled
* @param isClawbackEnabled Whether clawback is enabled
* @param isTokenEscrowEnabled Whether token escrow is enabled
* @param version The API version
* @param signerLists The signer lists
*/
Output(
uint32_t ledgerId,
std::string ledgerHash,
ripple::STLedgerEntry sle,
bool isDisallowIncomingEnabled,
bool isClawbackEnabled,
bool isTokenEscrowEnabled,
uint32_t version,
std::optional<std::vector<ripple::STLedgerEntry>> signerLists = std::nullopt
)
: ledgerIndex(ledgerId)
, ledgerHash(std::move(ledgerHash))
, accountData(std::move(sle))
, isDisallowIncomingEnabled(isDisallowIncomingEnabled)
, isClawbackEnabled(isClawbackEnabled)
, isTokenEscrowEnabled(isTokenEscrowEnabled)
, apiVersion(version)
, signerLists(std::move(signerLists))
{
}
};
/**

8
src/rpc/handlers/GetAggregatePrice.cpp
View File

@@ -124,7 +124,13 @@ GetAggregatePriceHandler::process(GetAggregatePriceHandler::Input const& input,
auto const latestTime = timestampPricesBiMap.left.begin()->first;
Output out(latestTime, ripple::to_string(lgrInfo.hash), lgrInfo.seq);
Output out{
.time = latestTime,
.trimStats = std::nullopt,
.ledgerHash = ripple::to_string(lgrInfo.hash),
.ledgerIndex = lgrInfo.seq,
.median = ""
};
if (input.timeThreshold) {
auto const oldestTime = timestampPricesBiMap.left.rbegin()->first;

19
src/rpc/handlers/GetAggregatePrice.hpp
View File

@@ -43,7 +43,6 @@
#include <optional>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
namespace rpc {
@@ -59,8 +58,9 @@ public:
* @brief A struct to hold the statistics
*/
struct Stats {
ripple::STAmount avg;
ripple::Number sd; // standard deviation
ripple::STAmount avg{}; // NOLINT(readability-redundant-member-init)
// standard deviation
ripple::Number sd{}; // NOLINT(readability-redundant-member-init)
uint32_t size{0};
};
@@ -69,23 +69,12 @@ public:
*/
struct Output {
uint32_t time;
Stats extireStats;
Stats extireStats{};
std::optional<Stats> trimStats;
std::string ledgerHash;
uint32_t ledgerIndex;
std::string median;
bool validated = true;
/**
* @brief Construct a new Output object
* @param time The time of the latest oracle data
* @param ledgerHash The hash of the ledger
* @param ledgerIndex The index of the ledger
*/
Output(uint32_t time, std::string ledgerHash, uint32_t ledgerIndex)
: time(time), ledgerHash(std::move(ledgerHash)), ledgerIndex(ledgerIndex)
{
}
};
/**

6
src/util/config/ConfigDefinition.cpp
View File

@@ -237,6 +237,12 @@ ClioConfigDefinition::parse(ConfigFileInterface const& config)
});
}
for (auto const& key : config.getAllKeys()) {
if (!map_.contains(key) && !arrayPrefixesToKeysMap.contains(key)) {
listOfErrors.emplace_back("Unknown key: " + key);
}
}
if (!listOfErrors.empty())
return listOfErrors;

4
src/util/config/ConfigDefinition.hpp
View File

@@ -19,7 +19,6 @@
#pragma once
#include "rpc/common/APIVersion.hpp"
#include "util/Assert.hpp"
#include "util/config/Array.hpp"
#include "util/config/ConfigConstraints.hpp"
@@ -27,18 +26,15 @@
#include "util/config/ConfigValue.hpp"
#include "util/config/Error.hpp"
#include "util/config/ObjectView.hpp"
#include "util/config/Types.hpp"
#include "util/config/ValueView.hpp"
#include <algorithm>
#include <chrono>
#include <cstddef>
#include <cstdint>
#include <initializer_list>
#include <optional>
#include <string>
#include <string_view>
#include <thread>
#include <unordered_map>
#include <utility>
#include <variant>

41
src/util/config/ConfigDescription.hpp
View File

@@ -110,13 +110,7 @@ public:
static void
writeConfigDescriptionToFile(std::ostream& file)
{
file << "# Clio Config Description\n\n";
file << "This document provides a list of all available Clio configuration properties in detail.\n\n";
file << "> [!NOTE]\n";
file << "> Dot notation in configuration key names represents nested fields. For example, "
"**database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name "
"includes \"[]\", it indicates that the nested field is an array (e.g., etl_sources.[]).\n\n";
file << "## Configuration Details\n";
file << kCONFIG_DESCRIPTION_HEADER;
for (auto const& [key, val] : kCONFIG_DESCRIPTION) {
file << "\n### " << key << "\n\n";
@@ -133,6 +127,19 @@ public:
}
private:
static constexpr auto kCONFIG_DESCRIPTION_HEADER =
R"(# Clio Config Description
This document provides a list of all available Clio configuration properties in detail.
> [!NOTE]
> Dot notation in configuration key names represents nested fields.
> For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object.
> If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
## Configuration Details
)";
static constexpr auto kCONFIG_DESCRIPTION = std::array{
KV{
.key = "database.type",
@@ -266,9 +273,23 @@ private:
KV{.key = "log.level",
.value = "The general logging level of Clio. This level is applied to all log channels that do not have an "
"explicitly defined logging level."},
KV{.key = "log.format",
.value = "The format string for log messages using spdlog format patterns. Documentation can be found at: "
"<https://github.com/gabime/spdlog/wiki/Custom-formatting>."},
KV{.key = "log.format", .value = R"(The format string for log messages using spdlog format patterns.
Each of the variables expands like so:
- `%Y-%m-%d %H:%M:%S.%f`: The full date and time of the log entry with microsecond precision
- `%^`: Start color range
- `%3!l`: The severity (aka log level) the entry was sent at stripped to 3 characters
- `%n`: The logger name (channel) that this log entry was sent to
- `%$`: End color range
- `%v`: The actual log message
Some additional variables that might be useful:
- `%@`: A partial path to the C++ file and the line number in the said file (`src/file/path:linenumber`)
- `%t`: The ID of the thread the log entry is written from
Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.)"},
KV{.key = "log.is_async", .value = "Whether spdlog is asynchronous or not."},
KV{.key = "log.enable_console", .value = "Enables or disables logging to the console."},
KV{.key = "log.directory", .value = "The directory path for the log files."},

9
src/util/config/ConfigFileInterface.hpp
View File

@@ -22,6 +22,7 @@
#include "util/config/Types.hpp"
#include <optional>
#include <string>
#include <string_view>
#include <vector>
@@ -63,6 +64,14 @@ public:
*/
virtual bool
containsKey(std::string_view key) const = 0;
/**
* @brief Retrieves all keys in the configuration file.
*
* @return A vector of all keys in the configuration file.
*/
virtual std::vector<std::string>
getAllKeys() const = 0;
};
} // namespace util::config

10
src/util/config/ConfigFileJson.cpp
View File

@@ -142,6 +142,16 @@ ConfigFileJson::containsKey(std::string_view key) const
return jsonObject_.contains(key);
}
std::vector<std::string>
ConfigFileJson::getAllKeys() const
{
std::vector<std::string> keys;
for (auto const& [key, value] : jsonObject_) {
keys.push_back(key);
}
return keys;
}
boost::json::object const&
ConfigFileJson::inner() const
{

9
src/util/config/ConfigFileJson.hpp
View File

@@ -28,6 +28,7 @@
#include <expected>
#include <filesystem>
#include <optional>
#include <string>
#include <string_view>
#include <vector>
@@ -73,6 +74,14 @@ public:
[[nodiscard]] bool
containsKey(std::string_view key) const override;
/**
* @brief Retrieves all keys in the configuration file.
*
* @return A vector of all keys in the configuration file.
*/
[[nodiscard]] std::vector<std::string>
getAllKeys() const override;
/**
* @brief Creates a new ConfigFileJson by parsing the provided JSON file and
* stores the values in the object.

49
src/util/config/ConfigFileYaml.hpp
View File

@@ -1,49 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "util/config/ConfigFileInterface.hpp"
#include "util/config/Types.hpp"
#include <boost/filesystem/path.hpp>
#include <string_view>
#include <vector>
// TODO: implement when we support yaml
namespace util::config {
/** @brief Yaml representation of config */
class ConfigFileYaml final : public ConfigFileInterface {
public:
ConfigFileYaml() = default;
Value
getValue(std::string_view key) const override;
std::vector<Value>
getArray(std::string_view key) const override;
bool
containsKey(std::string_view key) const override;
};
} // namespace util::config

55
src/util/log/Logger.cpp
View File

@@ -31,7 +31,10 @@
#include <spdlog/async.h>
#include <spdlog/async_logger.h>
#include <spdlog/common.h>
#include <spdlog/details/log_msg.h>
#include <spdlog/formatter.h>
#include <spdlog/logger.h>
#include <spdlog/pattern_formatter.h>
#include <spdlog/sinks/rotating_file_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#include <spdlog/spdlog.h>
@@ -41,6 +44,7 @@
#include <cstddef>
#include <cstdint>
#include <filesystem>
#include <iostream>
#include <memory>
#include <optional>
#include <string>
@@ -120,26 +124,63 @@ getSeverityLevel(std::string_view logLevel)
std::unreachable();
}
/**
* @brief Custom formatter that filters out critical messages
*
* This formatter only processes and formats messages with severity level less than critical.
* Critical messages will be handled separately.
*/
class NonCriticalFormatter : public spdlog::formatter {
public:
NonCriticalFormatter(std::unique_ptr<spdlog::formatter> wrappedFormatter)
: wrapped_formatter_(std::move(wrappedFormatter))
{
}
void
format(spdlog::details::log_msg const& msg, spdlog::memory_buf_t& dest) override
{
// Only format messages with severity less than critical
if (msg.level != spdlog::level::critical) {
wrapped_formatter_->format(msg, dest);
}
}
std::unique_ptr<formatter>
clone() const override
{
return std::make_unique<NonCriticalFormatter>(wrapped_formatter_->clone());
}
private:
std::unique_ptr<spdlog::formatter> wrapped_formatter_;
};
/**
* @brief Initializes console logging.
*
* @param logToConsole A boolean indicating whether to log to console.
* @param format A string representing the log format.
* @return Vector of sinks for console logging.
*/
static std::vector<spdlog::sink_ptr>
createConsoleSinks(bool logToConsole)
createConsoleSinks(bool logToConsole, std::string const& format)
{
std::vector<spdlog::sink_ptr> sinks;
if (logToConsole) {
auto consoleSink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
consoleSink->set_level(spdlog::level::trace);
consoleSink->set_formatter(
std::make_unique<NonCriticalFormatter>(std::make_unique<spdlog::pattern_formatter>(format))
);
sinks.push_back(std::move(consoleSink));
}
// Always add stderr sink for fatal logs
auto stderrSink = std::make_shared<spdlog::sinks::stderr_color_sink_mt>();
stderrSink->set_level(spdlog::level::critical);
stderrSink->set_formatter(std::make_unique<spdlog::pattern_formatter>(format));
sinks.push_back(std::move(stderrSink));
return sinks;
@@ -153,7 +194,7 @@ createConsoleSinks(bool logToConsole)
* @return File sink for logging.
*/
spdlog::sink_ptr
LogService::createFileSink(FileLoggingParams const& params)
LogService::createFileSink(FileLoggingParams const& params, std::string const& format)
{
std::filesystem::path const dirPath(params.logDir);
// the below are taken from user in MB, but spdlog needs it to be in bytes
@@ -163,6 +204,7 @@ LogService::createFileSink(FileLoggingParams const& params)
(dirPath / "clio.log").string(), rotationSize, params.dirMaxFiles
);
fileSink->set_level(spdlog::level::trace);
fileSink->set_formatter(std::make_unique<spdlog::pattern_formatter>(format));
return fileSink;
}
@@ -229,11 +271,13 @@ LogService::init(config::ClioConfigDefinition const& config)
data.isAsync = config.get<bool>("log.is_async");
data.defaultSeverity = getSeverityLevel(config.get<std::string>("log.level"));
std::string const format = config.get<std::string>("log.format");
if (data.isAsync) {
spdlog::init_thread_pool(8192, 1);
}
data.allSinks = createConsoleSinks(config.get<bool>("log.enable_console"));
data.allSinks = createConsoleSinks(config.get<bool>("log.enable_console"), format);
if (auto const logDir = config.maybeValue<std::string>("log.directory"); logDir.has_value()) {
std::filesystem::path const dirPath{logDir.value()};
@@ -250,7 +294,7 @@ LogService::init(config::ClioConfigDefinition const& config)
.rotationSizeMB = config.get<uint32_t>("log.rotation_size"),
.dirMaxFiles = config.get<uint32_t>("log.directory_max_files"),
};
data.allSinks.push_back(createFileSink(params));
data.allSinks.push_back(createFileSink(params, format));
}
// get min severity per channel, can be overridden using the `log.channels` array
@@ -269,9 +313,6 @@ LogService::init(config::ClioConfigDefinition const& config)
spdlog::set_default_logger(spdlog::get("General"));
std::string const format = config.get<std::string>("log.format");
spdlog::set_pattern(format);
LOG(LogService::info()) << "Default log level = " << toString(data.defaultSeverity);
return {};
}

2
src/util/log/Logger.hpp
View File

@@ -346,7 +346,7 @@ private:
[[nodiscard]]
static std::shared_ptr<spdlog::sinks::sink>
createFileSink(FileLoggingParams const& params);
createFileSink(FileLoggingParams const& params, std::string const& format);
};
}; // namespace util

4
tests/common/util/LoggerFixtures.cpp
View File

@@ -22,6 +22,7 @@
#include "util/log/Logger.hpp"
#include <spdlog/common.h>
#include <spdlog/pattern_formatter.h>
#include <spdlog/sinks/ostream_sink.h>
#include <spdlog/spdlog.h>
@@ -35,6 +36,7 @@ LoggerFixture::LoggerFixture()
// Create ostream sink for testing
auto ostreamSink = std::make_shared<spdlog::sinks::ostream_sink_mt>(stream_);
ostreamSink->set_formatter(std::make_unique<spdlog::pattern_formatter>("%^%3!l:%n%$ - %v"));
// Create loggers for each channel
std::ranges::for_each(util::Logger::kCHANNELS, [&ostreamSink](char const* channel) {
@@ -50,8 +52,6 @@ LoggerFixture::LoggerFixture()
spdlog::register_logger(traceLogger);
spdlog::set_default_logger(spdlog::get("General"));
spdlog::set_pattern("%^%3!l:%n%$ - %v");
}
NoLoggerFixture::NoLoggerFixture()

4
tests/unit/LogServiceInitTests.cpp
View File

@@ -32,6 +32,7 @@
#include <fmt/format.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <spdlog/pattern_formatter.h>
#include <spdlog/sinks/ostream_sink.h>
#include <spdlog/spdlog.h>
@@ -84,6 +85,7 @@ protected:
replaceSinks()
{
auto ostreamSink = std::make_shared<spdlog::sinks::ostream_sink_mt>(stream_);
ostreamSink->set_formatter(std::make_unique<spdlog::pattern_formatter>("%^%3!l:%n%$ - %v"));
for (auto const& channel : Logger::kCHANNELS) {
auto logger = spdlog::get(channel);
@@ -93,8 +95,6 @@ protected:
logger->sinks().clear();
logger->sinks().push_back(ostreamSink);
}
spdlog::set_pattern("%^%3!l:%n%$ - %v");
}
private:

3
tests/unit/data/cassandra/SettingsProviderTests.cpp
View File

@@ -82,7 +82,8 @@ class SettingsProviderTest : public NoLoggerFixture {};
TEST_F(SettingsProviderTest, Defaults)
{
auto const cfg = getParseSettingsConfig(json::parse(R"JSON({"contact_points": "127.0.0.1"})JSON"));
auto const cfg =
getParseSettingsConfig(json::parse(R"JSON({"database.cassandra.contact_points": "127.0.0.1"})JSON"));
SettingsProvider const provider{cfg.getObject("database.cassandra")};
auto const settings = provider.getSettings();

2
tests/unit/etl/LoadBalancerTests.cpp
View File

@@ -160,7 +160,7 @@ TEST_F(LoadBalancerConstructorTests, construct)
TEST_F(LoadBalancerConstructorTests, forwardingTimeoutPassedToSourceFactory)
{
auto const forwardingTimeout = 10;
configJson_.as_object()["forwarding"] = boost::json::object{{"timeout", float{forwardingTimeout}}};
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", float{forwardingTimeout}}};
EXPECT_CALL(
sourceFactory_,
makeSource(

2
tests/unit/etlng/LoadBalancerTests.cpp
View File

@@ -182,7 +182,7 @@ TEST_F(LoadBalancerConstructorNgTests, construct)
TEST_F(LoadBalancerConstructorNgTests, forwardingTimeoutPassedToSourceFactory)
{
auto const forwardingTimeout = 10;
configJson_.as_object()["forwarding"] = boost::json::object{{"timeout", float{forwardingTimeout}}};
configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", float{forwardingTimeout}}};
EXPECT_CALL(
sourceFactory_,
makeSource(

32
tests/unit/util/config/ClioConfigDefinitionTests.cpp
View File

@@ -465,3 +465,35 @@ TEST_F(ClioConfigDefinitionParseArrayTest, missingAllRequiredFields)
EXPECT_EQ(result->size(), 1);
EXPECT_THAT(result->at(0).error, testing::StartsWith("array.[].int"));
}
TEST(ClioConfigDefinitionParse, unexpectedFields)
{
ClioConfigDefinition config{
{"expected", ConfigValue{ConfigType::String}.optional()},
};
auto const configJson = boost::json::parse(R"JSON({
"expected": "present",
"unexpected_string": "",
"unexpected_non_empty_array": [
{"string": ""},
{"string": ""}
],
"unexpected_empty_array": [],
"unexpected_object": {
"string": ""
}
})JSON")
.as_object();
auto const configFile = ConfigFileJson{configJson};
auto result = config.parse(configFile);
std::ranges::sort(*result, [](auto const& lhs, auto const& rhs) { return lhs.error < rhs.error; });
ASSERT_TRUE(result.has_value());
ASSERT_EQ(result->size(), 4);
EXPECT_EQ(result->at(0).error, "Unknown key: unexpected_empty_array.[]");
EXPECT_EQ(result->at(1).error, "Unknown key: unexpected_non_empty_array.[].string");
EXPECT_EQ(result->at(2).error, "Unknown key: unexpected_object.string");
EXPECT_EQ(result->at(3).error, "Unknown key: unexpected_string");
}

26
tests/unit/util/config/ConfigFileJsonTests.cpp
View File

@@ -34,6 +34,7 @@
#include <string>
#include <unordered_map>
#include <variant>
#include <vector>
using namespace util::config;
@@ -488,6 +489,31 @@ TEST_F(ConfigFileJsonTest, containsKey)
EXPECT_FALSE(jsonFileObj.containsKey("array_of_objects.[].object"));
}
TEST_F(ConfigFileJsonTest, getAllKeys)
{
auto const jsonStr = R"JSON({
"int": 42,
"object": { "string": "some string", "array": [1, 2, 3] },
"array2": [1, 2, 3],
"array_of_objects": [ {"int": 42}, {"string": "some string"} ]
})JSON";
auto const jsonFileObj = ConfigFileJson{boost::json::parse(jsonStr).as_object()};
auto allKeys = jsonFileObj.getAllKeys();
std::ranges::sort(allKeys);
EXPECT_EQ(allKeys.size(), 6);
std::vector<std::string> const expectedKeys{
{"array2.[]",
"array_of_objects.[].int",
"array_of_objects.[].string",
"int",
"object.array.[]",
"object.string"}
};
EXPECT_EQ(allKeys, expectedKeys);
}
struct ConfigFileJsonMakeTest : ConfigFileJsonTest {};
TEST_F(ConfigFileJsonMakeTest, invalidFile)