merge develop

.github/actions/generate/action.yml

@@ -37,6 +37,10 @@ inputs:
     description: Whether to use mold linker
     required: true
     default: "false"
+  package:
+    description: Whether to generate Debian package
+    required: true
+    default: "false"
 
 runs:
   using: composite
@@ -55,6 +59,7 @@ runs:
         BENCHMARK_OPTION: "${{ inputs.build_benchmark == 'true' && 'True' || 'False' }}"
         TIME_TRACE: "${{ inputs.time_trace == 'true' && 'True' || 'False' }}"
         USE_MOLD: "${{ inputs.use_mold == 'true' && 'True' || 'False' }}"
+        PACKAGE: "${{ inputs.package == 'true' && 'True' || 'False' }}"
       run: |
         cd build
         conan \
@@ -70,6 +75,7 @@ runs:
           -o "&:coverage=${CODE_COVERAGE}" \
           -o "&:time_trace=${TIME_TRACE}" \
           -o "&:use_mold=${USE_MOLD}" \
+          -o "&:package=${PACKAGE}" \
           --profile:all "${{ inputs.conan_profile }}"
 
     - name: Run cmake

.github/workflows/build.yml

@@ -85,6 +85,23 @@ jobs:
     secrets:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
+  package:
+    name: Build packages
+
+    uses: ./.github/workflows/build_impl.yml
+    with:
+      runs_on: heavy
+      container: '{ "image": "ghcr.io/xrplf/clio-ci:8ad111655c4d04bfedb7e7cb3bbfba6d4204852d" }'
+      conan_profile: gcc
+      build_type: Release
+      disable_cache: false
+      code_coverage: false
+      static: true
+      upload_clio_server: false
+      package: true
+      targets: package
+      analyze_build_time: false
+
   check_config:
     name: Check Config Description
     needs: build-and-test

.github/workflows/build_and_test.yml

@@ -63,6 +63,12 @@ on:
         type: string
         default: ""
 
+      package:
+        description: Whether to generate Debian package
+        required: false
+        type: boolean
+        default: false
+
 jobs:
   build:
     uses: ./.github/workflows/build_impl.yml
@@ -78,6 +84,7 @@ jobs:
       targets: ${{ inputs.targets }}
       analyze_build_time: false
       expected_version: ${{ inputs.expected_version }}
+      package: ${{ inputs.package }}
 
   test:
     needs: build

.github/workflows/build_impl.yml

@@ -59,6 +59,11 @@ on:
         type: string
         default: ""
 
+      package:
+        description: Whether to generate Debian package
+        required: false
+        type: boolean
+
     secrets:
       CODECOV_TOKEN:
         required: false
@@ -111,6 +116,7 @@ jobs:
           static: ${{ inputs.static }}
           time_trace: ${{ inputs.analyze_build_time }}
           use_mold: ${{ runner.os != 'macOS' }}
+          package: ${{ inputs.package }}
 
       - name: Build Clio
         uses: ./.github/actions/build_clio
@@ -158,19 +164,26 @@ jobs:
           path: build/clio_server
 
       - name: Upload clio_tests
-        if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }}
+        if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }}
         uses: actions/upload-artifact@v4
         with:
           name: clio_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
           path: build/clio_tests
 
       - name: Upload clio_integration_tests
-        if: ${{ !inputs.code_coverage && !inputs.analyze_build_time }}
+        if: ${{ !inputs.code_coverage && !inputs.analyze_build_time && !inputs.package }}
         uses: actions/upload-artifact@v4
         with:
           name: clio_integration_tests_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
           path: build/clio_integration_tests
 
+      - name: Upload Clio Linux package
+        if: inputs.package
+        uses: actions/upload-artifact@v4
+        with:
+          name: clio_deb_package_${{ runner.os }}_${{ inputs.build_type }}_${{ inputs.conan_profile }}
+          path: build/*.deb
+
       - name: Save cache
         if: ${{ !inputs.disable_cache && github.ref == 'refs/heads/develop' }}
         uses: ./.github/actions/save_cache

CMakeLists.txt

@@ -94,8 +94,8 @@ if (docs)
 endif ()
 
 include(install/install)
-if (packaging)
+if (package)
-  include(cmake/packaging.cmake) # This file exists only in build runner
+  include(ClioPackage)
 endif ()
 
 if (snapshot)

benchmarks/util/log/LoggerBenchmark.cpp

@@ -45,7 +45,7 @@ struct BenchmarkLoggingInitializer {
     static std::shared_ptr<spdlog::sinks::sink>
     createFileSink(LogService::FileLoggingParams const& params)
     {
-        return LogService::createFileSink(params);
+        return LogService::createFileSink(params, kLOG_FORMAT);
     }
 
     static Logger
@@ -107,7 +107,6 @@ benchmarkConcurrentFileLogging(benchmark::State& state)
                 auto logger = std::make_shared<spdlog::async_logger>(
                     channel, fileSink, spdlog::thread_pool(), spdlog::async_overflow_policy::block
                 );
-                logger->set_pattern(kLOG_FORMAT);
                 spdlog::register_logger(logger);
                 Logger const threadLogger = BenchmarkLoggingInitializer::getLogger(std::move(logger));
 

cmake/ClioPackage.cmake

@@ -0,0 +1,8 @@
+include("${CMAKE_CURRENT_LIST_DIR}/ClioVersion.cmake")
+
+set(CPACK_PACKAGING_INSTALL_PREFIX "/opt/clio")
+set(CPACK_PACKAGE_VERSION "${CLIO_VERSION}")
+set(CPACK_STRIP_FILES TRUE)
+
+include(pkg/deb)
+include(CPack)

cmake/install/clio.service.in

@@ -1,17 +0,0 @@
-[Unit]
-Description=Clio XRPL API server
-Documentation=https://github.com/XRPLF/clio.git
-
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-ExecStart=@CLIO_INSTALL_DIR@/bin/clio_server @CLIO_INSTALL_DIR@/etc/config.json
-Restart=on-failure
-User=clio
-Group=clio
-LimitNOFILE=65536
-
-[Install]
-WantedBy=multi-user.target

cmake/install/install.cmake

@@ -1,13 +1,13 @@
 set(CLIO_INSTALL_DIR "/opt/clio")
-set(CMAKE_INSTALL_PREFIX ${CLIO_INSTALL_DIR})
+set(CMAKE_INSTALL_PREFIX "${CLIO_INSTALL_DIR}" CACHE PATH "Install prefix" FORCE)
 
-install(TARGETS clio_server DESTINATION bin)
+set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}")
+
+include(GNUInstallDirs)
+
+install(TARGETS clio_server DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 file(READ docs/examples/config/example-config.json config)
 string(REGEX REPLACE "./clio_log" "/var/log/clio/" config "${config}")
 file(WRITE ${CMAKE_BINARY_DIR}/install-config.json "${config}")
 install(FILES ${CMAKE_BINARY_DIR}/install-config.json DESTINATION etc RENAME config.json)
-
-configure_file("${CMAKE_SOURCE_DIR}/cmake/install/clio.service.in" "${CMAKE_BINARY_DIR}/clio.service")
-
-install(FILES "${CMAKE_BINARY_DIR}/clio.service" DESTINATION /lib/systemd/system)

cmake/pkg/deb.cmake

@@ -0,0 +1,12 @@
+set(CPACK_GENERATOR "DEB")
+set(CPACK_DEBIAN_PACKAGE_HOMEPAGE "https://github.com/XRPLF/clio")
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Ripple Labs Inc. <support@ripple.com>")
+
+set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
+
+set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
+
+set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA ${CMAKE_SOURCE_DIR}/cmake/pkg/postinst)
+
+# We must replace "-" with "~" otherwise dpkg will sort "X.Y.Z-b1" as greater than "X.Y.Z"
+string(REPLACE "-" "~" git "${CPACK_PACKAGE_VERSION}")

cmake/pkg/postinst

@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -e
+
+USER_NAME=clio
+GROUP_NAME="${USER_NAME}"
+CLIO_EXECUTABLE="clio_server"
+CLIO_PREFIX="/opt/clio"
+CLIO_BIN="$CLIO_PREFIX/bin/${CLIO_EXECUTABLE}"
+CLIO_CONFIG="$CLIO_PREFIX/etc/config.json"
+
+case "$1" in
+    configure)
+        if ! id -u "$USER_NAME" >/dev/null 2>&1; then
+            # Users who should not have a home directory should have their home directory set to /nonexistent
+            # https://www.debian.org/doc/debian-policy/ch-opersys.html#non-existent-home-directories
+            useradd \
+                --system \
+                --home-dir /nonexistent \
+                --no-create-home \
+                --shell /usr/sbin/nologin \
+                --comment "system user for ${CLIO_EXECUTABLE}" \
+                --user-group \
+                ${USER_NAME}
+        fi
+
+        install -d -o "$USER_NAME" -g "$GROUP_NAME" /var/log/clio
+
+        if [ -f "$CLIO_CONFIG" ]; then
+            chown "$USER_NAME:$GROUP_NAME" "$CLIO_CONFIG"
+        fi
+
+        chown -R "$USER_NAME:$GROUP_NAME" "$CLIO_PREFIX"
+
+        ln -sf "$CLIO_BIN" "/usr/bin/${CLIO_EXECUTABLE}"
+
+        ;;
+    abort-upgrade|abort-remove|abort-deconfigure)
+        ;;
+    *)
+        echo "postinst called with unknown argument \`$1'" >&2
+        exit 1
+        ;;
+esac
+
+exit 0

conanfile.py

@@ -16,7 +16,7 @@ class ClioConan(ConanFile):
         'integration_tests': [True, False],   # build integration tests; create `clio_integration_tests` binary
         'benchmark': [True, False],           # build benchmarks; create `clio_benchmarks` binary
         'docs': [True, False],                # doxygen API docs; create custom target 'docs'
-        'packaging': [True, False],           # create distribution packages
+        'package': [True, False],             # create distribution packages
         'coverage': [True, False],            # build for test coverage report; create custom target `clio_tests-ccov`
         'lint': [True, False],                # run clang-tidy checks during compilation
         'snapshot': [True, False],            # build export/import snapshot tool
@@ -43,7 +43,7 @@ class ClioConan(ConanFile):
         'tests': False,
         'integration_tests': False,
         'benchmark': False,
-        'packaging': False,
+        'package': False,
         'coverage': False,
         'lint': False,
         'docs': False,

docs/config-description.md

@@ -3,7 +3,9 @@
 This document provides a list of all available Clio configuration properties in detail.
 
 > [!NOTE]
-> Dot notation in configuration key names represents nested fields. For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
+> Dot notation in configuration key names represents nested fields.
+> For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object.
+> If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
 
 ## Configuration Details
 
@@ -453,7 +455,23 @@ This document provides a list of all available Clio configuration properties in
 - **Type**: string
 - **Default value**: `%Y-%m-%d %H:%M:%S.%f %^%3!l:%n%$ - %v`
 - **Constraints**: None
-- **Description**: The format string for log messages using spdlog format patterns. Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.
+- **Description**: The format string for log messages using spdlog format patterns.
+
+Each of the variables expands like so:
+
+- `%Y-%m-%d %H:%M:%S.%f`: The full date and time of the log entry with microsecond precision
+- `%^`: Start color range
+- `%3!l`: The severity (aka log level) the entry was sent at stripped to 3 characters
+- `%n`: The logger name (channel) that this log entry was sent to
+- `%$`: End color range
+- `%v`: The actual log message
+
+Some additional variables that might be useful:
+
+- `%@`: A partial path to the C++ file and the line number in the said file (`src/file/path:linenumber`)
+- `%t`: The ID of the thread the log entry is written from
+
+Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.
 
 ### log.is_async
 

docs/configure-clio.md

@@ -88,13 +88,15 @@ Exactly equal password gains admin rights for the request or a websocket connect
 Clio can cache requests to ETL sources to reduce the load on the ETL source.
 Only following commands are cached: `server_info`, `server_state`, `server_definitions`, `fee`, `ledger_closed`.
 By default the forwarding cache is off.
-To enable the caching for a source, `forwarding_cache_timeout` value should be added to the configuration file, e.g.:
+To enable the caching for a source, `forwarding.cache_timeout` value should be added to the configuration file, e.g.:
 
 ```json
-"forwarding_cache_timeout": 0.250,
+"forwarding": {
+    "cache_timeout": 0.250,
+}
 ```
 
-`forwarding_cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
+`forwarding.cache_timeout` defines for how long (in seconds) a cache entry will be valid after being placed into the cache.
 Zero value turns off the cache feature.
 
 ## Graceful shutdown (not fully implemented yet)

src/app/CliArgs.cpp

@@ -23,6 +23,7 @@
 #include "util/build/Build.hpp"
 #include "util/config/ConfigDescription.hpp"
 
+#include <boost/program_options/errors.hpp>
 #include <boost/program_options/options_description.hpp>
 #include <boost/program_options/parsers.hpp>
 #include <boost/program_options/positional_options.hpp>
@@ -56,12 +57,22 @@ CliArgs::parse(int argc, char const* argv[])
     po::positional_options_description positional;
     positional.add("conf", 1);
 
+    auto const printHelp = [&description]() {
+        std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
+    };
+
     po::variables_map parsed;
-    po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
+    try {
-    po::notify(parsed);
+        po::store(po::command_line_parser(argc, argv).options(description).positional(positional).run(), parsed);
+        po::notify(parsed);
+    } catch (po::error const& e) {
+        std::cerr << "Error: " << e.what() << std::endl << std::endl;
+        printHelp();
+        return Action{Action::Exit{EXIT_FAILURE}};
+    }
 
     if (parsed.contains("help")) {
-        std::cout << "Clio server " << util::build::getClioFullVersionString() << "\n\n" << description;
+        printHelp();
         return Action{Action::Exit{EXIT_SUCCESS}};
     }
 

src/main/Main.cpp

@@ -34,19 +34,16 @@
 
 using namespace util::config;
 
+[[nodiscard]]
 int
-main(int argc, char const* argv[])
+runApp(int argc, char const* argv[])
-try {
+{
-    util::setTerminationHandler();
-    util::ScopeGuard const loggerShutdownGuard{[]() { util::LogService::shutdown(); }};
-
     auto const action = app::CliArgs::parse(argc, argv);
     return action.apply(
         [](app::CliArgs::Action::Exit const& exit) { return exit.exitCode; },
         [](app::CliArgs::Action::VerifyConfig const& verify) {
             if (app::parseConfig(verify.configPath)) {
-                std::cout << "Config " << verify.configPath << " is correct"
+                std::cout << "Config " << verify.configPath << " is correct" << "\n";
-                          << "\n";
                 return EXIT_SUCCESS;
             }
             return EXIT_FAILURE;
@@ -76,10 +73,22 @@ try {
             return migrator.run();
         }
     );
-} catch (std::exception const& e) {
+}
-    LOG(util::LogService::fatal()) << "Exit on exception: " << e.what();
+
-    return EXIT_FAILURE;
+int
-} catch (...) {
+main(int argc, char const* argv[])
-    LOG(util::LogService::fatal()) << "Exit on exception: unknown";
+{
-    return EXIT_FAILURE;
+    util::setTerminationHandler();
+
+    util::ScopeGuard const loggerShutdownGuard{[] { util::LogService::shutdown(); }};
+
+    try {
+        return runApp(argc, argv);
+    } catch (std::exception const& e) {
+        LOG(util::LogService::fatal()) << "Exit on exception: " << e.what();
+        return EXIT_FAILURE;
+    } catch (...) {
+        LOG(util::LogService::fatal()) << "Exit on exception: unknown";
+        return EXIT_FAILURE;
+    }
 }

src/rpc/handlers/AccountInfo.cpp

@@ -43,6 +43,7 @@
 
 #include <algorithm>
 #include <iterator>
+#include <optional>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -90,6 +91,17 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
     auto const isClawbackEnabled = isEnabled(Amendments::Clawback);
     auto const isTokenEscrowEnabled = isEnabled(Amendments::TokenEscrow);
 
+    Output out{
+        .ledgerIndex = lgrInfo.seq,
+        .ledgerHash = ripple::strHex(lgrInfo.hash),
+        .accountData = sle,
+        .isDisallowIncomingEnabled = isDisallowIncomingEnabled,
+        .isClawbackEnabled = isClawbackEnabled,
+        .isTokenEscrowEnabled = isTokenEscrowEnabled,
+        .apiVersion = ctx.apiVersion,
+        .signerLists = std::nullopt
+    };
+
     // Return SignerList(s) if that is requested.
     if (input.signerLists) {
         // We put the SignerList in an array because of an anticipated
@@ -99,7 +111,6 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
         // This code will need to be revisited if in the future we
         // support multiple SignerLists on one account.
         auto const signers = sharedPtrBackend_->fetchLedgerObject(signersKey.key, lgrInfo.seq, ctx.yield);
-        std::vector<ripple::STLedgerEntry> signerList;
 
         if (signers) {
             ripple::STLedgerEntry const sleSigners{
@@ -109,30 +120,11 @@ AccountInfoHandler::process(AccountInfoHandler::Input const& input, Context cons
             if (!signersKey.check(sleSigners))
                 return Error{Status{RippledError::rpcDB_DESERIALIZATION}};
 
-            signerList.push_back(sleSigners);
+            out.signerLists = std::vector<ripple::STLedgerEntry>{sleSigners};
         }
-
-        return Output(
-            lgrInfo.seq,
-            ripple::strHex(lgrInfo.hash),
-            sle,
-            isDisallowIncomingEnabled,
-            isClawbackEnabled,
-            isTokenEscrowEnabled,
-            ctx.apiVersion,
-            signerList
-        );
     }
 
-    return Output(
+    return out;
-        lgrInfo.seq,
-        ripple::strHex(lgrInfo.hash),
-        sle,
-        isDisallowIncomingEnabled,
-        isClawbackEnabled,
-        isTokenEscrowEnabled,
-        ctx.apiVersion
-    );
 }
 
 void

src/rpc/handlers/AccountInfo.hpp

@@ -66,39 +66,6 @@ public:
         std::optional<std::vector<ripple::STLedgerEntry>> signerLists;
         // validated should be sent via framework
         bool validated = true;
-
-        /**
-         * @brief Construct a new Output object
-         *
-         * @param ledgerId The ledger index
-         * @param ledgerHash The ledger hash
-         * @param sle The account data
-         * @param isDisallowIncomingEnabled Whether disallow incoming is enabled
-         * @param isClawbackEnabled Whether clawback is enabled
-         * @param isTokenEscrowEnabled Whether token escrow is enabled
-         * @param version The API version
-         * @param signerLists The signer lists
-         */
-        Output(
-            uint32_t ledgerId,
-            std::string ledgerHash,
-            ripple::STLedgerEntry sle,
-            bool isDisallowIncomingEnabled,
-            bool isClawbackEnabled,
-            bool isTokenEscrowEnabled,
-            uint32_t version,
-            std::optional<std::vector<ripple::STLedgerEntry>> signerLists = std::nullopt
-        )
-            : ledgerIndex(ledgerId)
-            , ledgerHash(std::move(ledgerHash))
-            , accountData(std::move(sle))
-            , isDisallowIncomingEnabled(isDisallowIncomingEnabled)
-            , isClawbackEnabled(isClawbackEnabled)
-            , isTokenEscrowEnabled(isTokenEscrowEnabled)
-            , apiVersion(version)
-            , signerLists(std::move(signerLists))
-        {
-        }
     };
 
     /**

src/rpc/handlers/GetAggregatePrice.cpp

@@ -124,7 +124,13 @@ GetAggregatePriceHandler::process(GetAggregatePriceHandler::Input const& input,
 
     auto const latestTime = timestampPricesBiMap.left.begin()->first;
 
-    Output out(latestTime, ripple::to_string(lgrInfo.hash), lgrInfo.seq);
+    Output out{
+        .time = latestTime,
+        .trimStats = std::nullopt,
+        .ledgerHash = ripple::to_string(lgrInfo.hash),
+        .ledgerIndex = lgrInfo.seq,
+        .median = ""
+    };
 
     if (input.timeThreshold) {
         auto const oldestTime = timestampPricesBiMap.left.rbegin()->first;

src/rpc/handlers/GetAggregatePrice.hpp

@@ -43,7 +43,6 @@
 #include <optional>
 #include <string>
 #include <string_view>
-#include <utility>
 #include <vector>
 
 namespace rpc {
@@ -59,8 +58,9 @@ public:
      * @brief A struct to hold the statistics
      */
     struct Stats {
-        ripple::STAmount avg;
+        ripple::STAmount avg{};  // NOLINT(readability-redundant-member-init)
-        ripple::Number sd;  // standard deviation
+        // standard deviation
+        ripple::Number sd{};  // NOLINT(readability-redundant-member-init)
         uint32_t size{0};
     };
 
@@ -69,23 +69,12 @@ public:
      */
     struct Output {
         uint32_t time;
-        Stats extireStats;
+        Stats extireStats{};
         std::optional<Stats> trimStats;
         std::string ledgerHash;
         uint32_t ledgerIndex;
         std::string median;
         bool validated = true;
-
-        /**
-         * @brief Construct a new Output object
-         * @param time The time of the latest oracle data
-         * @param ledgerHash The hash of the ledger
-         * @param ledgerIndex The index of the ledger
-         */
-        Output(uint32_t time, std::string ledgerHash, uint32_t ledgerIndex)
-            : time(time), ledgerHash(std::move(ledgerHash)), ledgerIndex(ledgerIndex)
-        {
-        }
     };
 
     /**

src/util/config/ConfigDefinition.cpp

@@ -237,6 +237,12 @@ ClioConfigDefinition::parse(ConfigFileInterface const& config)
         });
     }
 
+    for (auto const& key : config.getAllKeys()) {
+        if (!map_.contains(key) && !arrayPrefixesToKeysMap.contains(key)) {
+            listOfErrors.emplace_back("Unknown key: " + key);
+        }
+    }
+
     if (!listOfErrors.empty())
         return listOfErrors;
 

src/util/config/ConfigDefinition.hpp

@@ -19,7 +19,6 @@
 
 #pragma once
 
-#include "rpc/common/APIVersion.hpp"
 #include "util/Assert.hpp"
 #include "util/config/Array.hpp"
 #include "util/config/ConfigConstraints.hpp"
@@ -27,18 +26,15 @@
 #include "util/config/ConfigValue.hpp"
 #include "util/config/Error.hpp"
 #include "util/config/ObjectView.hpp"
-#include "util/config/Types.hpp"
 #include "util/config/ValueView.hpp"
 
 #include <algorithm>
 #include <chrono>
 #include <cstddef>
-#include <cstdint>
 #include <initializer_list>
 #include <optional>
 #include <string>
 #include <string_view>
-#include <thread>
 #include <unordered_map>
 #include <utility>
 #include <variant>

src/util/config/ConfigDescription.hpp

@@ -110,13 +110,7 @@ public:
     static void
     writeConfigDescriptionToFile(std::ostream& file)
     {
-        file << "# Clio Config Description\n\n";
+        file << kCONFIG_DESCRIPTION_HEADER;
-        file << "This document provides a list of all available Clio configuration properties in detail.\n\n";
-        file << "> [!NOTE]\n";
-        file << "> Dot notation in configuration key names represents nested fields. For example, "
-                "**database.scylladb** refers to the _scylladb_ field inside the _database_ object. If a key name "
-                "includes \"[]\", it indicates that the nested field is an array (e.g., etl_sources.[]).\n\n";
-        file << "## Configuration Details\n";
 
         for (auto const& [key, val] : kCONFIG_DESCRIPTION) {
             file << "\n### " << key << "\n\n";
@@ -133,6 +127,19 @@ public:
     }
 
 private:
+    static constexpr auto kCONFIG_DESCRIPTION_HEADER =
+        R"(# Clio Config Description
+
+This document provides a list of all available Clio configuration properties in detail.
+
+> [!NOTE]
+> Dot notation in configuration key names represents nested fields.
+> For example, **database.scylladb** refers to the _scylladb_ field inside the _database_ object.
+> If a key name includes "[]", it indicates that the nested field is an array (e.g., etl_sources.[]).
+
+## Configuration Details
+)";
+
     static constexpr auto kCONFIG_DESCRIPTION = std::array{
         KV{
             .key = "database.type",
@@ -266,9 +273,23 @@ private:
         KV{.key = "log.level",
            .value = "The general logging level of Clio. This level is applied to all log channels that do not have an "
                     "explicitly defined logging level."},
-        KV{.key = "log.format",
+        KV{.key = "log.format", .value = R"(The format string for log messages using spdlog format patterns.
-           .value = "The format string for log messages using spdlog format patterns. Documentation can be found at: "
+
-                    "<https://github.com/gabime/spdlog/wiki/Custom-formatting>."},
+Each of the variables expands like so:
+
+- `%Y-%m-%d %H:%M:%S.%f`: The full date and time of the log entry with microsecond precision
+- `%^`: Start color range
+- `%3!l`: The severity (aka log level) the entry was sent at stripped to 3 characters
+- `%n`: The logger name (channel) that this log entry was sent to
+- `%$`: End color range
+- `%v`: The actual log message
+
+Some additional variables that might be useful:
+
+- `%@`: A partial path to the C++ file and the line number in the said file (`src/file/path:linenumber`)
+- `%t`: The ID of the thread the log entry is written from
+
+Documentation can be found at: <https://github.com/gabime/spdlog/wiki/Custom-formatting>.)"},
         KV{.key = "log.is_async", .value = "Whether spdlog is asynchronous or not."},
         KV{.key = "log.enable_console", .value = "Enables or disables logging to the console."},
         KV{.key = "log.directory", .value = "The directory path for the log files."},

src/util/config/ConfigFileInterface.hpp

@@ -22,6 +22,7 @@
 #include "util/config/Types.hpp"
 
 #include <optional>
+#include <string>
 #include <string_view>
 #include <vector>
 
@@ -63,6 +64,14 @@ public:
      */
     virtual bool
     containsKey(std::string_view key) const = 0;
+
+    /**
+     * @brief Retrieves all keys in the configuration file.
+     *
+     * @return A vector of all keys in the configuration file.
+     */
+    virtual std::vector<std::string>
+    getAllKeys() const = 0;
 };
 
 }  // namespace util::config

src/util/config/ConfigFileJson.cpp

@@ -142,6 +142,16 @@ ConfigFileJson::containsKey(std::string_view key) const
     return jsonObject_.contains(key);
 }
 
+std::vector<std::string>
+ConfigFileJson::getAllKeys() const
+{
+    std::vector<std::string> keys;
+    for (auto const& [key, value] : jsonObject_) {
+        keys.push_back(key);
+    }
+    return keys;
+}
+
 boost::json::object const&
 ConfigFileJson::inner() const
 {

src/util/config/ConfigFileJson.hpp

@@ -28,6 +28,7 @@
 #include <expected>
 #include <filesystem>
 #include <optional>
+#include <string>
 #include <string_view>
 #include <vector>
 
@@ -73,6 +74,14 @@ public:
     [[nodiscard]] bool
     containsKey(std::string_view key) const override;
 
+    /**
+     * @brief Retrieves all keys in the configuration file.
+     *
+     * @return A vector of all keys in the configuration file.
+     */
+    [[nodiscard]] std::vector<std::string>
+    getAllKeys() const override;
+
     /**
      * @brief Creates a new ConfigFileJson by parsing the provided JSON file and
      * stores the values in the object.

src/util/config/ConfigFileYaml.hpp

@@ -1,49 +0,0 @@
-//------------------------------------------------------------------------------
-/*
-    This file is part of clio: https://github.com/XRPLF/clio
-    Copyright (c) 2024, the clio developers.
-
-    Permission to use, copy, modify, and distribute this software for any
-    purpose with or without fee is hereby granted, provided that the above
-    copyright notice and this permission notice appear in all copies.
-
-    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
-    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
-    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
-    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
-    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
-    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-*/
-//==============================================================================
-
-#pragma once
-
-#include "util/config/ConfigFileInterface.hpp"
-#include "util/config/Types.hpp"
-
-#include <boost/filesystem/path.hpp>
-
-#include <string_view>
-#include <vector>
-
-// TODO: implement when we support yaml
-
-namespace util::config {
-
-/** @brief Yaml representation of config */
-class ConfigFileYaml final : public ConfigFileInterface {
-public:
-    ConfigFileYaml() = default;
-
-    Value
-    getValue(std::string_view key) const override;
-
-    std::vector<Value>
-    getArray(std::string_view key) const override;
-
-    bool
-    containsKey(std::string_view key) const override;
-};
-
-}  // namespace util::config

src/util/log/Logger.cpp

@@ -31,7 +31,10 @@
 #include <spdlog/async.h>
 #include <spdlog/async_logger.h>
 #include <spdlog/common.h>
+#include <spdlog/details/log_msg.h>
+#include <spdlog/formatter.h>
 #include <spdlog/logger.h>
+#include <spdlog/pattern_formatter.h>
 #include <spdlog/sinks/rotating_file_sink.h>
 #include <spdlog/sinks/stdout_color_sinks.h>
 #include <spdlog/spdlog.h>
@@ -41,6 +44,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <filesystem>
+#include <iostream>
 #include <memory>
 #include <optional>
 #include <string>
@@ -120,26 +124,63 @@ getSeverityLevel(std::string_view logLevel)
     std::unreachable();
 }
 
+/**
+ * @brief Custom formatter that filters out critical messages
+ *
+ * This formatter only processes and formats messages with severity level less than critical.
+ * Critical messages will be handled separately.
+ */
+class NonCriticalFormatter : public spdlog::formatter {
+public:
+    NonCriticalFormatter(std::unique_ptr<spdlog::formatter> wrappedFormatter)
+        : wrapped_formatter_(std::move(wrappedFormatter))
+    {
+    }
+
+    void
+    format(spdlog::details::log_msg const& msg, spdlog::memory_buf_t& dest) override
+    {
+        // Only format messages with severity less than critical
+        if (msg.level != spdlog::level::critical) {
+            wrapped_formatter_->format(msg, dest);
+        }
+    }
+
+    std::unique_ptr<formatter>
+    clone() const override
+    {
+        return std::make_unique<NonCriticalFormatter>(wrapped_formatter_->clone());
+    }
+
+private:
+    std::unique_ptr<spdlog::formatter> wrapped_formatter_;
+};
+
 /**
  * @brief Initializes console logging.
  *
  * @param logToConsole A boolean indicating whether to log to console.
+ * @param format A string representing the log format.
  * @return Vector of sinks for console logging.
  */
 static std::vector<spdlog::sink_ptr>
-createConsoleSinks(bool logToConsole)
+createConsoleSinks(bool logToConsole, std::string const& format)
 {
     std::vector<spdlog::sink_ptr> sinks;
 
     if (logToConsole) {
         auto consoleSink = std::make_shared<spdlog::sinks::stdout_color_sink_mt>();
         consoleSink->set_level(spdlog::level::trace);
+        consoleSink->set_formatter(
+            std::make_unique<NonCriticalFormatter>(std::make_unique<spdlog::pattern_formatter>(format))
+        );
         sinks.push_back(std::move(consoleSink));
     }
 
     // Always add stderr sink for fatal logs
     auto stderrSink = std::make_shared<spdlog::sinks::stderr_color_sink_mt>();
     stderrSink->set_level(spdlog::level::critical);
+    stderrSink->set_formatter(std::make_unique<spdlog::pattern_formatter>(format));
     sinks.push_back(std::move(stderrSink));
 
     return sinks;
@@ -153,7 +194,7 @@ createConsoleSinks(bool logToConsole)
  * @return File sink for logging.
  */
 spdlog::sink_ptr
-LogService::createFileSink(FileLoggingParams const& params)
+LogService::createFileSink(FileLoggingParams const& params, std::string const& format)
 {
     std::filesystem::path const dirPath(params.logDir);
     // the below are taken from user in MB, but spdlog needs it to be in bytes
@@ -163,6 +204,7 @@ LogService::createFileSink(FileLoggingParams const& params)
         (dirPath / "clio.log").string(), rotationSize, params.dirMaxFiles
     );
     fileSink->set_level(spdlog::level::trace);
+    fileSink->set_formatter(std::make_unique<spdlog::pattern_formatter>(format));
 
     return fileSink;
 }
@@ -229,11 +271,13 @@ LogService::init(config::ClioConfigDefinition const& config)
     data.isAsync = config.get<bool>("log.is_async");
     data.defaultSeverity = getSeverityLevel(config.get<std::string>("log.level"));
 
+    std::string const format = config.get<std::string>("log.format");
+
     if (data.isAsync) {
         spdlog::init_thread_pool(8192, 1);
     }
 
-    data.allSinks = createConsoleSinks(config.get<bool>("log.enable_console"));
+    data.allSinks = createConsoleSinks(config.get<bool>("log.enable_console"), format);
 
     if (auto const logDir = config.maybeValue<std::string>("log.directory"); logDir.has_value()) {
         std::filesystem::path const dirPath{logDir.value()};
@@ -250,7 +294,7 @@ LogService::init(config::ClioConfigDefinition const& config)
             .rotationSizeMB = config.get<uint32_t>("log.rotation_size"),
             .dirMaxFiles = config.get<uint32_t>("log.directory_max_files"),
         };
-        data.allSinks.push_back(createFileSink(params));
+        data.allSinks.push_back(createFileSink(params, format));
     }
 
     // get min severity per channel, can be overridden using the `log.channels` array
@@ -269,9 +313,6 @@ LogService::init(config::ClioConfigDefinition const& config)
 
     spdlog::set_default_logger(spdlog::get("General"));
 
-    std::string const format = config.get<std::string>("log.format");
-    spdlog::set_pattern(format);
-
     LOG(LogService::info()) << "Default log level = " << toString(data.defaultSeverity);
     return {};
 }

src/util/log/Logger.hpp

@@ -346,7 +346,7 @@ private:
 
     [[nodiscard]]
     static std::shared_ptr<spdlog::sinks::sink>
-    createFileSink(FileLoggingParams const& params);
+    createFileSink(FileLoggingParams const& params, std::string const& format);
 };
 
 };  // namespace util

tests/common/util/LoggerFixtures.cpp

@@ -22,6 +22,7 @@
 #include "util/log/Logger.hpp"
 
 #include <spdlog/common.h>
+#include <spdlog/pattern_formatter.h>
 #include <spdlog/sinks/ostream_sink.h>
 #include <spdlog/spdlog.h>
 
@@ -35,6 +36,7 @@ LoggerFixture::LoggerFixture()
 
     // Create ostream sink for testing
     auto ostreamSink = std::make_shared<spdlog::sinks::ostream_sink_mt>(stream_);
+    ostreamSink->set_formatter(std::make_unique<spdlog::pattern_formatter>("%^%3!l:%n%$ - %v"));
 
     // Create loggers for each channel
     std::ranges::for_each(util::Logger::kCHANNELS, [&ostreamSink](char const* channel) {
@@ -50,8 +52,6 @@ LoggerFixture::LoggerFixture()
     spdlog::register_logger(traceLogger);
 
     spdlog::set_default_logger(spdlog::get("General"));
-
-    spdlog::set_pattern("%^%3!l:%n%$ - %v");
 }
 
 NoLoggerFixture::NoLoggerFixture()

tests/unit/LogServiceInitTests.cpp

@@ -32,6 +32,7 @@
 #include <fmt/format.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include <spdlog/pattern_formatter.h>
 #include <spdlog/sinks/ostream_sink.h>
 #include <spdlog/spdlog.h>
 
@@ -84,6 +85,7 @@ protected:
     replaceSinks()
     {
         auto ostreamSink = std::make_shared<spdlog::sinks::ostream_sink_mt>(stream_);
+        ostreamSink->set_formatter(std::make_unique<spdlog::pattern_formatter>("%^%3!l:%n%$ - %v"));
 
         for (auto const& channel : Logger::kCHANNELS) {
             auto logger = spdlog::get(channel);
@@ -93,8 +95,6 @@ protected:
             logger->sinks().clear();
             logger->sinks().push_back(ostreamSink);
         }
-
-        spdlog::set_pattern("%^%3!l:%n%$ - %v");
     }
 
 private:

tests/unit/data/cassandra/SettingsProviderTests.cpp

@@ -82,7 +82,8 @@ class SettingsProviderTest : public NoLoggerFixture {};
 
 TEST_F(SettingsProviderTest, Defaults)
 {
-    auto const cfg = getParseSettingsConfig(json::parse(R"JSON({"contact_points": "127.0.0.1"})JSON"));
+    auto const cfg =
+        getParseSettingsConfig(json::parse(R"JSON({"database.cassandra.contact_points": "127.0.0.1"})JSON"));
     SettingsProvider const provider{cfg.getObject("database.cassandra")};
 
     auto const settings = provider.getSettings();

tests/unit/etl/LoadBalancerTests.cpp

@@ -160,7 +160,7 @@ TEST_F(LoadBalancerConstructorTests, construct)
 TEST_F(LoadBalancerConstructorTests, forwardingTimeoutPassedToSourceFactory)
 {
     auto const forwardingTimeout = 10;
-    configJson_.as_object()["forwarding"] = boost::json::object{{"timeout", float{forwardingTimeout}}};
+    configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", float{forwardingTimeout}}};
     EXPECT_CALL(
         sourceFactory_,
         makeSource(

tests/unit/etlng/LoadBalancerTests.cpp

@@ -182,7 +182,7 @@ TEST_F(LoadBalancerConstructorNgTests, construct)
 TEST_F(LoadBalancerConstructorNgTests, forwardingTimeoutPassedToSourceFactory)
 {
     auto const forwardingTimeout = 10;
-    configJson_.as_object()["forwarding"] = boost::json::object{{"timeout", float{forwardingTimeout}}};
+    configJson_.as_object()["forwarding"] = boost::json::object{{"cache_timeout", float{forwardingTimeout}}};
     EXPECT_CALL(
         sourceFactory_,
         makeSource(

tests/unit/util/config/ClioConfigDefinitionTests.cpp

@@ -465,3 +465,35 @@ TEST_F(ClioConfigDefinitionParseArrayTest, missingAllRequiredFields)
     EXPECT_EQ(result->size(), 1);
     EXPECT_THAT(result->at(0).error, testing::StartsWith("array.[].int"));
 }
+
+TEST(ClioConfigDefinitionParse, unexpectedFields)
+{
+    ClioConfigDefinition config{
+        {"expected", ConfigValue{ConfigType::String}.optional()},
+    };
+
+    auto const configJson = boost::json::parse(R"JSON({
+        "expected": "present",
+        "unexpected_string": "",
+        "unexpected_non_empty_array": [
+            {"string": ""},
+            {"string": ""}
+        ],
+        "unexpected_empty_array": [],
+        "unexpected_object": {
+            "string": ""
+        }
+    })JSON")
+                                .as_object();
+
+    auto const configFile = ConfigFileJson{configJson};
+    auto result = config.parse(configFile);
+    std::ranges::sort(*result, [](auto const& lhs, auto const& rhs) { return lhs.error < rhs.error; });
+    ASSERT_TRUE(result.has_value());
+    ASSERT_EQ(result->size(), 4);
+
+    EXPECT_EQ(result->at(0).error, "Unknown key: unexpected_empty_array.[]");
+    EXPECT_EQ(result->at(1).error, "Unknown key: unexpected_non_empty_array.[].string");
+    EXPECT_EQ(result->at(2).error, "Unknown key: unexpected_object.string");
+    EXPECT_EQ(result->at(3).error, "Unknown key: unexpected_string");
+}

tests/unit/util/config/ConfigFileJsonTests.cpp

@@ -34,6 +34,7 @@
 #include <string>
 #include <unordered_map>
 #include <variant>
+#include <vector>
 
 using namespace util::config;
 
@@ -488,6 +489,31 @@ TEST_F(ConfigFileJsonTest, containsKey)
     EXPECT_FALSE(jsonFileObj.containsKey("array_of_objects.[].object"));
 }
 
+TEST_F(ConfigFileJsonTest, getAllKeys)
+{
+    auto const jsonStr = R"JSON({
+        "int": 42,
+        "object": { "string": "some string", "array": [1, 2, 3] },
+        "array2": [1, 2, 3],
+        "array_of_objects": [ {"int": 42}, {"string": "some string"} ]
+    })JSON";
+    auto const jsonFileObj = ConfigFileJson{boost::json::parse(jsonStr).as_object()};
+
+    auto allKeys = jsonFileObj.getAllKeys();
+    std::ranges::sort(allKeys);
+    EXPECT_EQ(allKeys.size(), 6);
+
+    std::vector<std::string> const expectedKeys{
+        {"array2.[]",
+         "array_of_objects.[].int",
+         "array_of_objects.[].string",
+         "int",
+         "object.array.[]",
+         "object.string"}
+    };
+    EXPECT_EQ(allKeys, expectedKeys);
+}
+
 struct ConfigFileJsonMakeTest : ConfigFileJsonTest {};
 
 TEST_F(ConfigFileJsonMakeTest, invalidFile)