fix: Check config is valid

For #1596

.clang-tidy

@@ -8,6 +8,7 @@ Checks: '-*,
   bugprone-chained-comparison,
   bugprone-compare-pointer-to-member-virtual-function,
   bugprone-copy-constructor-init,
+  bugprone-crtp-constructor-accessibility,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
   bugprone-empty-catch,
@@ -33,9 +34,11 @@ Checks: '-*,
   bugprone-non-zero-enum-to-bool-conversion,
   bugprone-optional-value-conversion,
   bugprone-parent-virtual-call,
+  bugprone-pointer-arithmetic-on-polymorphic-object,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
   bugprone-reserved-identifier,
+  bugprone-return-const-ref-from-parameter,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -55,6 +58,7 @@ Checks: '-*,
   bugprone-suspicious-realloc-usage,
   bugprone-suspicious-semicolon,
   bugprone-suspicious-string-compare,
+  bugprone-suspicious-stringview-data-usage,
   bugprone-swapped-arguments,
   bugprone-switch-missing-default-case,
   bugprone-terminating-continue,
@@ -97,10 +101,12 @@ Checks: '-*,
   modernize-make-unique,
   modernize-pass-by-value,
   modernize-type-traits,
+  modernize-use-designated-initializers,
   modernize-use-emplace,
   modernize-use-equals-default,
   modernize-use-equals-delete,
   modernize-use-override,
+  modernize-use-ranges,
   modernize-use-starts-ends-with,
   modernize-use-std-numbers,
   modernize-use-using,
@@ -121,9 +127,12 @@ Checks: '-*,
   readability-convert-member-functions-to-static,
   readability-duplicate-include,
   readability-else-after-return,
+  readability-enum-initial-value,
   readability-implicit-bool-conversion,
   readability-inconsistent-declaration-parameter-name,
+  readability-identifier-naming,
   readability-make-member-function-const,
+  readability-math-missing-parentheses,
   readability-misleading-indentation,
   readability-non-const-parameter,
   readability-redundant-casting,
@@ -135,11 +144,45 @@ Checks: '-*,
   readability-simplify-boolean-expr,
   readability-static-accessed-through-instance,
   readability-static-definition-in-anonymous-namespace,
-  readability-suspicious-call-argument
+  readability-suspicious-call-argument,
+  readability-use-std-min-max
   '
 
 CheckOptions:
   readability-braces-around-statements.ShortStatementLines: 2
+  readability-identifier-naming.MacroDefinitionCase: UPPER_CASE
+  readability-identifier-naming.ClassCase: CamelCase
+  readability-identifier-naming.StructCase: CamelCase
+  readability-identifier-naming.UnionCase: CamelCase
+  readability-identifier-naming.EnumCase: CamelCase
+  readability-identifier-naming.EnumConstantCase: CamelCase
+  readability-identifier-naming.ScopedEnumConstantCase: CamelCase
+  readability-identifier-naming.GlobalConstantCase: UPPER_CASE
+  readability-identifier-naming.GlobalConstantPrefix: 'k'
+  readability-identifier-naming.GlobalVariableCase: CamelCase
+  readability-identifier-naming.GlobalVariablePrefix: 'g'
+  readability-identifier-naming.ConstexprFunctionCase: camelBack
+  readability-identifier-naming.ConstexprMethodCase: camelBack
+  readability-identifier-naming.ClassMethodCase: camelBack
+  readability-identifier-naming.ClassMemberCase: camelBack
+  readability-identifier-naming.ClassConstantCase: UPPER_CASE
+  readability-identifier-naming.ClassConstantPrefix: 'k'
+  readability-identifier-naming.StaticConstantCase: UPPER_CASE
+  readability-identifier-naming.StaticConstantPrefix: 'k'
+  readability-identifier-naming.StaticVariableCase: UPPER_CASE
+  readability-identifier-naming.StaticVariablePrefix: 'k'
+  readability-identifier-naming.ConstexprVariableCase: UPPER_CASE
+  readability-identifier-naming.ConstexprVariablePrefix: 'k'
+  readability-identifier-naming.LocalConstantCase: camelBack
+  readability-identifier-naming.LocalVariableCase: camelBack
+  readability-identifier-naming.TemplateParameterCase: CamelCase
+  readability-identifier-naming.ParameterCase: camelBack
+  readability-identifier-naming.FunctionCase: camelBack
+  readability-identifier-naming.MemberCase: camelBack
+  readability-identifier-naming.PrivateMemberSuffix: _
+  readability-identifier-naming.ProtectedMemberSuffix: _
+  readability-identifier-naming.PublicMemberSuffix: ''
+  readability-identifier-naming.FunctionIgnoredRegexp: '.*tag_invoke.*'
   bugprone-unsafe-functions.ReportMoreUnsafeFunctions: true
   bugprone-unused-return-value.CheckedReturnTypes: ::std::error_code;::std::error_condition;::std::errc
   misc-include-cleaner.IgnoreHeaders: '.*/(detail|impl)/.*;.*(expected|unexpected).*;.*ranges_lower_bound\.h;time.h;stdlib.h'

.github/actions/prepare_runner/action.yml

@@ -11,7 +11,7 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       run: |
-          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1
+          brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates
           echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
 
     - name: Fix git permissions on Linux

.github/actions/setup_conan/action.yml

@@ -15,10 +15,10 @@ runs:
       if: ${{ runner.os == 'macOS' }}
       shell: bash
       env:
-        CONAN_PROFILE: apple_clang_15
+        CONAN_PROFILE: apple_clang_16
       id: conan_setup_mac
       run: |
-        echo "Creating $CONAN_PROFILE conan profile";
+        echo "Creating $CONAN_PROFILE conan profile"
         conan profile new $CONAN_PROFILE --detect --force
         conan profile update settings.compiler.libcxx=libc++ $CONAN_PROFILE
         conan profile update settings.compiler.cppstd=20 $CONAN_PROFILE

.github/workflows/build.yml

@@ -74,7 +74,7 @@ jobs:
             conan_profile: clang
             code_coverage: false
             static: true
-          - os: macos14
+          - os: macos15
             build_type: Release
             code_coverage: false
             static: false
@@ -197,8 +197,8 @@ jobs:
               image: rippleci/clio_ci:latest
             conan_profile: clang
             build_type: Debug
-          - os: macos14
-            conan_profile: apple_clang_15
+          - os: macos15
+            conan_profile: apple_clang_16
             build_type: Release
     runs-on: [self-hosted, "${{ matrix.os }}"]
     container: ${{ matrix.container }}

.github/workflows/nightly.yml

@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos14
+          - os: macos15
             build_type: Release
             static: false
           - os: heavy
@@ -90,7 +90,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos14
+          - os: macos15
             build_type: Release
             integration_tests: false
           - os: heavy

.github/workflows/upload_coverage_report.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v3.7.2
+        uses: wandalen/wretry.action@v3.7.3
         with:
           action: codecov/codecov-action@v4
           with: |

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -188,10 +188,10 @@ public:
 static auto
 generateData()
 {
-    constexpr auto TOTAL = 10'000;
+    constexpr auto kTOTAL = 10'000;
     std::vector<uint64_t> data;
-    data.reserve(TOTAL);
-    for (auto i = 0; i < TOTAL; ++i)
+    data.reserve(kTOTAL);
+    for (auto i = 0; i < kTOTAL; ++i)
         data.push_back(util::Random::uniform(1, 100'000'000));
 
     return data;
@@ -208,7 +208,7 @@ benchmarkThreads(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();
@@ -219,7 +219,7 @@ benchmarkExecutionContextBatched(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkAnyExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();

cmake/Build.cpp.in

@@ -23,19 +23,19 @@
 
 namespace util::build {
 
-static constexpr char versionString[] = "@CLIO_VERSION@";
+static constexpr char versionString[] = "@CLIO_VERSION@"; // NOLINT(readability-identifier-naming)
 
 std::string const&
 getClioVersionString()
 {
-    static std::string const value = versionString;
+    static std::string const value = versionString; // NOLINT(readability-identifier-naming)
     return value;
 }
 
 std::string const&
 getClioFullVersionString()
 {
-    static std::string const value = "clio-" + getClioVersionString();
+    static std::string const value = "clio-" + getClioVersionString(); // NOLINT(readability-identifier-naming)
     return value;
 }
 

conanfile.py

@@ -28,7 +28,7 @@ class Clio(ConanFile):
         'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.3.0-rc2',
+        'xrpl/2.4.0-b1',
         'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]

docs/examples/config/cloud-example-config.json

@@ -1,43 +0,0 @@
-/*
- * This is an example configuration file. Please do not use without modifying to suit your needs.
- */
-{
-    "database": {
-        "type": "cassandra",
-        "cassandra": {
-            // This option can be used to setup a secure connect bundle connection
-            "secure_connect_bundle": "[path/to/zip. ignore if using contact_points]",
-            // The following options are used only if using contact_points
-            "contact_points": "[ip. ignore if using secure_connect_bundle]",
-            "port": "[port. ignore if using_secure_connect_bundle]",
-            // Authentication settings
-            "username": "[username, if any]",
-            "password": "[password, if any]",
-            // Other common settings
-            "keyspace": "clio",
-            "max_write_requests_outstanding": 25000,
-            "max_read_requests_outstanding": 30000,
-            "threads": 8
-        }
-    },
-    "etl_sources": [
-        {
-            "ip": "[rippled ip]",
-            "ws_port": "6006",
-            "grpc_port": "50051"
-        }
-    ],
-    "dos_guard": {
-        "whitelist": [
-            "127.0.0.1"
-        ]
-    },
-    "server": {
-        "ip": "0.0.0.0",
-        "port": 8080
-    },
-    "log_level": "debug",
-    "log_file": "./clio.log",
-    "extractor_threads": 8,
-    "read_only": false
-}

docs/examples/config/example-config.json

@@ -77,7 +77,8 @@
         // send a reply for each request whenever it is ready.
         "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
         // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
-        "ws_max_sending_queue_size": 1500
+        "ws_max_sending_queue_size": 1500,
+        "__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server
     },
     // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
     "graceful_period": 10.0,

src/CMakeLists.txt

@@ -5,5 +5,6 @@ add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(migration)
 add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
-target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
+target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp WebHandlers.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration)

src/app/CliArgs.cpp

@@ -19,6 +19,7 @@
 
 #include "app/CliArgs.hpp"
 
+#include "migration/MigrationApplication.hpp"
 #include "util/build/Build.hpp"
 
 #include <boost/program_options/options_description.hpp>
@@ -43,7 +44,10 @@ CliArgs::parse(int argc, char const* argv[])
     description.add_options()
         ("help,h", "print help message and exit")
         ("version,v", "print version and exit")
-        ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
+        ("conf,c", po::value<std::string>()->default_value(kDEFAULT_CONFIG_PATH), "configuration file")
+        ("ng-web-server,w", "Use ng-web-server")
+        ("migrate", po::value<std::string>(), "start migration helper")
+        ("verify", "Checks the validity of config values")
     ;
     // clang-format on
     po::positional_options_description positional;
@@ -64,7 +68,19 @@ CliArgs::parse(int argc, char const* argv[])
     }
 
     auto configPath = parsed["conf"].as<std::string>();
-    return Action{Action::Run{std::move(configPath)}};
+
+    if (parsed.count("migrate") != 0u) {
+        auto const opt = parsed["migrate"].as<std::string>();
+        if (opt == "status")
+            return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()}};
+        return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)}};
+    }
+
+    if (parsed.count("verify") != 0u)
+        return Action{Action::VerifyConfig{.configPath = std::move(configPath)}};
+
+    return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
+    };
 }
 
 }  // namespace app

src/app/CliArgs.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "migration/MigrationApplication.hpp"
 #include "util/OverloadSet.hpp"
 
 #include <string>
@@ -34,7 +35,7 @@ public:
     /**
      * @brief Default configuration path.
      */
-    static constexpr char defaultConfigPath[] = "/etc/opt/clio/config.json";
+    static constexpr char kDEFAULT_CONFIG_PATH[] = "/etc/opt/clio/config.json";
 
     /**
      * @brief An action parsed from the command line.
@@ -43,14 +44,24 @@ public:
     public:
         /** @brief Run action. */
         struct Run {
-            /** @brief Configuration file path. */
-            std::string configPath;
+            std::string configPath;  ///< Configuration file path.
+            bool useNgWebServer;     ///< Whether to use a ng web server
         };
 
         /** @brief Exit action. */
         struct Exit {
-            /** @brief Exit code. */
-            int exitCode;
+            int exitCode;  ///< Exit code.
+        };
+
+        /** @brief Migration action. */
+        struct Migrate {
+            std::string configPath;
+            MigrateSubCmd subCmd;
+        };
+
+        /** @brief Verify Config action. */
+        struct VerifyConfig {
+            std::string configPath;
         };
 
         /**
@@ -59,7 +70,8 @@ public:
          * @param action Run action.
          */
         template <typename ActionType>
-            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit> or
+            std::is_same_v<ActionType, Migrate> or std::is_same_v<ActionType, VerifyConfig>
         explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
         {
         }
@@ -79,7 +91,7 @@ public:
         }
 
     private:
-        std::variant<Run, Exit> action_;
+        std::variant<Run, Exit, Migrate, VerifyConfig> action_;
     };
 
     /**

src/app/ClioApplication.cpp

@@ -19,6 +19,7 @@
 
 #include "app/ClioApplication.hpp"
 
+#include "app/WebHandlers.hpp"
 #include "data/AmendmentCenter.hpp"
 #include "data/BackendFactory.hpp"
 #include "etl/ETLService.hpp"
@@ -30,21 +31,26 @@
 #include "rpc/WorkQueue.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
 #include "util/build/Build.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 #include "util/prometheus/Prometheus.hpp"
+#include "web/AdminVerificationStrategy.hpp"
 #include "web/RPCServerHandler.hpp"
 #include "web/Server.hpp"
 #include "web/dosguard/DOSGuard.hpp"
 #include "web/dosguard/IntervalSweepHandler.hpp"
 #include "web/dosguard/WhitelistHandler.hpp"
+#include "web/ng/RPCServerHandler.hpp"
+#include "web/ng/Server.hpp"
 
 #include <boost/asio/io_context.hpp>
 
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
+#include <optional>
 #include <thread>
+#include <utility>
 #include <vector>
 
 namespace app {
@@ -72,20 +78,17 @@ start(boost::asio::io_context& ioc, std::uint32_t numThreads)
 
 }  // namespace
 
-ClioApplication::ClioApplication(util::Config const& config) : config_(config), signalsHandler_{config_}
+ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& config)
+    : config_(config), signalsHandler_{config_}
 {
     LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
     PrometheusService::init(config);
 }
 
 int
-ClioApplication::run()
+ClioApplication::run(bool const useNgWebServer)
 {
-    auto const threads = config_.valueOr("io_threads", 2);
-    if (threads <= 0) {
-        LOG(util::LogService::fatal()) << "io_threads is less than 1";
-        return EXIT_FAILURE;
-    }
+    auto const threads = config_.get<uint16_t>("io_threads");
     LOG(util::LogService::info()) << "Number of io threads = " << threads;
 
     // IO context to handle all incoming requests, as well as other things.
@@ -98,25 +101,25 @@ ClioApplication::run()
     auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
 
     // Interface to the database
-    auto backend = data::make_Backend(config_);
+    auto backend = data::makeBackend(config_);
 
     // Manages clients subscribed to streams
-    auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config_, backend);
+    auto subscriptions = feed::SubscriptionManager::makeSubscriptionManager(config_, backend);
 
     // Tracks which ledgers have been validated by the network
-    auto ledgers = etl::NetworkValidatedLedgers::make_ValidatedLedgers();
+    auto ledgers = etl::NetworkValidatedLedgers::makeValidatedLedgers();
 
     // Handles the connection to one or more rippled nodes.
     // ETL uses the balancer to extract data.
     // The server uses the balancer to forward RPCs to a rippled node.
     // The balancer itself publishes to streams (transactions_proposed and accounts_proposed)
-    auto balancer = etl::LoadBalancer::make_LoadBalancer(config_, ioc, backend, subscriptions, ledgers);
+    auto balancer = etl::LoadBalancer::makeLoadBalancer(config_, ioc, backend, subscriptions, ledgers);
 
     // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
-    auto etl = etl::ETLService::make_ETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
+    auto etl = etl::ETLService::makeETLService(config_, ioc, backend, subscriptions, balancer, ledgers);
 
-    auto workQueue = rpc::WorkQueue::make_WorkQueue(config_);
-    auto counters = rpc::Counters::make_Counters(workQueue);
+    auto workQueue = rpc::WorkQueue::makeWorkQueue(config_);
+    auto counters = rpc::Counters::makeCounters(workQueue);
     auto const amendmentCenter = std::make_shared<data::AmendmentCenter const>(backend);
     auto const handlerProvider = std::make_shared<rpc::impl::ProductionHandlerProvider const>(
         config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
@@ -124,12 +127,50 @@ ClioApplication::run()
 
     using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
     auto const rpcEngine =
-        RPCEngineType::make_RPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+        RPCEngineType::makeRPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+
+    if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
+        web::ng::RPCServerHandler<RPCEngineType, etl::ETLService> handler{config_, backend, rpcEngine, etl};
+
+        auto expectedAdminVerifier = web::makeAdminVerificationStrategy(config_);
+        if (not expectedAdminVerifier.has_value()) {
+            LOG(util::LogService::error()) << "Error creating admin verifier: " << expectedAdminVerifier.error();
+            return EXIT_FAILURE;
+        }
+        auto const adminVerifier = std::move(expectedAdminVerifier).value();
+
+        auto httpServer = web::ng::makeServer(config_, OnConnectCheck{dosGuard}, DisconnectHook{dosGuard}, ioc);
+
+        if (not httpServer.has_value()) {
+            LOG(util::LogService::error()) << "Error creating web server: " << httpServer.error();
+            return EXIT_FAILURE;
+        }
+
+        httpServer->onGet("/metrics", MetricsHandler{adminVerifier});
+        httpServer->onGet("/health", HealthCheckHandler{});
+        auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard};
+        httpServer->onPost("/", requestHandler);
+        httpServer->onWs(std::move(requestHandler));
+
+        auto const maybeError = httpServer->run();
+        if (maybeError.has_value()) {
+            LOG(util::LogService::error()) << "Error starting web server: " << *maybeError;
+            return EXIT_FAILURE;
+        }
+
+        // Blocks until stopped.
+        // When stopped, shared_ptrs fall out of scope
+        // Calls destructors on all resources, and destructs in order
+        start(ioc, threads);
+
+        return EXIT_SUCCESS;
+    }
 
     // Init the web server
     auto handler =
         std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
-    auto const httpServer = web::make_HttpServer(config_, ioc, dosGuard, handler);
+
+    auto const httpServer = web::makeHttpServer(config_, ioc, dosGuard, handler);
 
     // Blocks until stopped.
     // When stopped, shared_ptrs fall out of scope

src/app/ClioApplication.hpp

@@ -20,7 +20,7 @@
 #pragma once
 
 #include "util/SignalsHandler.hpp"
-#include "util/config//Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 namespace app {
 
@@ -28,7 +28,7 @@ namespace app {
  * @brief The main application class
  */
 class ClioApplication {
-    util::Config const& config_;
+    util::config::ClioConfigDefinition const& config_;
     util::SignalsHandler signalsHandler_;
 
 public:
@@ -37,15 +37,17 @@ public:
      *
      * @param config The configuration of the application
      */
-    ClioApplication(util::Config const& config);
+    ClioApplication(util::config::ClioConfigDefinition const& config);
 
     /**
      * @brief Run the application
      *
+     * @param useNgWebServer Whether to use the new web server
+     *
      * @return exit code
      */
     int
-    run();
+    run(bool useNgWebServer);
 };
 
 }  // namespace app

src/app/VerifyConfig.hpp

@@ -0,0 +1,57 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigFileJson.hpp"
+
+#include <cstdlib>
+#include <iostream>
+#include <string_view>
+
+namespace app {
+
+/**
+ * @brief Verifies user's config values are correct
+ *
+ * @param configPath The path to config
+ * @return true if config values are all correct, false otherwise
+ */
+inline bool
+verifyConfig(std::string_view configPath)
+{
+    using namespace util::config;
+
+    auto const json = ConfigFileJson::makeConfigFileJson(configPath);
+    if (!json.has_value()) {
+        std::cerr << "Error parsing json from config: " << configPath << "\n" << json.error().error << std::endl;
+        return false;
+    }
+    auto const errors = gClioConfig.parse(json.value());
+    if (errors.has_value()) {
+        for (auto const& err : errors.value()) {
+            std::cerr << "Issues found in provided config '" << configPath << "':\n";
+            std::cerr << err.error << std::endl;
+        }
+        return false;
+    }
+    return true;
+}
+}  // namespace app

src/app/WebHandlers.cpp

@@ -0,0 +1,111 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/WebHandlers.hpp"
+
+#include "util/Assert.hpp"
+#include "util/prometheus/Http.hpp"
+#include "web/AdminVerificationStrategy.hpp"
+#include "web/SubscriptionContextInterface.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
+#include "web/ng/Connection.hpp"
+#include "web/ng/Request.hpp"
+#include "web/ng/Response.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/beast/http/status.hpp>
+
+#include <memory>
+#include <optional>
+#include <utility>
+
+namespace app {
+
+OnConnectCheck::OnConnectCheck(web::dosguard::DOSGuardInterface& dosguard) : dosguard_{dosguard}
+{
+}
+
+std::expected<void, web::ng::Response>
+OnConnectCheck::operator()(web::ng::Connection const& connection)
+{
+    dosguard_.get().increment(connection.ip());
+    if (not dosguard_.get().isOk(connection.ip())) {
+        return std::unexpected{
+            web::ng::Response{boost::beast::http::status::too_many_requests, "Too many requests", connection}
+        };
+    }
+
+    return {};
+}
+
+DisconnectHook::DisconnectHook(web::dosguard::DOSGuardInterface& dosguard) : dosguard_{dosguard}
+{
+}
+
+void
+DisconnectHook::operator()(web::ng::Connection const& connection)
+{
+    dosguard_.get().decrement(connection.ip());
+}
+
+MetricsHandler::MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier)
+    : adminVerifier_{std::move(adminVerifier)}
+{
+}
+
+web::ng::Response
+MetricsHandler::operator()(
+    web::ng::Request const& request,
+    web::ng::ConnectionMetadata& connectionMetadata,
+    web::SubscriptionContextPtr,
+    boost::asio::yield_context
+)
+{
+    auto const maybeHttpRequest = request.asHttpRequest();
+    ASSERT(maybeHttpRequest.has_value(), "Got not a http request in Get");
+    auto const& httpRequest = maybeHttpRequest->get();
+
+    // FIXME(#1702): Using veb server thread to handle prometheus request. Better to post on work queue.
+    auto maybeResponse = util::prometheus::handlePrometheusRequest(
+        httpRequest, adminVerifier_->isAdmin(httpRequest, connectionMetadata.ip())
+    );
+    ASSERT(maybeResponse.has_value(), "Got unexpected request for Prometheus");
+    return web::ng::Response{std::move(maybeResponse).value(), request};
+}
+
+web::ng::Response
+HealthCheckHandler::operator()(
+    web::ng::Request const& request,
+    web::ng::ConnectionMetadata&,
+    web::SubscriptionContextPtr,
+    boost::asio::yield_context
+)
+{
+    static auto constexpr kHEALTH_CHECK_HTML = R"html(
+    <!DOCTYPE html>
+    <html>
+        <head><title>Test page for Clio</title></head>
+        <body><h1>Clio Test</h1><p>This page shows Clio http(s) connectivity is working.</p></body>
+    </html>
+)html";
+
+    return web::ng::Response{boost::beast::http::status::ok, kHEALTH_CHECK_HTML, request};
+}
+
+}  // namespace app

src/app/WebHandlers.hpp

@@ -0,0 +1,234 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "rpc/Errors.hpp"
+#include "util/log/Logger.hpp"
+#include "web/AdminVerificationStrategy.hpp"
+#include "web/SubscriptionContextInterface.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
+#include "web/ng/Connection.hpp"
+#include "web/ng/Request.hpp"
+#include "web/ng/Response.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/beast/http/status.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/parse.hpp>
+
+#include <exception>
+#include <functional>
+#include <memory>
+#include <utility>
+
+namespace app {
+
+/**
+ * @brief A function object that checks if the connection is allowed to proceed.
+ */
+class OnConnectCheck {
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new OnConnectCheck object
+     *
+     * @param dosguard The DOSGuardInterface to use for checking the connection.
+     */
+    OnConnectCheck(web::dosguard::DOSGuardInterface& dosguard);
+
+    /**
+     * @brief Check if the connection is allowed to proceed.
+     *
+     * @param connection The connection to check.
+     * @return A response if the connection is not allowed to proceed or void otherwise.
+     */
+    std::expected<void, web::ng::Response>
+    operator()(web::ng::Connection const& connection);
+};
+
+/**
+ * @brief A function object to be called when a connection is disconnected.
+ */
+class DisconnectHook {
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new DisconnectHook object
+     *
+     * @param dosguard The DOSGuardInterface to use for disconnecting the connection.
+     */
+    DisconnectHook(web::dosguard::DOSGuardInterface& dosguard);
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param connection The connection which has disconnected.
+     */
+    void
+    operator()(web::ng::Connection const& connection);
+};
+
+/**
+ * @brief A function object that handles the metrics endpoint.
+ */
+class MetricsHandler {
+    std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
+
+public:
+    /**
+     * @brief Construct a new MetricsHandler object
+     *
+     * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
+     */
+    MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier);
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @param connectionMetadata The connection metadata.
+     * @return The response to the request.
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata& connectionMetadata,
+        web::SubscriptionContextPtr,
+        boost::asio::yield_context
+    );
+};
+
+/**
+ * @brief A function object that handles the health check endpoint.
+ */
+class HealthCheckHandler {
+public:
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @return The response to the request
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata&,
+        web::SubscriptionContextPtr,
+        boost::asio::yield_context
+    );
+};
+
+/**
+ * @brief A function object that handles the websocket endpoint.
+ *
+ * @tparam RpcHandlerType The type of the RPC handler.
+ */
+template <typename RpcHandlerType>
+class RequestHandler {
+    util::Logger webServerLog_{"WebServer"};
+    std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
+    std::reference_wrapper<RpcHandlerType> rpcHandler_;
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new RequestHandler object
+     *
+     * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
+     * @param rpcHandler The RPC handler to use for handling the request.
+     * @param dosguard The DOSGuardInterface to use for checking the connection.
+     */
+    RequestHandler(
+        std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
+        RpcHandlerType& rpcHandler,
+        web::dosguard::DOSGuardInterface& dosguard
+    )
+        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard)
+    {
+    }
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @param connectionMetadata The connection metadata.
+     * @param subscriptionContext The subscription context.
+     * @param yield The yield context.
+     * @return The response to the request.
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata& connectionMetadata,
+        web::SubscriptionContextPtr subscriptionContext,
+        boost::asio::yield_context yield
+    )
+    {
+        if (not dosguard_.get().request(connectionMetadata.ip())) {
+            auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN);
+
+            if (not request.isHttp()) {
+                try {
+                    auto requestJson = boost::json::parse(request.message());
+                    if (requestJson.is_object() && requestJson.as_object().contains("id"))
+                        error["id"] = requestJson.as_object().at("id");
+                    error["request"] = request.message();
+                } catch (std::exception const&) {
+                    error["request"] = request.message();
+                }
+            }
+            return web::ng::Response{boost::beast::http::status::service_unavailable, error, request};
+        }
+        LOG(webServerLog_.info()) << connectionMetadata.tag()
+                                  << "Received request from ip = " << connectionMetadata.ip()
+                                  << " - posting to WorkQueue";
+
+        connectionMetadata.setIsAdmin([this, &request, &connectionMetadata]() {
+            return adminVerifier_->isAdmin(request.httpHeaders(), connectionMetadata.ip());
+        });
+
+        try {
+            auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
+
+            if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) {
+                auto jsonResponse = boost::json::parse(response.message()).as_object();
+                jsonResponse["warning"] = "load";
+                if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) {
+                    jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::WarnRpcRateLimit));
+                } else {
+                    jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::WarnRpcRateLimit)};
+                }
+                response.setMessage(jsonResponse);
+            }
+
+            return response;
+        } catch (std::exception const&) {
+            return web::ng::Response{
+                boost::beast::http::status::internal_server_error,
+                rpc::makeError(rpc::RippledError::rpcINTERNAL),
+                request
+            };
+        }
+    }
+};
+
+}  // namespace app

src/data/AmendmentCenter.cpp

@@ -50,10 +50,10 @@
 namespace {
 
 std::unordered_set<std::string>&
-SUPPORTED_AMENDMENTS()
+supportedAmendments()
 {
-    static std::unordered_set<std::string> amendments = {};
-    return amendments;
+    static std::unordered_set<std::string> kAMENDMENTS = {};
+    return kAMENDMENTS;
 }
 
 bool
@@ -72,8 +72,8 @@ namespace impl {
 
 WritingAmendmentKey::WritingAmendmentKey(std::string amendmentName) : AmendmentKey{std::move(amendmentName)}
 {
-    ASSERT(not SUPPORTED_AMENDMENTS().contains(name), "Attempt to register the same amendment twice");
-    SUPPORTED_AMENDMENTS().insert(name);
+    ASSERT(not supportedAmendments().contains(name), "Attempt to register the same amendment twice");
+    supportedAmendments().insert(name);
 }
 
 }  // namespace impl
@@ -90,7 +90,7 @@ AmendmentKey::operator std::string_view() const
 
 AmendmentKey::operator ripple::uint256() const
 {
-    return Amendment::GetAmendmentId(name);
+    return Amendment::getAmendmentId(name);
 }
 
 AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const& backend) : backend_{backend}
@@ -103,9 +103,9 @@ AmendmentCenter::AmendmentCenter(std::shared_ptr<data::BackendInterface> const&
             auto const& [name, support] = p;
             return Amendment{
                 .name = name,
-                .feature = Amendment::GetAmendmentId(name),
+                .feature = Amendment::getAmendmentId(name),
                 .isSupportedByXRPL = support != ripple::AmendmentSupport::Unsupported,
-                .isSupportedByClio = rg::find(SUPPORTED_AMENDMENTS(), name) != rg::end(SUPPORTED_AMENDMENTS()),
+                .isSupportedByClio = rg::find(supportedAmendments(), name) != rg::end(supportedAmendments()),
                 .isRetired = support == ripple::AmendmentSupport::Retired
             };
         }),
@@ -180,7 +180,7 @@ AmendmentCenter::operator[](AmendmentKey const& key) const
 }
 
 ripple::uint256
-Amendment::GetAmendmentId(std::string_view name)
+Amendment::getAmendmentId(std::string_view name)
 {
     return ripple::sha512Half(ripple::Slice(name.data(), name.size()));
 }

src/data/AmendmentCenter.hpp

@@ -67,6 +67,7 @@ struct Amendments {
     // Most of the time it's going to be no changes at all.
 
     /** @cond */
+    // NOLINTBEGIN(readability-identifier-naming)
     REGISTER(OwnerPaysFee);
     REGISTER(Flow);
     REGISTER(FlowCross);
@@ -154,6 +155,7 @@ struct Amendments {
     REGISTER(fix1512);
     REGISTER(fix1523);
     REGISTER(fix1528);
+    // NOLINTEND(readability-identifier-naming)
     /** @endcond */
 };
 

src/data/BackendCounters.cpp

@@ -36,7 +36,7 @@ namespace data {
 
 namespace {
 
-std::vector<std::int64_t> const histogramBuckets{1, 2, 5, 10, 20, 50, 100, 200, 500, 700, 1000};
+std::vector<std::int64_t> const kHISTOGRAM_BUCKETS{1, 2, 5, 10, 20, 50, 100, 200, 500, 700, 1000};
 
 std::int64_t
 durationInMillisecondsSince(std::chrono::steady_clock::time_point const startTime)
@@ -69,13 +69,13 @@ BackendCounters::BackendCounters()
     , readDurationHistogram_(PrometheusService::histogramInt(
           "backend_duration_milliseconds_histogram",
           Labels({Label{"operation", "read"}}),
-          histogramBuckets,
+          kHISTOGRAM_BUCKETS,
           "The duration of backend read operations including retries"
       ))
     , writeDurationHistogram_(PrometheusService::histogramInt(
           "backend_duration_milliseconds_histogram",
           Labels({Label{"operation", "write"}}),
-          histogramBuckets,
+          kHISTOGRAM_BUCKETS,
           "The duration of backend write operations including retries"
       ))
 {

src/data/BackendFactory.hpp

@@ -22,8 +22,8 @@
 #include "data/BackendInterface.hpp"
 #include "data/CassandraBackend.hpp"
 #include "data/cassandra/SettingsProvider.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
@@ -41,18 +41,18 @@ namespace data {
  * @return A shared_ptr<BackendInterface> with the selected implementation
  */
 inline std::shared_ptr<BackendInterface>
-make_Backend(util::Config const& config)
+makeBackend(util::config::ClioConfigDefinition const& config)
 {
-    static util::Logger const log{"Backend"};
+    static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
     LOG(log.info()) << "Constructing BackendInterface";
 
-    auto const readOnly = config.valueOr("read_only", false);
+    auto const readOnly = config.get<bool>("read_only");
 
-    auto const type = config.value<std::string>("database.type");
+    auto const type = config.get<std::string>("database.type");
     std::shared_ptr<BackendInterface> backend = nullptr;
 
     if (boost::iequals(type, "cassandra")) {
-        auto cfg = config.section("database." + type);
+        auto const cfg = config.getObject("database." + type);
         backend = std::make_shared<data::cassandra::CassandraBackend>(data::cassandra::SettingsProvider{cfg}, readOnly);
     }
 

src/data/BackendInterface.cpp

@@ -176,9 +176,9 @@ BackendInterface::fetchSuccessorObject(
     if (succ) {
         auto obj = fetchLedgerObject(*succ, ledgerSequence, yield);
         if (!obj)
-            return {{*succ, {}}};
+            return {{.key = *succ, .blob = {}}};
 
-        return {{*succ, *obj}};
+        return {{.key = *succ, .blob = *obj}};
     }
     return {};
 }
@@ -267,7 +267,7 @@ std::optional<LedgerRange>
 BackendInterface::fetchLedgerRange() const
 {
     std::shared_lock const lck(rngMtx_);
-    return range;
+    return range_;
 }
 
 void
@@ -276,16 +276,16 @@ BackendInterface::updateRange(uint32_t newMax)
     std::scoped_lock const lck(rngMtx_);
 
     ASSERT(
-        !range || newMax >= range->maxSequence,
+        !range_ || newMax >= range_->maxSequence,
         "Range shouldn't exist yet or newMax should be greater. newMax = {}, range->maxSequence = {}",
         newMax,
-        range->maxSequence
+        range_->maxSequence
     );
 
-    if (!range) {
-        range = {newMax, newMax};
+    if (!range_) {
+        range_ = {.minSequence = newMax, .maxSequence = newMax};
     } else {
-        range->maxSequence = newMax;
+        range_->maxSequence = newMax;
     }
 }
 
@@ -296,10 +296,10 @@ BackendInterface::setRange(uint32_t min, uint32_t max, bool force)
 
     if (!force) {
         ASSERT(min <= max, "Range min must be less than or equal to max");
-        ASSERT(not range.has_value(), "Range was already set");
+        ASSERT(not range_.has_value(), "Range was already set");
     }
 
-    range = {min, max};
+    range_ = {.minSequence = min, .maxSequence = max};
 }
 
 LedgerPage
@@ -320,10 +320,10 @@ BackendInterface::fetchLedgerPage(
         ripple::uint256 const& curCursor = [&]() {
             if (!keys.empty())
                 return keys.back();
-            return (cursor ? *cursor : firstKey);
+            return (cursor ? *cursor : kFIRST_KEY);
         }();
 
-        std::uint32_t const seq = outOfOrder ? range->maxSequence : ledgerSequence;
+        std::uint32_t const seq = outOfOrder ? range_->maxSequence : ledgerSequence;
         auto succ = fetchSuccessorKey(curCursor, seq, yield);
 
         if (!succ) {

src/data/BackendInterface.hpp

@@ -65,7 +65,7 @@ public:
     }
 };
 
-static constexpr std::size_t DEFAULT_WAIT_BETWEEN_RETRY = 500;
+static constexpr std::size_t kDEFAULT_WAIT_BETWEEN_RETRY = 500;
 /**
  * @brief A helper function that catches DatabaseTimout exceptions and retries indefinitely.
  *
@@ -76,9 +76,9 @@ static constexpr std::size_t DEFAULT_WAIT_BETWEEN_RETRY = 500;
  */
 template <typename FnType>
 auto
-retryOnTimeout(FnType func, size_t waitMs = DEFAULT_WAIT_BETWEEN_RETRY)
+retryOnTimeout(FnType func, size_t waitMs = kDEFAULT_WAIT_BETWEEN_RETRY)
 {
-    static util::Logger const log{"Backend"};
+    static util::Logger const log{"Backend"};  // NOLINT(readability-identifier-naming)
 
     while (true) {
         try {
@@ -138,7 +138,7 @@ synchronousAndRetryOnTimeout(FnType&& func)
 class BackendInterface {
 protected:
     mutable std::shared_mutex rngMtx_;
-    std::optional<LedgerRange> range;
+    std::optional<LedgerRange> range_;
     LedgerCache cache_;
     std::optional<etl::CorruptionDetector<LedgerCache>> corruptionDetector_;
 
@@ -548,6 +548,16 @@ public:
         boost::asio::yield_context yield
     ) const;
 
+    /**
+     * @brief Fetches the status of migrator by name.
+     *
+     * @param migratorName The name of the migrator
+     * @param yield The coroutine context
+     * @return The status of the migrator if found; nullopt otherwise
+     */
+    virtual std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
+
     /**
      * @brief Synchronously fetches the ledger range from DB.
      *
@@ -673,6 +683,15 @@ public:
     bool
     finishWrites(std::uint32_t ledgerSequence);
 
+    /**
+     * @brief Mark the migration status of a migrator as Migrated in the database
+     *
+     * @param migratorName The name of the migrator
+     * @param status The status to set
+     */
+    virtual void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;
+
     /**
      * @return true if database is overwhelmed; false otherwise
      */

src/data/CassandraBackend.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
+#include "data/cassandra/Concepts.hpp"
 #include "data/cassandra/Handle.hpp"
 #include "data/cassandra/Schema.hpp"
 #include "data/cassandra/SettingsProvider.hpp"
@@ -72,13 +73,15 @@ class BasicCassandraBackend : public BackendInterface {
 
     SettingsProviderType settingsProvider_;
     Schema<SettingsProviderType> schema_;
+
+    std::atomic_uint32_t ledgerSequence_ = 0u;
+
+protected:
     Handle handle_;
 
     // have to be mutable because BackendInterface constness :(
     mutable ExecutionStrategyType executor_;
 
-    std::atomic_uint32_t ledgerSequence_ = 0u;
-
 public:
     /**
      * @brief Create a new cassandra/scylla backend instance.
@@ -93,7 +96,7 @@ public:
         , executor_{settingsProvider_.getSettings(), handle_}
     {
         if (auto const res = handle_.connect(); not res)
-            throw std::runtime_error("Could not connect to databse: " + res.error());
+            throw std::runtime_error("Could not connect to database: " + res.error());
 
         if (not readOnly) {
             if (auto const res = handle_.execute(schema_.createKeyspace); not res) {
@@ -128,7 +131,7 @@ public:
     {
         auto rng = fetchLedgerRange();
         if (!rng)
-            return {{}, {}};
+            return {.txns = {}, .cursor = {}};
 
         Statement const statement = [this, forward, &account]() {
             if (forward)
@@ -191,7 +194,7 @@ public:
         // wait for other threads to finish their writes
         executor_.sync();
 
-        if (!range) {
+        if (!range_) {
             executor_.writeSync(schema_->updateLedgerRange, ledgerSequence_, false, ledgerSequence_);
         }
 
@@ -399,7 +402,7 @@ public:
     {
         auto rng = fetchLedgerRange();
         if (!rng)
-            return {{}, {}};
+            return {.txns = {}, .cursor = {}};
 
         Statement const statement = [this, forward, &tokenID]() {
             if (forward)
@@ -647,7 +650,7 @@ public:
     {
         if (auto const res = executor_.read(yield, schema_->selectSuccessor, key, ledgerSequence); res) {
             if (auto const result = res->template get<ripple::uint256>(); result) {
-                if (*result == lastKey)
+                if (*result == kLAST_KEY)
                     return std::nullopt;
                 return result;
             }
@@ -835,12 +838,32 @@ public:
         return results;
     }
 
+    std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override
+    {
+        auto const res = executor_.read(yield, schema_->selectMigratorStatus, Text(migratorName));
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch migrator status: " << res.error();
+            return {};
+        }
+
+        auto const& results = res.value();
+        if (not results) {
+            return {};
+        }
+
+        for (auto [statusString] : extract<std::string>(results))
+            return statusString;
+
+        return {};
+    }
+
     void
     doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
     {
         LOG(log_.trace()) << " Writing ledger object " << key.size() << ":" << seq << " [" << blob.size() << " bytes]";
 
-        if (range)
+        if (range_)
             executor_.write(schema_->insertDiff, seq, key);
 
         executor_.write(schema_->insertObject, std::move(key), seq, std::move(blob));
@@ -949,7 +972,7 @@ public:
     {
         std::vector<Statement> statements;
         statements.reserve(data.size());
-for (auto [mptId, holder] : data)
+        for (auto [mptId, holder] : data)
             statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
 
         executor_.write(std::move(statements));
@@ -962,6 +985,14 @@ for (auto [mptId, holder] : data)
         // probably was used in PG to start a transaction or smth.
     }
 
+    void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) override
+    {
+        executor_.writeSync(
+            schema_->insertMigratorStatus, data::cassandra::Text{migratorName}, data::cassandra::Text(status)
+        );
+    }
+
     bool
     isTooBusy() const override
     {

src/data/DBHelpers.hpp

@@ -190,11 +190,11 @@ template <typename T>
 inline bool
 isOffer(T const& object)
 {
-    static constexpr short OFFER_OFFSET = 0x006f;
-    static constexpr short SHIFT = 8;
+    static constexpr short kOFFER_OFFSET = 0x006f;
+    static constexpr short kSHIFT = 8;
 
-    short offer_bytes = (object[1] << SHIFT) | object[2];
-    return offer_bytes == OFFER_OFFSET;
+    short offerBytes = (object[1] << kSHIFT) | object[2];
+    return offerBytes == kOFFER_OFFSET;
 }
 
 /**
@@ -223,9 +223,9 @@ template <typename T>
 inline bool
 isDirNode(T const& object)
 {
-    static constexpr short DIR_NODE_SPACE_KEY = 0x0064;
+    static constexpr short kDIR_NODE_SPACE_KEY = 0x0064;
     short const spaceKey = (object.data()[1] << 8) | object.data()[2];
-    return spaceKey == DIR_NODE_SPACE_KEY;
+    return spaceKey == kDIR_NODE_SPACE_KEY;
 }
 
 /**
@@ -273,12 +273,12 @@ template <typename T>
 inline ripple::uint256
 getBookBase(T const& key)
 {
-    static constexpr size_t KEY_SIZE = 24;
+    static constexpr size_t kEY_SIZE = 24;
 
     ASSERT(key.size() == ripple::uint256::size(), "Invalid key size {}", key.size());
 
     ripple::uint256 ret;
-    for (size_t i = 0; i < KEY_SIZE; ++i)
+    for (size_t i = 0; i < kEY_SIZE; ++i)
         ret.data()[i] = key.data()[i];
 
     return ret;
@@ -297,4 +297,4 @@ uint256ToString(ripple::uint256 const& input)
 }
 
 /** @brief The ripple epoch start timestamp. Midnight on 1st January 2000. */
-static constexpr std::uint32_t rippleEpochStart = 946684800;
+static constexpr std::uint32_t kRIPPLE_EPOCH_START = 946684800;

src/data/LedgerCache.cpp

@@ -75,7 +75,7 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
 
                 auto& e = map_[obj.key];
                 if (seq > e.seq) {
-                    e = {seq, obj.blob};
+                    e = {.seq = seq, .blob = obj.blob};
                 }
             } else {
                 map_.erase(obj.key);
@@ -101,7 +101,7 @@ LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
     if (e == map_.end())
         return {};
     ++successorHitCounter_.get();
-    return {{e->first, e->second.blob}};
+    return {{.key = e->first, .blob = e->second.blob}};
 }
 
 std::optional<LedgerObject>
@@ -117,7 +117,7 @@ LedgerCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
     if (e == map_.begin())
         return {};
     --e;
-    return {{e->first, e->second.blob}};
+    return {{.key = e->first, .blob = e->second.blob}};
 }
 
 std::optional<Blob>

src/data/README.md

@@ -262,3 +262,15 @@ CREATE TABLE clio.nf_token_transactions (
 ```
 
 The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, inspired by the same motivations and fulfilling a similar role within this context. It drives the `nft_history` API.
+
+### migrator_status
+
+```
+CREATE TABLE clio.migrator_status (
+    migrator_name TEXT,     # The name of the migrator
+    status TEXT,            # The status of the migrator
+    PRIMARY KEY (migrator_name)
+) 
+```
+
+The `migrator_status` table stores the status of the migratior in this database. If a migrator's status is `migrated`, it means this database has finished data migration for this migrator.

src/data/Types.hpp

@@ -266,7 +266,7 @@ struct Amendment {
      * @return The amendment Id as uint256
      */
     static ripple::uint256
-    GetAmendmentId(std::string_view const name);
+    getAmendmentId(std::string_view const name);
 
     /**
      * @brief Equality comparison operator
@@ -312,8 +312,8 @@ struct AmendmentKey {
     operator<=>(AmendmentKey const& other) const = default;
 };
 
-constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
-constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
-constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"};
+constexpr ripple::uint256 kFIRST_KEY{"0000000000000000000000000000000000000000000000000000000000000000"};
+constexpr ripple::uint256 kLAST_KEY{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
+constexpr ripple::uint256 kHI192{"0000000000000000000000000000000000000000000000001111111111111111"};
 
 }  // namespace data

src/data/cassandra/Handle.cpp

@@ -60,7 +60,7 @@ Handle::connect() const
 Handle::FutureType
 Handle::asyncConnect(std::string_view keyspace) const
 {
-    return cass_session_connect_keyspace(session_, cluster_, keyspace.data());
+    return cass_session_connect_keyspace_n(session_, cluster_, keyspace.data(), keyspace.size());
 }
 
 Handle::MaybeErrorType
@@ -155,7 +155,7 @@ Handle::asyncExecute(std::vector<StatementType> const& statements, std::function
 Handle::PreparedStatementType
 Handle::prepare(std::string_view query) const
 {
-    Handle::FutureType const future = cass_session_prepare(session_, query.data());
+    Handle::FutureType const future = cass_session_prepare_n(session_, query.data(), query.size());
     auto const rc = future.await();
     if (rc)
         return cass_future_get_prepared(future);

src/data/cassandra/Schema.hpp

@@ -74,7 +74,7 @@ public:
                      'class': 'SimpleStrategy',
                      'replication_factor': '{}'
                    }} 
-               AND durable_writes = true
+               AND durable_writes = True
             )",
             settingsProvider_.get().getKeyspace(),
             settingsProvider_.get().getReplicationFactor()
@@ -270,6 +270,18 @@ public:
             qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                   migrator_name TEXT,
+                          status TEXT,
+                         PRIMARY KEY (migrator_name)
+                  ) 
+            )",
+            qualifiedTableName(settingsProvider_.get(), "migrator_status")
+        ));
+
         return statements;
     }();
 
@@ -460,12 +472,23 @@ public:
                 R"(
                 UPDATE {} 
                    SET sequence = ?
-                 WHERE is_latest = false
+                 WHERE is_latest = False
                 )",
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
         }();
 
+        PreparedStatement insertMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {}
+                       (migrator_name, status)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
+
         //
         // Select queries
         //
@@ -753,7 +776,7 @@ public:
                 R"(
                 SELECT sequence
                   FROM {}    
-                 WHERE is_latest = true               
+                 WHERE is_latest = True
                 )",
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
@@ -764,10 +787,22 @@ public:
                 R"(
                 SELECT sequence
                   FROM {}
+                 WHERE is_latest in (True, False)
                 )",
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
         }();
+
+        PreparedStatement selectMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT status
+                  FROM {}
+                 WHERE migrator_name = ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
     };
 
     /**

src/data/cassandra/SettingsProvider.cpp

@@ -22,10 +22,7 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/Constants.hpp"
-#include "util/config/Config.hpp"
-
-#include <boost/json/conversion.hpp>
-#include <boost/json/value.hpp>
+#include "util/newconfig/ObjectView.hpp"
 
 #include <cerrno>
 #include <chrono>
@@ -36,43 +33,17 @@
 #include <ios>
 #include <iterator>
 #include <optional>
-#include <stdexcept>
 #include <string>
+#include <string_view>
 #include <system_error>
 
 namespace data::cassandra {
 
-namespace impl {
-inline Settings::ContactPoints
-tag_invoke(boost::json::value_to_tag<Settings::ContactPoints>, boost::json::value const& value)
-{
-    if (not value.is_object()) {
-        throw std::runtime_error("Feed entire Cassandra section to parse Settings::ContactPoints instead");
-    }
-
-    util::Config const obj{value};
-    Settings::ContactPoints out;
-
-    out.contactPoints = obj.valueOrThrow<std::string>("contact_points", "`contact_points` must be a string");
-    out.port = obj.maybeValue<uint16_t>("port");
-
-    return out;
-}
-
-inline Settings::SecureConnectionBundle
-tag_invoke(boost::json::value_to_tag<Settings::SecureConnectionBundle>, boost::json::value const& value)
-{
-    if (not value.is_string())
-        throw std::runtime_error("`secure_connect_bundle` must be a string");
-    return Settings::SecureConnectionBundle{value.as_string().data()};
-}
-}  // namespace impl
-
-SettingsProvider::SettingsProvider(util::Config const& cfg)
+SettingsProvider::SettingsProvider(util::config::ObjectView const& cfg)
     : config_{cfg}
-    , keyspace_{cfg.valueOr<std::string>("keyspace", "clio")}
+    , keyspace_{cfg.get<std::string>("keyspace")}
     , tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
-    , replicationFactor_{cfg.valueOr<uint16_t>("replication_factor", 3)}
+    , replicationFactor_{cfg.get<uint16_t>("replication_factor")}
     , settings_{parseSettings()}
 {
 }
@@ -86,8 +57,8 @@ SettingsProvider::getSettings() const
 std::optional<std::string>
 SettingsProvider::parseOptionalCertificate() const
 {
-    if (auto const certPath = config_.maybeValue<std::string>("certfile"); certPath) {
-        auto const path = std::filesystem::path(*certPath);
+    if (auto const certPath = config_.getValueView("certfile"); certPath.hasValue()) {
+        auto const path = std::filesystem::path(certPath.asString());
         std::ifstream fileStream(path.string(), std::ios::in);
         if (!fileStream) {
             throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string());
@@ -108,30 +79,34 @@ Settings
 SettingsProvider::parseSettings() const
 {
     auto settings = Settings::defaultSettings();
-    if (auto const bundle = config_.maybeValue<Settings::SecureConnectionBundle>("secure_connect_bundle"); bundle) {
-        settings.connectionInfo = *bundle;
+
+    // all config values used in settings is under "database.cassandra" prefix
+    if (config_.getValueView("secure_connect_bundle").hasValue()) {
+        auto const bundle = Settings::SecureConnectionBundle{(config_.get<std::string>("secure_connect_bundle"))};
+        settings.connectionInfo = bundle;
     } else {
-        settings.connectionInfo =
-            config_.valueOrThrow<Settings::ContactPoints>("Missing contact_points in Cassandra config");
+        Settings::ContactPoints out;
+        out.contactPoints = config_.get<std::string>("contact_points");
+        out.port = config_.maybeValue<uint32_t>("port");
+        settings.connectionInfo = out;
     }
 
-    settings.threads = config_.valueOr<uint32_t>("threads", settings.threads);
-    settings.maxWriteRequestsOutstanding =
-        config_.valueOr<uint32_t>("max_write_requests_outstanding", settings.maxWriteRequestsOutstanding);
-    settings.maxReadRequestsOutstanding =
-        config_.valueOr<uint32_t>("max_read_requests_outstanding", settings.maxReadRequestsOutstanding);
-    settings.coreConnectionsPerHost =
-        config_.valueOr<uint32_t>("core_connections_per_host", settings.coreConnectionsPerHost);
+    settings.threads = config_.get<uint32_t>("threads");
+    settings.maxWriteRequestsOutstanding = config_.get<uint32_t>("max_write_requests_outstanding");
+    settings.maxReadRequestsOutstanding = config_.get<uint32_t>("max_read_requests_outstanding");
+    settings.coreConnectionsPerHost = config_.get<uint32_t>("core_connections_per_host");
     settings.queueSizeIO = config_.maybeValue<uint32_t>("queue_size_io");
-    settings.writeBatchSize = config_.valueOr<std::size_t>("write_batch_size", settings.writeBatchSize);
+    settings.writeBatchSize = config_.get<std::size_t>("write_batch_size");
 
-    auto const connectTimeoutSecond = config_.maybeValue<uint32_t>("connect_timeout");
-    if (connectTimeoutSecond)
-        settings.connectionTimeout = std::chrono::milliseconds{*connectTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    if (config_.getValueView("connect_timeout").hasValue()) {
+        auto const connectTimeoutSecond = config_.get<uint32_t>("connect_timeout");
+        settings.connectionTimeout = std::chrono::milliseconds{connectTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
+    }
 
-    auto const requestTimeoutSecond = config_.maybeValue<uint32_t>("request_timeout");
-    if (requestTimeoutSecond)
-        settings.requestTimeout = std::chrono::milliseconds{*requestTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    if (config_.getValueView("request_timeout").hasValue()) {
+        auto const requestTimeoutSecond = config_.get<uint32_t>("request_timeout");
+        settings.requestTimeout = std::chrono::milliseconds{requestTimeoutSecond * util::kMILLISECONDS_PER_SECOND};
+    }
 
     settings.certificate = parseOptionalCertificate();
     settings.username = config_.maybeValue<std::string>("username");

src/data/cassandra/SettingsProvider.hpp

@@ -19,10 +19,9 @@
 
 #pragma once
 
-#include "data/cassandra/Handle.hpp"
 #include "data/cassandra/Types.hpp"
-#include "util/config/Config.hpp"
-#include "util/log/Logger.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <cstdint>
 #include <optional>
@@ -34,7 +33,7 @@ namespace data::cassandra {
  * @brief Provides settings for @ref BasicCassandraBackend.
  */
 class SettingsProvider {
-    util::Config config_;
+    util::config::ObjectView config_;
 
     std::string keyspace_;
     std::optional<std::string> tablePrefix_;
@@ -47,7 +46,7 @@ public:
      *
      * @param cfg The config of Clio to use
      */
-    explicit SettingsProvider(util::Config const& cfg);
+    explicit SettingsProvider(util::config::ObjectView const& cfg);
 
     /**
      * @return The cluster settings

src/data/cassandra/Types.hpp

@@ -21,6 +21,8 @@
 
 #include <cstdint>
 #include <expected>
+#include <string>
+#include <utility>
 
 namespace data::cassandra {
 
@@ -55,6 +57,26 @@ struct Limit {
     int32_t limit;
 };
 
+/**
+ * @brief A strong type wrapper for string
+ *
+ * This is unfortunately needed right now to support TEXT properly
+ * because clio uses string to represent BLOB
+ * If we want to bind TEXT with string, we need to use this type
+ */
+struct Text {
+    std::string text;
+
+    /**
+     * @brief Construct a new Text object from string type
+     *
+     * @param text The text to wrap
+     */
+    explicit Text(std::string text) : text{std::move(text)}
+    {
+    }
+};
+
 class Handle;
 class CassandraError;
 

src/data/cassandra/impl/Batch.cpp

@@ -31,14 +31,14 @@
 #include <vector>
 
 namespace {
-constexpr auto batchDeleter = [](CassBatch* ptr) { cass_batch_free(ptr); };
+constexpr auto kBATCH_DELETER = [](CassBatch* ptr) { cass_batch_free(ptr); };
 }  // namespace
 
 namespace data::cassandra::impl {
 
 // TODO: Use an appropriate value instead of CASS_BATCH_TYPE_LOGGED for different use cases
 Batch::Batch(std::vector<Statement> const& statements)
-    : ManagedObject{cass_batch_new(CASS_BATCH_TYPE_LOGGED), batchDeleter}
+    : ManagedObject{cass_batch_new(CASS_BATCH_TYPE_LOGGED), kBATCH_DELETER}
 {
     cass_batch_set_is_idempotent(*this, cass_true);
 

src/data/cassandra/impl/Cluster.cpp

@@ -33,13 +33,13 @@
 
 namespace {
 
-constexpr auto clusterDeleter = [](CassCluster* ptr) { cass_cluster_free(ptr); };
+constexpr auto kCLUSTER_DELETER = [](CassCluster* ptr) { cass_cluster_free(ptr); };
 
 };  // namespace
 
 namespace data::cassandra::impl {
 
-Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), clusterDeleter}
+Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), kCLUSTER_DELETER}
 {
     using std::to_string;
 

src/data/cassandra/impl/Cluster.hpp

@@ -25,6 +25,8 @@
 #include <cassandra.h>
 
 #include <chrono>
+#include <cstddef>
+#include <cstdint>
 #include <optional>
 #include <string>
 #include <string_view>
@@ -39,10 +41,10 @@ namespace data::cassandra::impl {
  * @brief Bundles all cassandra settings in one place.
  */
 struct Settings {
-    static constexpr std::size_t DEFAULT_CONNECTION_TIMEOUT = 10000;
-    static constexpr uint32_t DEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING = 10'000;
-    static constexpr uint32_t DEFAULT_MAX_READ_REQUESTS_OUTSTANDING = 100'000;
-    static constexpr std::size_t DEFAULT_BATCH_SIZE = 20;
+    static constexpr std::size_t kDEFAULT_CONNECTION_TIMEOUT = 10000;
+    static constexpr uint32_t kDEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING = 10'000;
+    static constexpr uint32_t kDEFAULT_MAX_READ_REQUESTS_OUTSTANDING = 100'000;
+    static constexpr std::size_t kDEFAULT_BATCH_SIZE = 20;
 
     /**
      * @brief Represents the configuration of contact points for cassandra.
@@ -63,7 +65,7 @@ struct Settings {
     bool enableLog = false;
 
     /** @brief Connect timeout specified in milliseconds */
-    std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{DEFAULT_CONNECTION_TIMEOUT};
+    std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{kDEFAULT_CONNECTION_TIMEOUT};
 
     /** @brief Request timeout specified in milliseconds */
     std::chrono::milliseconds requestTimeout = std::chrono::milliseconds{0};  // no timeout at all
@@ -75,16 +77,16 @@ struct Settings {
     uint32_t threads = std::thread::hardware_concurrency();
 
     /** @brief The maximum number of outstanding write requests at any given moment */
-    uint32_t maxWriteRequestsOutstanding = DEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING;
+    uint32_t maxWriteRequestsOutstanding = kDEFAULT_MAX_WRITE_REQUESTS_OUTSTANDING;
 
     /** @brief The maximum number of outstanding read requests at any given moment */
-    uint32_t maxReadRequestsOutstanding = DEFAULT_MAX_READ_REQUESTS_OUTSTANDING;
+    uint32_t maxReadRequestsOutstanding = kDEFAULT_MAX_READ_REQUESTS_OUTSTANDING;
 
     /** @brief The number of connection per host to always have active */
     uint32_t coreConnectionsPerHost = 1u;
 
     /** @brief Size of batches when writing */
-    std::size_t writeBatchSize = DEFAULT_BATCH_SIZE;
+    std::size_t writeBatchSize = kDEFAULT_BATCH_SIZE;
 
     /** @brief Size of the IO queue */
     std::optional<uint32_t> queueSizeIO = std::nullopt;  // NOLINT(readability-redundant-member-init)

src/data/cassandra/impl/Collection.hpp

@@ -33,7 +33,7 @@
 namespace data::cassandra::impl {
 
 class Collection : public ManagedObject<CassCollection> {
-    static constexpr auto deleter = [](CassCollection* ptr) { cass_collection_free(ptr); };
+    static constexpr auto kDELETER = [](CassCollection* ptr) { cass_collection_free(ptr); };
 
     static void
     throwErrorIfNeeded(CassError const rc, std::string_view const label)
@@ -49,7 +49,7 @@ public:
 
     template <typename Type>
     explicit Collection(std::vector<Type> const& value)
-        : ManagedObject{cass_collection_new(CASS_COLLECTION_TYPE_LIST, value.size()), deleter}
+        : ManagedObject{cass_collection_new(CASS_COLLECTION_TYPE_LIST, value.size()), kDELETER}
     {
         bind(value);
     }

src/data/cassandra/impl/Future.cpp

@@ -32,12 +32,12 @@
 #include <utility>
 
 namespace {
-constexpr auto futureDeleter = [](CassFuture* ptr) { cass_future_free(ptr); };
+constexpr auto kFUTURE_DELETER = [](CassFuture* ptr) { cass_future_free(ptr); };
 }  // namespace
 
 namespace data::cassandra::impl {
 
-/* implicit */ Future::Future(CassFuture* ptr) : ManagedObject{ptr, futureDeleter}
+/* implicit */ Future::Future(CassFuture* ptr) : ManagedObject{ptr, kFUTURE_DELETER}
 {
 }
 

src/data/cassandra/impl/ManagedObject.hpp

@@ -30,8 +30,8 @@ protected:
     std::unique_ptr<Managed, void (*)(Managed*)> ptr_;
 
 public:
-    template <typename deleterCallable>
-    ManagedObject(Managed* rawPtr, deleterCallable deleter) : ptr_{rawPtr, deleter}
+    template <typename DeleterCallable>
+    ManagedObject(Managed* rawPtr, DeleterCallable deleter) : ptr_{rawPtr, deleter}
     {
         if (rawPtr == nullptr)
             throw std::runtime_error("Could not create DB object - got nullptr");

src/data/cassandra/impl/Result.cpp

@@ -26,13 +26,13 @@
 #include <cstddef>
 
 namespace {
-constexpr auto resultDeleter = [](CassResult const* ptr) { cass_result_free(ptr); };
-constexpr auto resultIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
+constexpr auto kRESULT_DELETER = [](CassResult const* ptr) { cass_result_free(ptr); };
+constexpr auto kRESULT_ITERATOR_DELETER = [](CassIterator* ptr) { cass_iterator_free(ptr); };
 }  // namespace
 
 namespace data::cassandra::impl {
 
-/* implicit */ Result::Result(CassResult const* ptr) : ManagedObject{ptr, resultDeleter}
+/* implicit */ Result::Result(CassResult const* ptr) : ManagedObject{ptr, kRESULT_DELETER}
 {
 }
 
@@ -49,7 +49,7 @@ Result::hasRows() const
 }
 
 /* implicit */ ResultIterator::ResultIterator(CassIterator* ptr)
-    : ManagedObject{ptr, resultIteratorDeleter}, hasMore_{cass_iterator_next(ptr) != 0u}
+    : ManagedObject{ptr, kRESULT_ITERATOR_DELETER}, hasMore_{cass_iterator_next(ptr) != 0u}
 {
 }
 

src/data/cassandra/impl/Session.hpp

@@ -26,10 +26,10 @@
 namespace data::cassandra::impl {
 
 class Session : public ManagedObject<CassSession> {
-    static constexpr auto deleter = [](CassSession* ptr) { cass_session_free(ptr); };
+    static constexpr auto kDELETER = [](CassSession* ptr) { cass_session_free(ptr); };
 
 public:
-    Session() : ManagedObject{cass_session_new(), deleter}
+    Session() : ManagedObject{cass_session_new(), kDELETER}
     {
     }
 };

src/data/cassandra/impl/SslContext.cpp

@@ -27,12 +27,12 @@
 #include <string>
 
 namespace {
-constexpr auto contextDeleter = [](CassSsl* ptr) { cass_ssl_free(ptr); };
+constexpr auto kCONTEXT_DELETER = [](CassSsl* ptr) { cass_ssl_free(ptr); };
 }  // namespace
 
 namespace data::cassandra::impl {
 
-SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), contextDeleter}
+SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), kCONTEXT_DELETER}
 {
     cass_ssl_set_verify_flags(*this, CASS_SSL_VERIFY_NONE);
     if (auto const rc = cass_ssl_add_trusted_cert(*this, certificate.c_str()); rc != CASS_OK) {

src/data/cassandra/impl/Statement.hpp

@@ -43,7 +43,7 @@
 namespace data::cassandra::impl {
 
 class Statement : public ManagedObject<CassStatement> {
-    static constexpr auto deleter = [](CassStatement* ptr) { cass_statement_free(ptr); };
+    static constexpr auto kDELETER = [](CassStatement* ptr) { cass_statement_free(ptr); };
 
 public:
     /**
@@ -54,14 +54,14 @@ public:
      */
     template <typename... Args>
     explicit Statement(std::string_view query, Args&&... args)
-        : ManagedObject{cass_statement_new(query.data(), sizeof...(args)), deleter}
+        : ManagedObject{cass_statement_new_n(query.data(), query.size(), sizeof...(args)), kDELETER}
     {
         cass_statement_set_consistency(*this, CASS_CONSISTENCY_QUORUM);
         cass_statement_set_is_idempotent(*this, cass_true);
         bind<Args...>(std::forward<Args>(args)...);
     }
 
-    /* implicit */ Statement(CassStatement* ptr) : ManagedObject{ptr, deleter}
+    /* implicit */ Statement(CassStatement* ptr) : ManagedObject{ptr, kDELETER}
     {
         cass_statement_set_consistency(*this, CASS_CONSISTENCY_QUORUM);
         cass_statement_set_is_idempotent(*this, cass_true);
@@ -119,6 +119,9 @@ public:
             // reinterpret_cast is needed here :'(
             auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
             throwErrorIfNeeded(rc, "Bind string (as bytes)");
+        } else if constexpr (std::is_convertible_v<DecayedType, Text>) {
+            auto const rc = cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
+            throwErrorIfNeeded(rc, "Bind string (as TEXT)");
         } else if constexpr (std::is_same_v<DecayedType, UintTupleType> ||
                              std::is_same_v<DecayedType, UintByteTupleType>) {
             auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});
@@ -150,10 +153,10 @@ public:
  * This is used to produce Statement objects that can be executed.
  */
 class PreparedStatement : public ManagedObject<CassPrepared const> {
-    static constexpr auto deleter = [](CassPrepared const* ptr) { cass_prepared_free(ptr); };
+    static constexpr auto kDELETER = [](CassPrepared const* ptr) { cass_prepared_free(ptr); };
 
 public:
-    /* implicit */ PreparedStatement(CassPrepared const* ptr) : ManagedObject{ptr, deleter}
+    /* implicit */ PreparedStatement(CassPrepared const* ptr) : ManagedObject{ptr, kDELETER}
     {
     }
 

src/data/cassandra/impl/Tuple.cpp

@@ -24,17 +24,17 @@
 #include <cassandra.h>
 
 namespace {
-constexpr auto tupleDeleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
-constexpr auto tupleIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
+constexpr auto kTUPLE_DELETER = [](CassTuple* ptr) { cass_tuple_free(ptr); };
+constexpr auto kTUPLE_ITERATOR_DELETER = [](CassIterator* ptr) { cass_iterator_free(ptr); };
 }  // namespace
 
 namespace data::cassandra::impl {
 
-/* implicit */ Tuple::Tuple(CassTuple* ptr) : ManagedObject{ptr, tupleDeleter}
+/* implicit */ Tuple::Tuple(CassTuple* ptr) : ManagedObject{ptr, kTUPLE_DELETER}
 {
 }
 
-/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr) : ManagedObject{ptr, tupleIteratorDeleter}
+/* implicit */ TupleIterator::TupleIterator(CassIterator* ptr) : ManagedObject{ptr, kTUPLE_ITERATOR_DELETER}
 {
 }
 

src/data/cassandra/impl/Tuple.hpp

@@ -37,14 +37,14 @@
 namespace data::cassandra::impl {
 
 class Tuple : public ManagedObject<CassTuple> {
-    static constexpr auto deleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
+    static constexpr auto kDELETER = [](CassTuple* ptr) { cass_tuple_free(ptr); };
 
 public:
     /* implicit */ Tuple(CassTuple* ptr);
 
     template <typename... Types>
     explicit Tuple(std::tuple<Types...>&& value)
-        : ManagedObject{cass_tuple_new(std::tuple_size<std::tuple<Types...>>{}), deleter}
+        : ManagedObject{cass_tuple_new(std::tuple_size<std::tuple<Types...>>{}), kDELETER}
     {
         std::apply(std::bind_front(&Tuple::bind<Types...>, this), std::move(value));
     }

src/etl/CacheLoader.hpp

@@ -64,8 +64,12 @@ public:
      * @param backend The backend to use
      * @param cache The cache to load into
      */
-    CacheLoader(util::Config const& config, std::shared_ptr<BackendInterface> const& backend, CacheType& cache)
-        : backend_{backend}, cache_{cache}, settings_{make_CacheLoaderSettings(config)}, ctx_{settings_.numThreads}
+    CacheLoader(
+        util::config::ClioConfigDefinition const& config,
+        std::shared_ptr<BackendInterface> const& backend,
+        CacheType& cache
+    )
+        : backend_{backend}, cache_{cache}, settings_{makeCacheLoaderSettings(config)}, ctx_{settings_.numThreads}
     {
     }
 

src/etl/CacheLoaderSettings.cpp

@@ -19,11 +19,12 @@
 
 #include "etl/CacheLoaderSettings.hpp"
 
-#include "util/config/Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string/predicate.hpp>
 
 #include <cstddef>
+#include <cstdint>
 #include <string>
 
 namespace etl {
@@ -47,31 +48,29 @@ CacheLoaderSettings::isDisabled() const
 }
 
 [[nodiscard]] CacheLoaderSettings
-make_CacheLoaderSettings(util::Config const& config)
+makeCacheLoaderSettings(util::config::ClioConfigDefinition const& config)
 {
     CacheLoaderSettings settings;
-    settings.numThreads = config.valueOr("io_threads", settings.numThreads);
-    if (config.contains("cache")) {
-        auto const cache = config.section("cache");
-        // Given diff number to generate cursors
-        settings.numCacheDiffs = cache.valueOr<size_t>("num_diffs", settings.numCacheDiffs);
-        // Given cursors number fetching from diff
-        settings.numCacheCursorsFromDiff = cache.valueOr<size_t>("num_cursors_from_diff", 0);
-        // Given cursors number fetching from account
-        settings.numCacheCursorsFromAccount = cache.valueOr<size_t>("num_cursors_from_account", 0);
+    settings.numThreads = config.get<uint16_t>("io_threads");
+    auto const cache = config.getObject("cache");
+    // Given diff number to generate cursors
+    settings.numCacheDiffs = cache.get<std::size_t>("num_diffs");
+    // Given cursors number fetching from diff
+    settings.numCacheCursorsFromDiff = cache.get<std::size_t>("num_cursors_from_diff");
+    // Given cursors number fetching from account
+    settings.numCacheCursorsFromAccount = cache.get<std::size_t>("num_cursors_from_account");
 
-        settings.numCacheMarkers = cache.valueOr<size_t>("num_markers", settings.numCacheMarkers);
-        settings.cachePageFetchSize = cache.valueOr<size_t>("page_fetch_size", settings.cachePageFetchSize);
+    settings.numCacheMarkers = cache.get<std::size_t>("num_markers");
+    settings.cachePageFetchSize = cache.get<std::size_t>("page_fetch_size");
+
+    auto const entry = cache.get<std::string>("load");
+    if (boost::iequals(entry, "sync"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC;
+    if (boost::iequals(entry, "async"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC;
+    if (boost::iequals(entry, "none") or boost::iequals(entry, "no"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE;
 
-        if (auto entry = cache.maybeValue<std::string>("load"); entry) {
-            if (boost::iequals(*entry, "sync"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC;
-            if (boost::iequals(*entry, "async"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC;
-            if (boost::iequals(*entry, "none") or boost::iequals(*entry, "no"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE;
-        }
-    }
     return settings;
 }
 

src/etl/CacheLoaderSettings.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "util/config/Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <cstddef>
 
@@ -64,6 +64,6 @@ struct CacheLoaderSettings {
  * @returns The CacheLoaderSettings object
  */
 [[nodiscard]] CacheLoaderSettings
-make_CacheLoaderSettings(util::Config const& config);
+makeCacheLoaderSettings(util::config::ClioConfigDefinition const& config);
 
 }  // namespace etl

src/etl/ETLService.cpp

@@ -26,8 +26,8 @@
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/Constants.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <xrpl/beast/core/CurrentThreadName.h>
@@ -88,9 +88,9 @@ ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
 
     auto const end = std::chrono::system_clock::now();
     auto const lastPublishedSeq = ledgerPublisher_.getLastPublishedSequence();
-    static constexpr auto NANOSECONDS_PER_SECOND = 1'000'000'000.0;
+    static constexpr auto kNANOSECONDS_PER_SECOND = 1'000'000'000.0;
     LOG(log_.debug()) << "Extracted and wrote " << lastPublishedSeq.value_or(startSequence) - startSequence << " in "
-                      << ((end - begin).count()) / NANOSECONDS_PER_SECOND;
+                      << ((end - begin).count()) / kNANOSECONDS_PER_SECOND;
 
     state_.isWriting = false;
 
@@ -134,7 +134,7 @@ ETLService::monitor()
             }
         } catch (std::runtime_error const& e) {
             LOG(log_.fatal()) << "Failed to load initial ledger: " << e.what();
-            amendmentBlockHandler_.onAmendmentBlock();
+            amendmentBlockHandler_.notifyAmendmentBlocked();
             return;
         }
 
@@ -168,7 +168,7 @@ ETLService::publishNextSequence(uint32_t nextSequence)
     if (auto rng = backend_->hardFetchLedgerRangeNoThrow(); rng && rng->maxSequence >= nextSequence) {
         ledgerPublisher_.publish(nextSequence, {});
         ++nextSequence;
-    } else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(nextSequence, util::MILLISECONDS_PER_SECOND)) {
+    } else if (networkValidatedLedgers_->waitUntilValidatedByNetwork(nextSequence, util::kMILLISECONDS_PER_SECOND)) {
         LOG(log_.info()) << "Ledger with sequence = " << nextSequence << " has been validated by the network. "
                          << "Attempting to find in database and publish";
 
@@ -178,8 +178,8 @@ ETLService::publishNextSequence(uint32_t nextSequence)
         // database after the specified number of attempts. publishLedger()
         // waits one second between each attempt to read the ledger from the
         // database
-        constexpr size_t timeoutSeconds = 10;
-        bool const success = ledgerPublisher_.publish(nextSequence, timeoutSeconds);
+        constexpr size_t kTIMEOUT_SECONDS = 10;
+        bool const success = ledgerPublisher_.publish(nextSequence, kTIMEOUT_SECONDS);
 
         if (!success) {
             LOG(log_.warn()) << "Failed to publish ledger with sequence = " << nextSequence << " . Beginning ETL";
@@ -233,7 +233,7 @@ ETLService::monitorReadOnly()
             // if we can't, wait until it's validated by the network, or 1 second passes, whichever occurs
             // first. Even if we don't hear from rippled, if ledgers are being written to the db, we publish
             // them.
-            networkValidatedLedgers_->waitUntilValidatedByNetwork(latestSequence, util::MILLISECONDS_PER_SECOND);
+            networkValidatedLedgers_->waitUntilValidatedByNetwork(latestSequence, util::kMILLISECONDS_PER_SECOND);
         }
     }
 }
@@ -262,7 +262,7 @@ ETLService::doWork()
 }
 
 ETLService::ETLService(
-    util::Config const& config,
+    util::config::ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -280,9 +280,9 @@ ETLService::ETLService(
 {
     startSequence_ = config.maybeValue<uint32_t>("start_sequence");
     finishSequence_ = config.maybeValue<uint32_t>("finish_sequence");
-    state_.isReadOnly = config.valueOr("read_only", static_cast<bool>(state_.isReadOnly));
-    extractorThreads_ = config.valueOr<uint32_t>("extractor_threads", extractorThreads_);
-    txnThreshold_ = config.valueOr<size_t>("txn_threshold", txnThreshold_);
+    state_.isReadOnly = config.get<bool>("read_only");
+    extractorThreads_ = config.get<uint32_t>("extractor_threads");
+    txnThreshold_ = config.get<std::size_t>("txn_threshold");
 
     // This should probably be done in the backend factory but we don't have state available until here
     backend_->setCorruptionDetector(CorruptionDetector<data::LedgerCache>{state_, backend->cache()});

src/etl/ETLService.hpp

@@ -119,7 +119,7 @@ public:
      * @param ledgers The network validated ledgers datastructure
      */
     ETLService(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -141,8 +141,8 @@ public:
      * @return A shared pointer to a new instance of ETLService
      */
     static std::shared_ptr<ETLService>
-    make_ETLService(
-        util::Config const& config,
+    makeETLService(
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,

src/etl/LedgerFetcherInterface.hpp

@@ -0,0 +1,65 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+
+#include <cstdint>
+#include <optional>
+
+namespace etl {
+
+/**
+ * @brief An interface for LedgerFetcher
+ */
+struct LedgerFetcherInterface {
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+    virtual ~LedgerFetcherInterface() = default;
+
+    /**
+     * @brief Extract data for a particular ledger from an ETL source
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchData(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract diff data for a particular ledger from an ETL source.
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchDataAndDiff(uint32_t seq) = 0;
+};
+
+}  // namespace etl

src/etl/LoadBalancer.cpp

@@ -29,6 +29,9 @@
 #include "util/Random.hpp"
 #include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ArrayView.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
@@ -51,13 +54,13 @@
 #include <utility>
 #include <vector>
 
-using namespace util;
+using namespace util::config;
 
 namespace etl {
 
 std::shared_ptr<LoadBalancer>
-LoadBalancer::make_LoadBalancer(
-    Config const& config,
+LoadBalancer::makeLoadBalancer(
+    ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -71,7 +74,7 @@ LoadBalancer::make_LoadBalancer(
 }
 
 LoadBalancer::LoadBalancer(
-    Config const& config,
+    ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -79,23 +82,23 @@ LoadBalancer::LoadBalancer(
     SourceFactory sourceFactory
 )
 {
-    auto const forwardingCacheTimeout = config.valueOr<float>("forwarding.cache_timeout", 0.f);
+    auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
     if (forwardingCacheTimeout > 0.f) {
         forwardingCache_ = util::ResponseExpirationCache{
-            Config::toMilliseconds(forwardingCacheTimeout),
+            util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout),
             {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
         };
     }
 
-    static constexpr std::uint32_t MAX_DOWNLOAD = 256;
-    if (auto value = config.maybeValue<uint32_t>("num_markers"); value) {
-        ASSERT(*value > 0 and *value <= MAX_DOWNLOAD, "'num_markers' value in config must be in range 1-256");
-        downloadRanges_ = *value;
+    auto const numMarkers = config.getValueView("num_markers");
+    if (numMarkers.hasValue()) {
+        auto const value = numMarkers.asIntType<uint32_t>();
+        downloadRanges_ = value;
     } else if (backend->fetchLedgerRange()) {
         downloadRanges_ = 4;
     }
 
-    auto const allowNoEtl = config.valueOr("allow_no_etl", false);
+    auto const allowNoEtl = config.get<bool>("allow_no_etl");
 
     auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) {
         LOG(log_.warn()) << log;
@@ -106,10 +109,12 @@ LoadBalancer::LoadBalancer(
         }
     };
 
-    auto const forwardingTimeout = Config::toMilliseconds(config.valueOr<float>("forwarding.request_timeout", 10.));
-    for (auto const& entry : config.array("etl_sources")) {
+    auto const forwardingTimeout =
+        ClioConfigDefinition::toMilliseconds(config.get<float>("forwarding.request_timeout"));
+    auto const etlArray = config.getArray("etl_sources");
+    for (auto it = etlArray.begin<ObjectView>(); it != etlArray.end<ObjectView>(); ++it) {
         auto source = sourceFactory(
-            entry,
+            *it,
             ioc,
             backend,
             subscriptions,
@@ -230,7 +235,7 @@ LoadBalancer::forwardToRippled(
 )
 {
     if (not request.contains("command"))
-        return std::unexpected{rpc::ClioError::rpcCOMMAND_IS_MISSING};
+        return std::unexpected{rpc::ClioError::RpcCommandIsMissing};
 
     auto const cmd = boost::json::value_to<std::string>(request.at("command"));
     if (forwardingCache_) {
@@ -244,10 +249,10 @@ LoadBalancer::forwardToRippled(
 
     auto numAttempts = 0u;
 
-    auto xUserValue = isAdmin ? ADMIN_FORWARDING_X_USER_VALUE : USER_FORWARDING_X_USER_VALUE;
+    auto xUserValue = isAdmin ? kADMIN_FORWARDING_X_USER_VALUE : kUSER_FORWARDING_X_USER_VALUE;
 
     std::optional<boost::json::object> response;
-    rpc::ClioError error = rpc::ClioError::etlCONNECTION_ERROR;
+    rpc::ClioError error = rpc::ClioError::EtlConnectionError;
     while (numAttempts < sources_.size()) {
         auto res = sources_[sourceIdx]->forwardToRippled(request, clientIp, xUserValue, yield);
         if (res) {

src/etl/LoadBalancer.hpp

@@ -27,8 +27,8 @@
 #include "rpc/Errors.hpp"
 #include "util/Mutex.hpp"
 #include "util/ResponseExpirationCache.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/asio/io_context.hpp>
@@ -65,7 +65,7 @@ public:
     using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
 
 private:
-    static constexpr std::uint32_t DEFAULT_DOWNLOAD_RANGES = 16;
+    static constexpr std::uint32_t kDEFAULT_DOWNLOAD_RANGES = 16;
 
     util::Logger log_{"ETL"};
     // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
@@ -75,7 +75,7 @@ private:
     std::vector<SourcePtr> sources_;
     std::optional<ETLState> etlState_;
     std::uint32_t downloadRanges_ =
-        DEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
+        kDEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
 
     // Using mutext instead of atomic_bool because choosing a new source to
     // forward messages should be done with a mutual exclusion otherwise there will be a race condition
@@ -85,12 +85,12 @@ public:
     /**
      * @brief Value for the X-User header when forwarding admin requests
      */
-    static constexpr std::string_view ADMIN_FORWARDING_X_USER_VALUE = "clio_admin";
+    static constexpr std::string_view kADMIN_FORWARDING_X_USER_VALUE = "clio_admin";
 
     /**
      * @brief Value for the X-User header when forwarding user requests
      */
-    static constexpr std::string_view USER_FORWARDING_X_USER_VALUE = "clio_user";
+    static constexpr std::string_view kUSER_FORWARDING_X_USER_VALUE = "clio_user";
 
     /**
      * @brief Create an instance of the load balancer.
@@ -103,12 +103,12 @@ public:
      * @param sourceFactory A factory function to create a source
      */
     LoadBalancer(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
         std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
-        SourceFactory sourceFactory = make_Source
+        SourceFactory sourceFactory = makeSource
     );
 
     /**
@@ -123,13 +123,13 @@ public:
      * @return A shared pointer to a new instance of LoadBalancer
      */
     static std::shared_ptr<LoadBalancer>
-    make_LoadBalancer(
-        util::Config const& config,
+    makeLoadBalancer(
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
         std::shared_ptr<NetworkValidatedLedgersInterface> validatedLedgers,
-        SourceFactory sourceFactory = make_Source
+        SourceFactory sourceFactory = makeSource
     );
 
     ~LoadBalancer();

src/etl/MPTHelpers.cpp

@@ -27,6 +27,7 @@
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STLedgerEntry.h>
 #include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/TER.h>
 #include <xrpl/protocol/TxFormats.h>
 
@@ -41,7 +42,7 @@ namespace etl {
  * @param txMeta Transaction metadata
  * @return MPT and holder account pair
  */
-static std::optional<MPTHolderData>
+std::optional<MPTHolderData>
 getMPTokenAuthorize(ripple::TxMeta const& txMeta)
 {
     for (ripple::STObject const& node : txMeta.getNodes()) {
@@ -50,7 +51,9 @@ getMPTokenAuthorize(ripple::TxMeta const& txMeta)
 
         if (node.getFName() == ripple::sfCreatedNode) {
             auto const& newMPT = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
-            return MPTHolderData{newMPT[ripple::sfMPTokenIssuanceID], newMPT.getAccountID(ripple::sfAccount)};
+            return MPTHolderData{
+                .mptID = newMPT[ripple::sfMPTokenIssuanceID], .holder = newMPT.getAccountID(ripple::sfAccount)
+            };
         }
     }
     return {};
@@ -77,7 +80,7 @@ getMPTHolderFromObj(std::string const& key, std::string const& blob)
     auto const mptIssuanceID = sle[ripple::sfMPTokenIssuanceID];
     auto const holder = sle.getAccountID(ripple::sfAccount);
 
-    return MPTHolderData{mptIssuanceID, holder};
+    return MPTHolderData{.mptID = mptIssuanceID, .holder = holder};
 }
 
 }  // namespace etl

src/etl/NFTHelpers.cpp

@@ -73,9 +73,9 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
         if (node.getFName() == ripple::sfCreatedNode) {
             ripple::STArray const& toAddNFTs =
                 node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddNFTs.begin(),
-                toAddNFTs.end(),
+            std::ranges::transform(
+                toAddNFTs,
+
                 std::back_inserter(finalIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
@@ -98,18 +98,18 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
                 continue;
 
             ripple::STArray const& toAddNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddNFTs.begin(),
-                toAddNFTs.end(),
+            std::ranges::transform(
+                toAddNFTs,
+
                 std::back_inserter(prevIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
 
             ripple::STArray const& toAddFinalNFTs =
                 node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddFinalNFTs.begin(),
-                toAddFinalNFTs.end(),
+            std::ranges::transform(
+                toAddFinalNFTs,
+
                 std::back_inserter(finalIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
@@ -121,6 +121,7 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
 
     // Find the first NFT ID that doesn't match.  We're looking for an
     // added NFT, so the one we want will be the mismatch in finalIDs.
+    // NOLINTNEXTLINE(modernize-use-ranges)
     auto const diff = std::mismatch(finalIDs.begin(), finalIDs.end(), prevIDs.begin(), prevIDs.end());
 
     // There should always be a difference so the returned finalIDs
@@ -261,7 +262,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
                 .getFieldArray(ripple::sfNFTokens);
         }();
 
-        auto const nft = std::find_if(nfts.begin(), nfts.end(), [&tokenID](ripple::STObject const& candidate) {
+        auto const nft = std::ranges::find_if(nfts, [&tokenID](ripple::STObject const& candidate) {
             return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
         });
         if (nft != nfts.end()) {
@@ -298,10 +299,10 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
     std::ranges::sort(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
         return a.tokenID < b.tokenID;
     });
-    auto last = std::unique(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
+    auto [last, end] = std::ranges::unique(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
         return a.tokenID == b.tokenID;
     });
-    txs.erase(last, txs.end());
+    txs.erase(last, end);
     return {txs, {}};
 }
 
@@ -366,10 +367,9 @@ getUniqueNFTsDatas(std::vector<NFTsData> const& nfts)
         return a.tokenID == b.tokenID ? a.transactionIndex > b.transactionIndex : a.tokenID > b.tokenID;
     });
 
-    auto const last = std::unique(results.begin(), results.end(), [](NFTsData const& a, NFTsData const& b) {
-        return a.tokenID == b.tokenID;
-    });
-    results.erase(last, results.end());
+    auto const [last, end] =
+        std::ranges::unique(results, [](NFTsData const& a, NFTsData const& b) { return a.tokenID == b.tokenID; });
+    results.erase(last, end);
     return results;
 }
 

src/etl/NetworkValidatedLedgers.cpp

@@ -27,7 +27,7 @@
 
 namespace etl {
 std::shared_ptr<NetworkValidatedLedgers>
-NetworkValidatedLedgers::make_ValidatedLedgers()
+NetworkValidatedLedgers::makeValidatedLedgers()
 {
     return std::make_shared<NetworkValidatedLedgers>();
 }

src/etl/NetworkValidatedLedgers.hpp

@@ -51,7 +51,7 @@ public:
      * @return A shared pointer to a new instance of NetworkValidatedLedgers
      */
     static std::shared_ptr<NetworkValidatedLedgers>
-    make_ValidatedLedgers();
+    makeValidatedLedgers();
 
     /**
      * @brief Notify the datastructure that idx has been validated by the network.

src/etl/Source.cpp

@@ -26,7 +26,7 @@
 #include "etl/impl/SourceImpl.hpp"
 #include "etl/impl/SubscriptionSource.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
-#include "util/config/Config.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 
@@ -38,8 +38,8 @@
 namespace etl {
 
 SourcePtr
-make_Source(
-    util::Config const& config,
+makeSource(
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -50,9 +50,9 @@ make_Source(
     SourceBase::OnLedgerClosedHook onLedgerClosed
 )
 {
-    auto const ip = config.valueOr<std::string>("ip", {});
-    auto const wsPort = config.valueOr<std::string>("ws_port", {});
-    auto const grpcPort = config.valueOr<std::string>("grpc_port", {});
+    auto const ip = config.get<std::string>("ip");
+    auto const wsPort = config.get<std::string>("ws_port");
+    auto const grpcPort = config.get<std::string>("grpc_port");
 
     impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout};
     impl::GrpcSource grpcSource{ip, grpcPort, std::move(backend)};

src/etl/Source.hpp

@@ -23,8 +23,9 @@
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
@@ -147,7 +148,7 @@ public:
 using SourcePtr = std::unique_ptr<SourceBase>;
 
 using SourceFactory = std::function<SourcePtr(
-    util::Config const& config,
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -174,8 +175,8 @@ using SourceFactory = std::function<SourcePtr(
  * @return The created source
  */
 SourcePtr
-make_Source(
-    util::Config const& config,
+makeSource(
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,

src/etl/impl/AmendmentBlockHandler.cpp

@@ -30,8 +30,8 @@
 
 namespace etl::impl {
 
-AmendmentBlockHandler::ActionType const AmendmentBlockHandler::defaultAmendmentBlockAction = []() {
-    static util::Logger const log{"ETL"};
+AmendmentBlockHandler::ActionType const AmendmentBlockHandler::kDEFAULT_AMENDMENT_BLOCK_ACTION = []() {
+    static util::Logger const log{"ETL"};  // NOLINT(readability-identifier-naming)
     LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
                      << "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
 };
@@ -47,7 +47,7 @@ AmendmentBlockHandler::AmendmentBlockHandler(
 }
 
 void
-AmendmentBlockHandler::onAmendmentBlock()
+AmendmentBlockHandler::notifyAmendmentBlocked()
 {
     state_.get().isAmendmentBlocked = true;
     repeat_.start(interval_, action_);

src/etl/impl/AmendmentBlockHandler.hpp

@@ -43,17 +43,17 @@ private:
     ActionType action_;
 
 public:
-    static ActionType const defaultAmendmentBlockAction;
+    static ActionType const kDEFAULT_AMENDMENT_BLOCK_ACTION;
 
     AmendmentBlockHandler(
         boost::asio::io_context& ioc,
         SystemState& state,
         std::chrono::steady_clock::duration interval = std::chrono::seconds{1},
-        ActionType action = defaultAmendmentBlockAction
+        ActionType action = kDEFAULT_AMENDMENT_BLOCK_ACTION
     );
 
     void
-    onAmendmentBlock();
+    notifyAmendmentBlocked();
 };
 
 }  // namespace etl::impl

src/etl/impl/CursorFromAccountProvider.hpp

@@ -58,10 +58,10 @@ public:
         }();
 
         rg::sort(accountRoots);
-        std::vector<ripple::uint256> cursors{data::firstKey};
+        std::vector<ripple::uint256> cursors{data::kFIRST_KEY};
         rg::copy(accountRoots.begin(), accountRoots.end(), std::back_inserter(cursors));
         rg::sort(cursors);
-        cursors.push_back(data::lastKey);
+        cursors.push_back(data::kLAST_KEY);
 
         std::vector<CursorPair> pairs;
         pairs.reserve(cursors.size());

src/etl/impl/CursorFromDiffProvider.hpp

@@ -85,10 +85,10 @@ public:
             );
         }
 
-        std::vector<ripple::uint256> cursors{data::firstKey};
+        std::vector<ripple::uint256> cursors{data::kFIRST_KEY};
         rg::copy(liveCursors | vs::take(std::min(liveCursors.size(), numCursors_)), std::back_inserter(cursors));
         rg::sort(cursors);
-        cursors.push_back(data::lastKey);
+        cursors.push_back(data::kLAST_KEY);
 
         std::vector<CursorPair> pairs;
         pairs.reserve(cursors.size());

src/etl/impl/CursorFromFixDiffNumProvider.hpp

@@ -70,21 +70,18 @@ public:
             return a.key < b.key or (a.key == b.key and std::size(a.blob) < std::size(b.blob));
         });
 
-        diffs.erase(
-            std::unique(
-                std::begin(diffs), std::end(diffs), [](auto const& a, auto const& b) { return a.key == b.key; }
-            ),
-            std::end(diffs)
-        );
+        auto const [removalCursor, last] =
+            rg::unique(diffs, [](auto const& a, auto const& b) { return a.key == b.key; });
+        diffs.erase(removalCursor, last);
 
-        std::vector<ripple::uint256> cursors{data::firstKey};
+        std::vector<ripple::uint256> cursors{data::kFIRST_KEY};
         rg::copy(
             diffs                                                                   //
                 | vs::filter([](auto const& obj) { return not obj.blob.empty(); })  //
                 | vs::transform([](auto const& obj) { return obj.key; }),
             std::back_inserter(cursors)
         );
-        cursors.push_back(data::lastKey);  // last pair should cover the remaining range
+        cursors.push_back(data::kLAST_KEY);  // last pair should cover the remaining range
 
         std::vector<CursorPair> pairs;
         pairs.reserve(cursors.size());

src/etl/impl/ExtractionDataPipe.hpp

@@ -39,7 +39,7 @@ public:
     using DataType = std::optional<RawDataType>;
     using QueueType = ThreadSafeQueue<DataType>;  // TODO: probably should use boost::lockfree::queue instead?
 
-    constexpr static auto TOTAL_MAX_IN_QUEUE = 1000u;
+    static constexpr auto kTOTAL_MAX_IN_QUEUE = 1000u;
 
 private:
     util::Logger log_{"ETL"};
@@ -58,7 +58,7 @@ public:
      */
     ExtractionDataPipe(uint32_t stride, uint32_t startSequence) : stride_{stride}, startSequence_{startSequence}
     {
-        auto const maxQueueSize = TOTAL_MAX_IN_QUEUE / stride;
+        auto const maxQueueSize = kTOTAL_MAX_IN_QUEUE / stride;
         for (size_t i = 0; i < stride_; ++i)
             queues_.push_back(std::make_unique<QueueType>(maxQueueSize));
     }

src/etl/impl/Extractor.hpp

@@ -121,19 +121,19 @@ private:
         pipe_.get().finish(startSequence_);
     }
 
-    bool
+    [[nodiscard]] bool
     isStopping() const
     {
         return state_.get().isStopping;
     }
 
-    bool
+    [[nodiscard]] bool
     hasWriteConflict() const
     {
         return state_.get().writeConflict;
     }
 
-    bool
+    [[nodiscard]] bool
     shouldFinish(uint32_t seq) const
     {
         // Stopping conditions:

src/etl/impl/ForwardingSource.cpp

@@ -44,13 +44,13 @@ ForwardingSource::ForwardingSource(
     std::string ip,
     std::string wsPort,
     std::chrono::steady_clock::duration forwardingTimeout,
-    std::chrono::steady_clock::duration connectionTimeout
+    std::chrono::steady_clock::duration connTimeout
 )
     : log_(fmt::format("ForwardingSource[{}:{}]", ip, wsPort))
     , connectionBuilder_(std::move(ip), std::move(wsPort))
     , forwardingTimeout_{forwardingTimeout}
 {
-    connectionBuilder_.setConnectionTimeout(connectionTimeout)
+    connectionBuilder_.setConnectionTimeout(connTimeout)
         .addHeader(
             {boost::beast::http::field::user_agent, fmt::format("{} websocket-client-coro", BOOST_BEAST_VERSION_STRING)}
         );
@@ -76,14 +76,14 @@ ForwardingSource::forwardToRippled(
     auto expectedConnection = connectionBuilder.connect(yield);
     if (not expectedConnection) {
         LOG(log_.debug()) << "Couldn't connect to rippled to forward request.";
-        return std::unexpected{rpc::ClioError::etlCONNECTION_ERROR};
+        return std::unexpected{rpc::ClioError::EtlConnectionError};
     }
     auto& connection = expectedConnection.value();
 
     auto writeError = connection->write(boost::json::serialize(request), yield, forwardingTimeout_);
     if (writeError) {
         LOG(log_.debug()) << "Error sending request to rippled to forward request.";
-        return std::unexpected{rpc::ClioError::etlREQUEST_ERROR};
+        return std::unexpected{rpc::ClioError::EtlRequestError};
     }
 
     auto response = connection->read(yield, forwardingTimeout_);
@@ -91,10 +91,10 @@ ForwardingSource::forwardToRippled(
         if (auto errorCode = response.error().errorCode();
             errorCode.has_value() and errorCode->value() == boost::system::errc::timed_out) {
             LOG(log_.debug()) << "Request to rippled timed out";
-            return std::unexpected{rpc::ClioError::etlREQUEST_TIMEOUT};
+            return std::unexpected{rpc::ClioError::EtlRequestTimeout};
         }
         LOG(log_.debug()) << "Error sending request to rippled to forward request.";
-        return std::unexpected{rpc::ClioError::etlREQUEST_ERROR};
+        return std::unexpected{rpc::ClioError::EtlRequestError};
     }
 
     boost::json::value parsedResponse;
@@ -104,7 +104,7 @@ ForwardingSource::forwardToRippled(
             throw std::runtime_error("response is not an object");
     } catch (std::exception const& e) {
         LOG(log_.debug()) << "Error parsing response from rippled: " << e.what() << ". Response: " << *response;
-        return std::unexpected{rpc::ClioError::etlINVALID_RESPONSE};
+        return std::unexpected{rpc::ClioError::EtlInvalidResponse};
     }
 
     auto responseObject = parsedResponse.as_object();

src/etl/impl/ForwardingSource.hpp

@@ -39,14 +39,14 @@ class ForwardingSource {
     util::requests::WsConnectionBuilder connectionBuilder_;
     std::chrono::steady_clock::duration forwardingTimeout_;
 
-    static constexpr std::chrono::seconds CONNECTION_TIMEOUT{3};
+    static constexpr std::chrono::seconds kCONNECTION_TIMEOUT{3};
 
 public:
     ForwardingSource(
         std::string ip,
         std::string wsPort,
         std::chrono::steady_clock::duration forwardingTimeout,
-        std::chrono::steady_clock::duration connectionTimeout = CONNECTION_TIMEOUT
+        std::chrono::steady_clock::duration connTimeout = ForwardingSource::kCONNECTION_TIMEOUT
     );
 
     /**

src/etl/impl/LedgerFetcher.hpp

@@ -63,7 +63,7 @@ public:
      * @param sequence sequence of the ledger to extract
      * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchData(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
@@ -83,7 +83,7 @@ public:
      * @param sequence sequence of the ledger to extract
      * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchDataAndDiff(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;

src/etl/impl/LedgerLoader.hpp

@@ -131,8 +131,8 @@ public:
                 result.mptHoldersData.push_back(*maybeMPTHolder);
 
             result.accountTxData.emplace_back(txMeta, sttx.getTransactionID());
-            static constexpr std::size_t KEY_SIZE = 32;
-            std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), KEY_SIZE};
+            static constexpr std::size_t kEY_SIZE = 32;
+            std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), kEY_SIZE};
             backend_->writeTransaction(
                 std::move(keyStr),
                 ledger.seq,
@@ -204,10 +204,10 @@ public:
                         backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
                 }
 
-                ripple::uint256 prev = data::firstKey;
+                ripple::uint256 prev = data::kFIRST_KEY;
                 while (auto cur = backend_->cache().getSuccessor(prev, sequence)) {
                     ASSERT(cur.has_value(), "Succesor for key {} must exist", ripple::strHex(prev));
-                    if (prev == data::firstKey)
+                    if (prev == data::kFIRST_KEY)
                         backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
 
                     if (isBookDir(cur->key, cur->blob)) {
@@ -228,12 +228,12 @@ public:
                     }
 
                     prev = cur->key;
-                    static constexpr std::size_t LOG_INTERVAL = 100000;
-                    if (numWrites % LOG_INTERVAL == 0 && numWrites != 0)
+                    static constexpr std::size_t kLOG_INTERVAL = 100000;
+                    if (numWrites % kLOG_INTERVAL == 0 && numWrites != 0)
                         LOG(log_.info()) << "Wrote " << numWrites << " book successors";
                 }
 
-                backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
+                backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::kLAST_KEY));
                 ++numWrites;
             });
 

src/etl/impl/LedgerPublisher.hpp

@@ -184,8 +184,8 @@ public:
 
             // if the ledger closed over MAX_LEDGER_AGE_SECONDS ago, assume we are still catching up and don't publish
             // TODO: this probably should be a strategy
-            static constexpr std::uint32_t MAX_LEDGER_AGE_SECONDS = 600;
-            if (age < MAX_LEDGER_AGE_SECONDS) {
+            static constexpr std::uint32_t kMAX_LEDGER_AGE_SECONDS = 600;
+            if (age < kMAX_LEDGER_AGE_SECONDS) {
                 std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout([&](auto yield) {
                     return backend_->fetchFees(lgrInfo.seq, yield);
                 });
@@ -260,9 +260,9 @@ public:
         auto now = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch())
                        .count();
         auto closeTime = lastCloseTime_.time_since_epoch().count();
-        if (now < (rippleEpochStart + closeTime))
+        if (now < (kRIPPLE_EPOCH_START + closeTime))
             return 0;
-        return now - (rippleEpochStart + closeTime);
+        return now - (kRIPPLE_EPOCH_START + closeTime);
     }
 
     /**

src/etl/impl/SubscriptionSource.cpp

@@ -77,7 +77,7 @@ SubscriptionSource::SubscriptionSource(
     , subscriptions_(std::move(subscriptions))
     , strand_(boost::asio::make_strand(ioContext))
     , wsTimeout_(wsTimeout)
-    , retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
+    , retry_(util::makeRetryExponentialBackoff(retryDelay, kRETRY_MAX_DELAY, strand_))
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))
@@ -222,9 +222,9 @@ SubscriptionSource::handleMessage(std::string const& message)
         auto const object = raw.as_object();
         uint32_t ledgerIndex = 0;
 
-        static constexpr char const* const JS_LedgerClosed = "ledgerClosed";
-        static constexpr char const* const JS_ValidationReceived = "validationReceived";
-        static constexpr char const* const JS_ManifestReceived = "manifestReceived";
+        static constexpr auto kJS_LEDGER_CLOSED = "ledgerClosed";
+        static constexpr auto kJS_VALIDATION_RECEIVED = "validationReceived";
+        static constexpr auto kJS_MANIFEST_RECEIVED = "manifestReceived";
 
         if (object.contains(JS(result))) {
             auto const& result = object.at(JS(result)).as_object();
@@ -237,7 +237,7 @@ SubscriptionSource::handleMessage(std::string const& message)
             }
             LOG(log_.debug()) << "Received a message on ledger subscription stream. Message: " << object;
 
-        } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_LedgerClosed) {
+        } else if (object.contains(JS(type)) && object.at(JS(type)) == kJS_LEDGER_CLOSED) {
             LOG(log_.debug()) << "Received a message of type 'ledgerClosed' on ledger subscription stream. Message: "
                               << object;
             if (object.contains(JS(ledger_index))) {
@@ -259,10 +259,10 @@ SubscriptionSource::handleMessage(std::string const& message)
                 if (object.contains(JS(transaction)) and !object.contains(JS(meta))) {
                     LOG(log_.debug()) << "Forwarding proposed transaction: " << object;
                     subscriptions_->forwardProposedTransaction(object);
-                } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ValidationReceived) {
+                } else if (object.contains(JS(type)) && object.at(JS(type)) == kJS_VALIDATION_RECEIVED) {
                     LOG(log_.debug()) << "Forwarding validation: " << object;
                     subscriptions_->forwardValidation(object);
-                } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ManifestReceived) {
+                } else if (object.contains(JS(type)) && object.at(JS(type)) == kJS_MANIFEST_RECEIVED) {
                     LOG(log_.debug()) << "Forwarding manifest: " << object;
                     subscriptions_->forwardManifest(object);
                 }
@@ -353,7 +353,7 @@ SubscriptionSource::setValidatedRange(std::string range)
             pairs.emplace_back(min, max);
         }
     }
-    std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; });
+    std::ranges::sort(pairs, [](auto left, auto right) { return left.first < right.first; });
 
     auto dataLock = validatedLedgersData_.lock();
     dataLock->validatedLedgers = std::move(pairs);
@@ -363,12 +363,12 @@ SubscriptionSource::setValidatedRange(std::string range)
 std::string const&
 SubscriptionSource::getSubscribeCommandJson()
 {
-    static boost::json::object const jsonValue{
+    static boost::json::object const kJSON_VALUE{
         {"command", "subscribe"},
         {"streams", {"ledger", "manifests", "validations", "transactions_proposed"}},
     };
-    static std::string const jsonString = boost::json::serialize(jsonValue);
-    return jsonString;
+    static std::string const kJSON_STRING = boost::json::serialize(kJSON_VALUE);
+    return kJSON_STRING;
 }
 
 }  // namespace etl::impl

src/etl/impl/SubscriptionSource.hpp

@@ -91,9 +91,9 @@ private:
 
     std::future<void> runFuture_;
 
-    static constexpr std::chrono::seconds WS_TIMEOUT{30};
-    static constexpr std::chrono::seconds RETRY_MAX_DELAY{30};
-    static constexpr std::chrono::seconds RETRY_DELAY{1};
+    static constexpr std::chrono::seconds kWS_TIMEOUT{30};
+    static constexpr std::chrono::seconds kRETRY_MAX_DELAY{30};
+    static constexpr std::chrono::seconds kRETRY_DELAY{1};
 
 public:
     /**
@@ -107,8 +107,7 @@ public:
      * @param subscriptions The subscription manager object
      * @param onConnect The onConnect hook. Called when the connection is established
      * @param onDisconnect The onDisconnect hook. Called when the connection is lost
-     * @param onLedgerClosed The onLedgerClosed hook. Called when the ledger is closed but only if the source is
-     * forwarding
+     * @param onLedgerClosed The onLedgerClosed hook. Called when the ledger is closed if the source is forwarding
      * @param wsTimeout A timeout for websocket operations. Defaults to 30 seconds
      * @param retryDelay The retry delay. Defaults to 1 second
      */
@@ -121,8 +120,8 @@ public:
         OnConnectHook onConnect,
         OnDisconnectHook onDisconnect,
         OnLedgerClosedHook onLedgerClosed,
-        std::chrono::steady_clock::duration const wsTimeout = WS_TIMEOUT,
-        std::chrono::steady_clock::duration const retryDelay = RETRY_DELAY
+        std::chrono::steady_clock::duration const wsTimeout = SubscriptionSource::kWS_TIMEOUT,
+        std::chrono::steady_clock::duration const retryDelay = SubscriptionSource::kRETRY_DELAY
     );
 
     /**

src/etl/impl/Transformer.hpp

@@ -203,7 +203,7 @@ private:
         } catch (std::runtime_error const& e) {
             LOG(log_.fatal()) << "Failed to build next ledger: " << e.what();
 
-            amendmentBlockHandler_.get().onAmendmentBlock();
+            amendmentBlockHandler_.get().notifyAmendmentBlocked();
             return {ripple::LedgerHeader{}, false};
         }
 
@@ -308,11 +308,11 @@ private:
 
                 auto lb = backend_->cache().getPredecessor(obj.key, lgrInfo.seq);
                 if (!lb)
-                    lb = {data::firstKey, {}};
+                    lb = {.key = data::kFIRST_KEY, .blob = {}};
 
                 auto ub = backend_->cache().getSuccessor(obj.key, lgrInfo.seq);
                 if (!ub)
-                    ub = {data::lastKey, {}};
+                    ub = {.key = data::kLAST_KEY, .blob = {}};
 
                 if (obj.blob.empty()) {
                     LOG(log_.debug()) << "writing successor for deleted object " << ripple::strHex(obj.key) << " - "
@@ -336,10 +336,10 @@ private:
                     LOG(log_.debug()) << "Updating book successor " << ripple::strHex(base) << " - "
                                       << ripple::strHex(succ->key);
                 } else {
-                    backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(data::lastKey));
+                    backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(data::kLAST_KEY));
 
                     LOG(log_.debug()) << "Updating book successor " << ripple::strHex(base) << " - "
-                                      << ripple::strHex(data::lastKey);
+                                      << ripple::strHex(data::kLAST_KEY);
                 }
             }
         }
@@ -361,7 +361,7 @@ private:
             for (auto& obj : *(rawData.mutable_book_successors())) {
                 auto firstBook = std::move(*obj.mutable_first_book());
                 if (!firstBook.size())
-                    firstBook = uint256ToString(data::lastKey);
+                    firstBook = uint256ToString(data::kLAST_KEY);
                 LOG(log_.debug()) << "writing book successor " << ripple::strHex(obj.book_base()) << " - "
                                   << ripple::strHex(firstBook);
 
@@ -372,10 +372,10 @@ private:
                 if (obj.mod_type() != RawLedgerObjectType::MODIFIED) {
                     std::string* predPtr = obj.mutable_predecessor();
                     if (predPtr->empty())
-                        *predPtr = uint256ToString(data::firstKey);
+                        *predPtr = uint256ToString(data::kFIRST_KEY);
                     std::string* succPtr = obj.mutable_successor();
                     if (succPtr->empty())
-                        *succPtr = uint256ToString(data::lastKey);
+                        *succPtr = uint256ToString(data::kLAST_KEY);
 
                     if (obj.mod_type() == RawLedgerObjectType::DELETED) {
                         LOG(log_.debug()) << "Modifying successors for deleted object " << ripple::strHex(obj.key())

src/etlng/AmendmentBlockHandlerInterface.hpp

@@ -0,0 +1,37 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+namespace etlng {
+
+/**
+ * @brief The interface of a handler for amendment blocking
+ */
+struct AmendmentBlockHandlerInterface {
+    virtual ~AmendmentBlockHandlerInterface() = default;
+
+    /**
+     * @brief The function to call once an amendment block has been discovered
+     */
+    virtual void
+    notifyAmendmentBlocked() = 0;
+};
+
+}  // namespace etlng

src/etlng/CMakeLists.txt

@@ -1,5 +1,8 @@
-add_library(clio_etlng INTERFACE)
+add_library(clio_etlng)
 
-# target_sources(clio_etlng PRIVATE )
+target_sources(
+  clio_etlng PRIVATE impl/AmendmentBlockHandler.cpp impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp
+                     impl/Loading.cpp
+)
 
-target_link_libraries(clio_etlng INTERFACE clio_data)
+target_link_libraries(clio_etlng PUBLIC clio_data)

src/etlng/ExtractorInterface.hpp

@@ -0,0 +1,54 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief An interface for the Extractor
+ */
+struct ExtractorInterface {
+    virtual ~ExtractorInterface() = default;
+
+    /**
+     * @brief Extract diff data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequence and parent if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerWithDiff(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerOnly(uint32_t seq) = 0;
+};
+
+}  // namespace etlng

src/etlng/InitialLoadObserverInterface.hpp

@@ -0,0 +1,54 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <xrpl/protocol/LedgerHeader.h>
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief The interface for observing the initial ledger load
+ */
+struct InitialLoadObserverInterface {
+    virtual ~InitialLoadObserverInterface() = default;
+
+    /**
+     * @brief Callback for each incoming batch of objects during initial ledger load
+     *
+     * @param seq The sequence for this batch of objects
+     * @param data The batch of objects
+     * @param lastKey The last key of the previous batch if there was one
+     */
+    virtual void
+    onInitialLoadGotMoreObjects(
+        uint32_t seq,
+        std::vector<model::Object> const& data,
+        std::optional<std::string> lastKey = std::nullopt
+    ) = 0;
+};
+
+}  // namespace etlng

src/etlng/LoadBalancerInterface.hpp

@@ -0,0 +1,134 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include "etl/ETLState.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "rpc/Errors.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/json/object.hpp>
+#include <boost/json/value.hpp>
+#include <org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+
+#include <chrono>
+#include <cstdint>
+#include <expected>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief An interface for LoadBalancer
+ */
+class LoadBalancerInterface {
+public:
+    using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+    virtual ~LoadBalancerInterface() = default;
+
+    /**
+     * @brief Load the initial ledger, writing data to the queue.
+     * @note This function will retry indefinitely until the ledger is downloaded.
+     *
+     * @param sequence Sequence of ledger to download
+     * @param loader InitialLoadObserverInterface implementation
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return A std::vector<std::string> The ledger data
+     */
+    virtual std::vector<std::string>
+    loadInitialLedger(
+        uint32_t sequence,
+        etlng::InitialLoadObserverInterface& loader,
+        std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
+    ) = 0;
+
+    /**
+     * @brief Load the initial ledger, writing data to the queue.
+     * @note This function will retry indefinitely until the ledger is downloaded.
+     *
+     * @param sequence Sequence of ledger to download
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return A std::vector<std::string> The ledger data
+     */
+    virtual std::vector<std::string>
+    loadInitialLedger(uint32_t sequence, std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}) = 0;
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param ledgerSequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one
+     * @param getObjectNeighbors Whether to request object neighbors
+     * @param retryAfter Time to wait between retries (2 seconds by default)
+     * @return The extracted data, if extraction was successful. If the ledger was found
+     * in the database or the server is shutting down, the optional will be empty
+     */
+    virtual OptionalGetLedgerResponseType
+    fetchLedger(
+        uint32_t ledgerSequence,
+        bool getObjects,
+        bool getObjectNeighbors,
+        std::chrono::steady_clock::duration retryAfter = std::chrono::seconds{2}
+    ) = 0;
+
+    /**
+     * @brief Represent the state of this load balancer as a JSON object
+     *
+     * @return JSON representation of the state of this load balancer.
+     */
+    virtual boost::json::value
+    toJson() const = 0;
+
+    /**
+     * @brief Forward a JSON RPC request to a randomly selected rippled node.
+     *
+     * @param request JSON-RPC request to forward
+     * @param clientIp The IP address of the peer, if known
+     * @param isAdmin Whether the request is from an admin
+     * @param yield The coroutine context
+     * @return Response received from rippled node as JSON object on success or error on failure
+     */
+    virtual std::expected<boost::json::object, rpc::ClioError>
+    forwardToRippled(
+        boost::json::object const& request,
+        std::optional<std::string> const& clientIp,
+        bool isAdmin,
+        boost::asio::yield_context yield
+    ) = 0;
+
+    /**
+     * @brief Return state of ETL nodes.
+     * @return ETL state, nullopt if etl nodes not available
+     */
+    virtual std::optional<etl::ETLState>
+    getETLState() noexcept = 0;
+};
+
+}  // namespace etlng

src/etlng/LoaderInterface.hpp

@@ -0,0 +1,52 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <xrpl/protocol/LedgerHeader.h>
+
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief An interface for a ETL Loader
+ */
+struct LoaderInterface {
+    virtual ~LoaderInterface() = default;
+
+    /**
+     * @brief Load ledger data
+     * @param data The data to load
+     */
+    virtual void
+    load(model::LedgerData const& data) = 0;
+
+    /**
+     * @brief Load the initial ledger
+     * @param data The data to load
+     * @return Optional ledger header
+     */
+    virtual std::optional<ripple::LedgerHeader>
+    loadInitialLedger(model::LedgerData const& data) = 0;
+};
+
+}  // namespace etlng

src/etlng/Models.hpp

@@ -29,6 +29,7 @@
 #include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
 #include <xrpl/protocol/LedgerHeader.h>
 #include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
 #include <xrpl/protocol/TxFormats.h>
 #include <xrpl/protocol/TxMeta.h>
 
@@ -49,7 +50,7 @@ namespace etlng::model {
 template <ripple::TxType... Types>
     requires(util::hasNoDuplicates(Types...))
 struct Spec {
-    static constexpr bool SpecTag = true;
+    static constexpr bool kSPEC_TAG = true;
 
     /**
      * @brief Checks if the transaction type was requested.
@@ -57,7 +58,7 @@ struct Spec {
      * @param type The transaction type
      * @return true if the transaction was requested; false otherwise
      */
-    [[nodiscard]] constexpr static bool
+    [[nodiscard]] static constexpr bool
     wants(ripple::TxType type) noexcept
     {
         return ((Types == type) || ...);
@@ -79,6 +80,23 @@ struct Transaction {
     ripple::uint256 id;
     std::string key;  // key is the above id as a string of 32 characters
     ripple::TxType type;
+
+    /**
+     * @brief Compares Transaction objects to each other without considering sttx and meta fields
+     * @param other The Transaction to compare to
+     * @return true if transaction is equivalent; false otherwise
+     */
+    bool
+    operator==(Transaction const& other) const
+    {
+        return raw == other.raw                                           //
+            and metaRaw == other.metaRaw                                  //
+            and sttx.getTransactionID() == other.sttx.getTransactionID()  //
+            and meta.getTxID() == other.meta.getTxID()                    //
+            and id == other.id                                            //
+            and key == other.key                                          //
+            and type == other.type;
+    }
 };
 
 /**
@@ -103,6 +121,9 @@ struct Object {
     std::string predecessor;
 
     ModType type;
+
+    bool
+    operator==(Object const&) const = default;
 };
 
 /**
@@ -111,6 +132,9 @@ struct Object {
 struct BookSuccessor {
     std::string firstBook;
     std::string bookBase;
+
+    bool
+    operator==(BookSuccessor const&) const = default;
 };
 
 /**
@@ -120,10 +144,50 @@ struct LedgerData {
     std::vector<Transaction> transactions;
     std::vector<Object> objects;
     std::optional<std::vector<BookSuccessor>> successors;
+    std::optional<std::vector<std::string>> edgeKeys;
 
     ripple::LedgerHeader header;
     std::string rawHeader;
     uint32_t seq;
+
+    /**
+     * @brief Compares LedgerData objects to each other without considering the header field
+     * @param other The LedgerData to compare to
+     * @return true if data is equivalent; false otherwise
+     */
+    bool
+    operator==(LedgerData const& other) const
+    {
+        auto const serialized = [](auto const& hdr) {
+            ripple::Serializer ser;
+            ripple::addRaw(hdr, ser);
+            return ser.getString();
+        };
+
+        return transactions == other.transactions               //
+            and objects == other.objects                        //
+            and successors == other.successors                  //
+            and edgeKeys == other.edgeKeys                      //
+            and serialized(header) == serialized(other.header)  //
+            and rawHeader == other.rawHeader                    //
+            and seq == other.seq;
+    }
+};
+
+/**
+ * @brief Represents a task for the extractors
+ */
+struct Task {
+    /**
+     * @brief Represents the priority of the task
+     */
+    enum class Priority : uint8_t {
+        Lower = 0u,
+        Higher = 1u,
+    };
+
+    Priority priority;
+    uint32_t seq;
 };
 
 }  // namespace etlng::model

src/etlng/SchedulerInterface.hpp

@@ -0,0 +1,42 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief The interface of a scheduler for the extraction proccess
+ */
+struct SchedulerInterface {
+    virtual ~SchedulerInterface() = default;
+
+    /**
+     * @brief Attempt to obtain the next task
+     * @return A task if one exists; std::nullopt otherwise
+     */
+    [[nodiscard]] virtual std::optional<model::Task>
+    next() = 0;
+};
+
+}  // namespace etlng

src/etlng/impl/AmendmentBlockHandler.cpp

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/AmendmentBlockHandler.hpp"
+
+#include "etl/SystemState.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/log/Logger.hpp"
+
+#include <chrono>
+#include <functional>
+#include <utility>
+
+namespace etlng::impl {
+
+AmendmentBlockHandler::ActionType const AmendmentBlockHandler::kDEFAULT_AMENDMENT_BLOCK_ACTION = []() {
+    static util::Logger const log{"ETL"};  // NOLINT(readability-identifier-naming)
+    LOG(log.fatal()) << "Can't process new ledgers: The current ETL source is not compatible with the version of "
+                     << "the libxrpl Clio is currently using. Please upgrade Clio to a newer version.";
+};
+
+AmendmentBlockHandler::AmendmentBlockHandler(
+    util::async::AnyExecutionContext&& ctx,
+    etl::SystemState& state,
+    std::chrono::steady_clock::duration interval,
+    ActionType action
+)
+    : state_{std::ref(state)}, interval_{interval}, ctx_{std::move(ctx)}, action_{std::move(action)}
+{
+}
+
+void
+AmendmentBlockHandler::notifyAmendmentBlocked()
+{
+    state_.get().isAmendmentBlocked = true;
+    if (not operation_.has_value())
+        operation_.emplace(ctx_.executeRepeatedly(interval_, action_));
+}
+
+}  // namespace etlng::impl

src/etlng/impl/AmendmentBlockHandler.hpp

@@ -0,0 +1,69 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/SystemState.hpp"
+#include "etlng/AmendmentBlockHandlerInterface.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/async/AnyOperation.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/steady_timer.hpp>
+
+#include <chrono>
+#include <functional>
+#include <optional>
+
+namespace etlng::impl {
+
+class AmendmentBlockHandler : public AmendmentBlockHandlerInterface {
+public:
+    using ActionType = std::function<void()>;
+
+private:
+    std::reference_wrapper<etl::SystemState> state_;
+    std::chrono::steady_clock::duration interval_;
+    util::async::AnyExecutionContext ctx_;
+    std::optional<util::async::AnyOperation<void>> operation_;
+
+    ActionType action_;
+
+public:
+    static ActionType const kDEFAULT_AMENDMENT_BLOCK_ACTION;
+
+    AmendmentBlockHandler(
+        util::async::AnyExecutionContext&& ctx,
+        etl::SystemState& state,
+        std::chrono::steady_clock::duration interval = std::chrono::seconds{1},
+        ActionType action = kDEFAULT_AMENDMENT_BLOCK_ACTION
+    );
+
+    ~AmendmentBlockHandler() override
+    {
+        if (operation_.has_value())
+            operation_.value().abort();
+    }
+
+    void
+    notifyAmendmentBlocked() override;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/AsyncGrpcCall.cpp

@@ -0,0 +1,188 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/AsyncGrpcCall.hpp"
+
+#include "etl/ETLHelpers.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/Models.hpp"
+#include "etlng/impl/Extraction.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/client_context.h>
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/support/status.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+AsyncGrpcCall::AsyncGrpcCall(
+    uint32_t seq,
+    ripple::uint256 const& marker,
+    std::optional<ripple::uint256> const& nextMarker
+)
+{
+    request_.set_user("ETL");
+    request_.mutable_ledger()->set_sequence(seq);
+
+    if (marker.isNonZero())
+        request_.set_marker(marker.data(), ripple::uint256::size());
+
+    nextPrefix_ = nextMarker ? nextMarker->data()[0] : 0x00;
+    auto const prefix = marker.data()[0];
+
+    LOG(log_.debug()) << "Setting up AsyncGrpcCall. marker = " << ripple::strHex(marker)
+                      << ". prefix = " << ripple::strHex(std::string(1, prefix))
+                      << ". nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
+
+    ASSERT(
+        nextPrefix_ > prefix or nextPrefix_ == 0x00,
+        "Next prefix must be greater than current prefix. Got: nextPrefix_ = {}, prefix = {}",
+        nextPrefix_,
+        prefix
+    );
+
+    cur_ = std::make_unique<ResponseType>();
+    next_ = std::make_unique<ResponseType>();
+    context_ = std::make_unique<grpc::ClientContext>();
+}
+
+AsyncGrpcCall::CallStatus
+AsyncGrpcCall::process(
+    std::unique_ptr<AsyncGrpcCall::StubType>& stub,
+    grpc::CompletionQueue& cq,
+    etlng::InitialLoadObserverInterface& loader,
+    bool abort
+)
+{
+    LOG(log_.trace()) << "Processing response. "
+                      << "Marker prefix = " << getMarkerPrefix();
+
+    if (abort) {
+        LOG(log_.error()) << "AsyncGrpcCall aborted";
+        return CallStatus::Errored;
+    }
+
+    if (!status_.ok()) {
+        LOG(log_.error()) << "AsyncGrpcCall status_ not ok: code = " << status_.error_code()
+                          << " message = " << status_.error_message();
+
+        return CallStatus::Errored;
+    }
+
+    if (!next_->is_unlimited()) {
+        LOG(log_.warn()) << "AsyncGrpcCall is_unlimited is false. "
+                         << "Make sure secure_gateway is set correctly at the ETL source";
+    }
+
+    std::swap(cur_, next_);
+    auto more = true;
+
+    // if no marker returned, we are done
+    if (cur_->marker().empty())
+        more = false;
+
+    // if returned marker is greater than our end, we are done
+    auto const prefix = cur_->marker()[0];
+    if (nextPrefix_ != 0x00 && prefix >= nextPrefix_)
+        more = false;
+
+    // if we are not done, make the next async call
+    if (more) {
+        request_.set_marker(cur_->marker());
+        call(stub, cq);
+    }
+
+    auto const numObjects = cur_->ledger_objects().objects_size();
+    std::vector<etlng::model::Object> data;
+    data.reserve(numObjects);
+
+    for (int i = 0; i < numObjects; ++i) {
+        auto obj = std::move(*(cur_->mutable_ledger_objects()->mutable_objects(i)));
+        if (!more && nextPrefix_ != 0x00) {
+            if (static_cast<unsigned char>(obj.key()[0]) >= nextPrefix_)
+                continue;
+        }
+
+        lastKey_ = obj.key();  // this will end up the last key we actually touched eventually
+        data.push_back(etlng::impl::extractObj(std::move(obj)));
+    }
+
+    if (not data.empty())
+        loader.onInitialLoadGotMoreObjects(request_.ledger().sequence(), data, predecessorKey_);
+
+    predecessorKey_ = lastKey_;  // but for ongoing onInitialObjects calls we need to pass along the key we left
+                                 // off at so that we can link the two lists correctly
+
+    return more ? CallStatus::More : CallStatus::Done;
+}
+
+void
+AsyncGrpcCall::call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
+{
+    context_ = std::make_unique<grpc::ClientContext>();
+    auto rpc = stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq);
+
+    rpc->StartCall();
+    rpc->Finish(next_.get(), &status_, this);
+}
+
+std::string
+AsyncGrpcCall::getMarkerPrefix()
+{
+    return next_->marker().empty() ? std::string{} : ripple::strHex(std::string{next_->marker().data()[0]});
+}
+
+// this is used to generate edgeKeys - keys that were the last one in the onInitialObjects list
+// then we write them all in one go getting the successor from the cache once it's full
+std::string
+AsyncGrpcCall::getLastKey()
+{
+    return lastKey_;
+}
+
+std::vector<AsyncGrpcCall>
+AsyncGrpcCall::makeAsyncCalls(uint32_t const sequence, uint32_t const numMarkers)
+{
+    auto const markers = etl::getMarkers(numMarkers);
+
+    std::vector<AsyncGrpcCall> result;
+    result.reserve(markers.size());
+
+    for (size_t i = 0; i + 1 < markers.size(); ++i)
+        result.emplace_back(sequence, markers[i], markers[i + 1]);
+
+    if (not markers.empty())
+        result.emplace_back(sequence, markers.back(), std::nullopt);
+
+    return result;
+}
+
+}  // namespace etlng::impl

src/etlng/impl/AsyncGrpcCall.hpp

@@ -0,0 +1,85 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/client_context.h>
+#include <grpcpp/support/status.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger_data.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+class AsyncGrpcCall {
+public:
+    enum class CallStatus { More, Done, Errored };
+    using RequestType = org::xrpl::rpc::v1::GetLedgerDataRequest;
+    using ResponseType = org::xrpl::rpc::v1::GetLedgerDataResponse;
+    using StubType = org::xrpl::rpc::v1::XRPLedgerAPIService::Stub;
+
+private:
+    util::Logger log_{"ETL"};
+
+    std::unique_ptr<ResponseType> cur_;
+    std::unique_ptr<ResponseType> next_;
+
+    RequestType request_;
+    std::unique_ptr<grpc::ClientContext> context_;
+
+    grpc::Status status_;
+    unsigned char nextPrefix_;
+
+    std::string lastKey_;
+    std::optional<std::string> predecessorKey_;
+
+public:
+    AsyncGrpcCall(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker);
+
+    static std::vector<AsyncGrpcCall>
+    makeAsyncCalls(uint32_t const sequence, uint32_t const numMarkers);
+
+    CallStatus
+    process(
+        std::unique_ptr<StubType>& stub,
+        grpc::CompletionQueue& cq,
+        etlng::InitialLoadObserverInterface& loader,
+        bool abort
+    );
+
+    void
+    call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq);
+
+    std::string
+    getMarkerPrefix();
+
+    std::string
+    getLastKey();
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Extraction.cpp

@@ -0,0 +1,219 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/Extraction.hpp"
+
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/Models.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+model::Object::ModType
+extractModType(PBModType type)
+{
+    switch (type) {
+        case PBObjType::UNSPECIFIED:
+            return model::Object::ModType::Unspecified;
+        case PBObjType::CREATED:
+            return model::Object::ModType::Created;
+        case PBObjType::MODIFIED:
+            return model::Object::ModType::Modified;
+        case PBObjType::DELETED:
+            return model::Object::ModType::Deleted;
+        default:  // some gRPC system values that we don't care about
+            ASSERT(false, "Tried to extract bogus mod type '{}'", PBObjType::ModificationType_Name(type));
+    }
+
+    std::unreachable();
+}
+
+model::Transaction
+extractTx(PBTxType tx, uint32_t seq)
+{
+    auto raw = std::move(*tx.mutable_transaction_blob());
+    ripple::SerialIter it{raw.data(), raw.size()};
+    ripple::STTx const sttx{it};
+    ripple::TxMeta meta{sttx.getTransactionID(), seq, tx.metadata_blob()};
+
+    return {
+        .raw = std::move(raw),
+        .metaRaw = std::move(*tx.mutable_metadata_blob()),
+        .sttx = sttx,  // trivially copyable
+        .meta = std::move(meta),
+        .id = sttx.getTransactionID(),
+        .key = uint256ToString(sttx.getTransactionID()),
+        .type = sttx.getTxnType()
+    };
+}
+
+std::vector<model::Transaction>
+extractTxs(PBTxListType transactions, uint32_t seq)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::Transaction> output;
+    output.reserve(transactions.size());
+
+    rg::move(transactions | vs::transform([seq](auto&& tx) { return extractTx(tx, seq); }), std::back_inserter(output));
+    return output;
+}
+
+model::Object
+extractObj(PBObjType obj)
+{
+    auto const key = ripple::uint256::fromVoidChecked(obj.key());
+    ASSERT(key.has_value(), "Failed to deserialize key from void");
+
+    auto const valueOr = [](std::string const& maybe, std::string fallback) -> std::string {
+        if (maybe.empty())
+            return fallback;
+        return maybe;
+    };
+
+    return {
+        .key = *key,  // trivially copyable
+        .keyRaw = std::move(*obj.mutable_key()),
+        .data = {obj.mutable_data()->begin(), obj.mutable_data()->end()},
+        .dataRaw = std::move(*obj.mutable_data()),
+        .successor = valueOr(obj.successor(), uint256ToString(data::kFIRST_KEY)),
+        .predecessor = valueOr(obj.predecessor(), uint256ToString(data::kLAST_KEY)),
+        .type = extractModType(obj.mod_type()),
+    };
+}
+
+std::vector<model::Object>
+extractObjs(PBObjListType objects)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::Object> output;
+    output.reserve(objects.size());
+
+    rg::move(objects | vs::transform([](auto&& obj) { return extractObj(obj); }), std::back_inserter(output));
+    return output;
+}
+
+model::BookSuccessor
+extractSuccessor(PBBookSuccessorType successor)
+{
+    return {
+        .firstBook = successor.first_book(),
+        .bookBase = successor.book_base(),
+    };
+}
+
+std::optional<std::vector<model::BookSuccessor>>
+maybeExtractSuccessors(PBLedgerResponseType const& data)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    if (not data.object_neighbors_included())
+        return std::nullopt;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::BookSuccessor> output;
+    output.reserve(data.book_successors_size());
+
+    rg::copy(
+        data.book_successors() | vs::transform([](auto&& obj) { return extractSuccessor(obj); }),
+        std::back_inserter(output)
+    );
+    return output;
+}
+
+auto
+Extractor::unpack()
+{
+    return [](auto&& data) {
+        auto header = ::util::deserializeHeader(ripple::makeSlice(data.ledger_header()));
+
+        return std::make_optional<model::LedgerData>({
+            .transactions =
+                extractTxs(std::move(*data.mutable_transactions_list()->mutable_transactions()), header.seq),
+            .objects = extractObjs(std::move(*data.mutable_ledger_objects()->mutable_objects())),
+            .successors = maybeExtractSuccessors(data),
+            .edgeKeys = std::nullopt,
+            .header = header,
+            .rawHeader = std::move(*data.mutable_ledger_header()),
+            .seq = header.seq,
+        });
+    };
+}
+
+std::optional<model::LedgerData>
+Extractor::extractLedgerWithDiff(uint32_t seq)
+{
+    LOG(log_.debug()) << "Extracting DIFF " << seq;
+
+    auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
+        return fetcher_->fetchDataAndDiff(seq).and_then(unpack());
+    });
+
+    LOG(log_.debug()) << "Extracted and Transformed diff for " << seq << " in " << time << "ms";
+
+    // can be nullopt. this means that either the server is stopping or another node took over ETL writing.
+    return batch;
+}
+
+std::optional<model::LedgerData>
+Extractor::extractLedgerOnly(uint32_t seq)
+{
+    LOG(log_.debug()) << "Extracting FULL " << seq;
+
+    auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
+        return fetcher_->fetchData(seq).and_then(unpack());
+    });
+
+    LOG(log_.debug()) << "Extracted and Transformed full ledger for " << seq << " in " << time << "ms";
+
+    // can be nullopt. this means that either the server is stopping or another node took over ETL writing.
+    return batch;
+}
+
+}  // namespace etlng::impl

src/etlng/impl/Extraction.hpp

@@ -0,0 +1,100 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/ExtractorInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <google/protobuf/repeated_ptr_field.h>
+#include <sys/types.h>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+using PBObjType = org::xrpl::rpc::v1::RawLedgerObject;
+using PBModType = PBObjType::ModificationType;
+using PBTxType = org::xrpl::rpc::v1::TransactionAndMetadata;
+using PBTxListType = google::protobuf::RepeatedPtrField<PBTxType>;
+using PBObjListType = google::protobuf::RepeatedPtrField<PBObjType>;
+using PBBookSuccessorType = org::xrpl::rpc::v1::BookSuccessor;
+using PBLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+
+[[nodiscard]] model::Object::ModType
+extractModType(PBModType type);
+
+[[nodiscard]] model::Transaction
+extractTx(PBTxType tx, uint32_t seq);
+
+[[nodiscard]] std::vector<model::Transaction>
+extractTxs(PBTxListType transactions, uint32_t seq);
+
+[[nodiscard]] model::Object
+extractObj(PBObjType obj);
+
+[[nodiscard]] std::vector<model::Object>
+extractObjs(PBObjListType objects);
+
+[[nodiscard]] model::BookSuccessor
+extractSuccessor(PBBookSuccessorType successor);
+
+[[nodiscard]] std::optional<std::vector<model::BookSuccessor>>
+maybeExtractSuccessors(PBLedgerResponseType const& data);
+
+// fetches the data in gRPC and transforms to local representation
+class Extractor : public ExtractorInterface {
+    std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
+
+    util::Logger log_{"ETL"};
+
+private:
+    [[nodiscard]] static auto
+    unpack();
+
+public:
+    Extractor(std::shared_ptr<etl::LedgerFetcherInterface> fetcher) : fetcher_(std::move(fetcher))
+    {
+    }
+
+    [[nodiscard]] std::optional<model::LedgerData>
+    extractLedgerWithDiff(uint32_t seq) override;
+
+    [[nodiscard]] std::optional<model::LedgerData>
+    extractLedgerOnly(uint32_t seq) override;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/GrpcSource.cpp

@@ -0,0 +1,157 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/GrpcSource.hpp"
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/impl/AsyncGrpcCall.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+#include "web/Resolver.hpp"
+
+#include <fmt/core.h>
+#include <grpcpp/client_context.h>
+#include <grpcpp/security/credentials.h>
+#include <grpcpp/support/channel_arguments.h>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+std::string
+resolve(std::string const& ip, std::string const& port)
+{
+    web::Resolver resolver;
+
+    if (auto const results = resolver.resolve(ip, port); not results.empty())
+        return results.at(0);
+
+    throw std::runtime_error("Failed to resolve " + ip + ":" + port);
+}
+
+}  // namespace
+
+namespace etlng::impl {
+
+GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort)
+    : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort))
+{
+    try {
+        grpc::ChannelArguments chArgs;
+        chArgs.SetMaxReceiveMessageSize(-1);
+
+        stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
+            grpc::CreateCustomChannel(resolve(ip, grpcPort), grpc::InsecureChannelCredentials(), chArgs)
+        );
+
+        LOG(log_.debug()) << "Made stub for remote.";
+    } catch (std::exception const& e) {
+        LOG(log_.warn()) << "Exception while creating stub: " << e.what() << ".";
+    }
+}
+
+std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors)
+{
+    org::xrpl::rpc::v1::GetLedgerResponse response;
+    if (!stub_)
+        return {{grpc::StatusCode::INTERNAL, "No Stub"}, response};
+
+    // Ledger header with txns and metadata
+    org::xrpl::rpc::v1::GetLedgerRequest request;
+    grpc::ClientContext context;
+
+    request.mutable_ledger()->set_sequence(sequence);
+    request.set_transactions(true);
+    request.set_expand(true);
+    request.set_get_objects(getObjects);
+    request.set_get_object_neighbors(getObjectNeighbors);
+    request.set_user("ETL");
+
+    grpc::Status const status = stub_->GetLedger(&context, request, &response);
+
+    if (status.ok() and not response.is_unlimited()) {
+        log_.warn() << "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. Status = "
+                    << status.error_message();
+    }
+
+    return {status, std::move(response)};
+}
+
+std::pair<std::vector<std::string>, bool>
+GrpcSource::loadInitialLedger(
+    uint32_t const sequence,
+    uint32_t const numMarkers,
+    etlng::InitialLoadObserverInterface& observer
+)
+{
+    if (!stub_)
+        return {{}, false};
+
+    std::vector<AsyncGrpcCall> calls = AsyncGrpcCall::makeAsyncCalls(sequence, numMarkers);
+
+    LOG(log_.debug()) << "Starting data download for ledger " << sequence << ".";
+
+    grpc::CompletionQueue queue;
+    for (auto& call : calls)
+        call.call(stub_, queue);
+
+    std::vector<std::string> edgeKeys;
+    void* tag = nullptr;
+    bool ok = false;
+    bool abort = false;
+    size_t numFinished = 0;
+
+    while (numFinished < calls.size() && queue.Next(&tag, &ok)) {
+        ASSERT(tag != nullptr, "Tag can't be null.");
+        auto ptr = static_cast<AsyncGrpcCall*>(tag);
+
+        if (!ok) {
+            LOG(log_.error()) << "loadInitialLedger - ok is false";
+            return {{}, false};  // cancelled
+        }
+
+        LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix();
+
+        auto result = ptr->process(stub_, queue, observer, abort);
+        if (result != AsyncGrpcCall::CallStatus::More) {
+            ++numFinished;
+            LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished;
+
+            if (auto lastKey = ptr->getLastKey(); !lastKey.empty())
+                edgeKeys.push_back(std::move(lastKey));
+        }
+
+        if (result == AsyncGrpcCall::CallStatus::Errored)
+            abort = true;
+    }
+
+    return {std::move(edgeKeys), !abort};
+}
+
+}  // namespace etlng::impl

src/etlng/impl/GrpcSource.hpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+class GrpcSource {
+    util::Logger log_;
+    std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub> stub_;
+
+public:
+    GrpcSource(std::string const& ip, std::string const& grpcPort);
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param sequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
+     * @param getObjectNeighbors Whether to request object neighbors; defaults to false
+     * @return A std::pair of the response status and the response itself
+     */
+    std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+    fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false);
+
+    /**
+     * @brief Download a ledger in full.
+     *
+     * @param sequence Sequence of the ledger to download
+     * @param numMarkers Number of markers to generate for async calls
+     * @param observer InitialLoadObserverInterface implementation
+     * @return A std::pair of the data and a bool indicating whether the download was successful
+     */
+    std::pair<std::vector<std::string>, bool>
+    loadInitialLedger(uint32_t sequence, uint32_t numMarkers, etlng::InitialLoadObserverInterface& observer);
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Loading.cpp

@@ -0,0 +1,111 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/Loading.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "etlng/AmendmentBlockHandlerInterface.hpp"
+#include "etlng/Models.hpp"
+#include "etlng/RegistryInterface.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/protocol/LedgerHeader.h>
+
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+Loader::Loader(
+    std::shared_ptr<BackendInterface> backend,
+    std::shared_ptr<etl::LedgerFetcherInterface> fetcher,
+    std::shared_ptr<RegistryInterface> registry,
+    std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler
+)
+    : backend_(std::move(backend))
+    , fetcher_(std::move(fetcher))
+    , registry_(std::move(registry))
+    , amendmentBlockHandler_(std::move(amendmentBlockHandler))
+{
+}
+
+void
+Loader::load(model::LedgerData const& data)
+{
+    try {
+        // perform cache updates and all writes from extensions
+        registry_->dispatch(data);
+
+        auto [success, duration] =
+            ::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(data.seq); });
+        LOG(log_.info()) << "Finished writes to DB for " << data.seq << ": " << (success ? "YES" : "NO") << "; took "
+                         << duration;
+    } catch (std::runtime_error const& e) {
+        LOG(log_.fatal()) << "Failed to load " << data.seq << ": " << e.what();
+        amendmentBlockHandler_->notifyAmendmentBlocked();
+    }
+};
+
+void
+Loader::onInitialLoadGotMoreObjects(
+    uint32_t seq,
+    std::vector<model::Object> const& data,
+    std::optional<std::string> lastKey
+)
+{
+    LOG(log_.debug()) << "On initial load: got more objects for seq " << seq << ". size = " << data.size();
+    registry_->dispatchInitialObjects(
+        seq, data, std::move(lastKey).value_or(std::string{})  // TODO: perhaps use optional all the way to extensions?
+    );
+}
+
+std::optional<ripple::LedgerHeader>
+Loader::loadInitialLedger(model::LedgerData const& data)
+{
+    // check that database is actually empty
+    auto rng = backend_->hardFetchLedgerRangeNoThrow();
+    if (rng) {
+        ASSERT(false, "Database is not empty");
+        return std::nullopt;
+    }
+
+    LOG(log_.debug()) << "Deserialized ledger header. " << ::util::toString(data.header);
+
+    auto seconds = ::util::timed<std::chrono::seconds>([this, &data]() { registry_->dispatchInitialData(data); });
+    LOG(log_.info()) << "Dispatching initial data and submitting all writes took " << seconds << " seconds.";
+
+    backend_->finishWrites(data.seq);
+    LOG(log_.debug()) << "Loaded initial ledger";
+
+    return {data.header};
+}
+
+}  // namespace etlng::impl

src/etlng/impl/Loading.hpp

@@ -0,0 +1,85 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerLoader.hpp"
+#include "etlng/AmendmentBlockHandlerInterface.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/LoaderInterface.hpp"
+#include "etlng/Models.hpp"
+#include "etlng/RegistryInterface.hpp"
+#include "util/log/Logger.hpp"
+
+#include <org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+class Loader : public LoaderInterface, public InitialLoadObserverInterface {
+    std::shared_ptr<BackendInterface> backend_;
+    std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
+    std::shared_ptr<RegistryInterface> registry_;
+    std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler_;
+
+    util::Logger log_{"ETL"};
+
+public:
+    using RawLedgerObjectType = org::xrpl::rpc::v1::RawLedgerObject;
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+    Loader(
+        std::shared_ptr<BackendInterface> backend,
+        std::shared_ptr<etl::LedgerFetcherInterface> fetcher,
+        std::shared_ptr<RegistryInterface> registry,
+        std::shared_ptr<AmendmentBlockHandlerInterface> amendmentBlockHandler
+    );
+
+    void
+    load(model::LedgerData const& data) override;
+
+    void
+    onInitialLoadGotMoreObjects(
+        uint32_t seq,
+        std::vector<model::Object> const& data,
+        std::optional<std::string> lastKey
+    ) override;
+
+    std::optional<ripple::LedgerHeader>
+    loadInitialLedger(model::LedgerData const& data) override;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Registry.hpp

@@ -72,7 +72,7 @@ concept HasInitialObjectHook = requires(T p) {
 };
 
 template <typename T>
-concept ContainsSpec = std::decay_t<T>::spec::SpecTag;
+concept ContainsSpec = std::decay_t<T>::spec::kSPEC_TAG;
 
 template <typename T>
 concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or

src/etlng/impl/Scheduling.hpp

@@ -0,0 +1,151 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2025, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/NetworkValidatedLedgersInterface.hpp"
+#include "etlng/Models.hpp"
+#include "etlng/SchedulerInterface.hpp"
+
+#include <sys/types.h>
+
+#include <atomic>
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+namespace etlng::impl {
+
+template <typename T>
+concept SomeScheduler = std::is_base_of_v<SchedulerInterface, std::decay_t<T>>;
+
+class ForwardScheduler : public SchedulerInterface {
+    std::reference_wrapper<etl::NetworkValidatedLedgersInterface> ledgers_;
+
+    uint32_t startSeq_;
+    std::optional<uint32_t> maxSeq_;
+    std::atomic_uint32_t seq_;
+
+public:
+    ForwardScheduler(ForwardScheduler const& other)
+        : ledgers_(other.ledgers_), startSeq_(other.startSeq_), maxSeq_(other.maxSeq_), seq_(other.seq_.load())
+    {
+    }
+
+    ForwardScheduler(
+        std::reference_wrapper<etl::NetworkValidatedLedgersInterface> ledgers,
+        uint32_t startSeq,
+        std::optional<uint32_t> maxSeq = std::nullopt
+    )
+        : ledgers_(ledgers), startSeq_(startSeq), maxSeq_(maxSeq), seq_(startSeq)
+    {
+    }
+
+    [[nodiscard]] std::optional<model::Task>
+    next() override
+    {
+        static constexpr auto kMAX = std::numeric_limits<uint32_t>::max();
+        uint32_t currentSeq = seq_;
+
+        if (ledgers_.get().getMostRecent() >= currentSeq) {
+            while (currentSeq < maxSeq_.value_or(kMAX)) {
+                if (seq_.compare_exchange_weak(currentSeq, currentSeq + 1u, std::memory_order_acq_rel)) {
+                    return {{.priority = model::Task::Priority::Higher, .seq = currentSeq}};
+                }
+            }
+        }
+
+        return std::nullopt;
+    }
+};
+
+class BackfillScheduler : public SchedulerInterface {
+    uint32_t startSeq_;
+    uint32_t minSeq_ = 0u;
+
+    std::atomic_uint32_t seq_;
+
+public:
+    BackfillScheduler(BackfillScheduler const& other)
+        : startSeq_(other.startSeq_), minSeq_(other.minSeq_), seq_(other.seq_.load())
+    {
+    }
+
+    BackfillScheduler(uint32_t startSeq, std::optional<uint32_t> minSeq = std::nullopt)
+        : startSeq_(startSeq), minSeq_(minSeq.value_or(0)), seq_(startSeq)
+    {
+    }
+
+    [[nodiscard]] std::optional<model::Task>
+    next() override
+    {
+        uint32_t currentSeq = seq_;
+        while (currentSeq > minSeq_) {
+            if (seq_.compare_exchange_weak(currentSeq, currentSeq - 1u, std::memory_order_acq_rel)) {
+                return {{.priority = model::Task::Priority::Lower, .seq = currentSeq}};
+            }
+        }
+
+        return std::nullopt;
+    }
+};
+
+template <SomeScheduler... Schedulers>
+class SchedulerChain : public SchedulerInterface {
+    std::tuple<Schedulers...> schedulers_;
+
+public:
+    template <SomeScheduler... Ts>
+        requires(std::is_same_v<Ts, Schedulers> and ...)
+    SchedulerChain(Ts&&... schedulers) : schedulers_(std::forward<Ts>(schedulers)...)
+    {
+    }
+
+    [[nodiscard]] std::optional<model::Task>
+    next() override
+    {
+        std::optional<model::Task> task;
+        auto const expand = [&](auto& s) {
+            if (task.has_value())
+                return false;
+
+            task = s.next();
+            return task.has_value();
+        };
+
+        std::apply([&expand](auto&&... xs) { (... || expand(xs)); }, schedulers_);
+
+        return task;
+    }
+};
+
+static auto
+makeScheduler(SomeScheduler auto&&... schedulers)
+{
+    return std::make_unique<SchedulerChain<std::decay_t<decltype(schedulers)>...>>(
+        std::forward<decltype(schedulers)>(schedulers)...
+    );
+}
+
+}  // namespace etlng::impl

src/feed/SubscriptionManager.hpp

@@ -30,8 +30,8 @@
 #include "feed/impl/TransactionFeed.hpp"
 #include "util/async/AnyExecutionContext.hpp"
 #include "util/async/context/BasicExecutionContext.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/io_context.hpp>
@@ -77,9 +77,12 @@ public:
      * @return A shared pointer to a new instance of SubscriptionManager
      */
     static std::shared_ptr<SubscriptionManager>
-    make_SubscriptionManager(util::Config const& config, std::shared_ptr<data::BackendInterface const> const& backend)
+    makeSubscriptionManager(
+        util::config::ClioConfigDefinition const& config,
+        std::shared_ptr<data::BackendInterface const> const& backend
+    )
     {
-        auto const workersNum = config.valueOr<std::uint64_t>("subscription_workers", 1);
+        auto const workersNum = config.get<uint64_t>("subscription_workers");
 
         util::Logger const logger{"Subscriptions"};
         LOG(logger.info()) << "Starting subscription manager with " << workersNum << " workers";

src/feed/Types.hpp

@@ -19,12 +19,13 @@
 
 #pragma once
 
-#include "web/interface/ConnectionBase.hpp"
+#include "web/SubscriptionContextInterface.hpp"
 
 #include <memory>
 
 namespace feed {
-using Subscriber = web::ConnectionBase;
+
+using Subscriber = web::SubscriptionContextInterface;
 using SubscriberPtr = Subscriber*;
 using SubscriberSharedPtr = std::shared_ptr<Subscriber>;
 

src/feed/impl/ProposedTransactionFeed.cpp

@@ -48,7 +48,7 @@ ProposedTransactionFeed::sub(SubscriberSharedPtr const& subscriber)
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed tx_proposed";
         ++subAllCount_.get();
-        subscriber->onDisconnect.connect([this](SubscriberPtr connection) { unsubInternal(connection); });
+        subscriber->onDisconnect([this](SubscriberPtr connection) { unsubInternal(connection); });
     }
 }
 
@@ -73,9 +73,7 @@ ProposedTransactionFeed::sub(ripple::AccountID const& account, SubscriberSharedP
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed accounts_proposed " << account;
         ++subAccountCount_.get();
-        subscriber->onDisconnect.connect([this, account](SubscriberPtr connection) {
-            unsubInternal(account, connection);
-        });
+        subscriber->onDisconnect([this, account](SubscriberPtr connection) { unsubInternal(account, connection); });
     }
 }