feat: Repeating operations for util::async (#1776)

Async framework needed a way to do repeating operations (think simplest
cases like AmendmentBlockHandler).
This PR implements the **absolute minimum**, barebones repeating
operations that
- Can't return any values (void)
- Do not take any arguments in the user-provided function
- Can't be scheduled (i.e. a delay before starting repeating)
- Can't be stopped from inside the user block of code (i.e. does not
have stop token or anything like that)
- Can be stopped through the RepeatedOperation's `abort()` function (but
not from the user-provided repeating function)

.clang-tidy

@@ -8,6 +8,7 @@ Checks: '-*,
   bugprone-chained-comparison,
   bugprone-compare-pointer-to-member-virtual-function,
   bugprone-copy-constructor-init,
+  bugprone-crtp-constructor-accessibility,
   bugprone-dangling-handle,
   bugprone-dynamic-static-initializers,
   bugprone-empty-catch,
@@ -33,9 +34,11 @@ Checks: '-*,
   bugprone-non-zero-enum-to-bool-conversion,
   bugprone-optional-value-conversion,
   bugprone-parent-virtual-call,
+  bugprone-pointer-arithmetic-on-polymorphic-object,
   bugprone-posix-return,
   bugprone-redundant-branch-condition,
   bugprone-reserved-identifier,
+  bugprone-return-const-ref-from-parameter,
   bugprone-shared-ptr-array-mismatch,
   bugprone-signal-handler,
   bugprone-signed-char-misuse,
@@ -55,6 +58,7 @@ Checks: '-*,
   bugprone-suspicious-realloc-usage,
   bugprone-suspicious-semicolon,
   bugprone-suspicious-string-compare,
+  bugprone-suspicious-stringview-data-usage,
   bugprone-swapped-arguments,
   bugprone-switch-missing-default-case,
   bugprone-terminating-continue,
@@ -97,10 +101,12 @@ Checks: '-*,
   modernize-make-unique,
   modernize-pass-by-value,
   modernize-type-traits,
+  modernize-use-designated-initializers,
   modernize-use-emplace,
   modernize-use-equals-default,
   modernize-use-equals-delete,
   modernize-use-override,
+  modernize-use-ranges,
   modernize-use-starts-ends-with,
   modernize-use-std-numbers,
   modernize-use-using,
@@ -121,9 +127,11 @@ Checks: '-*,
   readability-convert-member-functions-to-static,
   readability-duplicate-include,
   readability-else-after-return,
+  readability-enum-initial-value,
   readability-implicit-bool-conversion,
   readability-inconsistent-declaration-parameter-name,
   readability-make-member-function-const,
+  readability-math-missing-parentheses,
   readability-misleading-indentation,
   readability-non-const-parameter,
   readability-redundant-casting,
@@ -135,7 +143,8 @@ Checks: '-*,
   readability-simplify-boolean-expr,
   readability-static-accessed-through-instance,
   readability-static-definition-in-anonymous-namespace,
-  readability-suspicious-call-argument
+  readability-suspicious-call-argument,
+  readability-use-std-min-max
   '
 
 CheckOptions:

.githooks/check-format

@@ -26,12 +26,12 @@ sources="src tests"
 formatter="clang-format -i"
 version=$($formatter --version | grep -o '[0-9\.]*')
 
-if [[ "18.0.0" > "$version" ]]; then
+if [[ "19.0.0" > "$version" ]]; then
     cat <<EOF
 
                                     ERROR
 -----------------------------------------------------------------------------
-            A minimum of version 18 of `which clang-format` is required.
+            A minimum of version 19 of `which clang-format` is required.
             Your version is $version.
             Please fix paths and run again.
 -----------------------------------------------------------------------------

.githooks/pre-push

@@ -42,7 +42,7 @@ verify_tag_signed() {
 while read local_ref local_oid remote_ref remote_oid; do
     # Check some things if we're pushing a branch called "release/"
     if echo "$remote_ref" | grep ^refs\/heads\/release\/ &> /dev/null ; then
-        version=$(echo $remote_ref | awk -F/ '{print $NF}')
+        version=$(git tag --points-at HEAD)
         echo "Looks like you're trying to push a $version release..."
         echo "Making sure you've signed and tagged it."
         if verify_commit_signed && verify_tag && verify_tag_signed ; then

.github/workflows/build.yml

@@ -149,13 +149,6 @@ jobs:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
           path: build/clio_*tests
 
-      - name: Upload test data
-        if: ${{ !matrix.code_coverage }}
-        uses: actions/upload-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}_${{ steps.conan.outputs.conan_profile }}
-          path: build/tests/unit/test_data
-
       - name: Save cache
         uses: ./.github/actions/save_cache
         with:
@@ -219,11 +212,6 @@ jobs:
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}_${{ matrix.conan_profile }}
-          path: tests/unit/test_data
-
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/check_pr_title.yml

@@ -10,7 +10,7 @@ jobs:
     # permissions:
     #   pull-requests: write
     steps:
-      - uses: ytanikin/PRConventionalCommits@1.2.0
+      - uses: ytanikin/PRConventionalCommits@1.3.0
         with:
           task_types: '["build","feat","fix","docs","test","ci","style","refactor","perf","chore"]'
           add_label: false

.github/workflows/clang-tidy.yml

@@ -60,7 +60,7 @@ jobs:
         shell: bash
         id: run_clang_tidy
         run: |
-          run-clang-tidy-18 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
+          run-clang-tidy-19 -p build -j ${{ steps.number_of_threads.outputs.threads_number }} -fix -quiet 1>output.txt
 
       - name: Check format
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
@@ -99,7 +99,7 @@ jobs:
 
       - name: Create PR with fixes
         if: ${{ steps.run_clang_tidy.outcome != 'success' }}
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         env:
           GH_REPO: ${{ github.repository }}
           GH_TOKEN: ${{ github.token }}

.github/workflows/nightly.yml

@@ -71,12 +71,6 @@ jobs:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
           path: build/clio_*tests
 
-      - name: Upload test data
-        uses: actions/upload-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}
-          path: build/tests/unit/test_data
-
       - name: Compress clio_server
         shell: bash
         run: |
@@ -130,11 +124,6 @@ jobs:
         with:
           name: clio_tests_${{ runner.os }}_${{ matrix.build_type }}
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: clio_test_data_${{ runner.os }}_${{ matrix.build_type }}
-          path: tests/unit/test_data
-
       - name: Run clio_tests
         run: |
           chmod +x ./clio_tests

.github/workflows/upload_coverage_report.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Upload coverage report
         if: ${{ hashFiles('build/coverage_report.xml') != '' }}
-        uses: wandalen/wretry.action@v3.5.0
+        uses: wandalen/wretry.action@v3.7.3
         with:
           action: codecov/codecov-action@v4
           with: |

CONTRIBUTING.md

@@ -21,7 +21,7 @@ git config --local core.hooksPath .githooks
 ```
 
 ## Git hooks dependencies
-The pre-commit hook requires `clang-format >= 18.0.0` and `cmake-format` to be installed on your machine.
+The pre-commit hook requires `clang-format >= 19.0.0` and `cmake-format` to be installed on your machine.
 `clang-format` can be installed using `brew` on macOS and default package manager on Linux.
 `cmake-format` can be installed using `pip`.
 The hook will also attempt to automatically use `doxygen` to verify that everything public in the codebase is covered by doc comments. If `doxygen` is not installed, the hook will raise a warning suggesting to install `doxygen` for future commits.
@@ -105,7 +105,7 @@ The button for that is near the bottom of the PR's page on GitHub.
 This is a non-exhaustive list of recommended style guidelines. These are not always strictly enforced and serve as a way to keep the codebase coherent.
 
 ## Formatting
-Code must conform to `clang-format` version 18, unless the result would be unreasonably difficult to read or maintain.
+Code must conform to `clang-format` version 19, unless the result would be unreasonably difficult to read or maintain.
 In most cases the pre-commit hook will take care of formatting and will fix any issues automatically.
 To manually format your code, use `clang-format -i <your changed files>` for C++ files and `cmake-format -i <your changed files>` for CMake files.
 

benchmarks/util/async/ExecutionContextBenchmarks.cpp

@@ -208,7 +208,7 @@ benchmarkThreads(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();
@@ -219,7 +219,7 @@ benchmarkExecutionContextBatched(benchmark::State& state)
 }
 
 template <typename CtxType>
-void
+static void
 benchmarkAnyExecutionContextBatched(benchmark::State& state)
 {
     auto data = generateData();

cmake/ClangTidy.cmake

@@ -8,7 +8,7 @@ if (lint)
     endif ()
     message(STATUS "Using clang-tidy from CLIO_CLANG_TIDY_BIN")
   else ()
-    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-18" "clang-tidy" REQUIRED)
+    find_program(_CLANG_TIDY_BIN NAMES "clang-tidy-19" "clang-tidy" REQUIRED)
   endif ()
 
   if (NOT _CLANG_TIDY_BIN)

conanfile.py

@@ -28,7 +28,8 @@ class Clio(ConanFile):
         'protobuf/3.21.9',
         'grpc/1.50.1',
         'openssl/1.1.1u',
-        'xrpl/2.3.0-b1',
+        'xrpl/2.4.0-b1',
+        'zlib/1.3.1',
         'libbacktrace/cci.20210118'
     ]
 

docker/ci/dockerfile

@@ -7,7 +7,7 @@ USER root
 WORKDIR /root
 
 ENV CCACHE_VERSION=4.10.2 \
-    LLVM_TOOLS_VERSION=18 \
+    LLVM_TOOLS_VERSION=19 \
     GH_VERSION=2.40.0 \
     DOXYGEN_VERSION=1.12.0
  

docs/examples/config/cloud-example-config.json

@@ -1,43 +0,0 @@
-/*
- * This is an example configuration file. Please do not use without modifying to suit your needs.
- */
-{
-    "database": {
-        "type": "cassandra",
-        "cassandra": {
-            // This option can be used to setup a secure connect bundle connection
-            "secure_connect_bundle": "[path/to/zip. ignore if using contact_points]",
-            // The following options are used only if using contact_points
-            "contact_points": "[ip. ignore if using secure_connect_bundle]",
-            "port": "[port. ignore if using_secure_connect_bundle]",
-            // Authentication settings
-            "username": "[username, if any]",
-            "password": "[password, if any]",
-            // Other common settings
-            "keyspace": "clio",
-            "max_write_requests_outstanding": 25000,
-            "max_read_requests_outstanding": 30000,
-            "threads": 8
-        }
-    },
-    "etl_sources": [
-        {
-            "ip": "[rippled ip]",
-            "ws_port": "6006",
-            "grpc_port": "50051"
-        }
-    ],
-    "dos_guard": {
-        "whitelist": [
-            "127.0.0.1"
-        ]
-    },
-    "server": {
-        "ip": "0.0.0.0",
-        "port": 8080
-    },
-    "log_level": "debug",
-    "log_file": "./clio.log",
-    "extractor_threads": 8,
-    "read_only": false
-}

docs/examples/config/example-config.json

@@ -39,6 +39,9 @@
         "cache_timeout": 0.250, // in seconds, could be 0, which means no cache
         "request_timeout": 10.0 // time for Clio to wait for rippled to reply on a forwarded request (default is 10 seconds)
     },
+    "rpc": {
+        "cache_timeout": 0.5 // in seconds, could be 0, which means no cache for rpc
+    },
     "dos_guard": {
         // Comma-separated list of IPs to exclude from rate limiting
         "whitelist": [
@@ -67,7 +70,15 @@
         "admin_password": "xrp",
         // If local_admin is true, Clio will consider requests come from 127.0.0.1 as admin requests
         // It's true by default unless admin_password is set,'local_admin' : true and 'admin_password' can not be set at the same time
-        "local_admin": false
+        "local_admin": false,
+        "processing_policy": "parallel", // Could be "sequent" or "parallel".
+        // For sequent policy request from one client connection will be processed one by one and the next one will not be read before
+        // the previous one is processed. For parallel policy Clio will take all requests and process them in parallel and
+        // send a reply for each request whenever it is ready.
+        "parallel_requests_limit": 10, // Optional parameter, used only if "processing_strategy" is "parallel". It limits the number of requests for one client connection processed in parallel. Infinite if not specified.
+        // Max number of responses to queue up before sent successfully. If a client's waiting queue is too long, the server will close the connection.
+        "ws_max_sending_queue_size": 1500,
+        "__ng_web_server": false // Use ng web server. This is a temporary setting which will be deleted after switching to ng web server
     },
     // Time in seconds for graceful shutdown. Defaults to 10 seconds. Not fully implemented yet.
     "graceful_period": 10.0,

docs/metrics-and-static-analysis.md

@@ -14,7 +14,7 @@ You can find an example docker-compose file, with Prometheus and Grafana configs
 
 ## Using `clang-tidy` for static analysis
 
-The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 17.0.
+The minimum [clang-tidy](https://clang.llvm.org/extra/clang-tidy/) version required is 19.0.
 
 Clang-tidy can be run by Cmake when building the project. To achieve this, you just need to provide the option `-o lint=True` for the `conan install` command:
 
@@ -26,5 +26,5 @@ By default Cmake will try to find `clang-tidy` automatically in your system.
 To force Cmake to use your desired binary, set the `CLIO_CLANG_TIDY_BIN` environment variable to the path of the `clang-tidy` binary. For example:
 
 ```sh
-export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@17/bin/clang-tidy
+export CLIO_CLANG_TIDY_BIN=/opt/homebrew/opt/llvm@19/bin/clang-tidy
 ```

src/CMakeLists.txt

@@ -1,8 +1,10 @@
 add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(etl)
+add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(migration)
 add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
-target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
+target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp WebHandlers.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration)

src/app/CliArgs.cpp

@@ -19,6 +19,7 @@
 
 #include "app/CliArgs.hpp"
 
+#include "migration/MigrationApplication.hpp"
 #include "util/build/Build.hpp"
 
 #include <boost/program_options/options_description.hpp>
@@ -44,6 +45,8 @@ CliArgs::parse(int argc, char const* argv[])
         ("help,h", "print help message and exit")
         ("version,v", "print version and exit")
         ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
+        ("ng-web-server,w", "Use ng-web-server")
+        ("migrate", po::value<std::string>(), "start migration helper")
     ;
     // clang-format on
     po::positional_options_description positional;
@@ -64,7 +67,16 @@ CliArgs::parse(int argc, char const* argv[])
     }
 
     auto configPath = parsed["conf"].as<std::string>();
-    return Action{Action::Run{std::move(configPath)}};
+
+    if (parsed.count("migrate") != 0u) {
+        auto const opt = parsed["migrate"].as<std::string>();
+        if (opt == "status")
+            return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::status()}};
+        return Action{Action::Migrate{.configPath = std::move(configPath), .subCmd = MigrateSubCmd::migration(opt)}};
+    }
+
+    return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
+    };
 }
 
 }  // namespace app

src/app/CliArgs.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "migration/MigrationApplication.hpp"
 #include "util/OverloadSet.hpp"
 
 #include <string>
@@ -43,14 +44,19 @@ public:
     public:
         /** @brief Run action. */
         struct Run {
-            /** @brief Configuration file path. */
-            std::string configPath;
+            std::string configPath;  ///< Configuration file path.
+            bool useNgWebServer;     ///< Whether to use a ng web server
         };
 
         /** @brief Exit action. */
         struct Exit {
-            /** @brief Exit code. */
-            int exitCode;
+            int exitCode;  ///< Exit code.
+        };
+
+        /** @brief Migration action. */
+        struct Migrate {
+            std::string configPath;
+            MigrateSubCmd subCmd;
         };
 
         /**
@@ -59,7 +65,8 @@ public:
          * @param action Run action.
          */
         template <typename ActionType>
-            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit> or
+            std::is_same_v<ActionType, Migrate>
         explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
         {
         }
@@ -79,7 +86,7 @@ public:
         }
 
     private:
-        std::variant<Run, Exit> action_;
+        std::variant<Run, Exit, Migrate> action_;
     };
 
     /**

src/app/ClioApplication.cpp

@@ -19,6 +19,7 @@
 
 #include "app/ClioApplication.hpp"
 
+#include "app/WebHandlers.hpp"
 #include "data/AmendmentCenter.hpp"
 #include "data/BackendFactory.hpp"
 #include "etl/ETLService.hpp"
@@ -30,21 +31,26 @@
 #include "rpc/WorkQueue.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
 #include "util/build/Build.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 #include "util/prometheus/Prometheus.hpp"
+#include "web/AdminVerificationStrategy.hpp"
 #include "web/RPCServerHandler.hpp"
 #include "web/Server.hpp"
 #include "web/dosguard/DOSGuard.hpp"
 #include "web/dosguard/IntervalSweepHandler.hpp"
 #include "web/dosguard/WhitelistHandler.hpp"
+#include "web/ng/RPCServerHandler.hpp"
+#include "web/ng/Server.hpp"
 
 #include <boost/asio/io_context.hpp>
 
 #include <cstdint>
 #include <cstdlib>
 #include <memory>
+#include <optional>
 #include <thread>
+#include <utility>
 #include <vector>
 
 namespace app {
@@ -72,20 +78,17 @@ start(boost::asio::io_context& ioc, std::uint32_t numThreads)
 
 }  // namespace
 
-ClioApplication::ClioApplication(util::Config const& config) : config_(config), signalsHandler_{config_}
+ClioApplication::ClioApplication(util::config::ClioConfigDefinition const& config)
+    : config_(config), signalsHandler_{config_}
 {
     LOG(util::LogService::info()) << "Clio version: " << util::build::getClioFullVersionString();
     PrometheusService::init(config);
 }
 
 int
-ClioApplication::run()
+ClioApplication::run(bool const useNgWebServer)
 {
-    auto const threads = config_.valueOr("io_threads", 2);
-    if (threads <= 0) {
-        LOG(util::LogService::fatal()) << "io_threads is less than 1";
-        return EXIT_FAILURE;
-    }
+    auto const threads = config_.get<uint16_t>("io_threads");
     LOG(util::LogService::info()) << "Number of io threads = " << threads;
 
     // IO context to handle all incoming requests, as well as other things.
@@ -101,9 +104,7 @@ ClioApplication::run()
     auto backend = data::make_Backend(config_);
 
     // Manages clients subscribed to streams
-    auto subscriptionsRunner = feed::SubscriptionManagerRunner(config_, backend);
-
-    auto const subscriptions = subscriptionsRunner.getManager();
+    auto subscriptions = feed::SubscriptionManager::make_SubscriptionManager(config_, backend);
 
     // Tracks which ledgers have been validated by the network
     auto ledgers = etl::NetworkValidatedLedgers::make_ValidatedLedgers();
@@ -123,12 +124,52 @@ ClioApplication::run()
     auto const handlerProvider = std::make_shared<rpc::impl::ProductionHandlerProvider const>(
         config_, backend, subscriptions, balancer, etl, amendmentCenter, counters
     );
+
+    using RPCEngineType = rpc::RPCEngine<etl::LoadBalancer, rpc::Counters>;
     auto const rpcEngine =
-        rpc::RPCEngine::make_RPCEngine(backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+        RPCEngineType::make_RPCEngine(config_, backend, balancer, dosGuard, workQueue, counters, handlerProvider);
+
+    if (useNgWebServer or config_.get<bool>("server.__ng_web_server")) {
+        web::ng::RPCServerHandler<RPCEngineType, etl::ETLService> handler{config_, backend, rpcEngine, etl};
+
+        auto expectedAdminVerifier = web::make_AdminVerificationStrategy(config_);
+        if (not expectedAdminVerifier.has_value()) {
+            LOG(util::LogService::error()) << "Error creating admin verifier: " << expectedAdminVerifier.error();
+            return EXIT_FAILURE;
+        }
+        auto const adminVerifier = std::move(expectedAdminVerifier).value();
+
+        auto httpServer = web::ng::make_Server(config_, OnConnectCheck{dosGuard}, DisconnectHook{dosGuard}, ioc);
+
+        if (not httpServer.has_value()) {
+            LOG(util::LogService::error()) << "Error creating web server: " << httpServer.error();
+            return EXIT_FAILURE;
+        }
+
+        httpServer->onGet("/metrics", MetricsHandler{adminVerifier});
+        httpServer->onGet("/health", HealthCheckHandler{});
+        auto requestHandler = RequestHandler{adminVerifier, handler, dosGuard};
+        httpServer->onPost("/", requestHandler);
+        httpServer->onWs(std::move(requestHandler));
+
+        auto const maybeError = httpServer->run();
+        if (maybeError.has_value()) {
+            LOG(util::LogService::error()) << "Error starting web server: " << *maybeError;
+            return EXIT_FAILURE;
+        }
+
+        // Blocks until stopped.
+        // When stopped, shared_ptrs fall out of scope
+        // Calls destructors on all resources, and destructs in order
+        start(ioc, threads);
+
+        return EXIT_SUCCESS;
+    }
 
     // Init the web server
     auto handler =
-        std::make_shared<web::RPCServerHandler<rpc::RPCEngine, etl::ETLService>>(config_, backend, rpcEngine, etl);
+        std::make_shared<web::RPCServerHandler<RPCEngineType, etl::ETLService>>(config_, backend, rpcEngine, etl);
+
     auto const httpServer = web::make_HttpServer(config_, ioc, dosGuard, handler);
 
     // Blocks until stopped.

src/app/ClioApplication.hpp

@@ -18,9 +18,8 @@
 //==============================================================================
 
 #pragma once
-
 #include "util/SignalsHandler.hpp"
-#include "util/config//Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 namespace app {
 
@@ -28,7 +27,7 @@ namespace app {
  * @brief The main application class
  */
 class ClioApplication {
-    util::Config const& config_;
+    util::config::ClioConfigDefinition const& config_;
     util::SignalsHandler signalsHandler_;
 
 public:
@@ -37,15 +36,17 @@ public:
      *
      * @param config The configuration of the application
      */
-    ClioApplication(util::Config const& config);
+    ClioApplication(util::config::ClioConfigDefinition const& config);
 
     /**
      * @brief Run the application
      *
+     * @param useNgWebServer Whether to use the new web server
+     *
      * @return exit code
      */
     int
-    run();
+    run(bool useNgWebServer);
 };
 
 }  // namespace app

src/app/WebHandlers.cpp

@@ -0,0 +1,111 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "app/WebHandlers.hpp"
+
+#include "util/Assert.hpp"
+#include "util/prometheus/Http.hpp"
+#include "web/AdminVerificationStrategy.hpp"
+#include "web/SubscriptionContextInterface.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
+#include "web/ng/Connection.hpp"
+#include "web/ng/Request.hpp"
+#include "web/ng/Response.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/beast/http/status.hpp>
+
+#include <memory>
+#include <optional>
+#include <utility>
+
+namespace app {
+
+OnConnectCheck::OnConnectCheck(web::dosguard::DOSGuardInterface& dosguard) : dosguard_{dosguard}
+{
+}
+
+std::expected<void, web::ng::Response>
+OnConnectCheck::operator()(web::ng::Connection const& connection)
+{
+    dosguard_.get().increment(connection.ip());
+    if (not dosguard_.get().isOk(connection.ip())) {
+        return std::unexpected{
+            web::ng::Response{boost::beast::http::status::too_many_requests, "Too many requests", connection}
+        };
+    }
+
+    return {};
+}
+
+DisconnectHook::DisconnectHook(web::dosguard::DOSGuardInterface& dosguard) : dosguard_{dosguard}
+{
+}
+
+void
+DisconnectHook::operator()(web::ng::Connection const& connection)
+{
+    dosguard_.get().decrement(connection.ip());
+}
+
+MetricsHandler::MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier)
+    : adminVerifier_{std::move(adminVerifier)}
+{
+}
+
+web::ng::Response
+MetricsHandler::operator()(
+    web::ng::Request const& request,
+    web::ng::ConnectionMetadata& connectionMetadata,
+    web::SubscriptionContextPtr,
+    boost::asio::yield_context
+)
+{
+    auto const maybeHttpRequest = request.asHttpRequest();
+    ASSERT(maybeHttpRequest.has_value(), "Got not a http request in Get");
+    auto const& httpRequest = maybeHttpRequest->get();
+
+    // FIXME(#1702): Using veb server thread to handle prometheus request. Better to post on work queue.
+    auto maybeResponse = util::prometheus::handlePrometheusRequest(
+        httpRequest, adminVerifier_->isAdmin(httpRequest, connectionMetadata.ip())
+    );
+    ASSERT(maybeResponse.has_value(), "Got unexpected request for Prometheus");
+    return web::ng::Response{std::move(maybeResponse).value(), request};
+}
+
+web::ng::Response
+HealthCheckHandler::operator()(
+    web::ng::Request const& request,
+    web::ng::ConnectionMetadata&,
+    web::SubscriptionContextPtr,
+    boost::asio::yield_context
+)
+{
+    static auto constexpr HealthCheckHTML = R"html(
+    <!DOCTYPE html>
+    <html>
+        <head><title>Test page for Clio</title></head>
+        <body><h1>Clio Test</h1><p>This page shows Clio http(s) connectivity is working.</p></body>
+    </html>
+)html";
+
+    return web::ng::Response{boost::beast::http::status::ok, HealthCheckHTML, request};
+}
+
+}  // namespace app

src/app/WebHandlers.hpp

@@ -0,0 +1,234 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "rpc/Errors.hpp"
+#include "util/log/Logger.hpp"
+#include "web/AdminVerificationStrategy.hpp"
+#include "web/SubscriptionContextInterface.hpp"
+#include "web/dosguard/DOSGuardInterface.hpp"
+#include "web/ng/Connection.hpp"
+#include "web/ng/Request.hpp"
+#include "web/ng/Response.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <boost/beast/http/status.hpp>
+#include <boost/json/array.hpp>
+#include <boost/json/parse.hpp>
+
+#include <exception>
+#include <functional>
+#include <memory>
+#include <utility>
+
+namespace app {
+
+/**
+ * @brief A function object that checks if the connection is allowed to proceed.
+ */
+class OnConnectCheck {
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new OnConnectCheck object
+     *
+     * @param dosguard The DOSGuardInterface to use for checking the connection.
+     */
+    OnConnectCheck(web::dosguard::DOSGuardInterface& dosguard);
+
+    /**
+     * @brief Check if the connection is allowed to proceed.
+     *
+     * @param connection The connection to check.
+     * @return A response if the connection is not allowed to proceed or void otherwise.
+     */
+    std::expected<void, web::ng::Response>
+    operator()(web::ng::Connection const& connection);
+};
+
+/**
+ * @brief A function object to be called when a connection is disconnected.
+ */
+class DisconnectHook {
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new DisconnectHook object
+     *
+     * @param dosguard The DOSGuardInterface to use for disconnecting the connection.
+     */
+    DisconnectHook(web::dosguard::DOSGuardInterface& dosguard);
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param connection The connection which has disconnected.
+     */
+    void
+    operator()(web::ng::Connection const& connection);
+};
+
+/**
+ * @brief A function object that handles the metrics endpoint.
+ */
+class MetricsHandler {
+    std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
+
+public:
+    /**
+     * @brief Construct a new MetricsHandler object
+     *
+     * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
+     */
+    MetricsHandler(std::shared_ptr<web::AdminVerificationStrategy> adminVerifier);
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @param connectionMetadata The connection metadata.
+     * @return The response to the request.
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata& connectionMetadata,
+        web::SubscriptionContextPtr,
+        boost::asio::yield_context
+    );
+};
+
+/**
+ * @brief A function object that handles the health check endpoint.
+ */
+class HealthCheckHandler {
+public:
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @return The response to the request
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata&,
+        web::SubscriptionContextPtr,
+        boost::asio::yield_context
+    );
+};
+
+/**
+ * @brief A function object that handles the websocket endpoint.
+ *
+ * @tparam RpcHandlerType The type of the RPC handler.
+ */
+template <typename RpcHandlerType>
+class RequestHandler {
+    util::Logger webServerLog_{"WebServer"};
+    std::shared_ptr<web::AdminVerificationStrategy> adminVerifier_;
+    std::reference_wrapper<RpcHandlerType> rpcHandler_;
+    std::reference_wrapper<web::dosguard::DOSGuardInterface> dosguard_;
+
+public:
+    /**
+     * @brief Construct a new RequestHandler object
+     *
+     * @param adminVerifier The AdminVerificationStrategy to use for verifying the connection for admin access.
+     * @param rpcHandler The RPC handler to use for handling the request.
+     * @param dosguard The DOSGuardInterface to use for checking the connection.
+     */
+    RequestHandler(
+        std::shared_ptr<web::AdminVerificationStrategy> adminVerifier,
+        RpcHandlerType& rpcHandler,
+        web::dosguard::DOSGuardInterface& dosguard
+    )
+        : adminVerifier_(std::move(adminVerifier)), rpcHandler_(rpcHandler), dosguard_(dosguard)
+    {
+    }
+
+    /**
+     * @brief The call of the function object.
+     *
+     * @param request The request to handle.
+     * @param connectionMetadata The connection metadata.
+     * @param subscriptionContext The subscription context.
+     * @param yield The yield context.
+     * @return The response to the request.
+     */
+    web::ng::Response
+    operator()(
+        web::ng::Request const& request,
+        web::ng::ConnectionMetadata& connectionMetadata,
+        web::SubscriptionContextPtr subscriptionContext,
+        boost::asio::yield_context yield
+    )
+    {
+        if (not dosguard_.get().request(connectionMetadata.ip())) {
+            auto error = rpc::makeError(rpc::RippledError::rpcSLOW_DOWN);
+
+            if (not request.isHttp()) {
+                try {
+                    auto requestJson = boost::json::parse(request.message());
+                    if (requestJson.is_object() && requestJson.as_object().contains("id"))
+                        error["id"] = requestJson.as_object().at("id");
+                    error["request"] = request.message();
+                } catch (std::exception const&) {
+                    error["request"] = request.message();
+                }
+            }
+            return web::ng::Response{boost::beast::http::status::service_unavailable, error, request};
+        }
+        LOG(webServerLog_.info()) << connectionMetadata.tag()
+                                  << "Received request from ip = " << connectionMetadata.ip()
+                                  << " - posting to WorkQueue";
+
+        connectionMetadata.setIsAdmin([this, &request, &connectionMetadata]() {
+            return adminVerifier_->isAdmin(request.httpHeaders(), connectionMetadata.ip());
+        });
+
+        try {
+            auto response = rpcHandler_(request, connectionMetadata, std::move(subscriptionContext), yield);
+
+            if (not dosguard_.get().add(connectionMetadata.ip(), response.message().size())) {
+                auto jsonResponse = boost::json::parse(response.message()).as_object();
+                jsonResponse["warning"] = "load";
+                if (jsonResponse.contains("warnings") && jsonResponse["warnings"].is_array()) {
+                    jsonResponse["warnings"].as_array().push_back(rpc::makeWarning(rpc::warnRPC_RATE_LIMIT));
+                } else {
+                    jsonResponse["warnings"] = boost::json::array{rpc::makeWarning(rpc::warnRPC_RATE_LIMIT)};
+                }
+                response.setMessage(jsonResponse);
+            }
+
+            return response;
+        } catch (std::exception const&) {
+            return web::ng::Response{
+                boost::beast::http::status::internal_server_error,
+                rpc::makeError(rpc::RippledError::rpcINTERNAL),
+                request
+            };
+        }
+    }
+};
+
+}  // namespace app

src/data/AmendmentCenter.hpp

@@ -124,6 +124,13 @@ struct Amendments {
     REGISTER(NFTokenMintOffer);
     REGISTER(fixReducedOffersV2);
     REGISTER(fixEnforceNFTokenTrustline);
+    REGISTER(fixInnerObjTemplate2);
+    REGISTER(fixNFTokenPageLinks);
+    REGISTER(InvariantsV1_1);
+    REGISTER(MPTokensV1);
+    REGISTER(fixAMMv1_2);
+    REGISTER(AMMClawback);
+    REGISTER(Credentials);
 
     // Obsolete but supported by libxrpl
     REGISTER(CryptoConditionsSuite);

src/data/BackendFactory.hpp

@@ -22,8 +22,8 @@
 #include "data/BackendInterface.hpp"
 #include "data/CassandraBackend.hpp"
 #include "data/cassandra/SettingsProvider.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string.hpp>
 #include <boost/algorithm/string/predicate.hpp>
@@ -41,18 +41,18 @@ namespace data {
  * @return A shared_ptr<BackendInterface> with the selected implementation
  */
 inline std::shared_ptr<BackendInterface>
-make_Backend(util::Config const& config)
+make_Backend(util::config::ClioConfigDefinition const& config)
 {
     static util::Logger const log{"Backend"};
     LOG(log.info()) << "Constructing BackendInterface";
 
-    auto const readOnly = config.valueOr("read_only", false);
+    auto const readOnly = config.get<bool>("read_only");
 
-    auto const type = config.value<std::string>("database.type");
+    auto const type = config.get<std::string>("database.type");
     std::shared_ptr<BackendInterface> backend = nullptr;
 
     if (boost::iequals(type, "cassandra")) {
-        auto cfg = config.section("database." + type);
+        auto const cfg = config.getObject("database." + type);
         backend = std::make_shared<data::cassandra::CassandraBackend>(data::cassandra::SettingsProvider{cfg}, readOnly);
     }
 

src/data/BackendInterface.cpp

@@ -176,9 +176,9 @@ BackendInterface::fetchSuccessorObject(
     if (succ) {
         auto obj = fetchLedgerObject(*succ, ledgerSequence, yield);
         if (!obj)
-            return {{*succ, {}}};
+            return {{.key = *succ, .blob = {}}};
 
-        return {{*succ, *obj}};
+        return {{.key = *succ, .blob = *obj}};
     }
     return {};
 }
@@ -283,7 +283,7 @@ BackendInterface::updateRange(uint32_t newMax)
     );
 
     if (!range) {
-        range = {newMax, newMax};
+        range = {.minSequence = newMax, .maxSequence = newMax};
     } else {
         range->maxSequence = newMax;
     }
@@ -299,7 +299,7 @@ BackendInterface::setRange(uint32_t min, uint32_t max, bool force)
         ASSERT(not range.has_value(), "Range was already set");
     }
 
-    range = {min, max};
+    range = {.minSequence = min, .maxSequence = max};
 }
 
 LedgerPage

src/data/BackendInterface.hpp

@@ -364,6 +364,25 @@ public:
         boost::asio::yield_context yield
     ) const = 0;
 
+    /**
+     * @brief Fetches all holders' balances for a MPTIssuanceID
+     *
+     * @param mptID MPTIssuanceID you wish you query.
+     * @param limit Paging limit.
+     * @param cursorIn Optional cursor to allow us to pick up from where we last left off.
+     * @param ledgerSequence The ledger sequence to fetch for
+     * @param yield Currently executing coroutine.
+     * @return std::vector<Blob> of MPToken balances and an optional marker
+     */
+    virtual MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const = 0;
+
     /**
      * @brief Fetches a specific ledger object.
      *
@@ -529,6 +548,16 @@ public:
         boost::asio::yield_context yield
     ) const;
 
+    /**
+     * @brief Fetches the status of migrator by name.
+     *
+     * @param migratorName The name of the migrator
+     * @param yield The coroutine context
+     * @return The status of the migrator if found; nullopt otherwise
+     */
+    virtual std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
+
     /**
      * @brief Synchronously fetches the ledger range from DB.
      *
@@ -617,6 +646,14 @@ public:
     virtual void
     writeNFTTransactions(std::vector<NFTTransactionsData> const& data) = 0;
 
+    /**
+     * @brief Write accounts that started holding onto a MPT.
+     *
+     * @param data A vector of MPT ID and account pairs
+     */
+    virtual void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) = 0;
+
     /**
      * @brief Write a new successor.
      *
@@ -646,6 +683,15 @@ public:
     bool
     finishWrites(std::uint32_t ledgerSequence);
 
+    /**
+     * @brief Mark the migration status of a migrator as Migrated in the database
+     *
+     * @param migratorName The name of the migrator
+     * @param status The status to set
+     */
+    virtual void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;
+
     /**
      * @return true if database is overwhelmed; false otherwise
      */

src/data/CassandraBackend.hpp

@@ -72,13 +72,15 @@ class BasicCassandraBackend : public BackendInterface {
 
     SettingsProviderType settingsProvider_;
     Schema<SettingsProviderType> schema_;
+
+    std::atomic_uint32_t ledgerSequence_ = 0u;
+
+protected:
     Handle handle_;
 
     // have to be mutable because BackendInterface constness :(
     mutable ExecutionStrategyType executor_;
 
-    std::atomic_uint32_t ledgerSequence_ = 0u;
-
 public:
     /**
      * @brief Create a new cassandra/scylla backend instance.
@@ -93,7 +95,7 @@ public:
         , executor_{settingsProvider_.getSettings(), handle_}
     {
         if (auto const res = handle_.connect(); not res)
-            throw std::runtime_error("Could not connect to databse: " + res.error());
+            throw std::runtime_error("Could not connect to database: " + res.error());
 
         if (not readOnly) {
             if (auto const res = handle_.execute(schema_.createKeyspace); not res) {
@@ -128,7 +130,7 @@ public:
     {
         auto rng = fetchLedgerRange();
         if (!rng)
-            return {{}, {}};
+            return {.txns = {}, .cursor = {}};
 
         Statement const statement = [this, forward, &account]() {
             if (forward)
@@ -399,7 +401,7 @@ public:
     {
         auto rng = fetchLedgerRange();
         if (!rng)
-            return {{}, {}};
+            return {.txns = {}, .cursor = {}};
 
         Statement const statement = [this, forward, &tokenID]() {
             if (forward)
@@ -547,6 +549,45 @@ public:
         return ret;
     }
 
+    MPTHoldersAndCursor
+    fetchMPTHolders(
+        ripple::uint192 const& mptID,
+        std::uint32_t const limit,
+        std::optional<ripple::AccountID> const& cursorIn,
+        std::uint32_t const ledgerSequence,
+        boost::asio::yield_context yield
+    ) const override
+    {
+        auto const holderEntries = executor_.read(
+            yield, schema_->selectMPTHolders, mptID, cursorIn.value_or(ripple::AccountID(0)), Limit{limit}
+        );
+
+        auto const& holderResults = holderEntries.value();
+        if (not holderResults.hasRows()) {
+            LOG(log_.debug()) << "No rows returned";
+            return {};
+        }
+
+        std::vector<ripple::uint256> mptKeys;
+        std::optional<ripple::AccountID> cursor;
+        for (auto const [holder] : extract<ripple::AccountID>(holderResults)) {
+            mptKeys.push_back(ripple::keylet::mptoken(mptID, holder).key);
+            cursor = holder;
+        }
+
+        auto mptObjects = doFetchLedgerObjects(mptKeys, ledgerSequence, yield);
+
+        auto it = std::remove_if(mptObjects.begin(), mptObjects.end(), [](Blob const& mpt) { return mpt.empty(); });
+
+        mptObjects.erase(it, mptObjects.end());
+
+        ASSERT(mptKeys.size() <= limit, "Number of keys can't exceed the limit");
+        if (mptKeys.size() == limit)
+            return {mptObjects, cursor};
+
+        return {mptObjects, {}};
+    }
+
     std::optional<Blob>
     doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
         const override
@@ -796,6 +837,26 @@ public:
         return results;
     }
 
+    std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override
+    {
+        auto const res = executor_.read(yield, schema_->selectMigratorStatus, Text(migratorName));
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch migrator status: " << res.error();
+            return {};
+        }
+
+        auto const& results = res.value();
+        if (not results) {
+            return {};
+        }
+
+        for (auto [statusString] : extract<std::string>(results))
+            return statusString;
+
+        return {};
+    }
+
     void
     doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
     {
@@ -905,6 +966,17 @@ public:
         executor_.write(std::move(statements));
     }
 
+    void
+    writeMPTHolders(std::vector<MPTHolderData> const& data) override
+    {
+        std::vector<Statement> statements;
+        statements.reserve(data.size());
+        for (auto [mptId, holder] : data)
+            statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
+
+        executor_.write(std::move(statements));
+    }
+
     void
     startWrites() const override
     {
@@ -912,6 +984,14 @@ public:
         // probably was used in PG to start a transaction or smth.
     }
 
+    void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) override
+    {
+        executor_.writeSync(
+            schema_->insertMigratorStatus, data::cassandra::Text{migratorName}, data::cassandra::Text(status)
+        );
+    }
+
     bool
     isTooBusy() const override
     {

src/data/DBHelpers.hpp

@@ -172,6 +172,14 @@ struct NFTsData {
     }
 };
 
+/**
+ * @brief Represents an MPT and holder pair
+ */
+struct MPTHolderData {
+    ripple::uint192 mptID;
+    ripple::AccountID holder;
+};
+
 /**
  * @brief Check whether the supplied object is an offer.
  *

src/data/LedgerCache.cpp

@@ -75,7 +75,7 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
 
                 auto& e = map_[obj.key];
                 if (seq > e.seq) {
-                    e = {seq, obj.blob};
+                    e = {.seq = seq, .blob = obj.blob};
                 }
             } else {
                 map_.erase(obj.key);
@@ -101,7 +101,7 @@ LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
     if (e == map_.end())
         return {};
     ++successorHitCounter_.get();
-    return {{e->first, e->second.blob}};
+    return {{.key = e->first, .blob = e->second.blob}};
 }
 
 std::optional<LedgerObject>
@@ -117,7 +117,7 @@ LedgerCache::getPredecessor(ripple::uint256 const& key, uint32_t seq) const
     if (e == map_.begin())
         return {};
     --e;
-    return {{e->first, e->second.blob}};
+    return {{.key = e->first, .blob = e->second.blob}};
 }
 
 std::optional<Blob>

src/data/README.md

@@ -262,3 +262,15 @@ CREATE TABLE clio.nf_token_transactions (
 ```
 
 The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, inspired by the same motivations and fulfilling a similar role within this context. It drives the `nft_history` API.
+
+### migrator_status
+
+```
+CREATE TABLE clio.migrator_status (
+    migrator_name TEXT,     # The name of the migrator
+    status TEXT,            # The status of the migrator
+    PRIMARY KEY (migrator_name)
+) 
+```
+
+The `migrator_status` table stores the status of the migratior in this database. If a migrator's status is `migrated`, it means this database has finished data migration for this migrator.

src/data/Types.hpp

@@ -233,6 +233,14 @@ struct NFTsAndCursor {
     std::optional<ripple::uint256> cursor;
 };
 
+/**
+ * @brief Represents an array of MPTokens
+ */
+struct MPTHoldersAndCursor {
+    std::vector<Blob> mptokens;
+    std::optional<ripple::AccountID> cursor;
+};
+
 /**
  * @brief Stores a range of sequences as a min and max pair.
  */

src/data/cassandra/Handle.cpp

@@ -60,7 +60,7 @@ Handle::connect() const
 Handle::FutureType
 Handle::asyncConnect(std::string_view keyspace) const
 {
-    return cass_session_connect_keyspace(session_, cluster_, keyspace.data());
+    return cass_session_connect_keyspace_n(session_, cluster_, keyspace.data(), keyspace.size());
 }
 
 Handle::MaybeErrorType
@@ -155,7 +155,7 @@ Handle::asyncExecute(std::vector<StatementType> const& statements, std::function
 Handle::PreparedStatementType
 Handle::prepare(std::string_view query) const
 {
-    Handle::FutureType const future = cass_session_prepare(session_, query.data());
+    Handle::FutureType const future = cass_session_prepare_n(session_, query.data(), query.size());
     auto const rc = future.await();
     if (rc)
         return cass_future_get_prepared(future);

src/data/cassandra/Schema.hpp

@@ -257,6 +257,31 @@ public:
             qualifiedTableName(settingsProvider_.get(), "nf_token_transactions")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                    mpt_id blob,
+                    holder blob,
+                   PRIMARY KEY (mpt_id, holder)
+                  ) 
+             WITH CLUSTERING ORDER BY (holder ASC)
+            )",
+            qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+        ));
+
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                   migrator_name TEXT,
+                          status TEXT,
+                         PRIMARY KEY (migrator_name)
+                  ) 
+            )",
+            qualifiedTableName(settingsProvider_.get(), "migrator_status")
+        ));
+
         return statements;
     }();
 
@@ -393,6 +418,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMPTHolder = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {} 
+                       (mpt_id, holder)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement insertLedgerHeader = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(
@@ -442,6 +478,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {}
+                       (migrator_name, status)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
+
         //
         // Select queries
         //
@@ -687,6 +734,20 @@ public:
             ));
         }();
 
+        PreparedStatement selectMPTHolders = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT holder
+                  FROM {}    
+                 WHERE mpt_id = ?
+                   AND holder > ?
+              ORDER BY holder ASC
+                 LIMIT ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
+            ));
+        }();
+
         PreparedStatement selectLedgerByHash = [this]() {
             return handle_.get().prepare(fmt::format(
                 R"(
@@ -730,6 +791,17 @@ public:
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
         }();
+
+        PreparedStatement selectMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT status
+                  FROM {}
+                 WHERE migrator_name = ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
     };
 
     /**

src/data/cassandra/SettingsProvider.cpp

@@ -22,10 +22,7 @@
 #include "data/cassandra/Types.hpp"
 #include "data/cassandra/impl/Cluster.hpp"
 #include "util/Constants.hpp"
-#include "util/config/Config.hpp"
-
-#include <boost/json/conversion.hpp>
-#include <boost/json/value.hpp>
+#include "util/newconfig/ObjectView.hpp"
 
 #include <cerrno>
 #include <chrono>
@@ -36,43 +33,17 @@
 #include <ios>
 #include <iterator>
 #include <optional>
-#include <stdexcept>
 #include <string>
+#include <string_view>
 #include <system_error>
 
 namespace data::cassandra {
 
-namespace impl {
-inline Settings::ContactPoints
-tag_invoke(boost::json::value_to_tag<Settings::ContactPoints>, boost::json::value const& value)
-{
-    if (not value.is_object()) {
-        throw std::runtime_error("Feed entire Cassandra section to parse Settings::ContactPoints instead");
-    }
-
-    util::Config const obj{value};
-    Settings::ContactPoints out;
-
-    out.contactPoints = obj.valueOrThrow<std::string>("contact_points", "`contact_points` must be a string");
-    out.port = obj.maybeValue<uint16_t>("port");
-
-    return out;
-}
-
-inline Settings::SecureConnectionBundle
-tag_invoke(boost::json::value_to_tag<Settings::SecureConnectionBundle>, boost::json::value const& value)
-{
-    if (not value.is_string())
-        throw std::runtime_error("`secure_connect_bundle` must be a string");
-    return Settings::SecureConnectionBundle{value.as_string().data()};
-}
-}  // namespace impl
-
-SettingsProvider::SettingsProvider(util::Config const& cfg)
+SettingsProvider::SettingsProvider(util::config::ObjectView const& cfg)
     : config_{cfg}
-    , keyspace_{cfg.valueOr<std::string>("keyspace", "clio")}
+    , keyspace_{cfg.get<std::string>("keyspace")}
     , tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
-    , replicationFactor_{cfg.valueOr<uint16_t>("replication_factor", 3)}
+    , replicationFactor_{cfg.get<uint16_t>("replication_factor")}
     , settings_{parseSettings()}
 {
 }
@@ -86,8 +57,8 @@ SettingsProvider::getSettings() const
 std::optional<std::string>
 SettingsProvider::parseOptionalCertificate() const
 {
-    if (auto const certPath = config_.maybeValue<std::string>("certfile"); certPath) {
-        auto const path = std::filesystem::path(*certPath);
+    if (auto const certPath = config_.getValueView("certfile"); certPath.hasValue()) {
+        auto const path = std::filesystem::path(certPath.asString());
         std::ifstream fileStream(path.string(), std::ios::in);
         if (!fileStream) {
             throw std::system_error(errno, std::generic_category(), "Opening certificate " + path.string());
@@ -108,30 +79,34 @@ Settings
 SettingsProvider::parseSettings() const
 {
     auto settings = Settings::defaultSettings();
-    if (auto const bundle = config_.maybeValue<Settings::SecureConnectionBundle>("secure_connect_bundle"); bundle) {
-        settings.connectionInfo = *bundle;
+
+    // all config values used in settings is under "database.cassandra" prefix
+    if (config_.getValueView("secure_connect_bundle").hasValue()) {
+        auto const bundle = Settings::SecureConnectionBundle{(config_.get<std::string>("secure_connect_bundle"))};
+        settings.connectionInfo = bundle;
     } else {
-        settings.connectionInfo =
-            config_.valueOrThrow<Settings::ContactPoints>("Missing contact_points in Cassandra config");
+        Settings::ContactPoints out;
+        out.contactPoints = config_.get<std::string>("contact_points");
+        out.port = config_.maybeValue<uint32_t>("port");
+        settings.connectionInfo = out;
     }
 
-    settings.threads = config_.valueOr<uint32_t>("threads", settings.threads);
-    settings.maxWriteRequestsOutstanding =
-        config_.valueOr<uint32_t>("max_write_requests_outstanding", settings.maxWriteRequestsOutstanding);
-    settings.maxReadRequestsOutstanding =
-        config_.valueOr<uint32_t>("max_read_requests_outstanding", settings.maxReadRequestsOutstanding);
-    settings.coreConnectionsPerHost =
-        config_.valueOr<uint32_t>("core_connections_per_host", settings.coreConnectionsPerHost);
+    settings.threads = config_.get<uint32_t>("threads");
+    settings.maxWriteRequestsOutstanding = config_.get<uint32_t>("max_write_requests_outstanding");
+    settings.maxReadRequestsOutstanding = config_.get<uint32_t>("max_read_requests_outstanding");
+    settings.coreConnectionsPerHost = config_.get<uint32_t>("core_connections_per_host");
     settings.queueSizeIO = config_.maybeValue<uint32_t>("queue_size_io");
-    settings.writeBatchSize = config_.valueOr<std::size_t>("write_batch_size", settings.writeBatchSize);
+    settings.writeBatchSize = config_.get<std::size_t>("write_batch_size");
 
-    auto const connectTimeoutSecond = config_.maybeValue<uint32_t>("connect_timeout");
-    if (connectTimeoutSecond)
-        settings.connectionTimeout = std::chrono::milliseconds{*connectTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    if (config_.getValueView("connect_timeout").hasValue()) {
+        auto const connectTimeoutSecond = config_.get<uint32_t>("connect_timeout");
+        settings.connectionTimeout = std::chrono::milliseconds{connectTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    }
 
-    auto const requestTimeoutSecond = config_.maybeValue<uint32_t>("request_timeout");
-    if (requestTimeoutSecond)
-        settings.requestTimeout = std::chrono::milliseconds{*requestTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    if (config_.getValueView("request_timeout").hasValue()) {
+        auto const requestTimeoutSecond = config_.get<uint32_t>("request_timeout");
+        settings.requestTimeout = std::chrono::milliseconds{requestTimeoutSecond * util::MILLISECONDS_PER_SECOND};
+    }
 
     settings.certificate = parseOptionalCertificate();
     settings.username = config_.maybeValue<std::string>("username");

src/data/cassandra/SettingsProvider.hpp

@@ -19,10 +19,9 @@
 
 #pragma once
 
-#include "data/cassandra/Handle.hpp"
 #include "data/cassandra/Types.hpp"
-#include "util/config/Config.hpp"
-#include "util/log/Logger.hpp"
+#include "data/cassandra/impl/Cluster.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <cstdint>
 #include <optional>
@@ -34,7 +33,7 @@ namespace data::cassandra {
  * @brief Provides settings for @ref BasicCassandraBackend.
  */
 class SettingsProvider {
-    util::Config config_;
+    util::config::ObjectView config_;
 
     std::string keyspace_;
     std::optional<std::string> tablePrefix_;
@@ -47,7 +46,7 @@ public:
      *
      * @param cfg The config of Clio to use
      */
-    explicit SettingsProvider(util::Config const& cfg);
+    explicit SettingsProvider(util::config::ObjectView const& cfg);
 
     /**
      * @return The cluster settings

src/data/cassandra/Types.hpp

@@ -21,6 +21,8 @@
 
 #include <cstdint>
 #include <expected>
+#include <string>
+#include <utility>
 
 namespace data::cassandra {
 
@@ -55,6 +57,26 @@ struct Limit {
     int32_t limit;
 };
 
+/**
+ * @brief A strong type wrapper for string
+ *
+ * This is unfortunately needed right now to support TEXT properly
+ * because clio uses string to represent BLOB
+ * If we want to bind TEXT with string, we need to use this type
+ */
+struct Text {
+    std::string text;
+
+    /**
+     * @brief Construct a new Text object from string type
+     *
+     * @param text The text to wrap
+     */
+    explicit Text(std::string text) : text{std::move(text)}
+    {
+    }
+};
+
 class Handle;
 class CassandraError;
 

src/data/cassandra/impl/Statement.hpp

@@ -54,7 +54,7 @@ public:
      */
     template <typename... Args>
     explicit Statement(std::string_view query, Args&&... args)
-        : ManagedObject{cass_statement_new(query.data(), sizeof...(args)), deleter}
+        : ManagedObject{cass_statement_new_n(query.data(), query.size(), sizeof...(args)), deleter}
     {
         cass_statement_set_consistency(*this, CASS_CONSISTENCY_QUORUM);
         cass_statement_set_is_idempotent(*this, cass_true);
@@ -106,9 +106,9 @@ public:
         using UintByteTupleType = std::tuple<uint32_t, ripple::uint256>;
         using ByteVectorType = std::vector<ripple::uint256>;
 
-        if constexpr (std::is_same_v<DecayedType, ripple::uint256>) {
+        if constexpr (std::is_same_v<DecayedType, ripple::uint256> || std::is_same_v<DecayedType, ripple::uint192>) {
             auto const rc = bindBytes(value.data(), value.size());
-            throwErrorIfNeeded(rc, "Bind ripple::uint256");
+            throwErrorIfNeeded(rc, "Bind ripple::base_uint");
         } else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>) {
             auto const rc = bindBytes(value.data(), value.size());
             throwErrorIfNeeded(rc, "Bind ripple::AccountID");
@@ -119,6 +119,9 @@ public:
             // reinterpret_cast is needed here :'(
             auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
             throwErrorIfNeeded(rc, "Bind string (as bytes)");
+        } else if constexpr (std::is_convertible_v<DecayedType, Text>) {
+            auto const rc = cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
+            throwErrorIfNeeded(rc, "Bind string (as TEXT)");
         } else if constexpr (std::is_same_v<DecayedType, UintTupleType> ||
                              std::is_same_v<DecayedType, UintByteTupleType>) {
             auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});

src/etl/CMakeLists.txt

@@ -10,8 +10,8 @@ target_sources(
           NetworkValidatedLedgers.cpp
           NFTHelpers.cpp
           Source.cpp
+          MPTHelpers.cpp
           impl/AmendmentBlockHandler.cpp
-          impl/ForwardingCache.cpp
           impl/ForwardingSource.cpp
           impl/GrpcSource.cpp
           impl/SubscriptionSource.cpp

src/etl/CacheLoader.hpp

@@ -64,7 +64,11 @@ public:
      * @param backend The backend to use
      * @param cache The cache to load into
      */
-    CacheLoader(util::Config const& config, std::shared_ptr<BackendInterface> const& backend, CacheType& cache)
+    CacheLoader(
+        util::config::ClioConfigDefinition const& config,
+        std::shared_ptr<BackendInterface> const& backend,
+        CacheType& cache
+    )
         : backend_{backend}, cache_{cache}, settings_{make_CacheLoaderSettings(config)}, ctx_{settings_.numThreads}
     {
     }

src/etl/CacheLoaderSettings.cpp

@@ -19,11 +19,12 @@
 
 #include "etl/CacheLoaderSettings.hpp"
 
-#include "util/config/Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/algorithm/string/predicate.hpp>
 
 #include <cstddef>
+#include <cstdint>
 #include <string>
 
 namespace etl {
@@ -47,31 +48,29 @@ CacheLoaderSettings::isDisabled() const
 }
 
 [[nodiscard]] CacheLoaderSettings
-make_CacheLoaderSettings(util::Config const& config)
+make_CacheLoaderSettings(util::config::ClioConfigDefinition const& config)
 {
     CacheLoaderSettings settings;
-    settings.numThreads = config.valueOr("io_threads", settings.numThreads);
-    if (config.contains("cache")) {
-        auto const cache = config.section("cache");
-        // Given diff number to generate cursors
-        settings.numCacheDiffs = cache.valueOr<size_t>("num_diffs", settings.numCacheDiffs);
-        // Given cursors number fetching from diff
-        settings.numCacheCursorsFromDiff = cache.valueOr<size_t>("num_cursors_from_diff", 0);
-        // Given cursors number fetching from account
-        settings.numCacheCursorsFromAccount = cache.valueOr<size_t>("num_cursors_from_account", 0);
+    settings.numThreads = config.get<uint16_t>("io_threads");
+    auto const cache = config.getObject("cache");
+    // Given diff number to generate cursors
+    settings.numCacheDiffs = cache.get<std::size_t>("num_diffs");
+    // Given cursors number fetching from diff
+    settings.numCacheCursorsFromDiff = cache.get<std::size_t>("num_cursors_from_diff");
+    // Given cursors number fetching from account
+    settings.numCacheCursorsFromAccount = cache.get<std::size_t>("num_cursors_from_account");
 
-        settings.numCacheMarkers = cache.valueOr<size_t>("num_markers", settings.numCacheMarkers);
-        settings.cachePageFetchSize = cache.valueOr<size_t>("page_fetch_size", settings.cachePageFetchSize);
+    settings.numCacheMarkers = cache.get<std::size_t>("num_markers");
+    settings.cachePageFetchSize = cache.get<std::size_t>("page_fetch_size");
+
+    auto const entry = cache.get<std::string>("load");
+    if (boost::iequals(entry, "sync"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC;
+    if (boost::iequals(entry, "async"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC;
+    if (boost::iequals(entry, "none") or boost::iequals(entry, "no"))
+        settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE;
 
-        if (auto entry = cache.maybeValue<std::string>("load"); entry) {
-            if (boost::iequals(*entry, "sync"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC;
-            if (boost::iequals(*entry, "async"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::ASYNC;
-            if (boost::iequals(*entry, "none") or boost::iequals(*entry, "no"))
-                settings.loadStyle = CacheLoaderSettings::LoadStyle::NONE;
-        }
-    }
     return settings;
 }
 

src/etl/CacheLoaderSettings.hpp

@@ -19,7 +19,7 @@
 
 #pragma once
 
-#include "util/config/Config.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <cstddef>
 
@@ -64,6 +64,6 @@ struct CacheLoaderSettings {
  * @returns The CacheLoaderSettings object
  */
 [[nodiscard]] CacheLoaderSettings
-make_CacheLoaderSettings(util::Config const& config);
+make_CacheLoaderSettings(util::config::ClioConfigDefinition const& config);
 
 }  // namespace etl

src/etl/ETLService.cpp

@@ -26,8 +26,8 @@
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "util/Assert.hpp"
 #include "util/Constants.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <xrpl/beast/core/CurrentThreadName.h>
@@ -262,7 +262,7 @@ ETLService::doWork()
 }
 
 ETLService::ETLService(
-    util::Config const& config,
+    util::config::ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -280,9 +280,9 @@ ETLService::ETLService(
 {
     startSequence_ = config.maybeValue<uint32_t>("start_sequence");
     finishSequence_ = config.maybeValue<uint32_t>("finish_sequence");
-    state_.isReadOnly = config.valueOr("read_only", static_cast<bool>(state_.isReadOnly));
-    extractorThreads_ = config.valueOr<uint32_t>("extractor_threads", extractorThreads_);
-    txnThreshold_ = config.valueOr<size_t>("txn_threshold", txnThreshold_);
+    state_.isReadOnly = config.get<bool>("read_only");
+    extractorThreads_ = config.get<uint32_t>("extractor_threads");
+    txnThreshold_ = config.get<std::size_t>("txn_threshold");
 
     // This should probably be done in the backend factory but we don't have state available until here
     backend_->setCorruptionDetector(CorruptionDetector<data::LedgerCache>{state_, backend->cache()});

src/etl/ETLService.hpp

@@ -119,7 +119,7 @@ public:
      * @param ledgers The network validated ledgers datastructure
      */
     ETLService(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -142,7 +142,7 @@ public:
      */
     static std::shared_ptr<ETLService>
     make_ETLService(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,

src/etl/LedgerFetcherInterface.hpp

@@ -0,0 +1,65 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+
+#include <cstdint>
+#include <optional>
+
+namespace etl {
+
+/**
+ * @brief An interface for LedgerFetcher
+ */
+struct LedgerFetcherInterface {
+    using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+    using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
+
+    virtual ~LedgerFetcherInterface() = default;
+
+    /**
+     * @brief Extract data for a particular ledger from an ETL source
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchData(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract diff data for a particular ledger from an ETL source.
+     *
+     * This function continously tries to extract the specified ledger (using all available ETL sources) until the
+     * extraction succeeds, or the server shuts down.
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
+     */
+    [[nodiscard]] virtual OptionalGetLedgerResponseType
+    fetchDataAndDiff(uint32_t seq) = 0;
+};
+
+}  // namespace etl

src/etl/LoadBalancer.cpp

@@ -27,13 +27,18 @@
 #include "rpc/Errors.hpp"
 #include "util/Assert.hpp"
 #include "util/Random.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ArrayView.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/json/array.hpp>
 #include <boost/json/object.hpp>
 #include <boost/json/value.hpp>
+#include <boost/json/value_to.hpp>
 #include <fmt/core.h>
 
 #include <algorithm>
@@ -49,13 +54,13 @@
 #include <utility>
 #include <vector>
 
-using namespace util;
+using namespace util::config;
 
 namespace etl {
 
 std::shared_ptr<LoadBalancer>
 LoadBalancer::make_LoadBalancer(
-    Config const& config,
+    ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -69,7 +74,7 @@ LoadBalancer::make_LoadBalancer(
 }
 
 LoadBalancer::LoadBalancer(
-    Config const& config,
+    ClioConfigDefinition const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -77,20 +82,23 @@ LoadBalancer::LoadBalancer(
     SourceFactory sourceFactory
 )
 {
-    auto const forwardingCacheTimeout = config.valueOr<float>("forwarding.cache_timeout", 0.f);
+    auto const forwardingCacheTimeout = config.get<float>("forwarding.cache_timeout");
     if (forwardingCacheTimeout > 0.f) {
-        forwardingCache_ = impl::ForwardingCache{Config::toMilliseconds(forwardingCacheTimeout)};
+        forwardingCache_ = util::ResponseExpirationCache{
+            util::config::ClioConfigDefinition::toMilliseconds(forwardingCacheTimeout),
+            {"server_info", "server_state", "server_definitions", "fee", "ledger_closed"}
+        };
     }
 
-    static constexpr std::uint32_t MAX_DOWNLOAD = 256;
-    if (auto value = config.maybeValue<uint32_t>("num_markers"); value) {
-        ASSERT(*value > 0 and *value <= MAX_DOWNLOAD, "'num_markers' value in config must be in range 1-256");
-        downloadRanges_ = *value;
+    auto const numMarkers = config.getValueView("num_markers");
+    if (numMarkers.hasValue()) {
+        auto const value = numMarkers.asIntType<uint32_t>();
+        downloadRanges_ = value;
     } else if (backend->fetchLedgerRange()) {
         downloadRanges_ = 4;
     }
 
-    auto const allowNoEtl = config.valueOr("allow_no_etl", false);
+    auto const allowNoEtl = config.get<bool>("allow_no_etl");
 
     auto const checkOnETLFailure = [this, allowNoEtl](std::string const& log) {
         LOG(log_.warn()) << log;
@@ -101,20 +109,25 @@ LoadBalancer::LoadBalancer(
         }
     };
 
-    auto const forwardingTimeout = Config::toMilliseconds(config.valueOr<float>("forwarding.request_timeout", 10.));
-    for (auto const& entry : config.array("etl_sources")) {
+    auto const forwardingTimeout =
+        ClioConfigDefinition::toMilliseconds(config.get<float>("forwarding.request_timeout"));
+    auto const etlArray = config.getArray("etl_sources");
+    for (auto it = etlArray.begin<ObjectView>(); it != etlArray.end<ObjectView>(); ++it) {
         auto source = sourceFactory(
-            entry,
+            *it,
             ioc,
             backend,
             subscriptions,
             validatedLedgers,
             forwardingTimeout,
             [this]() {
-                if (not hasForwardingSource_)
+                if (not hasForwardingSource_.lock().get())
+                    chooseForwardingSource();
+            },
+            [this](bool wasForwarding) {
+                if (wasForwarding)
                     chooseForwardingSource();
             },
-            [this]() { chooseForwardingSource(); },
             [this]() {
                 if (forwardingCache_.has_value())
                     forwardingCache_->invalidate();
@@ -221,8 +234,12 @@ LoadBalancer::forwardToRippled(
     boost::asio::yield_context yield
 )
 {
+    if (not request.contains("command"))
+        return std::unexpected{rpc::ClioError::rpcCOMMAND_IS_MISSING};
+
+    auto const cmd = boost::json::value_to<std::string>(request.at("command"));
     if (forwardingCache_) {
-        if (auto cachedResponse = forwardingCache_->get(request); cachedResponse) {
+        if (auto cachedResponse = forwardingCache_->get(cmd); cachedResponse) {
             return std::move(cachedResponse).value();
         }
     }
@@ -250,7 +267,7 @@ LoadBalancer::forwardToRippled(
 
     if (response) {
         if (forwardingCache_ and not response->contains("error"))
-            forwardingCache_->put(request, *response);
+            forwardingCache_->put(cmd, *response);
         return std::move(response).value();
     }
 
@@ -322,11 +339,13 @@ LoadBalancer::getETLState() noexcept
 void
 LoadBalancer::chooseForwardingSource()
 {
-    hasForwardingSource_ = false;
+    LOG(log_.info()) << "Choosing a new source to forward subscriptions";
+    auto hasForwardingSourceLock = hasForwardingSource_.lock();
+    hasForwardingSourceLock.get() = false;
     for (auto& source : sources_) {
-        if (not hasForwardingSource_ and source->isConnected()) {
+        if (not hasForwardingSourceLock.get() and source->isConnected()) {
             source->setForwarding(true);
-            hasForwardingSource_ = true;
+            hasForwardingSourceLock.get() = true;
         } else {
             source->setForwarding(false);
         }

src/etl/LoadBalancer.hpp

@@ -23,11 +23,12 @@
 #include "etl/ETLState.hpp"
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "etl/Source.hpp"
-#include "etl/impl/ForwardingCache.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
-#include "util/config/Config.hpp"
+#include "util/Mutex.hpp"
+#include "util/ResponseExpirationCache.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio.hpp>
 #include <boost/asio/io_context.hpp>
@@ -39,7 +40,6 @@
 #include <org/xrpl/rpc/v1/ledger.pb.h>
 #include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
 
-#include <atomic>
 #include <chrono>
 #include <cstdint>
 #include <expected>
@@ -69,14 +69,17 @@ private:
 
     util::Logger log_{"ETL"};
     // Forwarding cache must be destroyed after sources because sources have a callback to invalidate cache
-    std::optional<impl::ForwardingCache> forwardingCache_;
+    std::optional<util::ResponseExpirationCache> forwardingCache_;
     std::optional<std::string> forwardingXUserValue_;
 
     std::vector<SourcePtr> sources_;
     std::optional<ETLState> etlState_;
     std::uint32_t downloadRanges_ =
         DEFAULT_DOWNLOAD_RANGES; /*< The number of markers to use when downloading initial ledger */
-    std::atomic_bool hasForwardingSource_{false};
+
+    // Using mutext instead of atomic_bool because choosing a new source to
+    // forward messages should be done with a mutual exclusion otherwise there will be a race condition
+    util::Mutex<bool> hasForwardingSource_{false};
 
 public:
     /**
@@ -100,7 +103,7 @@ public:
      * @param sourceFactory A factory function to create a source
      */
     LoadBalancer(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -121,7 +124,7 @@ public:
      */
     static std::shared_ptr<LoadBalancer>
     make_LoadBalancer(
-        util::Config const& config,
+        util::config::ClioConfigDefinition const& config,
         boost::asio::io_context& ioc,
         std::shared_ptr<BackendInterface> backend,
         std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,

src/etl/MPTHelpers.cpp

@@ -0,0 +1,86 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/DBHelpers.hpp"
+
+#include <ripple/protocol/STBase.h>
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/LedgerFormats.h>
+#include <xrpl/protocol/SField.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TER.h>
+#include <xrpl/protocol/TxFormats.h>
+
+#include <optional>
+#include <string>
+
+namespace etl {
+
+/**
+ * @brief Get the MPToken created from a transaction
+ *
+ * @param txMeta Transaction metadata
+ * @return MPT and holder account pair
+ */
+std::optional<MPTHolderData>
+getMPTokenAuthorize(ripple::TxMeta const& txMeta)
+{
+    for (ripple::STObject const& node : txMeta.getNodes()) {
+        if (node.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+            continue;
+
+        if (node.getFName() == ripple::sfCreatedNode) {
+            auto const& newMPT = node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>();
+            return MPTHolderData{
+                .mptID = newMPT[ripple::sfMPTokenIssuanceID], .holder = newMPT.getAccountID(ripple::sfAccount)
+            };
+        }
+    }
+    return {};
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
+{
+    if (txMeta.getResultTER() != ripple::tesSUCCESS || sttx.getTxnType() != ripple::TxType::ttMPTOKEN_AUTHORIZE)
+        return {};
+
+    return getMPTokenAuthorize(txMeta);
+}
+
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob)
+{
+    ripple::STLedgerEntry const sle =
+        ripple::STLedgerEntry(ripple::SerialIter{blob.data(), blob.size()}, ripple::uint256::fromVoid(key.data()));
+
+    if (sle.getFieldU16(ripple::sfLedgerEntryType) != ripple::ltMPTOKEN)
+        return {};
+
+    auto const mptIssuanceID = sle[ripple::sfMPTokenIssuanceID];
+    auto const holder = sle.getAccountID(ripple::sfAccount);
+
+    return MPTHolderData{.mptID = mptIssuanceID, .holder = holder};
+}
+
+}  // namespace etl

src/etl/MPTHelpers.hpp

@@ -0,0 +1,50 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+/** @file */
+#pragma once
+
+#include "data/DBHelpers.hpp"
+
+#include <ripple/protocol/STTx.h>
+#include <ripple/protocol/TxMeta.h>
+
+namespace etl {
+
+/**
+ * @brief Pull MPT data from TX via ETLService.
+ *
+ * @param txMeta Transaction metadata
+ * @param sttx The transaction
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
+
+/**
+ * @brief Pull MPT data from ledger object via loadInitialLedger.
+ *
+ * @param key The owner key
+ * @param blob Object data as blob
+ * @return The MPTIssuanceID and holder pair as a optional
+ */
+std::optional<MPTHolderData>
+getMPTHolderFromObj(std::string const& key, std::string const& blob);
+
+}  // namespace etl

src/etl/NFTHelpers.cpp

@@ -73,9 +73,9 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
         if (node.getFName() == ripple::sfCreatedNode) {
             ripple::STArray const& toAddNFTs =
                 node.peekAtField(ripple::sfNewFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddNFTs.begin(),
-                toAddNFTs.end(),
+            std::ranges::transform(
+                toAddNFTs,
+
                 std::back_inserter(finalIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
@@ -98,18 +98,18 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
                 continue;
 
             ripple::STArray const& toAddNFTs = previousFields.getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddNFTs.begin(),
-                toAddNFTs.end(),
+            std::ranges::transform(
+                toAddNFTs,
+
                 std::back_inserter(prevIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
 
             ripple::STArray const& toAddFinalNFTs =
                 node.peekAtField(ripple::sfFinalFields).downcast<ripple::STObject>().getFieldArray(ripple::sfNFTokens);
-            std::transform(
-                toAddFinalNFTs.begin(),
-                toAddFinalNFTs.end(),
+            std::ranges::transform(
+                toAddFinalNFTs,
+
                 std::back_inserter(finalIDs),
                 [](ripple::STObject const& nft) { return nft.getFieldH256(ripple::sfNFTokenID); }
             );
@@ -121,6 +121,7 @@ getNFTokenMintData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx)
 
     // Find the first NFT ID that doesn't match.  We're looking for an
     // added NFT, so the one we want will be the mismatch in finalIDs.
+    // NOLINTNEXTLINE(modernize-use-ranges)
     auto const diff = std::mismatch(finalIDs.begin(), finalIDs.end(), prevIDs.begin(), prevIDs.end());
 
     // There should always be a difference so the returned finalIDs
@@ -261,7 +262,7 @@ getNFTokenAcceptOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
                 .getFieldArray(ripple::sfNFTokens);
         }();
 
-        auto const nft = std::find_if(nfts.begin(), nfts.end(), [&tokenID](ripple::STObject const& candidate) {
+        auto const nft = std::ranges::find_if(nfts, [&tokenID](ripple::STObject const& candidate) {
             return candidate.getFieldH256(ripple::sfNFTokenID) == tokenID;
         });
         if (nft != nfts.end()) {
@@ -298,10 +299,10 @@ getNFTokenCancelOfferData(ripple::TxMeta const& txMeta, ripple::STTx const& sttx
     std::ranges::sort(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
         return a.tokenID < b.tokenID;
     });
-    auto last = std::unique(txs.begin(), txs.end(), [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
+    auto [last, end] = std::ranges::unique(txs, [](NFTTransactionsData const& a, NFTTransactionsData const& b) {
         return a.tokenID == b.tokenID;
     });
-    txs.erase(last, txs.end());
+    txs.erase(last, end);
     return {txs, {}};
 }
 
@@ -366,10 +367,9 @@ getUniqueNFTsDatas(std::vector<NFTsData> const& nfts)
         return a.tokenID == b.tokenID ? a.transactionIndex > b.transactionIndex : a.tokenID > b.tokenID;
     });
 
-    auto const last = std::unique(results.begin(), results.end(), [](NFTsData const& a, NFTsData const& b) {
-        return a.tokenID == b.tokenID;
-    });
-    results.erase(last, results.end());
+    auto const [last, end] =
+        std::ranges::unique(results, [](NFTsData const& a, NFTsData const& b) { return a.tokenID == b.tokenID; });
+    results.erase(last, end);
     return results;
 }
 

src/etl/Source.cpp

@@ -26,7 +26,7 @@
 #include "etl/impl/SourceImpl.hpp"
 #include "etl/impl/SubscriptionSource.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
-#include "util/config/Config.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 
@@ -39,7 +39,7 @@ namespace etl {
 
 SourcePtr
 make_Source(
-    util::Config const& config,
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -50,9 +50,9 @@ make_Source(
     SourceBase::OnLedgerClosedHook onLedgerClosed
 )
 {
-    auto const ip = config.valueOr<std::string>("ip", {});
-    auto const wsPort = config.valueOr<std::string>("ws_port", {});
-    auto const grpcPort = config.valueOr<std::string>("grpc_port", {});
+    auto const ip = config.get<std::string>("ip");
+    auto const wsPort = config.get<std::string>("ws_port");
+    auto const grpcPort = config.get<std::string>("grpc_port");
 
     impl::ForwardingSource forwardingSource{ip, wsPort, forwardingTimeout};
     impl::GrpcSource grpcSource{ip, grpcPort, std::move(backend)};

src/etl/Source.hpp

@@ -23,8 +23,9 @@
 #include "etl/NetworkValidatedLedgersInterface.hpp"
 #include "feed/SubscriptionManagerInterface.hpp"
 #include "rpc/Errors.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ObjectView.hpp"
 
 #include <boost/asio/io_context.hpp>
 #include <boost/asio/spawn.hpp>
@@ -53,7 +54,7 @@ namespace etl {
 class SourceBase {
 public:
     using OnConnectHook = std::function<void()>;
-    using OnDisconnectHook = std::function<void()>;
+    using OnDisconnectHook = std::function<void(bool)>;
     using OnLedgerClosedHook = std::function<void()>;
 
     virtual ~SourceBase() = default;
@@ -147,7 +148,7 @@ public:
 using SourcePtr = std::unique_ptr<SourceBase>;
 
 using SourceFactory = std::function<SourcePtr(
-    util::Config const& config,
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,
@@ -175,7 +176,7 @@ using SourceFactory = std::function<SourcePtr(
  */
 SourcePtr
 make_Source(
-    util::Config const& config,
+    util::config::ObjectView const& config,
     boost::asio::io_context& ioc,
     std::shared_ptr<BackendInterface> backend,
     std::shared_ptr<feed::SubscriptionManagerInterface> subscriptions,

src/etl/impl/AsyncData.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/Types.hpp"
 #include "etl/ETLHelpers.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "util/Assert.hpp"
 #include "util/log/Logger.hpp"
@@ -154,6 +155,11 @@ public:
                     backend.writeSuccessor(std::move(lastKey_), request_.ledger().sequence(), std::string{obj.key()});
                 lastKey_ = obj.key();
                 backend.writeNFTs(getNFTDataFromObj(request_.ledger().sequence(), obj.key(), obj.data()));
+
+                auto const maybeMPTHolder = getMPTHolderFromObj(obj.key(), obj.data());
+                if (maybeMPTHolder)
+                    backend.writeMPTHolders({*maybeMPTHolder});
+
                 backend.writeLedgerObject(
                     std::move(*obj.mutable_key()), request_.ledger().sequence(), std::move(*obj.mutable_data())
                 );

src/etl/impl/CursorFromFixDiffNumProvider.hpp

@@ -70,12 +70,9 @@ public:
             return a.key < b.key or (a.key == b.key and std::size(a.blob) < std::size(b.blob));
         });
 
-        diffs.erase(
-            std::unique(
-                std::begin(diffs), std::end(diffs), [](auto const& a, auto const& b) { return a.key == b.key; }
-            ),
-            std::end(diffs)
-        );
+        auto const [removalCursor, last] =
+            rg::unique(diffs, [](auto const& a, auto const& b) { return a.key == b.key; });
+        diffs.erase(removalCursor, last);
 
         std::vector<ripple::uint256> cursors{data::firstKey};
         rg::copy(

src/etl/impl/Extractor.hpp

@@ -121,19 +121,19 @@ private:
         pipe_.get().finish(startSequence_);
     }
 
-    bool
+    [[nodiscard]] bool
     isStopping() const
     {
         return state_.get().isStopping;
     }
 
-    bool
+    [[nodiscard]] bool
     hasWriteConflict() const
     {
         return state_.get().writeConflict;
     }
 
-    bool
+    [[nodiscard]] bool
     shouldFinish(uint32_t seq) const
     {
         // Stopping conditions:

src/etl/impl/GrpcSource.cpp

@@ -47,7 +47,7 @@
 namespace etl::impl {
 
 GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort, std::shared_ptr<BackendInterface> backend)
-    : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
+    : log_(fmt::format("GrpcSource[{}:{}]", ip, grpcPort)), backend_(std::move(backend))
 {
     try {
         boost::asio::io_context ctx;

src/etl/impl/LedgerFetcher.hpp

@@ -63,7 +63,7 @@ public:
      * @param sequence sequence of the ledger to extract
      * @return Ledger header and transaction+metadata blobs; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchData(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;
@@ -83,7 +83,7 @@ public:
      * @param sequence sequence of the ledger to extract
      * @return Ledger data diff between sequance and parent; Empty optional if the server is shutting down
      */
-    OptionalGetLedgerResponseType
+    [[nodiscard]] OptionalGetLedgerResponseType
     fetchDataAndDiff(uint32_t sequence)
     {
         LOG(log_.debug()) << "Attempting to fetch ledger with sequence = " << sequence;

src/etl/impl/LedgerLoader.hpp

@@ -22,6 +22,7 @@
 #include "data/BackendInterface.hpp"
 #include "data/DBHelpers.hpp"
 #include "data/Types.hpp"
+#include "etl/MPTHelpers.hpp"
 #include "etl/NFTHelpers.hpp"
 #include "etl/SystemState.hpp"
 #include "etl/impl/LedgerFetcher.hpp"
@@ -55,6 +56,7 @@ struct FormattedTransactionsData {
     std::vector<AccountTransactionsData> accountTxData;
     std::vector<NFTTransactionsData> nfTokenTxData;
     std::vector<NFTsData> nfTokensData;
+    std::vector<MPTHolderData> mptHoldersData;
 };
 
 namespace etl::impl {
@@ -124,6 +126,10 @@ public:
             if (maybeNFT)
                 result.nfTokensData.push_back(*maybeNFT);
 
+            auto const maybeMPTHolder = getMPTHolderFromTx(txMeta, sttx);
+            if (maybeMPTHolder)
+                result.mptHoldersData.push_back(*maybeMPTHolder);
+
             result.accountTxData.emplace_back(txMeta, sttx.getTransactionID());
             static constexpr std::size_t KEY_SIZE = 32;
             std::string keyStr{reinterpret_cast<char const*>(sttx.getTransactionID().data()), KEY_SIZE};
@@ -240,6 +246,7 @@ public:
                 backend_->writeAccountTransactions(std::move(insertTxResult.accountTxData));
                 backend_->writeNFTs(insertTxResult.nfTokensData);
                 backend_->writeNFTTransactions(insertTxResult.nfTokenTxData);
+                backend_->writeMPTHolders(insertTxResult.mptHoldersData);
             }
 
             backend_->finishWrites(sequence);

src/etl/impl/SubscriptionSource.cpp

@@ -24,6 +24,8 @@
 #include "rpc/JS.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Label.hpp"
+#include "util/prometheus/Prometheus.hpp"
 #include "util/requests/Types.hpp"
 
 #include <boost/algorithm/string/classification.hpp>
@@ -66,22 +68,28 @@ SubscriptionSource::SubscriptionSource(
     OnConnectHook onConnect,
     OnDisconnectHook onDisconnect,
     OnLedgerClosedHook onLedgerClosed,
-    std::chrono::steady_clock::duration const connectionTimeout,
+    std::chrono::steady_clock::duration const wsTimeout,
     std::chrono::steady_clock::duration const retryDelay
 )
-    : log_(fmt::format("GrpcSource[{}:{}]", ip, wsPort))
+    : log_(fmt::format("SubscriptionSource[{}:{}]", ip, wsPort))
     , wsConnectionBuilder_(ip, wsPort)
     , validatedLedgers_(std::move(validatedLedgers))
     , subscriptions_(std::move(subscriptions))
     , strand_(boost::asio::make_strand(ioContext))
+    , wsTimeout_(wsTimeout)
     , retry_(util::makeRetryExponentialBackoff(retryDelay, RETRY_MAX_DELAY, strand_))
     , onConnect_(std::move(onConnect))
     , onDisconnect_(std::move(onDisconnect))
     , onLedgerClosed_(std::move(onLedgerClosed))
+    , lastMessageTimeSecondsSinceEpoch_(PrometheusService::gaugeInt(
+          "subscription_source_last_message_time",
+          util::prometheus::Labels({{"source", fmt::format("{}:{}", ip, wsPort)}}),
+          "Seconds since epoch of the last message received from rippled subscription streams"
+      ))
 {
     wsConnectionBuilder_.addHeader({boost::beast::http::field::user_agent, "clio-client"})
         .addHeader({"X-User", "clio-client"})
-        .setConnectionTimeout(connectionTimeout);
+        .setConnectionTimeout(wsTimeout_);
 }
 
 SubscriptionSource::~SubscriptionSource()
@@ -133,6 +141,7 @@ void
 SubscriptionSource::setForwarding(bool isForwarding)
 {
     isForwarding_ = isForwarding;
+    LOG(log_.info()) << "Forwarding set to " << isForwarding_;
 }
 
 std::chrono::steady_clock::time_point
@@ -166,20 +175,22 @@ SubscriptionSource::subscribe()
             }
 
             wsConnection_ = std::move(connection).value();
-            isConnected_ = true;
-            onConnect_();
 
             auto const& subscribeCommand = getSubscribeCommandJson();
-            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield);
+            auto const writeErrorOpt = wsConnection_->write(subscribeCommand, yield, wsTimeout_);
             if (writeErrorOpt) {
                 handleError(writeErrorOpt.value(), yield);
                 return;
             }
 
+            isConnected_ = true;
+            LOG(log_.info()) << "Connected";
+            onConnect_();
+
             retry_.reset();
 
             while (!stop_) {
-                auto const message = wsConnection_->read(yield);
+                auto const message = wsConnection_->read(yield, wsTimeout_);
                 if (not message) {
                     handleError(message.error(), yield);
                     return;
@@ -224,10 +235,11 @@ SubscriptionSource::handleMessage(std::string const& message)
                 auto validatedLedgers = boost::json::value_to<std::string>(result.at(JS(validated_ledgers)));
                 setValidatedRange(std::move(validatedLedgers));
             }
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message on ledger subscription stream. Message: " << object;
 
         } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_LedgerClosed) {
-            LOG(log_.info()) << "Received a message on ledger subscription stream. Message : " << object;
+            LOG(log_.debug()) << "Received a message of type 'ledgerClosed' on ledger subscription stream. Message: "
+                              << object;
             if (object.contains(JS(ledger_index))) {
                 ledgerIndex = object.at(JS(ledger_index)).as_int64();
             }
@@ -245,10 +257,13 @@ SubscriptionSource::handleMessage(std::string const& message)
                 // 2 - Validated transaction
                 // Only forward proposed transaction, validated transactions are sent by Clio itself
                 if (object.contains(JS(transaction)) and !object.contains(JS(meta))) {
+                    LOG(log_.debug()) << "Forwarding proposed transaction: " << object;
                     subscriptions_->forwardProposedTransaction(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ValidationReceived) {
+                    LOG(log_.debug()) << "Forwarding validation: " << object;
                     subscriptions_->forwardValidation(object);
                 } else if (object.contains(JS(type)) && object.at(JS(type)) == JS_ManifestReceived) {
+                    LOG(log_.debug()) << "Forwarding manifest: " << object;
                     subscriptions_->forwardManifest(object);
                 }
             }
@@ -261,7 +276,7 @@ SubscriptionSource::handleMessage(std::string const& message)
 
         return std::nullopt;
     } catch (std::exception const& e) {
-        LOG(log_.error()) << "Exception in handleMessage : " << e.what();
+        LOG(log_.error()) << "Exception in handleMessage: " << e.what();
         return util::requests::RequestError{fmt::format("Error handling message: {}", e.what())};
     }
 }
@@ -270,16 +285,14 @@ void
 SubscriptionSource::handleError(util::requests::RequestError const& error, boost::asio::yield_context yield)
 {
     isConnected_ = false;
-    isForwarding_ = false;
+    bool const wasForwarding = isForwarding_.exchange(false);
     if (not stop_) {
-        onDisconnect_();
+        LOG(log_.info()) << "Disconnected";
+        onDisconnect_(wasForwarding);
     }
 
     if (wsConnection_ != nullptr) {
-        auto const err = wsConnection_->close(yield);
-        if (err) {
-            LOG(log_.error()) << "Error closing websocket connection: " << err->message();
-        }
+        wsConnection_->close(yield);
         wsConnection_.reset();
     }
 
@@ -306,7 +319,11 @@ SubscriptionSource::logError(util::requests::RequestError const& error) const
 void
 SubscriptionSource::setLastMessageTime()
 {
-    lastMessageTime_.lock().get() = std::chrono::steady_clock::now();
+    lastMessageTimeSecondsSinceEpoch_.get().set(
+        std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count()
+    );
+    auto lock = lastMessageTime_.lock();
+    lock.get() = std::chrono::steady_clock::now();
 }
 
 void
@@ -336,7 +353,7 @@ SubscriptionSource::setValidatedRange(std::string range)
             pairs.emplace_back(min, max);
         }
     }
-    std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; });
+    std::ranges::sort(pairs, [](auto left, auto right) { return left.first < right.first; });
 
     auto dataLock = validatedLedgersData_.lock();
     dataLock->validatedLedgers = std::move(pairs);

src/etl/impl/SubscriptionSource.hpp

@@ -25,6 +25,7 @@
 #include "util/Mutex.hpp"
 #include "util/Retry.hpp"
 #include "util/log/Logger.hpp"
+#include "util/prometheus/Gauge.hpp"
 #include "util/requests/Types.hpp"
 #include "util/requests/WsConnection.hpp"
 
@@ -37,6 +38,7 @@
 #include <atomic>
 #include <chrono>
 #include <cstdint>
+#include <functional>
 #include <future>
 #include <memory>
 #include <optional>
@@ -71,6 +73,8 @@ private:
 
     boost::asio::strand<boost::asio::io_context::executor_type> strand_;
 
+    std::chrono::steady_clock::duration wsTimeout_;
+
     util::Retry retry_;
 
     OnConnectHook onConnect_;
@@ -83,9 +87,11 @@ private:
 
     util::Mutex<std::chrono::steady_clock::time_point> lastMessageTime_;
 
+    std::reference_wrapper<util::prometheus::GaugeInt> lastMessageTimeSecondsSinceEpoch_;
+
     std::future<void> runFuture_;
 
-    static constexpr std::chrono::seconds CONNECTION_TIMEOUT{30};
+    static constexpr std::chrono::seconds WS_TIMEOUT{30};
     static constexpr std::chrono::seconds RETRY_MAX_DELAY{30};
     static constexpr std::chrono::seconds RETRY_DELAY{1};
 
@@ -103,7 +109,7 @@ public:
      * @param onDisconnect The onDisconnect hook. Called when the connection is lost
      * @param onLedgerClosed The onLedgerClosed hook. Called when the ledger is closed but only if the source is
      * forwarding
-     * @param connectionTimeout The connection timeout. Defaults to 30 seconds
+     * @param wsTimeout A timeout for websocket operations. Defaults to 30 seconds
      * @param retryDelay The retry delay. Defaults to 1 second
      */
     SubscriptionSource(
@@ -115,7 +121,7 @@ public:
         OnConnectHook onConnect,
         OnDisconnectHook onDisconnect,
         OnLedgerClosedHook onLedgerClosed,
-        std::chrono::steady_clock::duration const connectionTimeout = CONNECTION_TIMEOUT,
+        std::chrono::steady_clock::duration const wsTimeout = WS_TIMEOUT,
         std::chrono::steady_clock::duration const retryDelay = RETRY_DELAY
     );
 

src/etl/impl/Transformer.hpp

@@ -213,6 +213,7 @@ private:
         backend_->writeAccountTransactions(std::move(insertTxResultOp->accountTxData));
         backend_->writeNFTs(insertTxResultOp->nfTokensData);
         backend_->writeNFTTransactions(insertTxResultOp->nfTokenTxData);
+        backend_->writeMPTHolders(insertTxResultOp->mptHoldersData);
 
         auto [success, duration] =
             ::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });
@@ -307,11 +308,11 @@ private:
 
                 auto lb = backend_->cache().getPredecessor(obj.key, lgrInfo.seq);
                 if (!lb)
-                    lb = {data::firstKey, {}};
+                    lb = {.key = data::firstKey, .blob = {}};
 
                 auto ub = backend_->cache().getSuccessor(obj.key, lgrInfo.seq);
                 if (!ub)
-                    ub = {data::lastKey, {}};
+                    ub = {.key = data::lastKey, .blob = {}};
 
                 if (obj.blob.empty()) {
                     LOG(log_.debug()) << "writing successor for deleted object " << ripple::strHex(obj.key) << " - "

src/etlng/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(clio_etlng)
+
+target_sources(clio_etlng PRIVATE impl/AsyncGrpcCall.cpp impl/Extraction.cpp impl/GrpcSource.cpp)
+
+target_link_libraries(clio_etlng PUBLIC clio_data)

src/etlng/ExtractorInterface.hpp

@@ -0,0 +1,54 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <optional>
+
+namespace etlng {
+
+/**
+ * @brief An interface for the Extractor
+ */
+struct ExtractorInterface {
+    virtual ~ExtractorInterface() = default;
+
+    /**
+     * @brief Extract diff data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger data diff between sequence and parent if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerWithDiff(uint32_t seq) = 0;
+
+    /**
+     * @brief Extract data for a particular ledger
+     *
+     * @param seq sequence of the ledger to extract
+     * @return Ledger header and transaction+metadata blobs if available
+     */
+    [[nodiscard]] virtual std::optional<model::LedgerData>
+    extractLedgerOnly(uint32_t seq) = 0;
+};
+
+}  // namespace etlng

src/etlng/InitialLoadObserverInterface.hpp

@@ -17,32 +17,38 @@
 */
 //==============================================================================
 
-#include "web/Server.hpp"
+#pragma once
 
-#include "util/config/Config.hpp"
+#include "etlng/Models.hpp"
 
-#include <boost/asio/ssl/context.hpp>
+#include <xrpl/protocol/LedgerHeader.h>
 
+#include <cstdint>
 #include <optional>
 #include <string>
+#include <vector>
 
-namespace web {
+namespace etlng {
 
-std::expected<std::optional<boost::asio::ssl::context>, std::string>
-makeServerSslContext(util::Config const& config)
-{
-    bool const configHasCertFile = config.contains("ssl_cert_file");
-    bool const configHasKeyFile = config.contains("ssl_key_file");
+/**
+ * @brief The interface for observing the initial ledger load
+ */
+struct InitialLoadObserverInterface {
+    virtual ~InitialLoadObserverInterface() = default;
 
-    if (configHasCertFile != configHasKeyFile)
-        return std::unexpected{"Config entries 'ssl_cert_file' and 'ssl_key_file' must be set or unset together."};
+    /**
+     * @brief Callback for each incoming batch of objects during initial ledger load
+     *
+     * @param seq The sequence for this batch of objects
+     * @param data The batch of objects
+     * @param lastKey The last key of the previous batch if there was one
+     */
+    virtual void
+    onInitialLoadGotMoreObjects(
+        uint32_t seq,
+        std::vector<model::Object> const& data,
+        std::optional<std::string> lastKey = std::nullopt
+    ) = 0;
+};
 
-    if (not configHasCertFile)
-        return std::nullopt;
-
-    auto const certFilename = config.value<std::string>("ssl_cert_file");
-    auto const keyFilename = config.value<std::string>("ssl_key_file");
-
-    return impl::makeServerSslContext(certFilename, keyFilename);
-}
-}  // namespace web
+}  // namespace etlng

src/etlng/Models.hpp

@@ -0,0 +1,130 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/Concepts.hpp"
+
+#include <boost/json/object.hpp>
+#include <fmt/core.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxFormats.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::model {
+
+/**
+ * @brief A specification for the Registry.
+ *
+ * This specification simply defines the transaction types that are to be filtered out from the incoming transactions by
+ * the Registry for its `onTransaction` and `onInitialTransaction` hooks.
+ * It's a compilation error to list the same transaction type more than once.
+ */
+template <ripple::TxType... Types>
+    requires(util::hasNoDuplicates(Types...))
+struct Spec {
+    static constexpr bool SpecTag = true;
+
+    /**
+     * @brief Checks if the transaction type was requested.
+     *
+     * @param type The transaction type
+     * @return true if the transaction was requested; false otherwise
+     */
+    [[nodiscard]] constexpr static bool
+    wants(ripple::TxType type) noexcept
+    {
+        return ((Types == type) || ...);
+    }
+};
+
+/**
+ * @brief Represents a single transaction on the ledger.
+ */
+struct Transaction {
+    std::string raw;  // raw binary blob
+    std::string metaRaw;
+
+    // unpacked blob and meta
+    ripple::STTx sttx;
+    ripple::TxMeta meta;
+
+    // commonly used stuff
+    ripple::uint256 id;
+    std::string key;  // key is the above id as a string of 32 characters
+    ripple::TxType type;
+};
+
+/**
+ * @brief Represents a single object on the ledger.
+ */
+struct Object {
+    /**
+     * @brief Modification type for the object.
+     */
+    enum class ModType : int {
+        Unspecified = 0,
+        Created = 1,
+        Modified = 2,
+        Deleted = 3,
+    };
+
+    ripple::uint256 key;
+    std::string keyRaw;
+    ripple::Blob data;
+    std::string dataRaw;
+    std::string successor;
+    std::string predecessor;
+
+    ModType type;
+};
+
+/**
+ * @brief Represents a book successor.
+ */
+struct BookSuccessor {
+    std::string firstBook;
+    std::string bookBase;
+};
+
+/**
+ * @brief Represents an entire ledger diff worth of transactions and objects.
+ */
+struct LedgerData {
+    std::vector<Transaction> transactions;
+    std::vector<Object> objects;
+    std::optional<std::vector<BookSuccessor>> successors;
+    std::optional<std::vector<std::string>> edgeKeys;
+
+    ripple::LedgerHeader header;
+    std::string rawHeader;
+    uint32_t seq;
+};
+
+}  // namespace etlng::model

src/etlng/RegistryInterface.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief The interface for a registry that can dispatch transactions and objects to extensions.
+ *
+ * This class defines the interface for dispatching data through to extensions.
+ *
+ * @note
+ * The registry itself consists of Extensions.
+ * Each extension must define at least one valid hook:
+ * - for ongoing ETL dispatch:
+ *   - void onLedgerData(etlng::model::LedgerData const&)
+ *   - void onTransaction(uint32_t, etlng::model::Transaction const&)
+ *   - void onObject(uint32_t, etlng::model::Object const&)
+ * - for initial ledger load
+ *   - void onInitialData(etlng::model::LedgerData const&)
+ *   - void onInitialTransaction(uint32_t, etlng::model::Transaction const&)
+ * - for initial objects (called for each downloaded batch)
+ *   - void onInitialObjects(uint32_t, std::vector<etlng::model::Object> const&, std::string)
+ *   - void onInitialObject(uint32_t, etlng::model::Object const&)
+ *
+ * When the registry dispatches (initial)data or objects, each of the above hooks will be called in order on each
+ * registered extension.
+ * This means that the order of execution is from left to right (hooks) and top to bottom (registered extensions).
+ *
+ * If either `onTransaction` or `onInitialTransaction` are defined, the extension will have to additionally define a
+ * Specification. The specification lists transaction types to filter from the incoming data such that `onTransaction`
+ * and `onInitialTransaction` are only called for the transactions that are of interest for the given extension.
+ *
+ * The specification is setup like so:
+ * @code{.cpp}
+ * struct Ext {
+ *   using spec = etlng::model::Spec<
+ *     ripple::TxType::ttNFTOKEN_BURN,
+ *     ripple::TxType::ttNFTOKEN_ACCEPT_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CREATE_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CANCEL_OFFER,
+ *     ripple::TxType::ttNFTOKEN_MINT>;
+ *
+ *   static void
+ *   onInitialTransaction(uint32_t, etlng::model::Transaction const&);
+ * };
+ * @endcode
+ */
+struct RegistryInterface {
+    virtual ~RegistryInterface() = default;
+
+    /**
+     * @brief Dispatch initial objects.
+     *
+     * These objects are received during initial ledger load.
+     *
+     * @param seq The sequence
+     * @param data The objects to dispatch
+     * @param lastKey The predcessor of the first object in data if known; an empty string otherwise
+     */
+    virtual void
+    dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) = 0;
+
+    /**
+     * @brief Dispatch initial ledger data.
+     *
+     * The transactions, header and edge keys are received during initial ledger load.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatchInitialData(model::LedgerData const& data) = 0;
+
+    /**
+     * @brief Dispatch an entire ledger diff.
+     *
+     * This is used to dispatch incoming diffs through the extensions.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatch(model::LedgerData const& data) = 0;
+};
+
+}  // namespace etlng

src/etlng/impl/AsyncGrpcCall.cpp

@@ -0,0 +1,188 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/AsyncGrpcCall.hpp"
+
+#include "etl/ETLHelpers.hpp"
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/Models.hpp"
+#include "etlng/impl/Extraction.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/client_context.h>
+#include <grpcpp/grpcpp.h>
+#include <grpcpp/support/status.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/basics/strHex.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+AsyncGrpcCall::AsyncGrpcCall(
+    uint32_t seq,
+    ripple::uint256 const& marker,
+    std::optional<ripple::uint256> const& nextMarker
+)
+{
+    request_.set_user("ETL");
+    request_.mutable_ledger()->set_sequence(seq);
+
+    if (marker.isNonZero())
+        request_.set_marker(marker.data(), ripple::uint256::size());
+
+    nextPrefix_ = nextMarker ? nextMarker->data()[0] : 0x00;
+    auto const prefix = marker.data()[0];
+
+    LOG(log_.debug()) << "Setting up AsyncGrpcCall. marker = " << ripple::strHex(marker)
+                      << ". prefix = " << ripple::strHex(std::string(1, prefix))
+                      << ". nextPrefix_ = " << ripple::strHex(std::string(1, nextPrefix_));
+
+    ASSERT(
+        nextPrefix_ > prefix or nextPrefix_ == 0x00,
+        "Next prefix must be greater than current prefix. Got: nextPrefix_ = {}, prefix = {}",
+        nextPrefix_,
+        prefix
+    );
+
+    cur_ = std::make_unique<ResponseType>();
+    next_ = std::make_unique<ResponseType>();
+    context_ = std::make_unique<grpc::ClientContext>();
+}
+
+AsyncGrpcCall::CallStatus
+AsyncGrpcCall::process(
+    std::unique_ptr<AsyncGrpcCall::StubType>& stub,
+    grpc::CompletionQueue& cq,
+    etlng::InitialLoadObserverInterface& loader,
+    bool abort
+)
+{
+    LOG(log_.trace()) << "Processing response. "
+                      << "Marker prefix = " << getMarkerPrefix();
+
+    if (abort) {
+        LOG(log_.error()) << "AsyncGrpcCall aborted";
+        return CallStatus::ERRORED;
+    }
+
+    if (!status_.ok()) {
+        LOG(log_.error()) << "AsyncGrpcCall status_ not ok: code = " << status_.error_code()
+                          << " message = " << status_.error_message();
+
+        return CallStatus::ERRORED;
+    }
+
+    if (!next_->is_unlimited()) {
+        LOG(log_.warn()) << "AsyncGrpcCall is_unlimited is false. "
+                         << "Make sure secure_gateway is set correctly at the ETL source";
+    }
+
+    std::swap(cur_, next_);
+    auto more = true;
+
+    // if no marker returned, we are done
+    if (cur_->marker().empty())
+        more = false;
+
+    // if returned marker is greater than our end, we are done
+    auto const prefix = cur_->marker()[0];
+    if (nextPrefix_ != 0x00 && prefix >= nextPrefix_)
+        more = false;
+
+    // if we are not done, make the next async call
+    if (more) {
+        request_.set_marker(cur_->marker());
+        call(stub, cq);
+    }
+
+    auto const numObjects = cur_->ledger_objects().objects_size();
+    std::vector<etlng::model::Object> data;
+    data.reserve(numObjects);
+
+    for (int i = 0; i < numObjects; ++i) {
+        auto obj = std::move(*(cur_->mutable_ledger_objects()->mutable_objects(i)));
+        if (!more && nextPrefix_ != 0x00) {
+            if (static_cast<unsigned char>(obj.key()[0]) >= nextPrefix_)
+                continue;
+        }
+
+        lastKey_ = obj.key();  // this will end up the last key we actually touched eventually
+        data.push_back(etlng::impl::extractObj(std::move(obj)));
+    }
+
+    if (not data.empty())
+        loader.onInitialLoadGotMoreObjects(request_.ledger().sequence(), data, predecessorKey_);
+
+    predecessorKey_ = lastKey_;  // but for ongoing onInitialObjects calls we need to pass along the key we left
+                                 // off at so that we can link the two lists correctly
+
+    return more ? CallStatus::MORE : CallStatus::DONE;
+}
+
+void
+AsyncGrpcCall::call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq)
+{
+    context_ = std::make_unique<grpc::ClientContext>();
+    auto rpc = stub->PrepareAsyncGetLedgerData(context_.get(), request_, &cq);
+
+    rpc->StartCall();
+    rpc->Finish(next_.get(), &status_, this);
+}
+
+std::string
+AsyncGrpcCall::getMarkerPrefix()
+{
+    return next_->marker().empty() ? std::string{} : ripple::strHex(std::string{next_->marker().data()[0]});
+}
+
+// this is used to generate edgeKeys - keys that were the last one in the onInitialObjects list
+// then we write them all in one go getting the successor from the cache once it's full
+std::string
+AsyncGrpcCall::getLastKey()
+{
+    return lastKey_;
+}
+
+std::vector<AsyncGrpcCall>
+AsyncGrpcCall::makeAsyncCalls(uint32_t const sequence, uint32_t const numMarkers)
+{
+    auto const markers = etl::getMarkers(numMarkers);
+
+    std::vector<AsyncGrpcCall> result;
+    result.reserve(markers.size());
+
+    for (size_t i = 0; i + 1 < markers.size(); ++i)
+        result.emplace_back(sequence, markers[i], markers[i + 1]);
+
+    if (not markers.empty())
+        result.emplace_back(sequence, markers.back(), std::nullopt);
+
+    return result;
+}
+
+}  // namespace etlng::impl

src/etlng/impl/AsyncGrpcCall.hpp

@@ -0,0 +1,85 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/client_context.h>
+#include <grpcpp/support/status.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger_data.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::impl {
+
+class AsyncGrpcCall {
+public:
+    enum class CallStatus { MORE, DONE, ERRORED };
+    using RequestType = org::xrpl::rpc::v1::GetLedgerDataRequest;
+    using ResponseType = org::xrpl::rpc::v1::GetLedgerDataResponse;
+    using StubType = org::xrpl::rpc::v1::XRPLedgerAPIService::Stub;
+
+private:
+    util::Logger log_{"ETL"};
+
+    std::unique_ptr<ResponseType> cur_;
+    std::unique_ptr<ResponseType> next_;
+
+    RequestType request_;
+    std::unique_ptr<grpc::ClientContext> context_;
+
+    grpc::Status status_;
+    unsigned char nextPrefix_;
+
+    std::string lastKey_;
+    std::optional<std::string> predecessorKey_;
+
+public:
+    AsyncGrpcCall(uint32_t seq, ripple::uint256 const& marker, std::optional<ripple::uint256> const& nextMarker);
+
+    static std::vector<AsyncGrpcCall>
+    makeAsyncCalls(uint32_t const sequence, uint32_t const numMarkers);
+
+    CallStatus
+    process(
+        std::unique_ptr<StubType>& stub,
+        grpc::CompletionQueue& cq,
+        etlng::InitialLoadObserverInterface& loader,
+        bool abort
+    );
+
+    void
+    call(std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>& stub, grpc::CompletionQueue& cq);
+
+    std::string
+    getMarkerPrefix();
+
+    std::string
+    getLastKey();
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Extraction.cpp

@@ -0,0 +1,219 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/Extraction.hpp"
+
+#include "data/DBHelpers.hpp"
+#include "data/Types.hpp"
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/Models.hpp"
+#include "util/Assert.hpp"
+#include "util/LedgerUtils.hpp"
+#include "util/Profiler.hpp"
+#include "util/log/Logger.hpp"
+
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+model::Object::ModType
+extractModType(PBModType type)
+{
+    switch (type) {
+        case PBObjType::UNSPECIFIED:
+            return model::Object::ModType::Unspecified;
+        case PBObjType::CREATED:
+            return model::Object::ModType::Created;
+        case PBObjType::MODIFIED:
+            return model::Object::ModType::Modified;
+        case PBObjType::DELETED:
+            return model::Object::ModType::Deleted;
+        default:  // some gRPC system values that we don't care about
+            ASSERT(false, "Tried to extract bogus mod type '{}'", PBObjType::ModificationType_Name(type));
+    }
+
+    std::unreachable();
+}
+
+model::Transaction
+extractTx(PBTxType tx, uint32_t seq)
+{
+    auto raw = std::move(*tx.mutable_transaction_blob());
+    ripple::SerialIter it{raw.data(), raw.size()};
+    ripple::STTx const sttx{it};
+    ripple::TxMeta meta{sttx.getTransactionID(), seq, tx.metadata_blob()};
+
+    return {
+        .raw = std::move(raw),
+        .metaRaw = std::move(*tx.mutable_metadata_blob()),
+        .sttx = sttx,  // trivially copyable
+        .meta = std::move(meta),
+        .id = sttx.getTransactionID(),
+        .key = uint256ToString(sttx.getTransactionID()),
+        .type = sttx.getTxnType()
+    };
+}
+
+std::vector<model::Transaction>
+extractTxs(PBTxListType transactions, uint32_t seq)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::Transaction> output;
+    output.reserve(transactions.size());
+
+    rg::move(transactions | vs::transform([seq](auto&& tx) { return extractTx(tx, seq); }), std::back_inserter(output));
+    return output;
+}
+
+model::Object
+extractObj(PBObjType obj)
+{
+    auto const key = ripple::uint256::fromVoidChecked(obj.key());
+    ASSERT(key.has_value(), "Failed to deserialize key from void");
+
+    auto const valueOr = [](std::string const& maybe, std::string fallback) -> std::string {
+        if (maybe.empty())
+            return fallback;
+        return maybe;
+    };
+
+    return {
+        .key = *key,  // trivially copyable
+        .keyRaw = std::move(*obj.mutable_key()),
+        .data = {obj.mutable_data()->begin(), obj.mutable_data()->end()},
+        .dataRaw = std::move(*obj.mutable_data()),
+        .successor = valueOr(obj.successor(), uint256ToString(data::firstKey)),
+        .predecessor = valueOr(obj.predecessor(), uint256ToString(data::lastKey)),
+        .type = extractModType(obj.mod_type()),
+    };
+}
+
+std::vector<model::Object>
+extractObjs(PBObjListType objects)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::Object> output;
+    output.reserve(objects.size());
+
+    rg::move(objects | vs::transform([](auto&& obj) { return extractObj(obj); }), std::back_inserter(output));
+    return output;
+}
+
+model::BookSuccessor
+extractSuccessor(PBBookSuccessorType successor)
+{
+    return {
+        .firstBook = successor.first_book(),
+        .bookBase = successor.book_base(),
+    };
+}
+
+std::optional<std::vector<model::BookSuccessor>>
+maybeExtractSuccessors(PBLedgerResponseType const& data)
+{
+    namespace rg = std::ranges;
+    namespace vs = std::views;
+
+    if (not data.object_neighbors_included())
+        return std::nullopt;
+
+    // TODO: should be simplified with ranges::to<> when available
+    std::vector<model::BookSuccessor> output;
+    output.reserve(data.book_successors_size());
+
+    rg::copy(
+        data.book_successors() | vs::transform([](auto&& obj) { return extractSuccessor(obj); }),
+        std::back_inserter(output)
+    );
+    return output;
+}
+
+auto
+Extractor::unpack()
+{
+    return [](auto&& data) {
+        auto header = ::util::deserializeHeader(ripple::makeSlice(data.ledger_header()));
+
+        return std::make_optional<model::LedgerData>({
+            .transactions =
+                extractTxs(std::move(*data.mutable_transactions_list()->mutable_transactions()), header.seq),
+            .objects = extractObjs(std::move(*data.mutable_ledger_objects()->mutable_objects())),
+            .successors = maybeExtractSuccessors(data),
+            .edgeKeys = std::nullopt,
+            .header = header,
+            .rawHeader = std::move(*data.mutable_ledger_header()),
+            .seq = header.seq,
+        });
+    };
+}
+
+std::optional<model::LedgerData>
+Extractor::extractLedgerWithDiff(uint32_t seq)
+{
+    LOG(log_.debug()) << "Extracting DIFF " << seq;
+
+    auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
+        return fetcher_->fetchDataAndDiff(seq).and_then(unpack());
+    });
+
+    LOG(log_.debug()) << "Extracted and Transformed diff for " << seq << " in " << time << "ms";
+
+    // can be nullopt. this means that either the server is stopping or another node took over ETL writing.
+    return batch;
+}
+
+std::optional<model::LedgerData>
+Extractor::extractLedgerOnly(uint32_t seq)
+{
+    LOG(log_.debug()) << "Extracting FULL " << seq;
+
+    auto const [batch, time] = ::util::timed<std::chrono::duration<double>>([this, seq] {
+        return fetcher_->fetchData(seq).and_then(unpack());
+    });
+
+    LOG(log_.debug()) << "Extracted and Transformed full ledger for " << seq << " in " << time << "ms";
+
+    // can be nullopt. this means that either the server is stopping or another node took over ETL writing.
+    return batch;
+}
+
+}  // namespace etlng::impl

src/etlng/impl/Extraction.hpp

@@ -0,0 +1,100 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/LedgerFetcherInterface.hpp"
+#include "etl/impl/LedgerFetcher.hpp"
+#include "etlng/ExtractorInterface.hpp"
+#include "etlng/Models.hpp"
+#include "util/log/Logger.hpp"
+
+#include <google/protobuf/repeated_ptr_field.h>
+#include <sys/types.h>
+#include <xrpl/basics/Slice.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+using PBObjType = org::xrpl::rpc::v1::RawLedgerObject;
+using PBModType = PBObjType::ModificationType;
+using PBTxType = org::xrpl::rpc::v1::TransactionAndMetadata;
+using PBTxListType = google::protobuf::RepeatedPtrField<PBTxType>;
+using PBObjListType = google::protobuf::RepeatedPtrField<PBObjType>;
+using PBBookSuccessorType = org::xrpl::rpc::v1::BookSuccessor;
+using PBLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
+
+[[nodiscard]] model::Object::ModType
+extractModType(PBModType type);
+
+[[nodiscard]] model::Transaction
+extractTx(PBTxType tx, uint32_t seq);
+
+[[nodiscard]] std::vector<model::Transaction>
+extractTxs(PBTxListType transactions, uint32_t seq);
+
+[[nodiscard]] model::Object
+extractObj(PBObjType obj);
+
+[[nodiscard]] std::vector<model::Object>
+extractObjs(PBObjListType objects);
+
+[[nodiscard]] model::BookSuccessor
+extractSuccessor(PBBookSuccessorType successor);
+
+[[nodiscard]] std::optional<std::vector<model::BookSuccessor>>
+maybeExtractSuccessors(PBLedgerResponseType const& data);
+
+// fetches the data in gRPC and transforms to local representation
+class Extractor : public ExtractorInterface {
+    std::shared_ptr<etl::LedgerFetcherInterface> fetcher_;
+
+    util::Logger log_{"ETL"};
+
+private:
+    [[nodiscard]] static auto
+    unpack();
+
+public:
+    Extractor(std::shared_ptr<etl::LedgerFetcherInterface> fetcher) : fetcher_(std::move(fetcher))
+    {
+    }
+
+    [[nodiscard]] std::optional<model::LedgerData>
+    extractLedgerWithDiff(uint32_t seq) override;
+
+    [[nodiscard]] std::optional<model::LedgerData>
+    extractLedgerOnly(uint32_t seq) override;
+};
+
+}  // namespace etlng::impl

src/etlng/impl/GrpcSource.cpp

@@ -0,0 +1,157 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "etlng/impl/GrpcSource.hpp"
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "etlng/impl/AsyncGrpcCall.hpp"
+#include "util/Assert.hpp"
+#include "util/log/Logger.hpp"
+#include "web/Resolver.hpp"
+
+#include <fmt/core.h>
+#include <grpcpp/client_context.h>
+#include <grpcpp/security/credentials.h>
+#include <grpcpp/support/channel_arguments.h>
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstddef>
+#include <cstdint>
+#include <exception>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace {
+
+std::string
+resolve(std::string const& ip, std::string const& port)
+{
+    web::Resolver resolver;
+
+    if (auto const results = resolver.resolve(ip, port); not results.empty())
+        return results.at(0);
+
+    throw std::runtime_error("Failed to resolve " + ip + ":" + port);
+}
+
+}  // namespace
+
+namespace etlng::impl {
+
+GrpcSource::GrpcSource(std::string const& ip, std::string const& grpcPort)
+    : log_(fmt::format("ETL_Grpc[{}:{}]", ip, grpcPort))
+{
+    try {
+        grpc::ChannelArguments chArgs;
+        chArgs.SetMaxReceiveMessageSize(-1);
+
+        stub_ = org::xrpl::rpc::v1::XRPLedgerAPIService::NewStub(
+            grpc::CreateCustomChannel(resolve(ip, grpcPort), grpc::InsecureChannelCredentials(), chArgs)
+        );
+
+        LOG(log_.debug()) << "Made stub for remote.";
+    } catch (std::exception const& e) {
+        LOG(log_.warn()) << "Exception while creating stub: " << e.what() << ".";
+    }
+}
+
+std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+GrpcSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors)
+{
+    org::xrpl::rpc::v1::GetLedgerResponse response;
+    if (!stub_)
+        return {{grpc::StatusCode::INTERNAL, "No Stub"}, response};
+
+    // Ledger header with txns and metadata
+    org::xrpl::rpc::v1::GetLedgerRequest request;
+    grpc::ClientContext context;
+
+    request.mutable_ledger()->set_sequence(sequence);
+    request.set_transactions(true);
+    request.set_expand(true);
+    request.set_get_objects(getObjects);
+    request.set_get_object_neighbors(getObjectNeighbors);
+    request.set_user("ETL");
+
+    grpc::Status const status = stub_->GetLedger(&context, request, &response);
+
+    if (status.ok() and not response.is_unlimited()) {
+        log_.warn() << "is_unlimited is false. Make sure secure_gateway is set correctly on the ETL source. Status = "
+                    << status.error_message();
+    }
+
+    return {status, std::move(response)};
+}
+
+std::pair<std::vector<std::string>, bool>
+GrpcSource::loadInitialLedger(
+    uint32_t const sequence,
+    uint32_t const numMarkers,
+    etlng::InitialLoadObserverInterface& observer
+)
+{
+    if (!stub_)
+        return {{}, false};
+
+    std::vector<AsyncGrpcCall> calls = AsyncGrpcCall::makeAsyncCalls(sequence, numMarkers);
+
+    LOG(log_.debug()) << "Starting data download for ledger " << sequence << ".";
+
+    grpc::CompletionQueue queue;
+    for (auto& call : calls)
+        call.call(stub_, queue);
+
+    std::vector<std::string> edgeKeys;
+    void* tag = nullptr;
+    bool ok = false;
+    bool abort = false;
+    size_t numFinished = 0;
+
+    while (numFinished < calls.size() && queue.Next(&tag, &ok)) {
+        ASSERT(tag != nullptr, "Tag can't be null.");
+        auto ptr = static_cast<AsyncGrpcCall*>(tag);
+
+        if (!ok) {
+            LOG(log_.error()) << "loadInitialLedger - ok is false";
+            return {{}, false};  // cancelled
+        }
+
+        LOG(log_.trace()) << "Marker prefix = " << ptr->getMarkerPrefix();
+
+        auto result = ptr->process(stub_, queue, observer, abort);
+        if (result != AsyncGrpcCall::CallStatus::MORE) {
+            ++numFinished;
+            LOG(log_.debug()) << "Finished a marker. Current number of finished = " << numFinished;
+
+            if (auto lastKey = ptr->getLastKey(); !lastKey.empty())
+                edgeKeys.push_back(std::move(lastKey));
+        }
+
+        if (result == AsyncGrpcCall::CallStatus::ERRORED)
+            abort = true;
+    }
+
+    return {std::move(edgeKeys), !abort};
+}
+
+}  // namespace etlng::impl

src/etlng/impl/GrpcSource.hpp

@@ -0,0 +1,70 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/InitialLoadObserverInterface.hpp"
+#include "util/log/Logger.hpp"
+
+#include <grpcpp/support/status.h>
+#include <org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+class GrpcSource {
+    util::Logger log_;
+    std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub> stub_;
+
+public:
+    GrpcSource(std::string const& ip, std::string const& grpcPort);
+
+    /**
+     * @brief Fetch data for a specific ledger.
+     *
+     * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
+     * is found in the database, or the server is shutting down.
+     *
+     * @param sequence Sequence of the ledger to fetch
+     * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
+     * @param getObjectNeighbors Whether to request object neighbors; defaults to false
+     * @return A std::pair of the response status and the response itself
+     */
+    std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
+    fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false);
+
+    /**
+     * @brief Download a ledger in full.
+     *
+     * @param sequence Sequence of the ledger to download
+     * @param numMarkers Number of markers to generate for async calls
+     * @param observer InitialLoadObserverInterface implementation
+     * @return A std::pair of the data and a bool indicating whether the download was successful
+     */
+    std::pair<std::vector<std::string>, bool>
+    loadInitialLedger(uint32_t sequence, uint32_t numMarkers, etlng::InitialLoadObserverInterface& observer);
+};
+
+}  // namespace etlng::impl

src/etlng/impl/Registry.hpp

@@ -0,0 +1,219 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+#include "etlng/RegistryInterface.hpp"
+
+#include <xrpl/protocol/TxFormats.h>
+
+#include <concepts>
+#include <cstdint>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace etlng::impl {
+
+template <typename T>
+concept HasLedgerDataHook = requires(T p) {
+    { p.onLedgerData(std::declval<etlng::model::LedgerData>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialDataHook = requires(T p) {
+    { p.onInitialData(std::declval<etlng::model::LedgerData>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasTransactionHook = requires(T p) {
+    { p.onTransaction(uint32_t{}, std::declval<etlng::model::Transaction>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasObjectHook = requires(T p) {
+    { p.onObject(uint32_t{}, std::declval<etlng::model::Object>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialTransactionHook = requires(T p) {
+    { p.onInitialTransaction(uint32_t{}, std::declval<etlng::model::Transaction>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialObjectsHook = requires(T p) {
+    {
+        p.onInitialObjects(uint32_t{}, std::declval<std::vector<etlng::model::Object>>(), std::string{})
+    } -> std::same_as<void>;
+};
+
+template <typename T>
+concept HasInitialObjectHook = requires(T p) {
+    { p.onInitialObject(uint32_t{}, std::declval<etlng::model::Object>()) } -> std::same_as<void>;
+};
+
+template <typename T>
+concept ContainsSpec = std::decay_t<T>::spec::SpecTag;
+
+template <typename T>
+concept ContainsValidHook = HasLedgerDataHook<T> or HasInitialDataHook<T> or
+    (HasTransactionHook<T> and ContainsSpec<T>) or (HasInitialTransactionHook<T> and ContainsSpec<T>) or
+    HasObjectHook<T> or HasInitialObjectsHook<T> or HasInitialObjectHook<T>;
+
+template <typename T>
+concept NoTwoOfKind = not(HasLedgerDataHook<T> and HasTransactionHook<T>) and
+    not(HasInitialDataHook<T> and HasInitialTransactionHook<T>) and not(HasInitialDataHook<T> and HasObjectHook<T>) and
+    not(HasInitialObjectsHook<T> and HasInitialObjectHook<T>);
+
+template <typename T>
+concept SomeExtension = NoTwoOfKind<T> and ContainsValidHook<T>;
+
+template <SomeExtension... Ps>
+class Registry : public RegistryInterface {
+    std::tuple<Ps...> store_;
+
+    static_assert(
+        (((not HasTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and ...),
+        "Spec must be specified when 'onTransaction' function exists."
+    );
+
+    static_assert(
+        (((not HasInitialTransactionHook<std::decay_t<Ps>>) or ContainsSpec<std::decay_t<Ps>>) and ...),
+        "Spec must be specified when 'onInitialTransaction' function exists."
+    );
+
+public:
+    explicit constexpr Registry(SomeExtension auto&&... exts)
+        requires(std::is_same_v<std::decay_t<decltype(exts)>, std::decay_t<Ps>> and ...)
+        : store_(std::forward<Ps>(exts)...)
+    {
+    }
+
+    ~Registry() override = default;
+    Registry(Registry const&) = delete;
+    Registry(Registry&&) = default;
+    Registry&
+    operator=(Registry const&) = delete;
+    Registry&
+    operator=(Registry&&) = default;
+
+    constexpr void
+    dispatch(model::LedgerData const& data) override
+    {
+        // send entire batch of data at once
+        {
+            auto const expand = [&](auto& p) {
+                if constexpr (requires { p.onLedgerData(data); }) {
+                    p.onLedgerData(data);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send filtered transactions
+        {
+            auto const expand = [&]<typename P>(P& p, model::Transaction const& t) {
+                if constexpr (requires { p.onTransaction(data.seq, t); }) {
+                    if (std::decay_t<P>::spec::wants(t.type))
+                        p.onTransaction(data.seq, t);
+                }
+            };
+
+            for (auto const& t : data.transactions) {
+                std::apply([&expand, &t](auto&&... xs) { (expand(xs, t), ...); }, store_);
+            }
+        }
+
+        // send per object path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
+                if constexpr (requires { p.onObject(data.seq, o); }) {
+                    p.onObject(data.seq, o);
+                }
+            };
+
+            for (auto const& obj : data.objects) {
+                std::apply([&expand, &obj](auto&&... xs) { (expand(xs, obj), ...); }, store_);
+            }
+        }
+    }
+
+    constexpr void
+    dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) override
+    {
+        // send entire vector path
+        {
+            auto const expand = [&](auto&& p) {
+                if constexpr (requires { p.onInitialObjects(seq, data, lastKey); }) {
+                    p.onInitialObjects(seq, data, lastKey);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send per object path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Object const& o) {
+                if constexpr (requires { p.onInitialObject(seq, o); }) {
+                    p.onInitialObject(seq, o);
+                }
+            };
+
+            for (auto const& obj : data) {
+                std::apply([&expand, &obj](auto&&... xs) { (expand(xs, obj), ...); }, store_);
+            }
+        }
+    }
+
+    constexpr void
+    dispatchInitialData(model::LedgerData const& data) override
+    {
+        // send entire batch path
+        {
+            auto const expand = [&](auto&& p) {
+                if constexpr (requires { p.onInitialData(data); }) {
+                    p.onInitialData(data);
+                }
+            };
+
+            std::apply([&expand](auto&&... xs) { (expand(xs), ...); }, store_);
+        }
+
+        // send per tx path
+        {
+            auto const expand = [&]<typename P>(P&& p, model::Transaction const& tx) {
+                if constexpr (requires { p.onInitialTransaction(data.seq, tx); }) {
+                    if (std::decay_t<P>::spec::wants(tx.type))
+                        p.onInitialTransaction(data.seq, tx);
+                }
+            };
+
+            for (auto const& tx : data.transactions) {
+                std::apply([&expand, &tx](auto&&... xs) { (expand(xs, tx), ...); }, store_);
+            }
+        }
+    }
+};
+
+}  // namespace etlng::impl

src/feed/SubscriptionManager.hpp

@@ -31,6 +31,7 @@
 #include "util/async/AnyExecutionContext.hpp"
 #include "util/async/context/BasicExecutionContext.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
 
 #include <boost/asio/executor_work_guard.hpp>
 #include <boost/asio/io_context.hpp>
@@ -44,6 +45,7 @@
 #include <cstdint>
 #include <memory>
 #include <string>
+#include <utility>
 #include <vector>
 
 /**
@@ -67,16 +69,39 @@ class SubscriptionManager : public SubscriptionManagerInterface {
     impl::ProposedTransactionFeed proposedTransactionFeed_;
 
 public:
+    /**
+     * @brief Factory function to create a new SubscriptionManager with a PoolExecutionContext.
+     *
+     * @param config The configuration to use
+     * @param backend The backend to use
+     * @return A shared pointer to a new instance of SubscriptionManager
+     */
+    static std::shared_ptr<SubscriptionManager>
+    make_SubscriptionManager(
+        util::config::ClioConfigDefinition const& config,
+        std::shared_ptr<data::BackendInterface const> const& backend
+    )
+    {
+        auto const workersNum = config.get<uint64_t>("subscription_workers");
+
+        util::Logger const logger{"Subscriptions"};
+        LOG(logger.info()) << "Starting subscription manager with " << workersNum << " workers";
+
+        return std::make_shared<feed::SubscriptionManager>(util::async::PoolExecutionContext(workersNum), backend);
+    }
+
     /**
      * @brief Construct a new Subscription Manager object
      *
      * @param executor The executor to use to publish the feeds
      * @param backend The backend to use
      */
-    template <class ExecutorCtx>
-    SubscriptionManager(ExecutorCtx& executor, std::shared_ptr<data::BackendInterface const> const& backend)
+    SubscriptionManager(
+        util::async::AnyExecutionContext&& executor,
+        std::shared_ptr<data::BackendInterface const> const& backend
+    )
         : backend_(backend)
-        , ctx_(executor)
+        , ctx_(std::move(executor))
         , manifestFeed_(ctx_, "manifest")
         , validationsFeed_(ctx_, "validations")
         , ledgerFeed_(ctx_)
@@ -291,41 +316,4 @@ public:
     report() const final;
 };
 
-/**
- * @brief The help class to run the subscription manager. The container of PoolExecutionContext which is used to publish
- * the feeds.
- */
-class SubscriptionManagerRunner {
-    std::uint64_t workersNum_;
-    using ActualExecutionCtx = util::async::PoolExecutionContext;
-    ActualExecutionCtx ctx_;
-    std::shared_ptr<SubscriptionManager> subscriptionManager_;
-    util::Logger logger_{"Subscriptions"};
-
-public:
-    /**
-     * @brief Construct a new Subscription Manager Runner object
-     *
-     * @param config The configuration
-     * @param backend The backend to use
-     */
-    SubscriptionManagerRunner(util::Config const& config, std::shared_ptr<data::BackendInterface> const& backend)
-        : workersNum_(config.valueOr<std::uint64_t>("subscription_workers", 1))
-        , ctx_(workersNum_)
-        , subscriptionManager_(std::make_shared<SubscriptionManager>(ctx_, backend))
-    {
-        LOG(logger_.info()) << "Starting subscription manager with " << workersNum_ << " workers";
-    }
-
-    /**
-     * @brief Get the subscription manager
-     *
-     * @return The subscription manager
-     */
-    std::shared_ptr<SubscriptionManager>
-    getManager()
-    {
-        return subscriptionManager_;
-    }
-};
 }  // namespace feed

src/feed/Types.hpp

@@ -19,12 +19,13 @@
 
 #pragma once
 
-#include "web/interface/ConnectionBase.hpp"
+#include "web/SubscriptionContextInterface.hpp"
 
 #include <memory>
 
 namespace feed {
-using Subscriber = web::ConnectionBase;
+
+using Subscriber = web::SubscriptionContextInterface;
 using SubscriberPtr = Subscriber*;
 using SubscriberSharedPtr = std::shared_ptr<Subscriber>;
 

src/feed/impl/ProposedTransactionFeed.cpp

@@ -48,7 +48,7 @@ ProposedTransactionFeed::sub(SubscriberSharedPtr const& subscriber)
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed tx_proposed";
         ++subAllCount_.get();
-        subscriber->onDisconnect.connect([this](SubscriberPtr connection) { unsubInternal(connection); });
+        subscriber->onDisconnect([this](SubscriberPtr connection) { unsubInternal(connection); });
     }
 }
 
@@ -73,9 +73,7 @@ ProposedTransactionFeed::sub(ripple::AccountID const& account, SubscriberSharedP
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed accounts_proposed " << account;
         ++subAccountCount_.get();
-        subscriber->onDisconnect.connect([this, account](SubscriberPtr connection) {
-            unsubInternal(account, connection);
-        });
+        subscriber->onDisconnect([this, account](SubscriberPtr connection) { unsubInternal(account, connection); });
     }
 }
 

src/feed/impl/SingleFeedBase.cpp

@@ -49,7 +49,7 @@ SingleFeedBase::sub(SubscriberSharedPtr const& subscriber)
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed " << name_;
         ++subCount_.get();
-        subscriber->onDisconnect.connect([this](SubscriberPtr connectionDisconnecting) {
+        subscriber->onDisconnect([this](SubscriberPtr connectionDisconnecting) {
             unsubInternal(connectionDisconnecting);
         });
     };

src/feed/impl/TrackableSignalMap.hpp

@@ -23,6 +23,7 @@
 
 #include <boost/signals2.hpp>
 
+#include <concepts>
 #include <cstddef>
 #include <functional>
 #include <memory>

src/feed/impl/TransactionFeed.cpp

@@ -53,14 +53,14 @@ namespace feed::impl {
 void
 TransactionFeed::TransactionSlot::operator()(AllVersionTransactionsType const& allVersionMsgs) const
 {
-    if (auto connection = connectionWeakPtr.lock(); connection) {
+    if (auto connection = subscriptionContextWeakPtr.lock(); connection) {
         // Check if this connection already sent
         if (feed.get().notified_.contains(connection.get()))
             return;
 
         feed.get().notified_.insert(connection.get());
 
-        if (connection->apiSubVersion < 2u) {
+        if (connection->apiSubversion() < 2u) {
             connection->send(allVersionMsgs[0]);
             return;
         }
@@ -75,7 +75,7 @@ TransactionFeed::sub(SubscriberSharedPtr const& subscriber)
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed transactions";
         ++subAllCount_.get();
-        subscriber->onDisconnect.connect([this](SubscriberPtr connection) { unsubInternal(connection); });
+        subscriber->onDisconnect([this](SubscriberPtr connection) { unsubInternal(connection); });
     }
 }
 
@@ -86,18 +86,16 @@ TransactionFeed::sub(ripple::AccountID const& account, SubscriberSharedPtr const
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed account " << account;
         ++subAccountCount_.get();
-        subscriber->onDisconnect.connect([this, account](SubscriberPtr connection) {
-            unsubInternal(account, connection);
-        });
+        subscriber->onDisconnect([this, account](SubscriberPtr connection) { unsubInternal(account, connection); });
     }
 }
 
 void
 TransactionFeed::subProposed(SubscriberSharedPtr const& subscriber)
 {
-    auto const added = txProposedsignal_.connectTrackableSlot(subscriber, TransactionSlot(*this, subscriber));
+    auto const added = txProposedSignal_.connectTrackableSlot(subscriber, TransactionSlot(*this, subscriber));
     if (added) {
-        subscriber->onDisconnect.connect([this](SubscriberPtr connection) { unsubProposedInternal(connection); });
+        subscriber->onDisconnect([this](SubscriberPtr connection) { unsubProposedInternal(connection); });
     }
 }
 
@@ -107,7 +105,7 @@ TransactionFeed::subProposed(ripple::AccountID const& account, SubscriberSharedP
     auto const added =
         accountProposedSignal_.connectTrackableSlot(subscriber, account, TransactionSlot(*this, subscriber));
     if (added) {
-        subscriber->onDisconnect.connect([this, account](SubscriberPtr connection) {
+        subscriber->onDisconnect([this, account](SubscriberPtr connection) {
             unsubProposedInternal(account, connection);
         });
     }
@@ -120,7 +118,7 @@ TransactionFeed::sub(ripple::Book const& book, SubscriberSharedPtr const& subscr
     if (added) {
         LOG(logger_.info()) << subscriber->tag() << "Subscribed book " << book;
         ++subBookCount_.get();
-        subscriber->onDisconnect.connect([this, book](SubscriberPtr connection) { unsubInternal(book, connection); });
+        subscriber->onDisconnect([this, book](SubscriberPtr connection) { unsubInternal(book, connection); });
     }
 }
 
@@ -203,6 +201,7 @@ TransactionFeed::pub(
         pubObj[JS(meta)] = rpc::toJson(*meta);
         rpc::insertDeliveredAmount(pubObj[JS(meta)].as_object(), tx, meta, txMeta.date);
         rpc::insertDeliverMaxAlias(pubObj[txKey].as_object(), version);
+        rpc::insertMPTIssuanceID(pubObj[JS(meta)].as_object(), tx, meta);
 
         pubObj[JS(type)] = "transaction";
         pubObj[JS(validated)] = true;
@@ -284,7 +283,7 @@ TransactionFeed::pub(
         // clear the notified set. If the same connection subscribes both transactions + proposed_transactions,
         // rippled SENDS the same message twice
         notified_.clear();
-        txProposedsignal_.emit(allVersionsMsgs);
+        txProposedSignal_.emit(allVersionsMsgs);
         notified_.clear();
         // check duplicate for account and proposed_account, this prevents sending the same message multiple times
         // if it affects multiple accounts watched by the same connection
@@ -322,7 +321,7 @@ TransactionFeed::unsubInternal(ripple::AccountID const& account, SubscriberPtr s
 void
 TransactionFeed::unsubProposedInternal(SubscriberPtr subscriber)
 {
-    txProposedsignal_.disconnect(subscriber);
+    txProposedSignal_.disconnect(subscriber);
 }
 
 void

src/feed/impl/TransactionFeed.hpp

@@ -52,10 +52,10 @@ class TransactionFeed {
 
     struct TransactionSlot {
         std::reference_wrapper<TransactionFeed> feed;
-        std::weak_ptr<Subscriber> connectionWeakPtr;
+        std::weak_ptr<Subscriber> subscriptionContextWeakPtr;
 
         TransactionSlot(TransactionFeed& feed, SubscriberSharedPtr const& connection)
-            : feed(feed), connectionWeakPtr(connection)
+            : feed(feed), subscriptionContextWeakPtr(connection)
         {
         }
 
@@ -76,7 +76,7 @@ class TransactionFeed {
 
     // Signals for proposed tx subscribers
     TrackableSignalMap<ripple::AccountID, Subscriber, AllVersionTransactionsType const&> accountProposedSignal_;
-    TrackableSignal<Subscriber, AllVersionTransactionsType const&> txProposedsignal_;
+    TrackableSignal<Subscriber, AllVersionTransactionsType const&> txProposedSignal_;
 
     std::unordered_set<SubscriberPtr>
         notified_;  // Used by slots to prevent double notifications if tx contains multiple subscribed accounts

src/main/Main.cpp

@@ -19,15 +19,19 @@
 
 #include "app/CliArgs.hpp"
 #include "app/ClioApplication.hpp"
+#include "migration/MigrationApplication.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
 #include "util/TerminationHandler.hpp"
-#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigFileJson.hpp"
 
 #include <cstdlib>
 #include <exception>
 #include <iostream>
 
+using namespace util::config;
+
 int
 main(int argc, char const* argv[])
 try {
@@ -37,14 +41,36 @@ try {
     return action.apply(
         [](app::CliArgs::Action::Exit const& exit) { return exit.exitCode; },
         [](app::CliArgs::Action::Run const& run) {
-            auto const config = util::ConfigReader::open(run.configPath);
-            if (!config) {
-                std::cerr << "Couldnt parse config '" << run.configPath << "'." << std::endl;
+            auto const json = ConfigFileJson::make_ConfigFileJson(run.configPath);
+            if (!json.has_value()) {
+                std::cerr << json.error().error << std::endl;
                 return EXIT_FAILURE;
             }
-            util::LogService::init(config);
-            app::ClioApplication clio{config};
-            return clio.run();
+            auto const errors = ClioConfig.parse(json.value());
+            if (errors.has_value()) {
+                for (auto const& err : errors.value())
+                    std::cerr << err.error << std::endl;
+                return EXIT_FAILURE;
+            }
+            util::LogService::init(ClioConfig);
+            app::ClioApplication clio{ClioConfig};
+            return clio.run(run.useNgWebServer);
+        },
+        [](app::CliArgs::Action::Migrate const& migrate) {
+            auto const json = ConfigFileJson::make_ConfigFileJson(migrate.configPath);
+            if (!json.has_value()) {
+                std::cerr << json.error().error << std::endl;
+                return EXIT_FAILURE;
+            }
+            auto const errors = ClioConfig.parse(json.value());
+            if (errors.has_value()) {
+                for (auto const& err : errors.value())
+                    std::cerr << err.error << std::endl;
+                return EXIT_FAILURE;
+            }
+            util::LogService::init(ClioConfig);
+            app::MigratorApplication migrator{ClioConfig, migrate.subCmd};
+            return migrator.run();
         }
     );
 } catch (std::exception const& e) {

src/migration/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_library(clio_migration)
+
+target_sources(
+  clio_migration PRIVATE MigrationApplication.cpp impl/MigrationManagerFactory.cpp MigratorStatus.cpp
+                         cassandra/impl/ObjectsAdapter.cpp cassandra/impl/TransactionsAdapter.cpp
+)
+
+target_link_libraries(clio_migration PRIVATE clio_util clio_etl)

src/migration/MigrationApplication.cpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigrationApplication.hpp"
+
+#include "migration/MigratiorStatus.hpp"
+#include "migration/impl/MigrationManagerFactory.hpp"
+#include "util/OverloadSet.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Prometheus.hpp"
+
+#include <cstdlib>
+#include <iostream>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <variant>
+
+namespace app {
+
+MigratorApplication::MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command)
+    : cmd_(std::move(command))
+{
+    PrometheusService::init(config);
+
+    auto expectedMigrationManager = migration::impl::makeMigrationManager(config);
+
+    if (not expectedMigrationManager) {
+        throw std::runtime_error("Failed to create migration manager: " + expectedMigrationManager.error());
+    }
+
+    migrationManager_ = std::move(expectedMigrationManager.value());
+}
+
+int
+MigratorApplication::run()
+{
+    return std::visit(
+        util::OverloadSet{
+            [this](MigrateSubCmd::Status const&) { return printStatus(); },
+            [this](MigrateSubCmd::Migration const& cmdBundle) { return migrate(cmdBundle.migratorName); }
+        },
+        cmd_.state
+    );
+}
+
+int
+MigratorApplication::printStatus()
+{
+    std::cout << "Current Migration Status:" << std::endl;
+    auto const allMigratorsStatusPairs = migrationManager_->allMigratorsStatusPairs();
+
+    if (allMigratorsStatusPairs.empty()) {
+        std::cout << "No migrator found" << std::endl;
+    }
+
+    for (auto const& [migrator, status] : allMigratorsStatusPairs) {
+        std::cout << "Migrator: " << migrator << " - " << migrationManager_->getMigratorDescriptionByName(migrator)
+                  << " - " << status.toString() << std::endl;
+    }
+    return EXIT_SUCCESS;
+}
+
+int
+MigratorApplication::migrate(std::string const& migratorName)
+{
+    auto const status = migrationManager_->getMigratorStatusByName(migratorName);
+    if (status == migration::MigratorStatus::Migrated) {
+        std::cout << "Migrator " << migratorName << " has already migrated" << std::endl;
+        printStatus();
+        return EXIT_SUCCESS;
+    }
+
+    if (status == migration::MigratorStatus::NotKnown) {
+        std::cout << "Migrator " << migratorName << " not found" << std::endl;
+        printStatus();
+        return EXIT_FAILURE;
+    }
+
+    std::cout << "Running migration for " << migratorName << std::endl;
+    migrationManager_->runMigration(migratorName);
+    std::cout << "Migration for " << migratorName << " has finished" << std::endl;
+    return EXIT_SUCCESS;
+}
+
+}  // namespace app

src/migration/MigrationApplication.hpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <memory>
+#include <string>
+#include <variant>
+
+namespace app {
+
+/**
+ * @brief The command to run for migration framework
+ */
+struct MigrateSubCmd {
+    /**
+     * @brief Check the status of the migrations
+     */
+    struct Status {};
+    /**
+     * @brief Run a migration
+     */
+    struct Migration {
+        std::string migratorName;
+    };
+
+    std::variant<Status, Migration> state;
+
+    /**
+     * @brief Helper function to create a status command
+     *
+     * @return Cmd object containing the status command
+     */
+    static MigrateSubCmd
+    status()
+    {
+        return MigrateSubCmd{Status{}};
+    }
+
+    /**
+     * @brief Helper function to create a migration command
+     *
+     * @param name The name of the migration to run
+     * @return Cmd object containing the migration command
+     */
+    static MigrateSubCmd
+    migration(std::string const& name)
+    {
+        return MigrateSubCmd{Migration{name}};
+    }
+};
+
+/**
+ * @brief The migration application class
+ */
+class MigratorApplication {
+    std::string option_;
+    std::shared_ptr<migration::MigrationManagerInterface> migrationManager_;
+    MigrateSubCmd cmd_;
+
+public:
+    /**
+     * @brief Construct a new MigratorApplication object
+     *
+     * @param config The configuration of the application
+     * @param command The command to run
+     */
+    MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command);
+
+    /**
+     * @brief Run the application
+     *
+     * @return exit code
+     */
+    int
+    run();
+
+private:
+    int
+    printStatus();
+
+    int
+    migrate(std::string const& name);
+};
+}  // namespace app

src/migration/MigrationManagerInterface.hpp

@@ -0,0 +1,77 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace migration {
+
+/**
+ * @brief The interface for the migration manager. This interface is tend to be implemented for specific database. The
+ * application layer will use this interface to run the migrations.
+ */
+struct MigrationManagerInterface {
+    virtual ~MigrationManagerInterface() = default;
+
+    /**
+     * @brief Run the the migration according to the given migrator's name
+     */
+    virtual void
+    runMigration(std::string const&) = 0;
+
+    /**
+     * @brief Get the status of all the migrators
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     */
+    virtual std::vector<std::tuple<std::string, MigratorStatus>>
+    allMigratorsStatusPairs() const = 0;
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A vector of migrators' names
+     */
+    virtual std::vector<std::string>
+    allMigratorsNames() const = 0;
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The status of the migrator
+     */
+    virtual MigratorStatus
+    getMigratorStatusByName(std::string const& name) const = 0;
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The description of the migrator
+     */
+    virtual std::string
+    getMigratorDescriptionByName(std::string const& name) const = 0;
+};
+
+}  // namespace migration

src/migration/MigratiorStatus.hpp

@@ -0,0 +1,90 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+/**
+ * @brief The status of a migrator, it provides the helper functions to convert the status to string and vice versa
+ */
+class MigratorStatus {
+public:
+    /**
+     * @brief The status of a migrator
+     */
+    enum Status { Migrated, NotMigrated, NotKnown, NumStatuses };
+
+    /**
+     * @brief Construct a new Migrator Status object with the given status
+     *
+     * @param status The status of the migrator
+     */
+    MigratorStatus(Status status) : status_(status)
+    {
+    }
+
+    /**
+     * @brief Compare the status with another MigratorStatus
+     *
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(MigratorStatus const& other) const;
+
+    /**
+     * @brief Compare the status with another status
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(Status const& other) const;
+
+    /**
+     * @brief Convert the status to string
+     *
+     * @return The string representation of the status
+     */
+    std::string
+    toString() const;
+
+    /**
+     * @brief Convert the string to status
+     *
+     * @param statusStr The string to convert
+     * @return The status representation of the string
+     */
+    static MigratorStatus
+    fromString(std::string const& statusStr);
+
+private:
+    static constexpr std::array<char const*, static_cast<size_t>(NumStatuses)> statusStrMap = {
+        "Migrated",
+        "NotMigrated",
+        "NotKnown"
+    };
+
+    Status status_;
+};
+}  // namespace migration

src/migration/MigratorStatus.cpp

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+bool
+MigratorStatus::operator==(MigratorStatus const& other) const
+{
+    return status_ == other.status_;
+}
+
+bool
+MigratorStatus::operator==(Status const& other) const
+{
+    return status_ == other;
+}
+
+std::string
+MigratorStatus::toString() const
+{
+    return statusStrMap[static_cast<size_t>(status_)];
+}
+
+MigratorStatus
+MigratorStatus::fromString(std::string const& statusStr)
+{
+    for (std::size_t i = 0; i < statusStrMap.size(); ++i) {
+        if (statusStr == statusStrMap[i]) {
+            return MigratorStatus(static_cast<Status>(i));
+        }
+    }
+    return MigratorStatus(Status::NotMigrated);
+}
+
+}  // namespace migration

src/migration/README.md

@@ -0,0 +1,93 @@
+
+# Clio Migration 
+
+Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly.
+
+
+## Command Line Usage
+
+Clio provides a migration command-line tool to migrate data in database. 
+
+
+> Note: We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session.
+
+
+### To query migration status:
+    
+    
+    ./clio_server --migrate status  ~/config/migrator.json
+    
+This command returns the current migration status of each migrator. The example output: 
+
+    
+    Current Migration Status:
+    Migrator: ExampleMigrator - Feature v1, Clio v3 - not migrated
+    
+
+### To start a migration:
+    
+    
+    ./clio_server --migrate ExampleMigrator  ~/config/migrator.json
+    
+    
+Migration will run if the migrator has not been migrated. The migrator will be marked as migrated after the migration is completed.
+
+## How to write a migrator
+
+> **Note** If you'd like to add new index table in Clio and old historical data needs to be migrated into new table, you'd need to write a migrator.
+
+A migrator satisfies the `MigratorSpec`(impl/Spec.hpp) concept.
+
+It contains:
+
+-  A `name` which will be used to identify the migrator. User will refer this migrator in command-line tool by this name. The name needs to be different with other migrators, otherwise a compilation error will be raised.
+
+-  A `description` which is the detail information of the migrator.
+
+- A static function `runMigration`, it will be called when user run `--migrate name`. It accepts two parameters: backend, which provides the DB operations interface, and cfg, which provides migration-related configuration. Each migrator can have its own configuration under `.migration` session.
+
+- A type name alias `Backend` which specifies the backend type it supports.
+
+> **Note** Each migrator is designed to work with a specific database.
+
+- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB.  Migrator needs to be registered in `CassandraSupportedMigrators`.
+
+
+## How to use full table scanner (Only for Cassandra/ScyllaDB)
+Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel. 
+
+Most indexes are based on either ledger states or transactions. We provide the `objects` and `transactions` scanner. Developers only need to implement the callback function to receive the historical data. Please find the examples in `tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp` and `tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp`.
+
+> **Note** The full table scanner splits the table into multiple ranges by token(https://opensource.docs.scylladb.com/stable/cql/functions.html#token). A few of rows maybe read 2 times if its token happens to be at the edge of ranges. **Deduplication is needed** in the callback function.
+
+## How to write a full table scan adapter (Only for Cassandra/ScyllaDB)
+
+If you need to do full scan against other table, you can follow below steps:
+- Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member:
+    - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
+    - `PARTITION_KEY`, it's the name of the partition key of the table.
+    - `TABLE_NAME`
+
+- Inherent from `FullTableScannerAdapterBase`.
+- Implement `onRowRead`, its parameter is the `Row` we defined. It's the callback function when a row is read.
+
+
+Please take ObjectsAdapter/TransactionsAdapter as example.
+
+## Examples:
+
+We have some example migrators under `tests/integration/migration/cassandra` folder.
+
+- ExampleDropTableMigrator
+
+    This migrator drops `diff` table. 
+- ExampleLedgerMigrator
+
+    This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash. 
+- ExampleObjectsMigrator
+
+    This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT.
+- ExampleTransactionsMigrator
+
+    This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type.
+

src/migration/cassandra/CassandraMigrationBackend.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/CassandraBackend.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/cassandra/impl/CassandraMigrationSchema.hpp"
+#include "migration/cassandra/impl/Spec.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+
+namespace migration::cassandra {
+
+/**
+ * @brief The backend for the migration. It is a subclass of the CassandraBackend and provides the migration specific
+ * functionalities.
+ */
+class CassandraMigrationBackend : public data::cassandra::CassandraBackend {
+    util::Logger log_{"Migration"};
+    data::cassandra::SettingsProvider settingsProvider_;
+    impl::CassandraMigrationSchema migrationSchema_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Backend object. The backend is not readonly.
+     *
+     * @param settingsProvider The settings provider
+     */
+    explicit CassandraMigrationBackend(data::cassandra::SettingsProvider settingsProvider)
+        : data::cassandra::CassandraBackend{auto{settingsProvider}, false /* not readonly */}
+        , settingsProvider_(std::move(settingsProvider))
+        , migrationSchema_{settingsProvider_}
+    {
+    }
+
+    /**
+     *@brief Scan a table in a token range and call the callback for each row
+     *
+     *@tparam TableDesc The table description of the table to scan
+     *@param start The start token
+     *@param end The end token
+     *@param callback The callback to call for each row
+     *@param yield The boost asio yield context
+     */
+    template <impl::TableSpec TableDesc>
+    void
+    migrateInTokenRange(
+        std::int64_t const start,
+        std::int64_t const end,
+        auto const& callback,
+        boost::asio::yield_context yield
+    )
+    {
+        LOG(log_.debug()) << "Travsering token range: " << start << " - " << end
+                          << " ; table: " << TableDesc::TABLE_NAME;
+        // for each table we only have one prepared statement
+        static auto statementPrepared =
+            migrationSchema_.getPreparedFullScanStatement(handle_, TableDesc::TABLE_NAME, TableDesc::PARTITION_KEY);
+
+        auto const statement = statementPrepared.bind(start, end);
+
+        auto const res = this->executor_.read(yield, statement);
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch data from table: " << TableDesc::TABLE_NAME << " range: " << start
+                              << " - " << end << ";" << res.error();
+            return;
+        }
+
+        auto const& results = res.value();
+        if (not results.hasRows()) {
+            LOG(log_.debug()) << "No rows returned  - table: " << TableDesc::TABLE_NAME << " range: " << start << " - "
+                              << end;
+            return;
+        }
+
+        for (auto const& row : std::apply(
+                 [&](auto... args) { return data::cassandra::extract<decltype(args)...>(results); },
+                 typename TableDesc::Row{}
+             )) {
+            callback(row);
+        }
+    }
+};
+}  // namespace migration::cassandra

src/migration/cassandra/CassandraMigrationManager.hpp

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/impl/MigrationManagerBase.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+
+namespace migration::cassandra {
+
+// Register migrators here
+// MigratorsRegister<BackendType, ExampleMigrator>
+template <typename BackendType>
+using CassandraSupportedMigrators = migration::impl::MigratorsRegister<BackendType>;
+
+// Register with MigrationBackend which proceeds the migration
+using MigrationProcesser = CassandraSupportedMigrators<CassandraMigrationBackend>;
+
+// The Cassandra migration manager
+using CassandraMigrationManager = migration::impl::MigrationManagerBase<MigrationProcesser>;
+}  // namespace migration::cassandra

src/migration/cassandra/impl/CassandraMigrationSchema.hpp

@@ -0,0 +1,98 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+
+#include <fmt/core.h>
+
+#include <functional>
+#include <string>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The schema for the migration process. It contains the prepared statements only used for the migration process.
+ */
+class CassandraMigrationSchema {
+    using SettingsProviderType = data::cassandra::SettingsProvider;
+    std::reference_wrapper<SettingsProviderType const> settingsProvider_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Schema object
+     *
+     * @param settings The settings provider of database
+     */
+    explicit CassandraMigrationSchema(SettingsProviderType const& settings) : settingsProvider_{settings}
+    {
+    }
+
+    /**
+     * @brief Get the prepared statement for the full scan of a table
+     *
+     * @param handler The database handler
+     * @param tableName The name of the table
+     * @param key The partition key of the table
+     * @return The prepared statement
+     */
+    data::cassandra::PreparedStatement
+    getPreparedFullScanStatement(
+        data::cassandra::Handle const& handler,
+        std::string const& tableName,
+        std::string const& key
+    )
+    {
+        return handler.prepare(fmt::format(
+            R"(
+            SELECT * 
+              FROM {} 
+             WHERE TOKEN({}) >= ? AND TOKEN({}) <= ?
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), tableName),
+            key,
+            key
+        ));
+    }
+
+    /**
+     * @brief Get the prepared statement for insertion of migrator_status table
+     *
+     * @param handler The database handler
+     * @return The prepared statement to insert into migrator_status table
+     */
+    data::cassandra::PreparedStatement const&
+    getPreparedInsertMigratedMigrator(data::cassandra::Handle const& handler)
+    {
+        static auto prepared = handler.prepare(fmt::format(
+            R"(
+            INSERT INTO {} 
+                   (migrator_name, status)
+            VALUES (?, ?)
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), "migrator_status")
+        ));
+        return prepared;
+    }
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScanner.hpp

@@ -0,0 +1,183 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/ETLHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/async/AnyOperation.hpp"
+#include "util/async/context/BasicExecutionContext.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <ranges>
+#include <vector>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The token range used to split the full table scan into multiple ranges.
+ */
+struct TokenRange {
+    std::int64_t start;
+    std::int64_t end;
+
+    /**
+     * @brief Construct a new Token Range object
+     *
+     * @param start The start token
+     * @param end The end token
+     */
+    TokenRange(std::int64_t start, std::int64_t end) : start{start}, end{end}
+    {
+    }
+};
+
+/**
+ * @brief The concept for an adapter.
+ */
+template <typename T>
+concept CanReadByTokenRange = requires(T obj, TokenRange const& range, boost::asio::yield_context yield) {
+    { obj.readByTokenRange(range, yield) } -> std::same_as<void>;
+};
+
+/**
+ * @brief The full table scanner. It will split the full table scan into multiple ranges and read the data in given
+ * executor.
+ *
+ * @tparam TableAdapter The table adapter type
+ */
+template <CanReadByTokenRange TableAdapter>
+class FullTableScanner {
+    /**
+     * @brief The helper to generate the token ranges.
+     */
+    struct TokenRangesProvider {
+        uint32_t numRanges_;
+
+        TokenRangesProvider(uint32_t numRanges) : numRanges_{numRanges}
+        {
+        }
+
+        [[nodiscard]] std::vector<TokenRange>
+        getRanges() const
+        {
+            auto const minValue = std::numeric_limits<std::int64_t>::min();
+            auto const maxValue = std::numeric_limits<std::int64_t>::max();
+            if (numRanges_ == 1)
+                return {TokenRange{minValue, maxValue}};
+
+            // Safely calculate the range size using uint64_t to avoid overflow
+            uint64_t const rangeSize = (static_cast<uint64_t>(maxValue) * 2) / numRanges_;
+
+            std::vector<TokenRange> ranges;
+            ranges.reserve(numRanges_);
+
+            for (std::int64_t i = 0; i < numRanges_; ++i) {
+                int64_t const start = minValue + (i * rangeSize);
+                int64_t const end = (i == numRanges_ - 1) ? maxValue : start + static_cast<int64_t>(rangeSize) - 1;
+                ranges.emplace_back(start, end);
+            }
+
+            return ranges;
+        }
+    };
+
+    [[nodiscard]] auto
+    spawnWorker()
+    {
+        return ctx_.execute([this](auto token) {
+            while (not token.isStopRequested()) {
+                auto cursor = queue_.tryPop();
+                if (not cursor.has_value()) {
+                    return;  // queue is empty
+                }
+                reader_.readByTokenRange(cursor.value(), token);
+            }
+        });
+    }
+
+    void
+    load(size_t workerNum)
+    {
+        namespace vs = std::views;
+
+        tasks_.reserve(workerNum);
+
+        for ([[maybe_unused]] auto taskId : vs::iota(0u, workerNum))
+            tasks_.push_back(spawnWorker());
+    }
+
+    util::async::AnyExecutionContext ctx_;
+    std::size_t cursorsNum_;
+    etl::ThreadSafeQueue<TokenRange> queue_;
+    std::vector<util::async::AnyOperation<void>> tasks_;
+    TableAdapter reader_;
+
+public:
+    /**
+     * @brief The full table scanner settings.
+     */
+    struct FullTableScannerSettings {
+        std::uint32_t ctxThreadsNum; /**< number of threads used in the execution context */
+        std::uint32_t jobsNum;       /**< number of coroutines to run, it is the number of concurrent database reads */
+        std::uint32_t cursorsPerJob; /**< number of cursors per coroutine */
+    };
+
+    /**
+     * @brief Construct a new Full Table Scanner object, it will run in a sync or async context according to the
+     * parameter. The scan process will immediately start.
+     *
+     * @tparam ExecutionContextType The execution context type
+     * @param settings The full table scanner settings
+     * @param reader The table adapter
+     */
+    template <typename ExecutionContextType = util::async::CoroExecutionContext>
+    FullTableScanner(FullTableScannerSettings settings, TableAdapter&& reader)
+        : ctx_(ExecutionContextType(settings.ctxThreadsNum))
+        , cursorsNum_(settings.jobsNum * settings.cursorsPerJob)
+        , queue_{cursorsNum_}
+        , reader_{std::move(reader)}
+    {
+        ASSERT(settings.jobsNum > 0, "jobsNum for full table scanner must be greater than 0");
+        ASSERT(settings.cursorsPerJob > 0, "cursorsPerJob for full table scanner must be greater than 0");
+
+        auto const cursors = TokenRangesProvider{cursorsNum_}.getRanges();
+        std::ranges::for_each(cursors, [this](auto const& cursor) { queue_.push(cursor); });
+        load(settings.jobsNum);
+    }
+
+    /**
+     * @brief Wait for all workers to finish.
+     */
+    void
+    wait()
+    {
+        for (auto& task : tasks_) {
+            task.wait();
+        }
+    }
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScannerAdapterBase.hpp

@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The base class for the full table scanner adapter. It is responsible for reading the rows from the full table
+ * scanner and call the callback when a row is read. With this base class, each table adapter can focus on the actual
+ * row data converting.
+ *
+ * @tparam TableDesc The table description, it has to be a TableSpec.
+ */
+template <TableSpec TableDesc>
+struct FullTableScannerAdapterBase {
+    static_assert(TableSpec<TableDesc>);
+
+protected:
+    /**
+     * @brief The backend to use
+     */
+    std::shared_ptr<CassandraMigrationBackend> backend_;
+
+public:
+    virtual ~FullTableScannerAdapterBase() = default;
+
+    /**
+     * @brief Construct a new Full Table Scanner Adapter Base object
+     *
+     * @param backend The backend
+     */
+    FullTableScannerAdapterBase(std::shared_ptr<CassandraMigrationBackend> backend) : backend_(std::move(backend))
+    {
+    }
+
+    /**
+     * @brief Read the row in the given token range from database, this is the adapt function for the FullTableScanner.
+     *
+     * @param range The token range to read
+     * @param yield The yield context
+     */
+    void
+    readByTokenRange(TokenRange const& range, boost::asio::yield_context yield)
+    {
+        backend_->migrateInTokenRange<TableDesc>(
+            range.start, range.end, [this](auto const& row) { onRowRead(row); }, yield
+        );
+    }
+
+    /**
+     * @brief Called when a row is read. The derived class should implement this function to convert the database blob
+     * to actual data type.
+     *
+     * @param row The row read
+     */
+    virtual void
+    onRowRead(TableDesc::Row const& row) = 0;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.cpp

@@ -0,0 +1,43 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+
+#include <optional>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+void
+ObjectsAdapter::onRowRead(TableObjectsDesc::Row const& row)
+{
+    auto const& [key, ledgerSeq, blob] = row;
+    // the blob can be empty which means the ledger state is deleted
+    if (blob.empty()) {
+        onStateRead_(ledgerSeq, std::nullopt);
+        return;
+    }
+    ripple::SLE sle{ripple::SerialIter{blob.data(), blob.size()}, key};
+    onStateRead_(ledgerSeq, std::make_optional(std::move(sle)));
+}
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/Types.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the objects table. It has to be a TableSpec.
+ */
+struct TableObjectsDesc {
+    using Row = std::tuple<ripple::uint256, std::uint32_t, data::Blob>;
+    static constexpr char const* PARTITION_KEY = "key";
+    static constexpr char const* TABLE_NAME = "objects";
+};
+
+/**
+ * @brief The adapter for the objects table. This class is responsible for reading the objects from the
+ * FullTableScanner and converting the blobs to the STObject.
+ */
+class ObjectsAdapter : public impl::FullTableScannerAdapterBase<TableObjectsDesc> {
+public:
+    using OnStateRead = std::function<void(std::uint32_t, std::optional<ripple::SLE>)>;
+
+    /**
+     * @brief Construct a new Objects Adapter object
+     *
+     * @param backend The backend to use
+     * @param onStateRead The callback to call when a state is read
+     */
+    explicit ObjectsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnStateRead onStateRead)
+        : FullTableScannerAdapterBase<TableObjectsDesc>(backend), onStateRead_{std::move(onStateRead)}
+    {
+    }
+
+    /**
+     * @brief Called when a row is read
+     *
+     * @param row The row to read
+     */
+    void
+    onRowRead(TableObjectsDesc::Row const& row) override;
+
+private:
+    OnStateRead onStateRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Spec.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/asio/spawn.hpp>
+
+#include <concepts>
+#include <tuple>
+#include <type_traits>
+
+namespace migration::cassandra::impl {
+// Define the concept for a class like TableObjectsDesc
+template <typename T>
+concept TableSpec = requires {
+    // Check that 'row' exists and is a tuple
+    // keys types are at the begining and the other fields types sort in alphabetical order
+    typename T::Row;
+    requires std::tuple_size<typename T::Row>::value >= 0;  // Ensures 'row' is a tuple
+
+    // Check that static constexpr members 'partitionKey' and 'tableName' exist
+    { T::PARTITION_KEY } -> std::convertible_to<char const*>;
+    { T::TABLE_NAME } -> std::convertible_to<char const*>;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.cpp

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+namespace migration::cassandra::impl {
+
+void
+TransactionsAdapter::onRowRead(TableTransactionsDesc::Row const& row)
+{
+    auto const& [txHash, date, ledgerSeq, metaBlob, txBlob] = row;
+
+    ripple::SerialIter it{txBlob.data(), txBlob.size()};
+    ripple::STTx const sttx{it};
+    ripple::TxMeta const txMeta{sttx.getTransactionID(), ledgerSeq, metaBlob};
+    onTransactionRead_(sttx, txMeta);
+}
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the transactions table. It has to be a TableSpec.
+ */
+struct TableTransactionsDesc {
+    // hash, date, ledger_seq, metadata, transaction
+    using Row = std::tuple<ripple::uint256, std::uint64_t, std::uint32_t, ripple::Blob, ripple::Blob>;
+    static constexpr char const* PARTITION_KEY = "hash";
+    static constexpr char const* TABLE_NAME = "transactions";
+};
+
+/**
+ * @brief The adapter for the transactions table. This class is responsible for reading the transactions from the
+ * FullTableScanner and converting the blobs to the STTx and TxMeta.
+ */
+class TransactionsAdapter : public impl::FullTableScannerAdapterBase<TableTransactionsDesc> {
+public:
+    using OnTransactionRead = std::function<void(ripple::STTx, ripple::TxMeta)>;
+
+    /**
+     * @brief Construct a new Transactions Adapter object
+     *
+     * @param backend The backend
+     * @param onTxRead The callback to call when a transaction is read
+     */
+    explicit TransactionsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnTransactionRead onTxRead)
+        : FullTableScannerAdapterBase<TableTransactionsDesc>(backend), onTransactionRead_{std::move(onTxRead)}
+    {
+    }
+
+    /**
+     *@brief The callback when a row is read.
+     *
+     *@param row The row to read
+     */
+    void
+    onRowRead(TableTransactionsDesc::Row const& row) override;
+
+private:
+    OnTransactionRead onTransactionRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Types.hpp

@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+namespace migration::cassandra::impl {
+
+using ObjectsScanner = impl::FullTableScanner<impl::ObjectsAdapter>;
+using TransactionsScanner = impl::FullTableScanner<impl::TransactionsAdapter>;
+
+}  // namespace migration::cassandra::impl