feat: Migration framework (#1768)

This PR implemented the migration framework, which contains the command
line interface to execute migration and helps to migrate data easily.
Please read README.md for more information about this framework.

src/CMakeLists.txt

@@ -5,5 +5,6 @@ add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(migration)
 add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
 target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp WebHandlers.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration)

src/app/CliArgs.cpp

@@ -19,6 +19,7 @@
 
 #include "app/CliArgs.hpp"
 
+#include "migration/MigrationApplication.hpp"
 #include "util/build/Build.hpp"
 
 #include <boost/program_options/options_description.hpp>
@@ -45,6 +46,7 @@ CliArgs::parse(int argc, char const* argv[])
         ("version,v", "print version and exit")
         ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
         ("ng-web-server,w", "Use ng-web-server")
+        ("migrate", po::value<std::string>(), "start migration helper")
     ;
     // clang-format on
     po::positional_options_description positional;
@@ -65,6 +67,14 @@ CliArgs::parse(int argc, char const* argv[])
     }
 
     auto configPath = parsed["conf"].as<std::string>();
+
+    if (parsed.count("migrate") != 0u) {
+        auto const opt = parsed["migrate"].as<std::string>();
+        if (opt == "status")
+            return Action{Action::Migrate{std::move(configPath), MigrateSubCmd::status()}};
+        return Action{Action::Migrate{std::move(configPath), MigrateSubCmd::migration(opt)}};
+    }
+
     return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
     };
 }

src/app/CliArgs.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "migration/MigrationApplication.hpp"
 #include "util/OverloadSet.hpp"
 
 #include <string>
@@ -52,13 +53,20 @@ public:
             int exitCode;  ///< Exit code.
         };
 
+        /** @brief Migration action. */
+        struct Migrate {
+            std::string configPath;
+            MigrateSubCmd subCmd;
+        };
+
         /**
          * @brief Construct an action from a Run.
          *
          * @param action Run action.
          */
         template <typename ActionType>
-            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit> or
+            std::is_same_v<ActionType, Migrate>
         explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
         {
         }
@@ -78,7 +86,7 @@ public:
         }
 
     private:
-        std::variant<Run, Exit> action_;
+        std::variant<Run, Exit, Migrate> action_;
     };
 
     /**

src/data/BackendInterface.hpp

@@ -548,6 +548,16 @@ public:
         boost::asio::yield_context yield
     ) const;
 
+    /**
+     * @brief Fetches the status of migrator by name.
+     *
+     * @param migratorName The name of the migrator
+     * @param yield The coroutine context
+     * @return The status of the migrator if found; nullopt otherwise
+     */
+    virtual std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
+
     /**
      * @brief Synchronously fetches the ledger range from DB.
      *
@@ -673,6 +683,15 @@ public:
     bool
     finishWrites(std::uint32_t ledgerSequence);
 
+    /**
+     * @brief Mark the migration status of a migrator as Migrated in the database
+     *
+     * @param migratorName The name of the migrator
+     * @param status The status to set
+     */
+    virtual void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;
+
     /**
      * @return true if database is overwhelmed; false otherwise
      */

src/data/CassandraBackend.hpp

@@ -72,13 +72,15 @@ class BasicCassandraBackend : public BackendInterface {
 
     SettingsProviderType settingsProvider_;
     Schema<SettingsProviderType> schema_;
+
+    std::atomic_uint32_t ledgerSequence_ = 0u;
+
+protected:
     Handle handle_;
 
     // have to be mutable because BackendInterface constness :(
     mutable ExecutionStrategyType executor_;
 
-    std::atomic_uint32_t ledgerSequence_ = 0u;
-
 public:
     /**
      * @brief Create a new cassandra/scylla backend instance.
@@ -835,6 +837,26 @@ public:
         return results;
     }
 
+    std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override
+    {
+        auto const res = executor_.read(yield, schema_->selectMigratorStatus, Text(migratorName));
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch migrator status: " << res.error();
+            return {};
+        }
+
+        auto const& results = res.value();
+        if (not results) {
+            return {};
+        }
+
+        for (auto [statusString] : extract<std::string>(results))
+            return statusString;
+
+        return {};
+    }
+
     void
     doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
     {
@@ -962,6 +984,14 @@ public:
         // probably was used in PG to start a transaction or smth.
     }
 
+    void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) override
+    {
+        executor_.writeSync(
+            schema_->insertMigratorStatus, data::cassandra::Text{migratorName}, data::cassandra::Text(status)
+        );
+    }
+
     bool
     isTooBusy() const override
     {

src/data/README.md

@@ -262,3 +262,15 @@ CREATE TABLE clio.nf_token_transactions (
 ```
 
 The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, inspired by the same motivations and fulfilling a similar role within this context. It drives the `nft_history` API.
+
+### migrator_status
+
+```
+CREATE TABLE clio.migrator_status (
+    migrator_name TEXT,     # The name of the migrator
+    status TEXT,            # The status of the migrator
+    PRIMARY KEY (migrator_name)
+) 
+```
+
+The `migrator_status` table stores the status of the migratior in this database. If a migrator's status is `migrated`, it means this database has finished data migration for this migrator.

src/data/cassandra/Schema.hpp

@@ -270,6 +270,18 @@ public:
             qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                   migrator_name TEXT,
+                          status TEXT,
+                         PRIMARY KEY (migrator_name)
+                  ) 
+            )",
+            qualifiedTableName(settingsProvider_.get(), "migrator_status")
+        ));
+
         return statements;
     }();
 
@@ -466,6 +478,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {}
+                       (migrator_name, status)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
+
         //
         // Select queries
         //
@@ -768,6 +791,17 @@ public:
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
         }();
+
+        PreparedStatement selectMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT status
+                  FROM {}
+                 WHERE migrator_name = ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
     };
 
     /**

src/data/cassandra/SettingsProvider.cpp

@@ -24,7 +24,6 @@
 #include "util/Constants.hpp"
 #include "util/newconfig/ObjectView.hpp"
 
-
 #include <cerrno>
 #include <chrono>
 #include <cstddef>

src/data/cassandra/Types.hpp

@@ -21,6 +21,8 @@
 
 #include <cstdint>
 #include <expected>
+#include <string>
+#include <utility>
 
 namespace data::cassandra {
 
@@ -55,6 +57,26 @@ struct Limit {
     int32_t limit;
 };
 
+/**
+ * @brief A strong type wrapper for string
+ *
+ * This is unfortunately needed right now to support TEXT properly
+ * because clio uses string to represent BLOB
+ * If we want to bind TEXT with string, we need to use this type
+ */
+struct Text {
+    std::string text;
+
+    /**
+     * @brief Construct a new Text object from string type
+     *
+     * @param text The text to wrap
+     */
+    explicit Text(std::string text) : text{std::move(text)}
+    {
+    }
+};
+
 class Handle;
 class CassandraError;
 

src/data/cassandra/impl/Statement.hpp

@@ -119,6 +119,9 @@ public:
             // reinterpret_cast is needed here :'(
             auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
             throwErrorIfNeeded(rc, "Bind string (as bytes)");
+        } else if constexpr (std::is_convertible_v<DecayedType, Text>) {
+            auto const rc = cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
+            throwErrorIfNeeded(rc, "Bind string (as TEXT)");
         } else if constexpr (std::is_same_v<DecayedType, UintTupleType> ||
                              std::is_same_v<DecayedType, UintByteTupleType>) {
             auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});

src/main/Main.cpp

@@ -19,8 +19,10 @@
 
 #include "app/CliArgs.hpp"
 #include "app/ClioApplication.hpp"
+#include "migration/MigrationApplication.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
 #include "util/TerminationHandler.hpp"
+#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
 #include "util/newconfig/ConfigFileJson.hpp"
@@ -54,6 +56,22 @@ try {
             util::LogService::init(ClioConfig);
             app::ClioApplication clio{ClioConfig};
             return clio.run(run.useNgWebServer);
+        },
+        [](app::CliArgs::Action::Migrate const& migrate) {
+            auto const json = ConfigFileJson::make_ConfigFileJson(migrate.configPath);
+            if (!json.has_value()) {
+                std::cerr << json.error().error << std::endl;
+                return EXIT_FAILURE;
+            }
+            auto const errors = ClioConfig.parse(json.value());
+            if (errors.has_value()) {
+                for (auto const& err : errors.value())
+                    std::cerr << err.error << std::endl;
+                return EXIT_FAILURE;
+            }
+            util::LogService::init(ClioConfig);
+            app::MigratorApplication migrator{ClioConfig, migrate.subCmd};
+            return migrator.run();
         }
     );
 } catch (std::exception const& e) {

src/migration/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_library(clio_migration)
+
+target_sources(
+  clio_migration PRIVATE MigrationApplication.cpp impl/MigrationManagerFactory.cpp MigratorStatus.cpp
+                         cassandra/impl/ObjectsAdapter.cpp cassandra/impl/TransactionsAdapter.cpp
+)
+
+target_link_libraries(clio_migration PRIVATE clio_util clio_etl)

src/migration/MigrationApplication.cpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigrationApplication.hpp"
+
+#include "migration/MigratiorStatus.hpp"
+#include "migration/impl/MigrationManagerFactory.hpp"
+#include "util/OverloadSet.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Prometheus.hpp"
+
+#include <cstdlib>
+#include <iostream>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <variant>
+
+namespace app {
+
+MigratorApplication::MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command)
+    : cmd_(std::move(command))
+{
+    PrometheusService::init(config);
+
+    auto expectedMigrationManager = migration::impl::makeMigrationManager(config);
+
+    if (not expectedMigrationManager) {
+        throw std::runtime_error("Failed to create migration manager: " + expectedMigrationManager.error());
+    }
+
+    migrationManager_ = std::move(expectedMigrationManager.value());
+}
+
+int
+MigratorApplication::run()
+{
+    return std::visit(
+        util::OverloadSet{
+            [this](MigrateSubCmd::Status const&) { return printStatus(); },
+            [this](MigrateSubCmd::Migration const& cmdBundle) { return migrate(cmdBundle.migratorName); }
+        },
+        cmd_.state
+    );
+}
+
+int
+MigratorApplication::printStatus()
+{
+    std::cout << "Current Migration Status:" << std::endl;
+    auto const allMigratorsStatusPairs = migrationManager_->allMigratorsStatusPairs();
+
+    if (allMigratorsStatusPairs.empty()) {
+        std::cout << "No migrator found" << std::endl;
+    }
+
+    for (auto const& [migrator, status] : allMigratorsStatusPairs) {
+        std::cout << "Migrator: " << migrator << " - " << migrationManager_->getMigratorDescriptionByName(migrator)
+                  << " - " << status.toString() << std::endl;
+    }
+    return EXIT_SUCCESS;
+}
+
+int
+MigratorApplication::migrate(std::string const& migratorName)
+{
+    auto const status = migrationManager_->getMigratorStatusByName(migratorName);
+    if (status == migration::MigratorStatus::Migrated) {
+        std::cout << "Migrator " << migratorName << " has already migrated" << std::endl;
+        printStatus();
+        return EXIT_SUCCESS;
+    }
+
+    if (status == migration::MigratorStatus::NotKnown) {
+        std::cout << "Migrator " << migratorName << " not found" << std::endl;
+        printStatus();
+        return EXIT_FAILURE;
+    }
+
+    std::cout << "Running migration for " << migratorName << std::endl;
+    migrationManager_->runMigration(migratorName);
+    std::cout << "Migration for " << migratorName << " has finished" << std::endl;
+    return EXIT_SUCCESS;
+}
+
+}  // namespace app

src/migration/MigrationApplication.hpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <memory>
+#include <string>
+#include <variant>
+
+namespace app {
+
+/**
+ * @brief The command to run for migration framework
+ */
+struct MigrateSubCmd {
+    /**
+     * @brief Check the status of the migrations
+     */
+    struct Status {};
+    /**
+     * @brief Run a migration
+     */
+    struct Migration {
+        std::string migratorName;
+    };
+
+    std::variant<Status, Migration> state;
+
+    /**
+     * @brief Helper function to create a status command
+     *
+     * @return Cmd object containing the status command
+     */
+    static MigrateSubCmd
+    status()
+    {
+        return MigrateSubCmd{Status{}};
+    }
+
+    /**
+     * @brief Helper function to create a migration command
+     *
+     * @param name The name of the migration to run
+     * @return Cmd object containing the migration command
+     */
+    static MigrateSubCmd
+    migration(std::string const& name)
+    {
+        return MigrateSubCmd{Migration{name}};
+    }
+};
+
+/**
+ * @brief The migration application class
+ */
+class MigratorApplication {
+    std::string option_;
+    std::shared_ptr<migration::MigrationManagerInterface> migrationManager_;
+    MigrateSubCmd cmd_;
+
+public:
+    /**
+     * @brief Construct a new MigratorApplication object
+     *
+     * @param config The configuration of the application
+     * @param command The command to run
+     */
+    MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command);
+
+    /**
+     * @brief Run the application
+     *
+     * @return exit code
+     */
+    int
+    run();
+
+private:
+    int
+    printStatus();
+
+    int
+    migrate(std::string const& name);
+};
+}  // namespace app

src/migration/MigrationManagerInterface.hpp

@@ -0,0 +1,77 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace migration {
+
+/**
+ * @brief The interface for the migration manager. This interface is tend to be implemented for specific database. The
+ * application layer will use this interface to run the migrations.
+ */
+struct MigrationManagerInterface {
+    virtual ~MigrationManagerInterface() = default;
+
+    /**
+     * @brief Run the the migration according to the given migrator's name
+     */
+    virtual void
+    runMigration(std::string const&) = 0;
+
+    /**
+     * @brief Get the status of all the migrators
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     */
+    virtual std::vector<std::tuple<std::string, MigratorStatus>>
+    allMigratorsStatusPairs() const = 0;
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A vector of migrators' names
+     */
+    virtual std::vector<std::string>
+    allMigratorsNames() const = 0;
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The status of the migrator
+     */
+    virtual MigratorStatus
+    getMigratorStatusByName(std::string const& name) const = 0;
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The description of the migrator
+     */
+    virtual std::string
+    getMigratorDescriptionByName(std::string const& name) const = 0;
+};
+
+}  // namespace migration

src/migration/MigratiorStatus.hpp

@@ -0,0 +1,90 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+/**
+ * @brief The status of a migrator, it provides the helper functions to convert the status to string and vice versa
+ */
+class MigratorStatus {
+public:
+    /**
+     * @brief The status of a migrator
+     */
+    enum Status { Migrated, NotMigrated, NotKnown, NumStatuses };
+
+    /**
+     * @brief Construct a new Migrator Status object with the given status
+     *
+     * @param status The status of the migrator
+     */
+    MigratorStatus(Status status) : status_(status)
+    {
+    }
+
+    /**
+     * @brief Compare the status with another MigratorStatus
+     *
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(MigratorStatus const& other) const;
+
+    /**
+     * @brief Compare the status with another status
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(Status const& other) const;
+
+    /**
+     * @brief Convert the status to string
+     *
+     * @return The string representation of the status
+     */
+    std::string
+    toString() const;
+
+    /**
+     * @brief Convert the string to status
+     *
+     * @param statusStr The string to convert
+     * @return The status representation of the string
+     */
+    static MigratorStatus
+    fromString(std::string const& statusStr);
+
+private:
+    static constexpr std::array<char const*, static_cast<size_t>(NumStatuses)> statusStrMap = {
+        "Migrated",
+        "NotMigrated",
+        "NotKnown"
+    };
+
+    Status status_;
+};
+}  // namespace migration

src/migration/MigratorStatus.cpp

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+bool
+MigratorStatus::operator==(MigratorStatus const& other) const
+{
+    return status_ == other.status_;
+}
+
+bool
+MigratorStatus::operator==(Status const& other) const
+{
+    return status_ == other;
+}
+
+std::string
+MigratorStatus::toString() const
+{
+    return statusStrMap[static_cast<size_t>(status_)];
+}
+
+MigratorStatus
+MigratorStatus::fromString(std::string const& statusStr)
+{
+    for (std::size_t i = 0; i < statusStrMap.size(); ++i) {
+        if (statusStr == statusStrMap[i]) {
+            return MigratorStatus(static_cast<Status>(i));
+        }
+    }
+    return MigratorStatus(Status::NotMigrated);
+}
+
+}  // namespace migration

src/migration/README.md

@@ -0,0 +1,93 @@
+
+# Clio Migration 
+
+Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly.
+
+
+## Command Line Usage
+
+Clio provides a migration command-line tool to migrate data in database. 
+
+
+> Note: We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session.
+
+
+### To query migration status:
+    
+    
+    ./clio_server --migrate status  ~/config/migrator.json
+    
+This command returns the current migration status of each migrator. The example output: 
+
+    
+    Current Migration Status:
+    Migrator: ExampleMigrator - Feature v1, Clio v3 - not migrated
+    
+
+### To start a migration:
+    
+    
+    ./clio_server --migrate ExampleMigrator  ~/config/migrator.json
+    
+    
+Migration will run if the migrator has not been migrated. The migrator will be marked as migrated after the migration is completed.
+
+## How to write a migrator
+
+> **Note** If you'd like to add new index table in Clio and old historical data needs to be migrated into new table, you'd need to write a migrator.
+
+A migrator satisfies the `MigratorSpec`(impl/Spec.hpp) concept.
+
+It contains:
+
+-  A `name` which will be used to identify the migrator. User will refer this migrator in command-line tool by this name. The name needs to be different with other migrators, otherwise a compilation error will be raised.
+
+-  A `description` which is the detail information of the migrator.
+
+- A static function `runMigration`, it will be called when user run `--migrate name`. It accepts two parameters: backend, which provides the DB operations interface, and cfg, which provides migration-related configuration. Each migrator can have its own configuration under `.migration` session.
+
+- A type name alias `Backend` which specifies the backend type it supports.
+
+> **Note** Each migrator is designed to work with a specific database.
+
+- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB.  Migrator needs to be registered in `CassandraSupportedMigrators`.
+
+
+## How to use full table scanner (Only for Cassandra/ScyllaDB)
+Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel. 
+
+Most indexes are based on either ledger states or transactions. We provide the `objects` and `transactions` scanner. Developers only need to implement the callback function to receive the historical data. Please find the examples in `tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp` and `tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp`.
+
+> **Note** The full table scanner splits the table into multiple ranges by token(https://opensource.docs.scylladb.com/stable/cql/functions.html#token). A few of rows maybe read 2 times if its token happens to be at the edge of ranges. **Deduplication is needed** in the callback function.
+
+## How to write a full table scan adapter (Only for Cassandra/ScyllaDB)
+
+If you need to do full scan against other table, you can follow below steps:
+- Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member:
+    - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
+    - `PARTITION_KEY`, it's the name of the partition key of the table.
+    - `TABLE_NAME`
+
+- Inherent from `FullTableScannerAdapterBase`.
+- Implement `onRowRead`, its parameter is the `Row` we defined. It's the callback function when a row is read.
+
+
+Please take ObjectsAdapter/TransactionsAdapter as example.
+
+## Examples:
+
+We have some example migrators under `tests/integration/migration/cassandra` folder.
+
+- ExampleDropTableMigrator
+
+    This migrator drops `diff` table. 
+- ExampleLedgerMigrator
+
+    This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash. 
+- ExampleObjectsMigrator
+
+    This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT.
+- ExampleTransactionsMigrator
+
+    This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type.
+

src/migration/cassandra/CassandraMigrationBackend.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/CassandraBackend.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/cassandra/impl/CassandraMigrationSchema.hpp"
+#include "migration/cassandra/impl/Spec.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+
+namespace migration::cassandra {
+
+/**
+ * @brief The backend for the migration. It is a subclass of the CassandraBackend and provides the migration specific
+ * functionalities.
+ */
+class CassandraMigrationBackend : public data::cassandra::CassandraBackend {
+    util::Logger log_{"Migration"};
+    data::cassandra::SettingsProvider settingsProvider_;
+    impl::CassandraMigrationSchema migrationSchema_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Backend object. The backend is not readonly.
+     *
+     * @param settingsProvider The settings provider
+     */
+    explicit CassandraMigrationBackend(data::cassandra::SettingsProvider settingsProvider)
+        : data::cassandra::CassandraBackend{auto{settingsProvider}, false /* not readonly */}
+        , settingsProvider_(std::move(settingsProvider))
+        , migrationSchema_{settingsProvider_}
+    {
+    }
+
+    /**
+     *@brief Scan a table in a token range and call the callback for each row
+     *
+     *@tparam TableDesc The table description of the table to scan
+     *@param start The start token
+     *@param end The end token
+     *@param callback The callback to call for each row
+     *@param yield The boost asio yield context
+     */
+    template <impl::TableSpec TableDesc>
+    void
+    migrateInTokenRange(
+        std::int64_t const start,
+        std::int64_t const end,
+        auto const& callback,
+        boost::asio::yield_context yield
+    )
+    {
+        LOG(log_.debug()) << "Travsering token range: " << start << " - " << end
+                          << " ; table: " << TableDesc::TABLE_NAME;
+        // for each table we only have one prepared statement
+        static auto statementPrepared =
+            migrationSchema_.getPreparedFullScanStatement(handle_, TableDesc::TABLE_NAME, TableDesc::PARTITION_KEY);
+
+        auto const statement = statementPrepared.bind(start, end);
+
+        auto const res = this->executor_.read(yield, statement);
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch data from table: " << TableDesc::TABLE_NAME << " range: " << start
+                              << " - " << end << ";" << res.error();
+            return;
+        }
+
+        auto const& results = res.value();
+        if (not results.hasRows()) {
+            LOG(log_.debug()) << "No rows returned  - table: " << TableDesc::TABLE_NAME << " range: " << start << " - "
+                              << end;
+            return;
+        }
+
+        for (auto const& row : std::apply(
+                 [&](auto... args) { return data::cassandra::extract<decltype(args)...>(results); },
+                 typename TableDesc::Row{}
+             )) {
+            callback(row);
+        }
+    }
+};
+}  // namespace migration::cassandra

src/migration/cassandra/CassandraMigrationManager.hpp

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/impl/MigrationManagerBase.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+
+namespace migration::cassandra {
+
+// Register migrators here
+// MigratorsRegister<BackendType, ExampleMigrator>
+template <typename BackendType>
+using CassandraSupportedMigrators = migration::impl::MigratorsRegister<BackendType>;
+
+// Register with MigrationBackend which proceeds the migration
+using MigrationProcesser = CassandraSupportedMigrators<CassandraMigrationBackend>;
+
+// The Cassandra migration manager
+using CassandraMigrationManager = migration::impl::MigrationManagerBase<MigrationProcesser>;
+}  // namespace migration::cassandra

src/migration/cassandra/impl/CassandraMigrationSchema.hpp

@@ -0,0 +1,98 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+
+#include <fmt/core.h>
+
+#include <functional>
+#include <string>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The schema for the migration process. It contains the prepared statements only used for the migration process.
+ */
+class CassandraMigrationSchema {
+    using SettingsProviderType = data::cassandra::SettingsProvider;
+    std::reference_wrapper<SettingsProviderType const> settingsProvider_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Schema object
+     *
+     * @param settings The settings provider of database
+     */
+    explicit CassandraMigrationSchema(SettingsProviderType const& settings) : settingsProvider_{settings}
+    {
+    }
+
+    /**
+     * @brief Get the prepared statement for the full scan of a table
+     *
+     * @param handler The database handler
+     * @param tableName The name of the table
+     * @param key The partition key of the table
+     * @return The prepared statement
+     */
+    data::cassandra::PreparedStatement
+    getPreparedFullScanStatement(
+        data::cassandra::Handle const& handler,
+        std::string const& tableName,
+        std::string const& key
+    )
+    {
+        return handler.prepare(fmt::format(
+            R"(
+            SELECT * 
+              FROM {} 
+             WHERE TOKEN({}) >= ? AND TOKEN({}) <= ?
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), tableName),
+            key,
+            key
+        ));
+    }
+
+    /**
+     * @brief Get the prepared statement for insertion of migrator_status table
+     *
+     * @param handler The database handler
+     * @return The prepared statement to insert into migrator_status table
+     */
+    data::cassandra::PreparedStatement const&
+    getPreparedInsertMigratedMigrator(data::cassandra::Handle const& handler)
+    {
+        static auto prepared = handler.prepare(fmt::format(
+            R"(
+            INSERT INTO {} 
+                   (migrator_name, status)
+            VALUES (?, ?)
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), "migrator_status")
+        ));
+        return prepared;
+    }
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScanner.hpp

@@ -0,0 +1,183 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/ETLHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/async/AnyOperation.hpp"
+#include "util/async/context/BasicExecutionContext.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <ranges>
+#include <vector>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The token range used to split the full table scan into multiple ranges.
+ */
+struct TokenRange {
+    std::int64_t start;
+    std::int64_t end;
+
+    /**
+     * @brief Construct a new Token Range object
+     *
+     * @param start The start token
+     * @param end The end token
+     */
+    TokenRange(std::int64_t start, std::int64_t end) : start{start}, end{end}
+    {
+    }
+};
+
+/**
+ * @brief The concept for an adapter.
+ */
+template <typename T>
+concept CanReadByTokenRange = requires(T obj, TokenRange const& range, boost::asio::yield_context yield) {
+    { obj.readByTokenRange(range, yield) } -> std::same_as<void>;
+};
+
+/**
+ * @brief The full table scanner. It will split the full table scan into multiple ranges and read the data in given
+ * executor.
+ *
+ * @tparam TableAdapter The table adapter type
+ */
+template <CanReadByTokenRange TableAdapter>
+class FullTableScanner {
+    /**
+     * @brief The helper to generate the token ranges.
+     */
+    struct TokenRangesProvider {
+        uint32_t numRanges_;
+
+        TokenRangesProvider(uint32_t numRanges) : numRanges_{numRanges}
+        {
+        }
+
+        [[nodiscard]] std::vector<TokenRange>
+        getRanges() const
+        {
+            auto const minValue = std::numeric_limits<std::int64_t>::min();
+            auto const maxValue = std::numeric_limits<std::int64_t>::max();
+            if (numRanges_ == 1)
+                return {TokenRange{minValue, maxValue}};
+
+            // Safely calculate the range size using uint64_t to avoid overflow
+            uint64_t rangeSize = (static_cast<uint64_t>(maxValue) * 2) / numRanges_;
+
+            std::vector<TokenRange> ranges;
+            ranges.reserve(numRanges_);
+
+            for (std::int64_t i = 0; i < numRanges_; ++i) {
+                int64_t start = minValue + i * rangeSize;
+                int64_t end = (i == numRanges_ - 1) ? maxValue : start + static_cast<int64_t>(rangeSize) - 1;
+                ranges.emplace_back(start, end);
+            }
+
+            return ranges;
+        }
+    };
+
+    [[nodiscard]] auto
+    spawnWorker()
+    {
+        return ctx_.execute([this](auto token) {
+            while (not token.isStopRequested()) {
+                auto cursor = queue_.tryPop();
+                if (not cursor.has_value()) {
+                    return;  // queue is empty
+                }
+                reader_.readByTokenRange(cursor.value(), token);
+            }
+        });
+    }
+
+    void
+    load(size_t workerNum)
+    {
+        namespace vs = std::views;
+
+        tasks_.reserve(workerNum);
+
+        for ([[maybe_unused]] auto taskId : vs::iota(0u, workerNum))
+            tasks_.push_back(spawnWorker());
+    }
+
+    util::async::AnyExecutionContext ctx_;
+    std::size_t cursorsNum_;
+    etl::ThreadSafeQueue<TokenRange> queue_;
+    std::vector<util::async::AnyOperation<void>> tasks_;
+    TableAdapter reader_;
+
+public:
+    /**
+     * @brief The full table scanner settings.
+     */
+    struct FullTableScannerSettings {
+        std::uint32_t ctxThreadsNum; /**< number of threads used in the execution context */
+        std::uint32_t jobsNum;       /**< number of coroutines to run, it is the number of concurrent database reads */
+        std::uint32_t cursorsPerJob; /**< number of cursors per coroutine */
+    };
+
+    /**
+     * @brief Construct a new Full Table Scanner object, it will run in a sync or async context according to the
+     * parameter. The scan process will immediately start.
+     *
+     * @tparam ExecutionContextType The execution context type
+     * @param settings The full table scanner settings
+     * @param reader The table adapter
+     */
+    template <typename ExecutionContextType = util::async::CoroExecutionContext>
+    FullTableScanner(FullTableScannerSettings settings, TableAdapter&& reader)
+        : ctx_(ExecutionContextType(settings.ctxThreadsNum))
+        , cursorsNum_(settings.jobsNum * settings.cursorsPerJob)
+        , queue_{cursorsNum_}
+        , reader_{std::move(reader)}
+    {
+        ASSERT(settings.jobsNum > 0, "jobsNum for full table scanner must be greater than 0");
+        ASSERT(settings.cursorsPerJob > 0, "cursorsPerJob for full table scanner must be greater than 0");
+
+        auto const cursors = TokenRangesProvider{cursorsNum_}.getRanges();
+        std::ranges::for_each(cursors, [this](auto const& cursor) { queue_.push(cursor); });
+        load(settings.jobsNum);
+    }
+
+    /**
+     * @brief Wait for all workers to finish.
+     */
+    void
+    wait()
+    {
+        for (auto& task : tasks_) {
+            task.wait();
+        }
+    }
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScannerAdapterBase.hpp

@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The base class for the full table scanner adapter. It is responsible for reading the rows from the full table
+ * scanner and call the callback when a row is read. With this base class, each table adapter can focus on the actual
+ * row data converting.
+ *
+ * @tparam TableDesc The table description, it has to be a TableSpec.
+ */
+template <TableSpec TableDesc>
+struct FullTableScannerAdapterBase {
+    static_assert(TableSpec<TableDesc>);
+
+protected:
+    /**
+     * @brief The backend to use
+     */
+    std::shared_ptr<CassandraMigrationBackend> backend_;
+
+public:
+    virtual ~FullTableScannerAdapterBase() = default;
+
+    /**
+     * @brief Construct a new Full Table Scanner Adapter Base object
+     *
+     * @param backend The backend
+     */
+    FullTableScannerAdapterBase(std::shared_ptr<CassandraMigrationBackend> backend) : backend_(std::move(backend))
+    {
+    }
+
+    /**
+     * @brief Read the row in the given token range from database, this is the adapt function for the FullTableScanner.
+     *
+     * @param range The token range to read
+     * @param yield The yield context
+     */
+    void
+    readByTokenRange(TokenRange const& range, boost::asio::yield_context yield)
+    {
+        backend_->migrateInTokenRange<TableDesc>(
+            range.start, range.end, [this](auto const& row) { onRowRead(row); }, yield
+        );
+    }
+
+    /**
+     * @brief Called when a row is read. The derived class should implement this function to convert the database blob
+     * to actual data type.
+     *
+     * @param row The row read
+     */
+    virtual void
+    onRowRead(TableDesc::Row const& row) = 0;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.cpp

@@ -0,0 +1,44 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+
+#include <optional>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+void
+ObjectsAdapter::onRowRead(TableObjectsDesc::Row const& row)
+{
+    auto const& [key, ledgerSeq, blob] = row;
+    // the blob can be empty which means the ledger state is deleted
+    if (blob.empty()) {
+        onStateRead_(ledgerSeq, std::nullopt);
+        return;
+    }
+    ripple::SLE sle{ripple::SerialIter{blob.data(), blob.size()}, key};
+    onStateRead_(ledgerSeq, std::make_optional(std::move(sle)));
+}
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/Types.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the objects table. It has to be a TableSpec.
+ */
+struct TableObjectsDesc {
+    using Row = std::tuple<ripple::uint256, std::uint32_t, data::Blob>;
+    static constexpr char const* PARTITION_KEY = "key";
+    static constexpr char const* TABLE_NAME = "objects";
+};
+
+/**
+ * @brief The adapter for the objects table. This class is responsible for reading the objects from the
+ * FullTableScanner and converting the blobs to the STObject.
+ */
+class ObjectsAdapter : public impl::FullTableScannerAdapterBase<TableObjectsDesc> {
+public:
+    using OnStateRead = std::function<void(std::uint32_t, std::optional<ripple::SLE>)>;
+
+    /**
+     * @brief Construct a new Objects Adapter object
+     *
+     * @param backend The backend to use
+     * @param onStateRead The callback to call when a state is read
+     */
+    explicit ObjectsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnStateRead onStateRead)
+        : FullTableScannerAdapterBase<TableObjectsDesc>(backend), onStateRead_{std::move(onStateRead)}
+    {
+    }
+
+    /**
+     * @brief Called when a row is read
+     *
+     * @param row The row to read
+     */
+    void
+    onRowRead(TableObjectsDesc::Row const& row) override;
+
+private:
+    OnStateRead onStateRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Spec.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/asio/spawn.hpp>
+
+#include <concepts>
+#include <tuple>
+#include <type_traits>
+
+namespace migration::cassandra::impl {
+// Define the concept for a class like TableObjectsDesc
+template <typename T>
+concept TableSpec = requires {
+    // Check that 'row' exists and is a tuple
+    // keys types are at the begining and the other fields types sort in alphabetical order
+    typename T::Row;
+    requires std::tuple_size<typename T::Row>::value >= 0;  // Ensures 'row' is a tuple
+
+    // Check that static constexpr members 'partitionKey' and 'tableName' exist
+    { T::PARTITION_KEY } -> std::convertible_to<char const*>;
+    { T::TABLE_NAME } -> std::convertible_to<char const*>;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.cpp

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+namespace migration::cassandra::impl {
+
+void
+TransactionsAdapter::onRowRead(TableTransactionsDesc::Row const& row)
+{
+    auto const& [txHash, date, ledgerSeq, metaBlob, txBlob] = row;
+
+    ripple::SerialIter it{txBlob.data(), txBlob.size()};
+    ripple::STTx const sttx{it};
+    ripple::TxMeta const txMeta{sttx.getTransactionID(), ledgerSeq, metaBlob};
+    onTransactionRead_(sttx, txMeta);
+}
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the transactions table. It has to be a TableSpec.
+ */
+struct TableTransactionsDesc {
+    // hash, date, ledger_seq, metadata, transaction
+    using Row = std::tuple<ripple::uint256, std::uint64_t, std::uint32_t, ripple::Blob, ripple::Blob>;
+    static constexpr char const* PARTITION_KEY = "hash";
+    static constexpr char const* TABLE_NAME = "transactions";
+};
+
+/**
+ * @brief The adapter for the transactions table. This class is responsible for reading the transactions from the
+ * FullTableScanner and converting the blobs to the STTx and TxMeta.
+ */
+class TransactionsAdapter : public impl::FullTableScannerAdapterBase<TableTransactionsDesc> {
+public:
+    using OnTransactionRead = std::function<void(ripple::STTx, ripple::TxMeta)>;
+
+    /**
+     * @brief Construct a new Transactions Adapter object
+     *
+     * @param backend The backend
+     * @param onTxRead The callback to call when a transaction is read
+     */
+    explicit TransactionsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnTransactionRead onTxRead)
+        : FullTableScannerAdapterBase<TableTransactionsDesc>(backend), onTransactionRead_{std::move(onTxRead)}
+    {
+    }
+
+    /**
+     *@brief The callback when a row is read.
+     *
+     *@param row The row to read
+     */
+    void
+    onRowRead(TableTransactionsDesc::Row const& row) override;
+
+private:
+    OnTransactionRead onTransactionRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Types.hpp

@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+namespace migration::cassandra::impl {
+
+using ObjectsScanner = impl::FullTableScanner<impl::ObjectsAdapter>;
+using TransactionsScanner = impl::FullTableScanner<impl::TransactionsAdapter>;
+
+}  // namespace migration::cassandra::impl

src/migration/impl/MigrationManagerBase.hpp

@@ -0,0 +1,120 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace migration::impl {
+
+/**
+ * @brief The migration manager implementation for Cassandra. It will run the migration for the Cassandra
+ * database.
+ *
+ * @tparam SupportedMigrators The migrators resgister that contains all the migrators
+ */
+template <typename SupportedMigrators>
+class MigrationManagerBase : public MigrationManagerInterface {
+    SupportedMigrators migrators_;
+    // contains only migration related settings
+    util::config::ObjectView config_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Manager object
+     *
+     * @param backend The backend of the Cassandra database
+     * @param config The configuration of the migration
+     */
+    explicit MigrationManagerBase(
+        std::shared_ptr<typename SupportedMigrators::BackendType> backend,
+        util::config::ObjectView const& config
+    )
+        : migrators_{backend}, config_{config}
+    {
+    }
+
+    /**
+     * @brief Run the the migration according to the given migrator's name
+     *
+     * @param name The name of the migrator
+     */
+    void
+    runMigration(std::string const& name) override
+    {
+        migrators_.runMigrator(name, config_);
+    }
+
+    /**
+     * @brief Get the status of all the migrators
+     *
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     * migrator
+     */
+    std::vector<std::tuple<std::string, MigratorStatus>>
+    allMigratorsStatusPairs() const override
+    {
+        return migrators_.getMigratorsStatus();
+    }
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The name of the migrator
+     * @return The status of the migrator
+     */
+    MigratorStatus
+    getMigratorStatusByName(std::string const& name) const override
+    {
+        return migrators_.getMigratorStatus(name);
+    }
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A vector of string, the names of all the migrators
+     */
+    std::vector<std::string>
+    allMigratorsNames() const override
+    {
+        auto const names = migrators_.getMigratorNames();
+        return std::vector<std::string>{names.begin(), names.end()};
+    }
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The name of the migrator
+     * @return The description of the migrator
+     */
+    std::string
+    getMigratorDescriptionByName(std::string const& name) const override
+    {
+        return migrators_.getMigratorDescription(name);
+    }
+};
+
+}  // namespace migration::impl

src/migration/impl/MigrationManagerFactory.cpp

@@ -0,0 +1,60 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/impl/MigrationManagerFactory.hpp"
+
+#include "data/cassandra/SettingsProvider.hpp"
+#include "migration/MigrationManagerInterface.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/CassandraMigrationManager.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace migration::impl {
+
+std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
+makeMigrationManager(util::config::ClioConfigDefinition const& config)
+{
+    static util::Logger const log{"Migration"};
+    LOG(log.info()) << "Constructing MigrationManager";
+
+    auto const type = config.get<std::string>("database.type");
+
+    if (not boost::iequals(type, "cassandra")) {
+        LOG(log.error()) << "Unknown database type to migrate: " << type;
+        return std::unexpected(std::string("Invalid database type"));
+    }
+
+    auto const cfg = config.getObject("database." + type);
+
+    auto migrationCfg = config.getObject("migration");
+
+    return std::make_shared<cassandra::CassandraMigrationManager>(
+        std::make_shared<cassandra::CassandraMigrationBackend>(data::cassandra::SettingsProvider{cfg}),
+        std::move(migrationCfg)
+    );
+}
+
+}  // namespace migration::impl

src/migration/impl/MigrationManagerFactory.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <expected>
+#include <memory>
+#include <string>
+
+namespace migration::impl {
+
+/**
+ * @brief The factory to create a MigrationManagerInferface
+ *
+ * @param config The configuration of the migration application, it contains the database connection configuration and
+ * other migration specific configurations
+ * @return A shared pointer to the MigrationManagerInterface if the creation was successful, otherwise an error message
+ */
+std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
+makeMigrationManager(util::config::ClioConfigDefinition const& config);
+
+}  // namespace migration::impl

src/migration/impl/MigratorsRegister.hpp

@@ -0,0 +1,184 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/impl/Spec.hpp"
+#include "util/Concepts.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <algorithm>
+#include <array>
+#include <iterator>
+#include <memory>
+#include <ranges>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+namespace migration::impl {
+
+/**
+ * The concept to check if BackendType is the same as the migrator's required backend type
+ */
+template <typename BackendType, typename MigratorType>
+concept MigrationBackend = requires { requires std::same_as<typename MigratorType::Backend, BackendType>; };
+
+template <typename Backend, typename... MigratorType>
+concept BackendMatchAllMigrators = (MigrationBackend<Backend, MigratorType> && ...);
+
+/**
+ *@brief The register of migrators. It will dispatch the migration to the corresponding migrator. It also
+ *hold the shared pointer of backend, which is used by the migrators.
+ *
+ *@tparam Backend The backend type
+ *@tparam MigratorType The migrator types. It should be a concept of MigratorSpec and not have duplicate names.
+ */
+template <typename Backend, typename... MigratorType>
+    requires AllMigratorSpec<MigratorType...>
+class MigratorsRegister {
+    static_assert(util::hasNoDuplicateNames<MigratorType...>());
+
+    util::Logger log_{"Migration"};
+    std::shared_ptr<Backend> backend_;
+
+    template <typename Migrator>
+    void
+    callMigration(std::string const& name, util::config::ObjectView const& config)
+    {
+        if (name == Migrator::name) {
+            LOG(log_.info()) << "Running migration: " << name;
+            Migrator::runMigration(backend_, config);
+            backend_->writeMigratorStatus(name, MigratorStatus(MigratorStatus::Migrated).toString());
+            LOG(log_.info()) << "Finished migration: " << name;
+        }
+    }
+
+    template <typename T>
+    static constexpr std::string_view
+    getDescriptionIfMatch(std::string_view targetName)
+    {
+        return (T::name == targetName) ? T::description : "";
+    }
+
+public:
+    /**
+     * @brief The backend type which is used by the migrators
+     */
+    using BackendType = Backend;
+
+    /**
+     * @brief Construct a new Migrators Register object
+     *
+     * @param backend The backend shared pointer
+     */
+    MigratorsRegister(std::shared_ptr<BackendType> backend) : backend_{std::move(backend)}
+    {
+    }
+
+    /**
+     * @brief Run the migration according to the given migrator's name
+     *
+     * @param name The migrator's name
+     * @param config The configuration of the migration
+     */
+    void
+    runMigrator(std::string const& name, util::config::ObjectView const& config)
+        requires BackendMatchAllMigrators<BackendType, MigratorType...>
+    {
+        (callMigration<MigratorType>(name, config), ...);
+    }
+
+    /**
+     * @brief Get the status of all the migrators
+     *
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     * migrator
+     */
+    std::vector<std::tuple<std::string, MigratorStatus>>
+    getMigratorsStatus() const
+    {
+        auto const fullList = getMigratorNames();
+
+        std::vector<std::tuple<std::string, MigratorStatus>> status;
+
+        std::ranges::transform(fullList, std::back_inserter(status), [&](auto const& migratorName) {
+            auto const migratorNameStr = std::string(migratorName);
+            return std::make_tuple(migratorNameStr, getMigratorStatus(migratorNameStr));
+        });
+        return status;
+    }
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The migrator's name to get the status
+     * @return The status of the migrator
+     */
+    MigratorStatus
+    getMigratorStatus(std::string const& name) const
+    {
+        auto const fullList = getMigratorNames();
+        if (std::ranges::find(fullList, name) == fullList.end()) {
+            return MigratorStatus::NotKnown;
+        }
+        auto const statusStringOpt =
+            data::synchronous([&](auto yield) { return backend_->fetchMigratorStatus(name, yield); });
+
+        return statusStringOpt ? MigratorStatus::fromString(statusStringOpt.value()) : MigratorStatus::NotMigrated;
+    }
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A array of migrator's names
+     */
+    constexpr auto
+    getMigratorNames() const
+    {
+        return std::array<std::string_view, sizeof...(MigratorType)>{MigratorType::name...};
+    }
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The description of the migrator
+     */
+    std::string
+    getMigratorDescription(std::string const& name) const
+    {
+        if constexpr (sizeof...(MigratorType) == 0) {
+            return "No Description";
+        } else {
+            // Fold expression to search through all types
+            std::string result = ([](std::string const& name) {
+                return std::string(getDescriptionIfMatch<MigratorType>(name));
+            }(name) + ...);
+
+            return result.empty() ? "No Description" : result;
+        }
+    }
+};
+
+}  // namespace migration::impl

src/migration/impl/Spec.hpp

@@ -0,0 +1,56 @@
+
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+#include <string>
+
+namespace migration::impl {
+
+/**
+ * @brief The migrator specification concept
+ */
+template <typename T, typename Backend>
+concept MigratorSpec = requires(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& cfg) {
+    // Check that 'name' exists and is a string
+    { T::name } -> std::convertible_to<std::string>;
+
+    // Check that 'description' exists and is a string
+    { T::description } -> std::convertible_to<std::string>;
+
+    // Check that the migrator specifies the backend type it supports
+    typename T::Backend;
+
+    // Check that 'runMigration' exists and is callable
+    { T::runMigration(backend, cfg) } -> std::same_as<void>;
+};
+
+/**
+ * @brief used by variadic template to check all migrators are MigratorSpec
+ */
+template <typename... Types>
+concept AllMigratorSpec = (MigratorSpec<Types, typename Types::Backend> && ...);
+
+}  // namespace migration::impl

src/util/Concepts.hpp

@@ -21,6 +21,8 @@
 
 #include <algorithm>
 #include <array>
+#include <cstddef>
+#include <string_view>
 #include <type_traits>
 
 namespace util {
@@ -46,4 +48,22 @@ hasNoDuplicates(auto&&... values)
     return (std::unique(std::begin(store), end) == end);
 }
 
+/**
+ * @brief Checks that the list of given type contains no duplicates
+ *
+ * @tparam Types The types to check
+ * @returns true if no duplicates exist; false otherwise
+ */
+template <typename... Types>
+constexpr bool
+hasNoDuplicateNames()
+{
+    constexpr std::array<std::string_view, sizeof...(Types)> names = {Types::name...};
+    return !std::ranges::any_of(names, [&](std::string_view const& name1) {
+        return std::ranges::any_of(names, [&](std::string_view const& name2) {
+            return &name1 != &name2 && name1 == name2;  // Ensure different elements are compared
+        });
+    });
+}
+
 }  // namespace util

src/util/log/Logger.hpp

@@ -169,7 +169,7 @@ class Logger final {
     };
 
 public:
-    static constexpr std::array<char const*, 7> CHANNELS = {
+    static constexpr std::array<char const*, 8> CHANNELS = {
         "General",
         "WebServer",
         "Backend",
@@ -177,6 +177,7 @@ public:
         "ETL",
         "Subscriptions",
         "Performance",
+        "Migration",
     };
 
     /**

src/util/newconfig/ConfigDefinition.hpp

@@ -380,7 +380,11 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
      {"api_version.min",
       ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MIN).withConstraint(validateApiVersion)},
      {"api_version.max",
-      ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MAX).withConstraint(validateApiVersion)}}
+      ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MAX).withConstraint(validateApiVersion)},
+     {"migration.full_scan_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateUint32)},
+     {"migration.full_scan_jobs", ConfigValue{ConfigType::Integer}.defaultValue(4).withConstraint(validateUint32)},
+     {"migration.cursors_per_job", ConfigValue{ConfigType::Integer}.defaultValue(100).withConstraint(validateUint32)}},
+
 };
 
 }  // namespace util::config

src/util/newconfig/ConfigDescription.hpp

@@ -163,7 +163,10 @@ private:
         KV{.key = "ssl_key_file", .value = "Path to the SSL key file."},
         KV{.key = "api_version.default", .value = "Default API version Clio will run on."},
         KV{.key = "api_version.min", .value = "Minimum API version."},
-        KV{.key = "api_version.max", .value = "Maximum API version."}
+        KV{.key = "api_version.max", .value = "Maximum API version."},
+        KV{.key = "migration.full_scan_threads", .value = "The number of threads used to scan table."},
+        KV{.key = "migration.full_scan_jobs", .value = "The number of coroutines used to scan table."},
+        KV{.key = "migration.cursors_per_job", .value = "The number of cursors each coroutine will scan."}
     };
 };
 

src/web/dosguard/DOSGuard.cpp

@@ -26,7 +26,6 @@
 #include "util/newconfig/ValueView.hpp"
 #include "web/dosguard/WhitelistHandlerInterface.hpp"
 
-
 #include <cstdint>
 #include <functional>
 #include <mutex>