feat: Migration framework (#1768)

This PR implemented the migration framework, which contains the command
line interface to execute migration and helps to migrate data easily.
Please read README.md for more information about this framework.

src/CMakeLists.txt

@@ -5,5 +5,6 @@ add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)
+add_subdirectory(migration)
 add_subdirectory(app)
 add_subdirectory(main)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
 target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp WebHandlers.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc clio_migration)

src/app/CliArgs.cpp

@@ -19,6 +19,7 @@
 
 #include "app/CliArgs.hpp"
 
+#include "migration/MigrationApplication.hpp"
 #include "util/build/Build.hpp"
 
 #include <boost/program_options/options_description.hpp>
@@ -45,6 +46,7 @@ CliArgs::parse(int argc, char const* argv[])
         ("version,v", "print version and exit")
         ("conf,c", po::value<std::string>()->default_value(defaultConfigPath), "configuration file")
         ("ng-web-server,w", "Use ng-web-server")
+        ("migrate", po::value<std::string>(), "start migration helper")
     ;
     // clang-format on
     po::positional_options_description positional;
@@ -65,6 +67,14 @@ CliArgs::parse(int argc, char const* argv[])
     }
 
     auto configPath = parsed["conf"].as<std::string>();
+
+    if (parsed.count("migrate") != 0u) {
+        auto const opt = parsed["migrate"].as<std::string>();
+        if (opt == "status")
+            return Action{Action::Migrate{std::move(configPath), MigrateSubCmd::status()}};
+        return Action{Action::Migrate{std::move(configPath), MigrateSubCmd::migration(opt)}};
+    }
+
     return Action{Action::Run{.configPath = std::move(configPath), .useNgWebServer = parsed.count("ng-web-server") != 0}
     };
 }

src/app/CliArgs.hpp

@@ -19,6 +19,7 @@
 
 #pragma once
 
+#include "migration/MigrationApplication.hpp"
 #include "util/OverloadSet.hpp"
 
 #include <string>
@@ -52,13 +53,20 @@ public:
             int exitCode;  ///< Exit code.
         };
 
+        /** @brief Migration action. */
+        struct Migrate {
+            std::string configPath;
+            MigrateSubCmd subCmd;
+        };
+
         /**
          * @brief Construct an action from a Run.
          *
          * @param action Run action.
          */
         template <typename ActionType>
-            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit>
+            requires std::is_same_v<ActionType, Run> or std::is_same_v<ActionType, Exit> or
+            std::is_same_v<ActionType, Migrate>
         explicit Action(ActionType&& action) : action_(std::forward<ActionType>(action))
         {
         }
@@ -78,7 +86,7 @@ public:
         }
 
     private:
-        std::variant<Run, Exit> action_;
+        std::variant<Run, Exit, Migrate> action_;
     };
 
     /**

src/data/BackendInterface.hpp

@@ -548,6 +548,16 @@ public:
         boost::asio::yield_context yield
     ) const;
 
+    /**
+     * @brief Fetches the status of migrator by name.
+     *
+     * @param migratorName The name of the migrator
+     * @param yield The coroutine context
+     * @return The status of the migrator if found; nullopt otherwise
+     */
+    virtual std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const = 0;
+
     /**
      * @brief Synchronously fetches the ledger range from DB.
      *
@@ -673,6 +683,15 @@ public:
     bool
     finishWrites(std::uint32_t ledgerSequence);
 
+    /**
+     * @brief Mark the migration status of a migrator as Migrated in the database
+     *
+     * @param migratorName The name of the migrator
+     * @param status The status to set
+     */
+    virtual void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) = 0;
+
     /**
      * @return true if database is overwhelmed; false otherwise
      */

src/data/CassandraBackend.hpp

@@ -72,13 +72,15 @@ class BasicCassandraBackend : public BackendInterface {
 
     SettingsProviderType settingsProvider_;
     Schema<SettingsProviderType> schema_;
+
+    std::atomic_uint32_t ledgerSequence_ = 0u;
+
+protected:
     Handle handle_;
 
     // have to be mutable because BackendInterface constness :(
     mutable ExecutionStrategyType executor_;
 
-    std::atomic_uint32_t ledgerSequence_ = 0u;
-
 public:
     /**
      * @brief Create a new cassandra/scylla backend instance.
@@ -835,6 +837,26 @@ public:
         return results;
     }
 
+    std::optional<std::string>
+    fetchMigratorStatus(std::string const& migratorName, boost::asio::yield_context yield) const override
+    {
+        auto const res = executor_.read(yield, schema_->selectMigratorStatus, Text(migratorName));
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch migrator status: " << res.error();
+            return {};
+        }
+
+        auto const& results = res.value();
+        if (not results) {
+            return {};
+        }
+
+        for (auto [statusString] : extract<std::string>(results))
+            return statusString;
+
+        return {};
+    }
+
     void
     doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) override
     {
@@ -962,6 +984,14 @@ public:
         // probably was used in PG to start a transaction or smth.
     }
 
+    void
+    writeMigratorStatus(std::string const& migratorName, std::string const& status) override
+    {
+        executor_.writeSync(
+            schema_->insertMigratorStatus, data::cassandra::Text{migratorName}, data::cassandra::Text(status)
+        );
+    }
+
     bool
     isTooBusy() const override
     {

src/data/README.md

@@ -262,3 +262,15 @@ CREATE TABLE clio.nf_token_transactions (
 ```
 
 The `nf_token_transactions` table serves as the NFT counterpart to `account_tx`, inspired by the same motivations and fulfilling a similar role within this context. It drives the `nft_history` API.
+
+### migrator_status
+
+```
+CREATE TABLE clio.migrator_status (
+    migrator_name TEXT,     # The name of the migrator
+    status TEXT,            # The status of the migrator
+    PRIMARY KEY (migrator_name)
+) 
+```
+
+The `migrator_status` table stores the status of the migratior in this database. If a migrator's status is `migrated`, it means this database has finished data migration for this migrator.

src/data/cassandra/Schema.hpp

@@ -270,6 +270,18 @@ public:
             qualifiedTableName(settingsProvider_.get(), "mp_token_holders")
         ));
 
+        statements.emplace_back(fmt::format(
+            R"(
+           CREATE TABLE IF NOT EXISTS {}
+                  ( 
+                   migrator_name TEXT,
+                          status TEXT,
+                         PRIMARY KEY (migrator_name)
+                  ) 
+            )",
+            qualifiedTableName(settingsProvider_.get(), "migrator_status")
+        ));
+
         return statements;
     }();
 
@@ -466,6 +478,17 @@ public:
             ));
         }();
 
+        PreparedStatement insertMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                INSERT INTO {}
+                       (migrator_name, status)
+                VALUES (?, ?)
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
+
         //
         // Select queries
         //
@@ -768,6 +791,17 @@ public:
                 qualifiedTableName(settingsProvider_.get(), "ledger_range")
             ));
         }();
+
+        PreparedStatement selectMigratorStatus = [this]() {
+            return handle_.get().prepare(fmt::format(
+                R"(
+                SELECT status
+                  FROM {}
+                 WHERE migrator_name = ?
+                )",
+                qualifiedTableName(settingsProvider_.get(), "migrator_status")
+            ));
+        }();
     };
 
     /**

src/data/cassandra/SettingsProvider.cpp

@@ -24,7 +24,6 @@
 #include "util/Constants.hpp"
 #include "util/newconfig/ObjectView.hpp"
 
-
 #include <cerrno>
 #include <chrono>
 #include <cstddef>

src/data/cassandra/Types.hpp

@@ -21,6 +21,8 @@
 
 #include <cstdint>
 #include <expected>
+#include <string>
+#include <utility>
 
 namespace data::cassandra {
 
@@ -55,6 +57,26 @@ struct Limit {
     int32_t limit;
 };
 
+/**
+ * @brief A strong type wrapper for string
+ *
+ * This is unfortunately needed right now to support TEXT properly
+ * because clio uses string to represent BLOB
+ * If we want to bind TEXT with string, we need to use this type
+ */
+struct Text {
+    std::string text;
+
+    /**
+     * @brief Construct a new Text object from string type
+     *
+     * @param text The text to wrap
+     */
+    explicit Text(std::string text) : text{std::move(text)}
+    {
+    }
+};
+
 class Handle;
 class CassandraError;
 

src/data/cassandra/impl/Statement.hpp

@@ -119,6 +119,9 @@ public:
             // reinterpret_cast is needed here :'(
             auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
             throwErrorIfNeeded(rc, "Bind string (as bytes)");
+        } else if constexpr (std::is_convertible_v<DecayedType, Text>) {
+            auto const rc = cass_statement_bind_string_n(*this, idx, value.text.c_str(), value.text.size());
+            throwErrorIfNeeded(rc, "Bind string (as TEXT)");
         } else if constexpr (std::is_same_v<DecayedType, UintTupleType> ||
                              std::is_same_v<DecayedType, UintByteTupleType>) {
             auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::forward<Type>(value)});

src/main/Main.cpp

@@ -19,8 +19,10 @@
 
 #include "app/CliArgs.hpp"
 #include "app/ClioApplication.hpp"
+#include "migration/MigrationApplication.hpp"
 #include "rpc/common/impl/HandlerProvider.hpp"
 #include "util/TerminationHandler.hpp"
+#include "util/config/Config.hpp"
 #include "util/log/Logger.hpp"
 #include "util/newconfig/ConfigDefinition.hpp"
 #include "util/newconfig/ConfigFileJson.hpp"
@@ -54,6 +56,22 @@ try {
             util::LogService::init(ClioConfig);
             app::ClioApplication clio{ClioConfig};
             return clio.run(run.useNgWebServer);
+        },
+        [](app::CliArgs::Action::Migrate const& migrate) {
+            auto const json = ConfigFileJson::make_ConfigFileJson(migrate.configPath);
+            if (!json.has_value()) {
+                std::cerr << json.error().error << std::endl;
+                return EXIT_FAILURE;
+            }
+            auto const errors = ClioConfig.parse(json.value());
+            if (errors.has_value()) {
+                for (auto const& err : errors.value())
+                    std::cerr << err.error << std::endl;
+                return EXIT_FAILURE;
+            }
+            util::LogService::init(ClioConfig);
+            app::MigratorApplication migrator{ClioConfig, migrate.subCmd};
+            return migrator.run();
         }
     );
 } catch (std::exception const& e) {

src/migration/CMakeLists.txt

@@ -0,0 +1,8 @@
+add_library(clio_migration)
+
+target_sources(
+  clio_migration PRIVATE MigrationApplication.cpp impl/MigrationManagerFactory.cpp MigratorStatus.cpp
+                         cassandra/impl/ObjectsAdapter.cpp cassandra/impl/TransactionsAdapter.cpp
+)
+
+target_link_libraries(clio_migration PRIVATE clio_util clio_etl)

src/migration/MigrationApplication.cpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigrationApplication.hpp"
+
+#include "migration/MigratiorStatus.hpp"
+#include "migration/impl/MigrationManagerFactory.hpp"
+#include "util/OverloadSet.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/prometheus/Prometheus.hpp"
+
+#include <cstdlib>
+#include <iostream>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <variant>
+
+namespace app {
+
+MigratorApplication::MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command)
+    : cmd_(std::move(command))
+{
+    PrometheusService::init(config);
+
+    auto expectedMigrationManager = migration::impl::makeMigrationManager(config);
+
+    if (not expectedMigrationManager) {
+        throw std::runtime_error("Failed to create migration manager: " + expectedMigrationManager.error());
+    }
+
+    migrationManager_ = std::move(expectedMigrationManager.value());
+}
+
+int
+MigratorApplication::run()
+{
+    return std::visit(
+        util::OverloadSet{
+            [this](MigrateSubCmd::Status const&) { return printStatus(); },
+            [this](MigrateSubCmd::Migration const& cmdBundle) { return migrate(cmdBundle.migratorName); }
+        },
+        cmd_.state
+    );
+}
+
+int
+MigratorApplication::printStatus()
+{
+    std::cout << "Current Migration Status:" << std::endl;
+    auto const allMigratorsStatusPairs = migrationManager_->allMigratorsStatusPairs();
+
+    if (allMigratorsStatusPairs.empty()) {
+        std::cout << "No migrator found" << std::endl;
+    }
+
+    for (auto const& [migrator, status] : allMigratorsStatusPairs) {
+        std::cout << "Migrator: " << migrator << " - " << migrationManager_->getMigratorDescriptionByName(migrator)
+                  << " - " << status.toString() << std::endl;
+    }
+    return EXIT_SUCCESS;
+}
+
+int
+MigratorApplication::migrate(std::string const& migratorName)
+{
+    auto const status = migrationManager_->getMigratorStatusByName(migratorName);
+    if (status == migration::MigratorStatus::Migrated) {
+        std::cout << "Migrator " << migratorName << " has already migrated" << std::endl;
+        printStatus();
+        return EXIT_SUCCESS;
+    }
+
+    if (status == migration::MigratorStatus::NotKnown) {
+        std::cout << "Migrator " << migratorName << " not found" << std::endl;
+        printStatus();
+        return EXIT_FAILURE;
+    }
+
+    std::cout << "Running migration for " << migratorName << std::endl;
+    migrationManager_->runMigration(migratorName);
+    std::cout << "Migration for " << migratorName << " has finished" << std::endl;
+    return EXIT_SUCCESS;
+}
+
+}  // namespace app

src/migration/MigrationApplication.hpp

@@ -0,0 +1,104 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <memory>
+#include <string>
+#include <variant>
+
+namespace app {
+
+/**
+ * @brief The command to run for migration framework
+ */
+struct MigrateSubCmd {
+    /**
+     * @brief Check the status of the migrations
+     */
+    struct Status {};
+    /**
+     * @brief Run a migration
+     */
+    struct Migration {
+        std::string migratorName;
+    };
+
+    std::variant<Status, Migration> state;
+
+    /**
+     * @brief Helper function to create a status command
+     *
+     * @return Cmd object containing the status command
+     */
+    static MigrateSubCmd
+    status()
+    {
+        return MigrateSubCmd{Status{}};
+    }
+
+    /**
+     * @brief Helper function to create a migration command
+     *
+     * @param name The name of the migration to run
+     * @return Cmd object containing the migration command
+     */
+    static MigrateSubCmd
+    migration(std::string const& name)
+    {
+        return MigrateSubCmd{Migration{name}};
+    }
+};
+
+/**
+ * @brief The migration application class
+ */
+class MigratorApplication {
+    std::string option_;
+    std::shared_ptr<migration::MigrationManagerInterface> migrationManager_;
+    MigrateSubCmd cmd_;
+
+public:
+    /**
+     * @brief Construct a new MigratorApplication object
+     *
+     * @param config The configuration of the application
+     * @param command The command to run
+     */
+    MigratorApplication(util::config::ClioConfigDefinition const& config, MigrateSubCmd command);
+
+    /**
+     * @brief Run the application
+     *
+     * @return exit code
+     */
+    int
+    run();
+
+private:
+    int
+    printStatus();
+
+    int
+    migrate(std::string const& name);
+};
+}  // namespace app

src/migration/MigrationManagerInterface.hpp

@@ -0,0 +1,77 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace migration {
+
+/**
+ * @brief The interface for the migration manager. This interface is tend to be implemented for specific database. The
+ * application layer will use this interface to run the migrations.
+ */
+struct MigrationManagerInterface {
+    virtual ~MigrationManagerInterface() = default;
+
+    /**
+     * @brief Run the the migration according to the given migrator's name
+     */
+    virtual void
+    runMigration(std::string const&) = 0;
+
+    /**
+     * @brief Get the status of all the migrators
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     */
+    virtual std::vector<std::tuple<std::string, MigratorStatus>>
+    allMigratorsStatusPairs() const = 0;
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A vector of migrators' names
+     */
+    virtual std::vector<std::string>
+    allMigratorsNames() const = 0;
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The status of the migrator
+     */
+    virtual MigratorStatus
+    getMigratorStatusByName(std::string const& name) const = 0;
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The description of the migrator
+     */
+    virtual std::string
+    getMigratorDescriptionByName(std::string const& name) const = 0;
+};
+
+}  // namespace migration

src/migration/MigratiorStatus.hpp

@@ -0,0 +1,90 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+/**
+ * @brief The status of a migrator, it provides the helper functions to convert the status to string and vice versa
+ */
+class MigratorStatus {
+public:
+    /**
+     * @brief The status of a migrator
+     */
+    enum Status { Migrated, NotMigrated, NotKnown, NumStatuses };
+
+    /**
+     * @brief Construct a new Migrator Status object with the given status
+     *
+     * @param status The status of the migrator
+     */
+    MigratorStatus(Status status) : status_(status)
+    {
+    }
+
+    /**
+     * @brief Compare the status with another MigratorStatus
+     *
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(MigratorStatus const& other) const;
+
+    /**
+     * @brief Compare the status with another status
+     * @param other The other status to compare
+     * @return true if the status is equal to the other status, false otherwise
+     */
+    bool
+    operator==(Status const& other) const;
+
+    /**
+     * @brief Convert the status to string
+     *
+     * @return The string representation of the status
+     */
+    std::string
+    toString() const;
+
+    /**
+     * @brief Convert the string to status
+     *
+     * @param statusStr The string to convert
+     * @return The status representation of the string
+     */
+    static MigratorStatus
+    fromString(std::string const& statusStr);
+
+private:
+    static constexpr std::array<char const*, static_cast<size_t>(NumStatuses)> statusStrMap = {
+        "Migrated",
+        "NotMigrated",
+        "NotKnown"
+    };
+
+    Status status_;
+};
+}  // namespace migration

src/migration/MigratorStatus.cpp

@@ -0,0 +1,56 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <cstddef>
+#include <string>
+
+namespace migration {
+
+bool
+MigratorStatus::operator==(MigratorStatus const& other) const
+{
+    return status_ == other.status_;
+}
+
+bool
+MigratorStatus::operator==(Status const& other) const
+{
+    return status_ == other;
+}
+
+std::string
+MigratorStatus::toString() const
+{
+    return statusStrMap[static_cast<size_t>(status_)];
+}
+
+MigratorStatus
+MigratorStatus::fromString(std::string const& statusStr)
+{
+    for (std::size_t i = 0; i < statusStrMap.size(); ++i) {
+        if (statusStr == statusStrMap[i]) {
+            return MigratorStatus(static_cast<Status>(i));
+        }
+    }
+    return MigratorStatus(Status::NotMigrated);
+}
+
+}  // namespace migration

src/migration/README.md

@@ -0,0 +1,93 @@
+
+# Clio Migration 
+
+Clio maintains the off-chain data of XRPL and multiple indexes tables to powering complex queries. To simplify the creation of index tables, this migration framework handles the process of database change and facilitates the migration of historical data seamlessly.
+
+
+## Command Line Usage
+
+Clio provides a migration command-line tool to migrate data in database. 
+
+
+> Note: We need a **configuration file** to run the migration tool. This configuration file has the same format as the configuration file of the Clio server, ensuring consistency and ease of use. It reads the database configuration from the same session as the server's configuration, eliminating the need for separate setup or additional configuration files. Be aware that migration-specific configuration is under `.migration` session.
+
+
+### To query migration status:
+    
+    
+    ./clio_server --migrate status  ~/config/migrator.json
+    
+This command returns the current migration status of each migrator. The example output: 
+
+    
+    Current Migration Status:
+    Migrator: ExampleMigrator - Feature v1, Clio v3 - not migrated
+    
+
+### To start a migration:
+    
+    
+    ./clio_server --migrate ExampleMigrator  ~/config/migrator.json
+    
+    
+Migration will run if the migrator has not been migrated. The migrator will be marked as migrated after the migration is completed.
+
+## How to write a migrator
+
+> **Note** If you'd like to add new index table in Clio and old historical data needs to be migrated into new table, you'd need to write a migrator.
+
+A migrator satisfies the `MigratorSpec`(impl/Spec.hpp) concept.
+
+It contains:
+
+-  A `name` which will be used to identify the migrator. User will refer this migrator in command-line tool by this name. The name needs to be different with other migrators, otherwise a compilation error will be raised.
+
+-  A `description` which is the detail information of the migrator.
+
+- A static function `runMigration`, it will be called when user run `--migrate name`. It accepts two parameters: backend, which provides the DB operations interface, and cfg, which provides migration-related configuration. Each migrator can have its own configuration under `.migration` session.
+
+- A type name alias `Backend` which specifies the backend type it supports.
+
+> **Note** Each migrator is designed to work with a specific database.
+
+- Register your migrator in MigrationManager. Currently we only support Cassandra/ScyllaDB.  Migrator needs to be registered in `CassandraSupportedMigrators`.
+
+
+## How to use full table scanner (Only for Cassandra/ScyllaDB)
+Sometimes migrator isn't able to query the historical data by table's partition key. For example, migrator of transactions needs the historical transaction data without knowing each transaction hash. Full table scanner can help to get all the rows in parallel. 
+
+Most indexes are based on either ledger states or transactions. We provide the `objects` and `transactions` scanner. Developers only need to implement the callback function to receive the historical data. Please find the examples in `tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp` and `tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp`.
+
+> **Note** The full table scanner splits the table into multiple ranges by token(https://opensource.docs.scylladb.com/stable/cql/functions.html#token). A few of rows maybe read 2 times if its token happens to be at the edge of ranges. **Deduplication is needed** in the callback function.
+
+## How to write a full table scan adapter (Only for Cassandra/ScyllaDB)
+
+If you need to do full scan against other table, you can follow below steps:
+- Describe the table which needs full scan in a struct. It has to satisfy the `TableSpec`(cassandra/Spec.hpp) concept, containing static member:
+    - Tuple type `Row`, it's the type of each field in a row. The order of types should match what database will return in a row. Key types should come first, followed by other field types sorted in alphabetical order.
+    - `PARTITION_KEY`, it's the name of the partition key of the table.
+    - `TABLE_NAME`
+
+- Inherent from `FullTableScannerAdapterBase`.
+- Implement `onRowRead`, its parameter is the `Row` we defined. It's the callback function when a row is read.
+
+
+Please take ObjectsAdapter/TransactionsAdapter as example.
+
+## Examples:
+
+We have some example migrators under `tests/integration/migration/cassandra` folder.
+
+- ExampleDropTableMigrator
+
+    This migrator drops `diff` table. 
+- ExampleLedgerMigrator
+
+    This migrator shows how to migrate data when we don't need to do full table scan. This migrator creates an index table `ledger_example` which maintains the map of ledger sequence and its account hash. 
+- ExampleObjectsMigrator
+
+    This migrator shows how to migrate ledger states related data. It uses `ObjectsScanner` to proceed the full scan in parallel. It counts the number of ACCOUNT_ROOT.
+- ExampleTransactionsMigrator
+
+    This migrator shows how to migrate transactions related data. It uses `TransactionsScanner` to proceed the `transactions` table full scan in parallel. It creates an index table `tx_index_example` which tracks the transaction hash and its according transaction type.
+

src/migration/cassandra/CassandraMigrationBackend.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/CassandraBackend.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/cassandra/impl/CassandraMigrationSchema.hpp"
+#include "migration/cassandra/impl/Spec.hpp"
+#include "util/log/Logger.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+
+namespace migration::cassandra {
+
+/**
+ * @brief The backend for the migration. It is a subclass of the CassandraBackend and provides the migration specific
+ * functionalities.
+ */
+class CassandraMigrationBackend : public data::cassandra::CassandraBackend {
+    util::Logger log_{"Migration"};
+    data::cassandra::SettingsProvider settingsProvider_;
+    impl::CassandraMigrationSchema migrationSchema_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Backend object. The backend is not readonly.
+     *
+     * @param settingsProvider The settings provider
+     */
+    explicit CassandraMigrationBackend(data::cassandra::SettingsProvider settingsProvider)
+        : data::cassandra::CassandraBackend{auto{settingsProvider}, false /* not readonly */}
+        , settingsProvider_(std::move(settingsProvider))
+        , migrationSchema_{settingsProvider_}
+    {
+    }
+
+    /**
+     *@brief Scan a table in a token range and call the callback for each row
+     *
+     *@tparam TableDesc The table description of the table to scan
+     *@param start The start token
+     *@param end The end token
+     *@param callback The callback to call for each row
+     *@param yield The boost asio yield context
+     */
+    template <impl::TableSpec TableDesc>
+    void
+    migrateInTokenRange(
+        std::int64_t const start,
+        std::int64_t const end,
+        auto const& callback,
+        boost::asio::yield_context yield
+    )
+    {
+        LOG(log_.debug()) << "Travsering token range: " << start << " - " << end
+                          << " ; table: " << TableDesc::TABLE_NAME;
+        // for each table we only have one prepared statement
+        static auto statementPrepared =
+            migrationSchema_.getPreparedFullScanStatement(handle_, TableDesc::TABLE_NAME, TableDesc::PARTITION_KEY);
+
+        auto const statement = statementPrepared.bind(start, end);
+
+        auto const res = this->executor_.read(yield, statement);
+        if (not res) {
+            LOG(log_.error()) << "Could not fetch data from table: " << TableDesc::TABLE_NAME << " range: " << start
+                              << " - " << end << ";" << res.error();
+            return;
+        }
+
+        auto const& results = res.value();
+        if (not results.hasRows()) {
+            LOG(log_.debug()) << "No rows returned  - table: " << TableDesc::TABLE_NAME << " range: " << start << " - "
+                              << end;
+            return;
+        }
+
+        for (auto const& row : std::apply(
+                 [&](auto... args) { return data::cassandra::extract<decltype(args)...>(results); },
+                 typename TableDesc::Row{}
+             )) {
+            callback(row);
+        }
+    }
+};
+}  // namespace migration::cassandra

src/migration/cassandra/CassandraMigrationManager.hpp

@@ -0,0 +1,38 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/impl/MigrationManagerBase.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+
+namespace migration::cassandra {
+
+// Register migrators here
+// MigratorsRegister<BackendType, ExampleMigrator>
+template <typename BackendType>
+using CassandraSupportedMigrators = migration::impl::MigratorsRegister<BackendType>;
+
+// Register with MigrationBackend which proceeds the migration
+using MigrationProcesser = CassandraSupportedMigrators<CassandraMigrationBackend>;
+
+// The Cassandra migration manager
+using CassandraMigrationManager = migration::impl::MigrationManagerBase<MigrationProcesser>;
+}  // namespace migration::cassandra

src/migration/cassandra/impl/CassandraMigrationSchema.hpp

@@ -0,0 +1,98 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+
+#include <fmt/core.h>
+
+#include <functional>
+#include <string>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The schema for the migration process. It contains the prepared statements only used for the migration process.
+ */
+class CassandraMigrationSchema {
+    using SettingsProviderType = data::cassandra::SettingsProvider;
+    std::reference_wrapper<SettingsProviderType const> settingsProvider_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Schema object
+     *
+     * @param settings The settings provider of database
+     */
+    explicit CassandraMigrationSchema(SettingsProviderType const& settings) : settingsProvider_{settings}
+    {
+    }
+
+    /**
+     * @brief Get the prepared statement for the full scan of a table
+     *
+     * @param handler The database handler
+     * @param tableName The name of the table
+     * @param key The partition key of the table
+     * @return The prepared statement
+     */
+    data::cassandra::PreparedStatement
+    getPreparedFullScanStatement(
+        data::cassandra::Handle const& handler,
+        std::string const& tableName,
+        std::string const& key
+    )
+    {
+        return handler.prepare(fmt::format(
+            R"(
+            SELECT * 
+              FROM {} 
+             WHERE TOKEN({}) >= ? AND TOKEN({}) <= ?
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), tableName),
+            key,
+            key
+        ));
+    }
+
+    /**
+     * @brief Get the prepared statement for insertion of migrator_status table
+     *
+     * @param handler The database handler
+     * @return The prepared statement to insert into migrator_status table
+     */
+    data::cassandra::PreparedStatement const&
+    getPreparedInsertMigratedMigrator(data::cassandra::Handle const& handler)
+    {
+        static auto prepared = handler.prepare(fmt::format(
+            R"(
+            INSERT INTO {} 
+                   (migrator_name, status)
+            VALUES (?, ?)
+            )",
+            data::cassandra::qualifiedTableName<SettingsProviderType>(settingsProvider_.get(), "migrator_status")
+        ));
+        return prepared;
+    }
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScanner.hpp

@@ -0,0 +1,183 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etl/ETLHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/async/AnyExecutionContext.hpp"
+#include "util/async/AnyOperation.hpp"
+#include "util/async/context/BasicExecutionContext.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <limits>
+#include <ranges>
+#include <vector>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The token range used to split the full table scan into multiple ranges.
+ */
+struct TokenRange {
+    std::int64_t start;
+    std::int64_t end;
+
+    /**
+     * @brief Construct a new Token Range object
+     *
+     * @param start The start token
+     * @param end The end token
+     */
+    TokenRange(std::int64_t start, std::int64_t end) : start{start}, end{end}
+    {
+    }
+};
+
+/**
+ * @brief The concept for an adapter.
+ */
+template <typename T>
+concept CanReadByTokenRange = requires(T obj, TokenRange const& range, boost::asio::yield_context yield) {
+    { obj.readByTokenRange(range, yield) } -> std::same_as<void>;
+};
+
+/**
+ * @brief The full table scanner. It will split the full table scan into multiple ranges and read the data in given
+ * executor.
+ *
+ * @tparam TableAdapter The table adapter type
+ */
+template <CanReadByTokenRange TableAdapter>
+class FullTableScanner {
+    /**
+     * @brief The helper to generate the token ranges.
+     */
+    struct TokenRangesProvider {
+        uint32_t numRanges_;
+
+        TokenRangesProvider(uint32_t numRanges) : numRanges_{numRanges}
+        {
+        }
+
+        [[nodiscard]] std::vector<TokenRange>
+        getRanges() const
+        {
+            auto const minValue = std::numeric_limits<std::int64_t>::min();
+            auto const maxValue = std::numeric_limits<std::int64_t>::max();
+            if (numRanges_ == 1)
+                return {TokenRange{minValue, maxValue}};
+
+            // Safely calculate the range size using uint64_t to avoid overflow
+            uint64_t rangeSize = (static_cast<uint64_t>(maxValue) * 2) / numRanges_;
+
+            std::vector<TokenRange> ranges;
+            ranges.reserve(numRanges_);
+
+            for (std::int64_t i = 0; i < numRanges_; ++i) {
+                int64_t start = minValue + i * rangeSize;
+                int64_t end = (i == numRanges_ - 1) ? maxValue : start + static_cast<int64_t>(rangeSize) - 1;
+                ranges.emplace_back(start, end);
+            }
+
+            return ranges;
+        }
+    };
+
+    [[nodiscard]] auto
+    spawnWorker()
+    {
+        return ctx_.execute([this](auto token) {
+            while (not token.isStopRequested()) {
+                auto cursor = queue_.tryPop();
+                if (not cursor.has_value()) {
+                    return;  // queue is empty
+                }
+                reader_.readByTokenRange(cursor.value(), token);
+            }
+        });
+    }
+
+    void
+    load(size_t workerNum)
+    {
+        namespace vs = std::views;
+
+        tasks_.reserve(workerNum);
+
+        for ([[maybe_unused]] auto taskId : vs::iota(0u, workerNum))
+            tasks_.push_back(spawnWorker());
+    }
+
+    util::async::AnyExecutionContext ctx_;
+    std::size_t cursorsNum_;
+    etl::ThreadSafeQueue<TokenRange> queue_;
+    std::vector<util::async::AnyOperation<void>> tasks_;
+    TableAdapter reader_;
+
+public:
+    /**
+     * @brief The full table scanner settings.
+     */
+    struct FullTableScannerSettings {
+        std::uint32_t ctxThreadsNum; /**< number of threads used in the execution context */
+        std::uint32_t jobsNum;       /**< number of coroutines to run, it is the number of concurrent database reads */
+        std::uint32_t cursorsPerJob; /**< number of cursors per coroutine */
+    };
+
+    /**
+     * @brief Construct a new Full Table Scanner object, it will run in a sync or async context according to the
+     * parameter. The scan process will immediately start.
+     *
+     * @tparam ExecutionContextType The execution context type
+     * @param settings The full table scanner settings
+     * @param reader The table adapter
+     */
+    template <typename ExecutionContextType = util::async::CoroExecutionContext>
+    FullTableScanner(FullTableScannerSettings settings, TableAdapter&& reader)
+        : ctx_(ExecutionContextType(settings.ctxThreadsNum))
+        , cursorsNum_(settings.jobsNum * settings.cursorsPerJob)
+        , queue_{cursorsNum_}
+        , reader_{std::move(reader)}
+    {
+        ASSERT(settings.jobsNum > 0, "jobsNum for full table scanner must be greater than 0");
+        ASSERT(settings.cursorsPerJob > 0, "cursorsPerJob for full table scanner must be greater than 0");
+
+        auto const cursors = TokenRangesProvider{cursorsNum_}.getRanges();
+        std::ranges::for_each(cursors, [this](auto const& cursor) { queue_.push(cursor); });
+        load(settings.jobsNum);
+    }
+
+    /**
+     * @brief Wait for all workers to finish.
+     */
+    void
+    wait()
+    {
+        for (auto& task : tasks_) {
+            task.wait();
+        }
+    }
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/FullTableScannerAdapterBase.hpp

@@ -0,0 +1,84 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The base class for the full table scanner adapter. It is responsible for reading the rows from the full table
+ * scanner and call the callback when a row is read. With this base class, each table adapter can focus on the actual
+ * row data converting.
+ *
+ * @tparam TableDesc The table description, it has to be a TableSpec.
+ */
+template <TableSpec TableDesc>
+struct FullTableScannerAdapterBase {
+    static_assert(TableSpec<TableDesc>);
+
+protected:
+    /**
+     * @brief The backend to use
+     */
+    std::shared_ptr<CassandraMigrationBackend> backend_;
+
+public:
+    virtual ~FullTableScannerAdapterBase() = default;
+
+    /**
+     * @brief Construct a new Full Table Scanner Adapter Base object
+     *
+     * @param backend The backend
+     */
+    FullTableScannerAdapterBase(std::shared_ptr<CassandraMigrationBackend> backend) : backend_(std::move(backend))
+    {
+    }
+
+    /**
+     * @brief Read the row in the given token range from database, this is the adapt function for the FullTableScanner.
+     *
+     * @param range The token range to read
+     * @param yield The yield context
+     */
+    void
+    readByTokenRange(TokenRange const& range, boost::asio::yield_context yield)
+    {
+        backend_->migrateInTokenRange<TableDesc>(
+            range.start, range.end, [this](auto const& row) { onRowRead(row); }, yield
+        );
+    }
+
+    /**
+     * @brief Called when a row is read. The derived class should implement this function to convert the database blob
+     * to actual data type.
+     *
+     * @param row The row read
+     */
+    virtual void
+    onRowRead(TableDesc::Row const& row) = 0;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.cpp

@@ -0,0 +1,44 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/Serializer.h>
+
+#include <optional>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+void
+ObjectsAdapter::onRowRead(TableObjectsDesc::Row const& row)
+{
+    auto const& [key, ledgerSeq, blob] = row;
+    // the blob can be empty which means the ledger state is deleted
+    if (blob.empty()) {
+        onStateRead_(ledgerSeq, std::nullopt);
+        return;
+    }
+    ripple::SLE sle{ripple::SerialIter{blob.data(), blob.size()}, key};
+    onStateRead_(ledgerSeq, std::make_optional(std::move(sle)));
+}
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/ObjectsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/Types.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <optional>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the objects table. It has to be a TableSpec.
+ */
+struct TableObjectsDesc {
+    using Row = std::tuple<ripple::uint256, std::uint32_t, data::Blob>;
+    static constexpr char const* PARTITION_KEY = "key";
+    static constexpr char const* TABLE_NAME = "objects";
+};
+
+/**
+ * @brief The adapter for the objects table. This class is responsible for reading the objects from the
+ * FullTableScanner and converting the blobs to the STObject.
+ */
+class ObjectsAdapter : public impl::FullTableScannerAdapterBase<TableObjectsDesc> {
+public:
+    using OnStateRead = std::function<void(std::uint32_t, std::optional<ripple::SLE>)>;
+
+    /**
+     * @brief Construct a new Objects Adapter object
+     *
+     * @param backend The backend to use
+     * @param onStateRead The callback to call when a state is read
+     */
+    explicit ObjectsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnStateRead onStateRead)
+        : FullTableScannerAdapterBase<TableObjectsDesc>(backend), onStateRead_{std::move(onStateRead)}
+    {
+    }
+
+    /**
+     * @brief Called when a row is read
+     *
+     * @param row The row to read
+     */
+    void
+    onRowRead(TableObjectsDesc::Row const& row) override;
+
+private:
+    OnStateRead onStateRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Spec.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <boost/asio/spawn.hpp>
+
+#include <concepts>
+#include <tuple>
+#include <type_traits>
+
+namespace migration::cassandra::impl {
+// Define the concept for a class like TableObjectsDesc
+template <typename T>
+concept TableSpec = requires {
+    // Check that 'row' exists and is a tuple
+    // keys types are at the begining and the other fields types sort in alphabetical order
+    typename T::Row;
+    requires std::tuple_size<typename T::Row>::value >= 0;  // Ensures 'row' is a tuple
+
+    // Check that static constexpr members 'partitionKey' and 'tableName' exist
+    { T::PARTITION_KEY } -> std::convertible_to<char const*>;
+    { T::TABLE_NAME } -> std::convertible_to<char const*>;
+};
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.cpp

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/Serializer.h>
+#include <xrpl/protocol/TxMeta.h>
+
+namespace migration::cassandra::impl {
+
+void
+TransactionsAdapter::onRowRead(TableTransactionsDesc::Row const& row)
+{
+    auto const& [txHash, date, ledgerSeq, metaBlob, txBlob] = row;
+
+    ripple::SerialIter it{txBlob.data(), txBlob.size()};
+    ripple::STTx const sttx{it};
+    ripple::TxMeta const txMeta{sttx.getTransactionID(), ledgerSeq, metaBlob};
+    onTransactionRead_(sttx, txMeta);
+}
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/TransactionsAdapter.hpp

@@ -0,0 +1,80 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/impl/FullTableScannerAdapterBase.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <functional>
+#include <memory>
+#include <tuple>
+#include <utility>
+
+namespace migration::cassandra::impl {
+
+/**
+ * @brief The description of the transactions table. It has to be a TableSpec.
+ */
+struct TableTransactionsDesc {
+    // hash, date, ledger_seq, metadata, transaction
+    using Row = std::tuple<ripple::uint256, std::uint64_t, std::uint32_t, ripple::Blob, ripple::Blob>;
+    static constexpr char const* PARTITION_KEY = "hash";
+    static constexpr char const* TABLE_NAME = "transactions";
+};
+
+/**
+ * @brief The adapter for the transactions table. This class is responsible for reading the transactions from the
+ * FullTableScanner and converting the blobs to the STTx and TxMeta.
+ */
+class TransactionsAdapter : public impl::FullTableScannerAdapterBase<TableTransactionsDesc> {
+public:
+    using OnTransactionRead = std::function<void(ripple::STTx, ripple::TxMeta)>;
+
+    /**
+     * @brief Construct a new Transactions Adapter object
+     *
+     * @param backend The backend
+     * @param onTxRead The callback to call when a transaction is read
+     */
+    explicit TransactionsAdapter(std::shared_ptr<CassandraMigrationBackend> backend, OnTransactionRead onTxRead)
+        : FullTableScannerAdapterBase<TableTransactionsDesc>(backend), onTransactionRead_{std::move(onTxRead)}
+    {
+    }
+
+    /**
+     *@brief The callback when a row is read.
+     *
+     *@param row The row to read
+     */
+    void
+    onRowRead(TableTransactionsDesc::Row const& row) override;
+
+private:
+    OnTransactionRead onTransactionRead_;
+};
+
+}  // namespace migration::cassandra::impl

src/migration/cassandra/impl/Types.hpp

@@ -0,0 +1,31 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+
+namespace migration::cassandra::impl {
+
+using ObjectsScanner = impl::FullTableScanner<impl::ObjectsAdapter>;
+using TransactionsScanner = impl::FullTableScanner<impl::TransactionsAdapter>;
+
+}  // namespace migration::cassandra::impl

src/migration/impl/MigrationManagerBase.hpp

@@ -0,0 +1,120 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace migration::impl {
+
+/**
+ * @brief The migration manager implementation for Cassandra. It will run the migration for the Cassandra
+ * database.
+ *
+ * @tparam SupportedMigrators The migrators resgister that contains all the migrators
+ */
+template <typename SupportedMigrators>
+class MigrationManagerBase : public MigrationManagerInterface {
+    SupportedMigrators migrators_;
+    // contains only migration related settings
+    util::config::ObjectView config_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Manager object
+     *
+     * @param backend The backend of the Cassandra database
+     * @param config The configuration of the migration
+     */
+    explicit MigrationManagerBase(
+        std::shared_ptr<typename SupportedMigrators::BackendType> backend,
+        util::config::ObjectView const& config
+    )
+        : migrators_{backend}, config_{config}
+    {
+    }
+
+    /**
+     * @brief Run the the migration according to the given migrator's name
+     *
+     * @param name The name of the migrator
+     */
+    void
+    runMigration(std::string const& name) override
+    {
+        migrators_.runMigrator(name, config_);
+    }
+
+    /**
+     * @brief Get the status of all the migrators
+     *
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     * migrator
+     */
+    std::vector<std::tuple<std::string, MigratorStatus>>
+    allMigratorsStatusPairs() const override
+    {
+        return migrators_.getMigratorsStatus();
+    }
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The name of the migrator
+     * @return The status of the migrator
+     */
+    MigratorStatus
+    getMigratorStatusByName(std::string const& name) const override
+    {
+        return migrators_.getMigratorStatus(name);
+    }
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A vector of string, the names of all the migrators
+     */
+    std::vector<std::string>
+    allMigratorsNames() const override
+    {
+        auto const names = migrators_.getMigratorNames();
+        return std::vector<std::string>{names.begin(), names.end()};
+    }
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The name of the migrator
+     * @return The description of the migrator
+     */
+    std::string
+    getMigratorDescriptionByName(std::string const& name) const override
+    {
+        return migrators_.getMigratorDescription(name);
+    }
+};
+
+}  // namespace migration::impl

src/migration/impl/MigrationManagerFactory.cpp

@@ -0,0 +1,60 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/impl/MigrationManagerFactory.hpp"
+
+#include "data/cassandra/SettingsProvider.hpp"
+#include "migration/MigrationManagerInterface.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+#include "migration/cassandra/CassandraMigrationManager.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <boost/algorithm/string/predicate.hpp>
+
+#include <memory>
+#include <string>
+#include <utility>
+
+namespace migration::impl {
+
+std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
+makeMigrationManager(util::config::ClioConfigDefinition const& config)
+{
+    static util::Logger const log{"Migration"};
+    LOG(log.info()) << "Constructing MigrationManager";
+
+    auto const type = config.get<std::string>("database.type");
+
+    if (not boost::iequals(type, "cassandra")) {
+        LOG(log.error()) << "Unknown database type to migrate: " << type;
+        return std::unexpected(std::string("Invalid database type"));
+    }
+
+    auto const cfg = config.getObject("database." + type);
+
+    auto migrationCfg = config.getObject("migration");
+
+    return std::make_shared<cassandra::CassandraMigrationManager>(
+        std::make_shared<cassandra::CassandraMigrationBackend>(data::cassandra::SettingsProvider{cfg}),
+        std::move(migrationCfg)
+    );
+}
+
+}  // namespace migration::impl

src/migration/impl/MigrationManagerFactory.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/MigrationManagerInterface.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <expected>
+#include <memory>
+#include <string>
+
+namespace migration::impl {
+
+/**
+ * @brief The factory to create a MigrationManagerInferface
+ *
+ * @param config The configuration of the migration application, it contains the database connection configuration and
+ * other migration specific configurations
+ * @return A shared pointer to the MigrationManagerInterface if the creation was successful, otherwise an error message
+ */
+std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
+makeMigrationManager(util::config::ClioConfigDefinition const& config);
+
+}  // namespace migration::impl

src/migration/impl/MigratorsRegister.hpp

@@ -0,0 +1,184 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/BackendInterface.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/impl/Spec.hpp"
+#include "util/Concepts.hpp"
+#include "util/log/Logger.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <algorithm>
+#include <array>
+#include <iterator>
+#include <memory>
+#include <ranges>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <vector>
+
+namespace migration::impl {
+
+/**
+ * The concept to check if BackendType is the same as the migrator's required backend type
+ */
+template <typename BackendType, typename MigratorType>
+concept MigrationBackend = requires { requires std::same_as<typename MigratorType::Backend, BackendType>; };
+
+template <typename Backend, typename... MigratorType>
+concept BackendMatchAllMigrators = (MigrationBackend<Backend, MigratorType> && ...);
+
+/**
+ *@brief The register of migrators. It will dispatch the migration to the corresponding migrator. It also
+ *hold the shared pointer of backend, which is used by the migrators.
+ *
+ *@tparam Backend The backend type
+ *@tparam MigratorType The migrator types. It should be a concept of MigratorSpec and not have duplicate names.
+ */
+template <typename Backend, typename... MigratorType>
+    requires AllMigratorSpec<MigratorType...>
+class MigratorsRegister {
+    static_assert(util::hasNoDuplicateNames<MigratorType...>());
+
+    util::Logger log_{"Migration"};
+    std::shared_ptr<Backend> backend_;
+
+    template <typename Migrator>
+    void
+    callMigration(std::string const& name, util::config::ObjectView const& config)
+    {
+        if (name == Migrator::name) {
+            LOG(log_.info()) << "Running migration: " << name;
+            Migrator::runMigration(backend_, config);
+            backend_->writeMigratorStatus(name, MigratorStatus(MigratorStatus::Migrated).toString());
+            LOG(log_.info()) << "Finished migration: " << name;
+        }
+    }
+
+    template <typename T>
+    static constexpr std::string_view
+    getDescriptionIfMatch(std::string_view targetName)
+    {
+        return (T::name == targetName) ? T::description : "";
+    }
+
+public:
+    /**
+     * @brief The backend type which is used by the migrators
+     */
+    using BackendType = Backend;
+
+    /**
+     * @brief Construct a new Migrators Register object
+     *
+     * @param backend The backend shared pointer
+     */
+    MigratorsRegister(std::shared_ptr<BackendType> backend) : backend_{std::move(backend)}
+    {
+    }
+
+    /**
+     * @brief Run the migration according to the given migrator's name
+     *
+     * @param name The migrator's name
+     * @param config The configuration of the migration
+     */
+    void
+    runMigrator(std::string const& name, util::config::ObjectView const& config)
+        requires BackendMatchAllMigrators<BackendType, MigratorType...>
+    {
+        (callMigration<MigratorType>(name, config), ...);
+    }
+
+    /**
+     * @brief Get the status of all the migrators
+     *
+     * @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
+     * migrator
+     */
+    std::vector<std::tuple<std::string, MigratorStatus>>
+    getMigratorsStatus() const
+    {
+        auto const fullList = getMigratorNames();
+
+        std::vector<std::tuple<std::string, MigratorStatus>> status;
+
+        std::ranges::transform(fullList, std::back_inserter(status), [&](auto const& migratorName) {
+            auto const migratorNameStr = std::string(migratorName);
+            return std::make_tuple(migratorNameStr, getMigratorStatus(migratorNameStr));
+        });
+        return status;
+    }
+
+    /**
+     * @brief Get the status of a migrator by its name
+     *
+     * @param name The migrator's name to get the status
+     * @return The status of the migrator
+     */
+    MigratorStatus
+    getMigratorStatus(std::string const& name) const
+    {
+        auto const fullList = getMigratorNames();
+        if (std::ranges::find(fullList, name) == fullList.end()) {
+            return MigratorStatus::NotKnown;
+        }
+        auto const statusStringOpt =
+            data::synchronous([&](auto yield) { return backend_->fetchMigratorStatus(name, yield); });
+
+        return statusStringOpt ? MigratorStatus::fromString(statusStringOpt.value()) : MigratorStatus::NotMigrated;
+    }
+
+    /**
+     * @brief Get all registered migrators' names
+     *
+     * @return A array of migrator's names
+     */
+    constexpr auto
+    getMigratorNames() const
+    {
+        return std::array<std::string_view, sizeof...(MigratorType)>{MigratorType::name...};
+    }
+
+    /**
+     * @brief Get the description of a migrator by its name
+     *
+     * @param name The migrator's name
+     * @return The description of the migrator
+     */
+    std::string
+    getMigratorDescription(std::string const& name) const
+    {
+        if constexpr (sizeof...(MigratorType) == 0) {
+            return "No Description";
+        } else {
+            // Fold expression to search through all types
+            std::string result = ([](std::string const& name) {
+                return std::string(getDescriptionIfMatch<MigratorType>(name));
+            }(name) + ...);
+
+            return result.empty() ? "No Description" : result;
+        }
+    }
+};
+
+}  // namespace migration::impl

src/migration/impl/Spec.hpp

@@ -0,0 +1,56 @@
+
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+#include <string>
+
+namespace migration::impl {
+
+/**
+ * @brief The migrator specification concept
+ */
+template <typename T, typename Backend>
+concept MigratorSpec = requires(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& cfg) {
+    // Check that 'name' exists and is a string
+    { T::name } -> std::convertible_to<std::string>;
+
+    // Check that 'description' exists and is a string
+    { T::description } -> std::convertible_to<std::string>;
+
+    // Check that the migrator specifies the backend type it supports
+    typename T::Backend;
+
+    // Check that 'runMigration' exists and is callable
+    { T::runMigration(backend, cfg) } -> std::same_as<void>;
+};
+
+/**
+ * @brief used by variadic template to check all migrators are MigratorSpec
+ */
+template <typename... Types>
+concept AllMigratorSpec = (MigratorSpec<Types, typename Types::Backend> && ...);
+
+}  // namespace migration::impl

src/util/Concepts.hpp

@@ -21,6 +21,8 @@
 
 #include <algorithm>
 #include <array>
+#include <cstddef>
+#include <string_view>
 #include <type_traits>
 
 namespace util {
@@ -46,4 +48,22 @@ hasNoDuplicates(auto&&... values)
     return (std::unique(std::begin(store), end) == end);
 }
 
+/**
+ * @brief Checks that the list of given type contains no duplicates
+ *
+ * @tparam Types The types to check
+ * @returns true if no duplicates exist; false otherwise
+ */
+template <typename... Types>
+constexpr bool
+hasNoDuplicateNames()
+{
+    constexpr std::array<std::string_view, sizeof...(Types)> names = {Types::name...};
+    return !std::ranges::any_of(names, [&](std::string_view const& name1) {
+        return std::ranges::any_of(names, [&](std::string_view const& name2) {
+            return &name1 != &name2 && name1 == name2;  // Ensure different elements are compared
+        });
+    });
+}
+
 }  // namespace util

src/util/log/Logger.hpp

@@ -169,7 +169,7 @@ class Logger final {
     };
 
 public:
-    static constexpr std::array<char const*, 7> CHANNELS = {
+    static constexpr std::array<char const*, 8> CHANNELS = {
         "General",
         "WebServer",
         "Backend",
@@ -177,6 +177,7 @@ public:
         "ETL",
         "Subscriptions",
         "Performance",
+        "Migration",
     };
 
     /**

src/util/newconfig/ConfigDefinition.hpp

@@ -380,7 +380,11 @@ static ClioConfigDefinition ClioConfig = ClioConfigDefinition{
      {"api_version.min",
       ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MIN).withConstraint(validateApiVersion)},
      {"api_version.max",
-      ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MAX).withConstraint(validateApiVersion)}}
+      ConfigValue{ConfigType::Integer}.defaultValue(rpc::API_VERSION_MAX).withConstraint(validateApiVersion)},
+     {"migration.full_scan_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateUint32)},
+     {"migration.full_scan_jobs", ConfigValue{ConfigType::Integer}.defaultValue(4).withConstraint(validateUint32)},
+     {"migration.cursors_per_job", ConfigValue{ConfigType::Integer}.defaultValue(100).withConstraint(validateUint32)}},
+
 };
 
 }  // namespace util::config

src/util/newconfig/ConfigDescription.hpp

@@ -163,7 +163,10 @@ private:
         KV{.key = "ssl_key_file", .value = "Path to the SSL key file."},
         KV{.key = "api_version.default", .value = "Default API version Clio will run on."},
         KV{.key = "api_version.min", .value = "Minimum API version."},
-        KV{.key = "api_version.max", .value = "Maximum API version."}
+        KV{.key = "api_version.max", .value = "Maximum API version."},
+        KV{.key = "migration.full_scan_threads", .value = "The number of threads used to scan table."},
+        KV{.key = "migration.full_scan_jobs", .value = "The number of coroutines used to scan table."},
+        KV{.key = "migration.cursors_per_job", .value = "The number of cursors each coroutine will scan."}
     };
 };
 

src/web/dosguard/DOSGuard.cpp

@@ -26,7 +26,6 @@
 #include "util/newconfig/ValueView.hpp"
 #include "web/dosguard/WhitelistHandlerInterface.hpp"
 
-
 #include <cstdint>
 #include <functional>
 #include <mutex>

tests/common/migration/TestMigrators.hpp

@@ -0,0 +1,53 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/MockMigrationBackend.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+
+struct SimpleTestMigrator {
+    using Backend = MockMigrationBackend;
+    static constexpr auto name = "SimpleTestMigrator";
+    static constexpr auto description = "The migrator for version 0 -> 1";
+    static void
+    runMigration(std::shared_ptr<MockMigrationBackend>, util::config::ObjectView const&)
+    {
+    }
+
+    static void
+    reset()
+    {
+    }
+};
+
+struct SimpleTestMigrator2 {
+    using Backend = MockMigrationBackend;
+    static constexpr auto name = "SimpleTestMigrator2";
+    static constexpr auto description = "The migrator for version 1 -> 2";
+    static void
+    runMigration(std::shared_ptr<MockMigrationBackend>, util::config::ObjectView const&)
+    {
+    }
+
+    static void
+    reset()
+    {
+    }
+};

tests/common/util/MockBackend.hpp

@@ -175,6 +175,13 @@ struct MockBackend : public BackendInterface {
         (const, override)
     );
 
+    MOCK_METHOD(
+        std::optional<std::string>,
+        fetchMigratorStatus,
+        (std::string const&, boost::asio::yield_context),
+        (const, override)
+    );
+
     MOCK_METHOD(std::optional<LedgerRange>, hardFetchLedgerRange, (boost::asio::yield_context), (const, override));
 
     MOCK_METHOD(void, writeLedger, (ripple::LedgerHeader const&, std::string&&), (override));
@@ -206,7 +213,7 @@ struct MockBackend : public BackendInterface {
 
     MOCK_METHOD(bool, doFinishWrites, (), (override));
 
-    MOCK_METHOD(void, writeMPTHolders, ((std::vector<MPTHolderData> const&)), (override));
+    MOCK_METHOD(void, writeMPTHolders, (std::vector<MPTHolderData> const&), (override));
 
     MOCK_METHOD(
         MPTHoldersAndCursor,
@@ -218,4 +225,6 @@ struct MockBackend : public BackendInterface {
          boost::asio::yield_context),
         (const, override)
     );
+
+    MOCK_METHOD(void, writeMigratorStatus, (std::string const&, std::string const&), (override));
 };

tests/common/util/MockMigrationBackend.hpp

@@ -0,0 +1,26 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/MockBackend.hpp"
+
+struct MockMigrationBackend : public MockBackend {
+    using MockBackend::MockBackend;
+};

tests/common/util/MockMigrationBackendFixture.hpp

@@ -0,0 +1,86 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/LoggerFixtures.hpp"
+#include "util/MockMigrationBackend.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+
+#include <gmock/gmock.h>
+
+#include <memory>
+
+template <template <typename> typename MockType = ::testing::NiceMock>
+struct MockMigrationBackendTestBase : virtual public NoLoggerFixture {
+    class BackendProxy {
+        std::shared_ptr<MockType<MockMigrationBackend>> backend =
+            std::make_shared<MockType<MockMigrationBackend>>(util::config::ClioConfigDefinition{});
+
+    public:
+        auto
+        operator->()
+        {
+            return backend.get();
+        }
+
+        operator std::shared_ptr<MockMigrationBackend>()
+        {
+            return backend;
+        }
+
+        operator std::shared_ptr<MockMigrationBackend const>() const
+        {
+            return backend;
+        }
+
+        MockType<MockMigrationBackend>&
+        operator*()
+        {
+            return *backend;
+        }
+    };
+
+protected:
+    BackendProxy backend_;
+};
+
+/**
+ * @brief Fixture with a "nice" mock backend.
+ *
+ * Use @see MockBackendTestNaggy during development to get unset call expectation warnings.
+ * Once the test is ready and you are happy you can switch to this fixture to mute the warnings.
+ *
+ * A fixture that is based off of this MockBackendTest or MockBackendTestNaggy get a `backend` member
+ * that is a `BackendProxy` that can be used to access the mock backend. It can be used wherever a
+ * `std::shared_ptr<BackendInterface>` is expected as well as `*backend` can be used with EXPECT_CALL and ON_CALL.
+ */
+using MockMigrationBackendTest = MockMigrationBackendTestBase<::testing::NiceMock>;
+
+/**
+ * @brief Fixture with a "naggy" mock backend.
+ *
+ * Use this during development to get unset call expectation warnings.
+ */
+using MockMigrationBackendTestNaggy = MockMigrationBackendTestBase<::testing::NaggyMock>;
+
+/**
+ * @brief Fixture with a "strict" mock backend.
+ */
+using MockMigrationBackendTestStrict = MockMigrationBackendTestBase<::testing::StrictMock>;

tests/integration/CMakeLists.txt

@@ -2,9 +2,19 @@ add_executable(clio_integration_tests)
 
 target_sources(
   clio_integration_tests
-  PRIVATE data/BackendFactoryTests.cpp data/cassandra/BackendTests.cpp data/cassandra/BaseTests.cpp
+  PRIVATE data/BackendFactoryTests.cpp
+          data/cassandra/BackendTests.cpp
+          data/cassandra/BaseTests.cpp
+          migration/cassandra/DBRawData.cpp
+          migration/cassandra/CassandraMigrationManagerTests.cpp
+          migration/cassandra/ExampleTransactionsMigrator.cpp
+          migration/cassandra/ExampleObjectsMigrator.cpp
+          migration/cassandra/ExampleLedgerMigrator.cpp
+          migration/cassandra/ExampleDropTableMigrator.cpp
+          util/CassandraDBHelper.cpp
           # Test runner
-          TestGlobals.cpp Main.cpp
+          TestGlobals.cpp
+          Main.cpp
 )
 
 # Fix for dwarf5 bug on ci. IS STILL NEEDED???

tests/integration/migration/cassandra/CassandraMigrationManagerTests.cpp

@@ -0,0 +1,348 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "migration/MigrationManagerInterface.hpp"
+#include "migration/MigratiorStatus.hpp"
+#include "migration/cassandra/CassandraMigrationTestBackend.hpp"
+#include "migration/cassandra/DBRawData.hpp"
+#include "migration/cassandra/ExampleDropTableMigrator.hpp"
+#include "migration/cassandra/ExampleLedgerMigrator.hpp"
+#include "migration/cassandra/ExampleObjectsMigrator.hpp"
+#include "migration/cassandra/ExampleTransactionsMigrator.hpp"
+#include "migration/impl/MigrationManagerBase.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+#include "util/CassandraDBHelper.hpp"
+#include "util/LoggerFixtures.hpp"
+#include "util/MockPrometheus.hpp"
+#include "util/newconfig/ConfigConstraints.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <TestGlobals.hpp>
+#include <boost/json/parse.hpp>
+#include <boost/json/value.hpp>
+#include <fmt/core.h>
+#include <gtest/gtest.h>
+#include <xrpl/basics/base_uint.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <string>
+#include <thread>
+#include <utility>
+
+using namespace util;
+using namespace std;
+using namespace prometheus;
+using namespace data::cassandra;
+using namespace migration;
+using namespace util::config;
+
+// Register the migrators
+using CassandraSupportedTestMigrators = migration::impl::MigratorsRegister<
+    CassandraMigrationTestBackend,
+    ExampleObjectsMigrator,
+    ExampleTransactionsMigrator,
+    ExampleLedgerMigrator,
+    ExampleDropTableMigrator>;
+
+// Define the test migration manager
+using CassandraMigrationTestManager = migration::impl::MigrationManagerBase<CassandraSupportedTestMigrators>;
+
+namespace {
+std::pair<std::shared_ptr<migration::MigrationManagerInterface>, std::shared_ptr<CassandraMigrationTestBackend>>
+make_MigrationTestManagerAndBackend(ClioConfigDefinition const& config)
+{
+    auto const cfg = config.getObject("database.cassandra");
+
+    auto const backendPtr = std::make_shared<CassandraMigrationTestBackend>(data::cassandra::SettingsProvider{cfg});
+
+    return std::make_pair(
+        std::make_shared<CassandraMigrationTestManager>(backendPtr, config.getObject("migration")), backendPtr
+    );
+}
+}  // namespace
+
+class MigrationCassandraSimpleTest : public WithPrometheus, public NoLoggerFixture {
+    // This function is used to prepare the database before running the tests
+    // It is called in the SetUp function. Different tests can override this function to prepare the database
+    // differently
+    virtual void
+    setupDatabase()
+    {
+    }
+
+protected:
+    ClioConfigDefinition cfg{
+
+        {{"database.type", ConfigValue{ConfigType::String}.defaultValue("cassandra")},
+         {"database.cassandra.contact_points",
+          ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendHost)},
+         {"database.cassandra.keyspace",
+          ConfigValue{ConfigType::String}.defaultValue(TestGlobals::instance().backendKeyspace)},
+         {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
+         {"database.cassandra.replication_factor", ConfigValue{ConfigType::Integer}.defaultValue(1)},
+         {"database.cassandra.connect_timeout", ConfigValue{ConfigType::Integer}.defaultValue(2)},
+         {"database.cassandra.secure_connect_bundle", ConfigValue{ConfigType::String}.optional()},
+         {"database.cassandra.port", ConfigValue{ConfigType::Integer}.withConstraint(validatePort).optional()},
+         {"database.cassandra.replication_factor",
+          ConfigValue{ConfigType::Integer}.defaultValue(3u).withConstraint(validateUint16)},
+         {"database.cassandra.table_prefix", ConfigValue{ConfigType::String}.optional()},
+         {"database.cassandra.max_write_requests_outstanding",
+          ConfigValue{ConfigType::Integer}.defaultValue(10'000).withConstraint(validateUint32)},
+         {"database.cassandra.max_read_requests_outstanding",
+          ConfigValue{ConfigType::Integer}.defaultValue(100'000).withConstraint(validateUint32)},
+         {"database.cassandra.threads",
+          ConfigValue{ConfigType::Integer}
+              .defaultValue(static_cast<uint32_t>(std::thread::hardware_concurrency()))
+              .withConstraint(validateUint32)},
+         {"database.cassandra.core_connections_per_host",
+          ConfigValue{ConfigType::Integer}.defaultValue(1).withConstraint(validateUint16)},
+         {"database.cassandra.queue_size_io", ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint16)
+         },
+         {"database.cassandra.write_batch_size",
+          ConfigValue{ConfigType::Integer}.defaultValue(20).withConstraint(validateUint16)},
+         {"database.cassandra.connect_timeout",
+          ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
+         {"database.cassandra.request_timeout",
+          ConfigValue{ConfigType::Integer}.optional().withConstraint(validateUint32)},
+         {"database.cassandra.username", ConfigValue{ConfigType::String}.optional()},
+         {"database.cassandra.password", ConfigValue{ConfigType::String}.optional()},
+         {"database.cassandra.certfile", ConfigValue{ConfigType::String}.optional()},
+         {"migration.full_scan_threads", ConfigValue{ConfigType::Integer}.defaultValue(2).withConstraint(validateUint32)
+         },
+         {"migration.full_scan_jobs", ConfigValue{ConfigType::Integer}.defaultValue(4).withConstraint(validateUint32)},
+         {"migration.cursors_per_job", ConfigValue{ConfigType::Integer}.defaultValue(100).withConstraint(validateUint32)
+         }}
+    };
+
+    std::shared_ptr<migration::MigrationManagerInterface> testMigrationManager;
+    std::shared_ptr<CassandraMigrationTestBackend> testMigrationBackend;
+
+    MigrationCassandraSimpleTest()
+    {
+        auto const testBundle = make_MigrationTestManagerAndBackend(cfg);
+        testMigrationManager = testBundle.first;
+        testMigrationBackend = testBundle.second;
+    }
+
+    void
+    SetUp() override
+    {
+        setupDatabase();
+    }
+
+    void
+    TearDown() override
+    {
+        // drop the keyspace
+        Handle const handle{TestGlobals::instance().backendHost};
+        EXPECT_TRUE(handle.connect());
+        handle.execute("DROP KEYSPACE " + TestGlobals::instance().backendKeyspace);
+    }
+};
+
+// The test suite for testing the migration manager without any data in the database
+struct MigrationCassandraManagerCleanDBTest : public MigrationCassandraSimpleTest {};
+
+TEST_F(MigrationCassandraManagerCleanDBTest, GetAllMigratorNames)
+{
+    auto const names = testMigrationManager->allMigratorsNames();
+    EXPECT_EQ(names.size(), 4);
+    EXPECT_EQ(names[0], "ExampleObjectsMigrator");
+    EXPECT_EQ(names[1], "ExampleTransactionsMigrator");
+    EXPECT_EQ(names[2], "ExampleLedgerMigrator");
+    EXPECT_EQ(names[3], "ExampleDropTableMigrator");
+}
+
+TEST_F(MigrationCassandraManagerCleanDBTest, AllMigratorStatusBeforeAnyMigration)
+{
+    auto const status = testMigrationManager->allMigratorsStatusPairs();
+    EXPECT_EQ(status.size(), 4);
+    EXPECT_EQ(std::get<1>(status[0]), MigratorStatus::NotMigrated);
+    EXPECT_EQ(std::get<1>(status[1]), MigratorStatus::NotMigrated);
+    EXPECT_EQ(std::get<1>(status[2]), MigratorStatus::NotMigrated);
+    EXPECT_EQ(std::get<1>(status[3]), MigratorStatus::NotMigrated);
+}
+
+TEST_F(MigrationCassandraManagerCleanDBTest, MigratorStatus)
+{
+    auto status = testMigrationManager->getMigratorStatusByName("ExampleObjectsMigrator");
+    EXPECT_EQ(status, MigratorStatus::NotMigrated);
+
+    status = testMigrationManager->getMigratorStatusByName("ExampleTransactionsMigrator");
+    EXPECT_EQ(status, MigratorStatus::NotMigrated);
+
+    status = testMigrationManager->getMigratorStatusByName("ExampleLedgerMigrator");
+    EXPECT_EQ(status, MigratorStatus::NotMigrated);
+
+    status = testMigrationManager->getMigratorStatusByName("ExampleDropTableMigrator");
+    EXPECT_EQ(status, MigratorStatus::NotMigrated);
+
+    status = testMigrationManager->getMigratorStatusByName("NonExistentMigrator");
+    EXPECT_EQ(status, MigratorStatus::NotKnown);
+}
+
+// The test suite for testing migration process for ExampleTransactionsMigrator. In this test suite, the transactions
+// are written to the database before running the migration
+class MigrationCassandraManagerTxTableTest : public MigrationCassandraSimpleTest {
+    void
+    setupDatabase() override
+    {
+        Handle const handle{TestGlobals::instance().backendHost};
+        EXPECT_TRUE(handle.connect());
+
+        std::for_each(std::begin(TransactionsRawData), std::end(TransactionsRawData), [&](auto const& value) {
+            writeTxFromCSVString(TestGlobals::instance().backendKeyspace, value, handle);
+        });
+    }
+};
+
+TEST_F(MigrationCassandraManagerTxTableTest, MigrateExampleTransactionsMigrator)
+{
+    auto constexpr TransactionsMigratorName = "ExampleTransactionsMigrator";
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(TransactionsMigratorName), MigratorStatus::NotMigrated);
+
+    ExampleTransactionsMigrator::count = 0;
+    testMigrationManager->runMigration(TransactionsMigratorName);
+    EXPECT_EQ(ExampleTransactionsMigrator::count, TransactionsRawData.size());
+
+    auto const newTableSize =
+        data::synchronous([&](auto ctx) { return testMigrationBackend->fetchTxIndexTableSize(ctx); });
+
+    EXPECT_TRUE(newTableSize.has_value());
+    EXPECT_EQ(newTableSize, TransactionsRawData.size());
+
+    // check a few tx types
+    auto const getTxType = [&](ripple::uint256 const& txHash) -> std::optional<std::string> {
+        return data::synchronous([&](auto ctx) {
+            return testMigrationBackend->fetchTxTypeViaID(uint256ToString(txHash), ctx);
+        });
+    };
+
+    auto txType = getTxType(ripple::uint256("CEECF7E516F8A53C5D32A357B737ED54D3186FDD510B1973D908AD8D93AD8E00"));
+    EXPECT_TRUE(txType.has_value());
+    EXPECT_EQ(txType.value(), "OracleSet");
+
+    txType = getTxType(ripple::uint256("35DBFB1A88DE17EBD2BCE37F6E1FD6D3B9887C92B7933ED2FCF2A84E9138B7CA"));
+    EXPECT_TRUE(txType.has_value());
+    EXPECT_EQ(txType.value(), "Payment");
+
+    txType = getTxType(ripple::uint256("FCACE9D00625FA3BCC5316078324EA153EC8551243AC1701D496CC1CA2B8A474"));
+    EXPECT_TRUE(txType.has_value());
+    EXPECT_EQ(txType.value(), "AMMCreate");
+
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(TransactionsMigratorName), MigratorStatus::Migrated);
+}
+
+// The test suite for testing migration process for ExampleObjectsMigrator. In this test suite, the objects are written
+// to the database before running the migration
+class MigrationCassandraManagerObjectsTableTest : public MigrationCassandraSimpleTest {
+    void
+    setupDatabase() override
+    {
+        Handle const handle{TestGlobals::instance().backendHost};
+        EXPECT_TRUE(handle.connect());
+        for (auto const& value : ObjectsRawData) {
+            writeObjectFromCSVString(TestGlobals::instance().backendKeyspace, value, handle);
+        }
+    }
+};
+
+TEST_F(MigrationCassandraManagerObjectsTableTest, MigrateExampleObjectsMigrator)
+{
+    auto constexpr ObjectsMigratorName = "ExampleObjectsMigrator";
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(ObjectsMigratorName), MigratorStatus::NotMigrated);
+
+    testMigrationManager->runMigration(ObjectsMigratorName);
+
+    EXPECT_EQ(ExampleObjectsMigrator::count, ObjectsRawData.size());
+    EXPECT_EQ(ExampleObjectsMigrator::accountCount, 37);
+
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(ObjectsMigratorName), MigratorStatus::Migrated);
+}
+
+// The test suite for testing migration process for ExampleLedgerMigrator. In this test suite, the ledger headers are
+// written to ledgers table before running the migration
+class MigrationCassandraManagerLedgerTableTest : public MigrationCassandraSimpleTest {
+    void
+    setupDatabase() override
+    {
+        Handle const handle{TestGlobals::instance().backendHost};
+        EXPECT_TRUE(handle.connect());
+        for (auto const& value : LedgerHeaderRawData) {
+            writeLedgerFromCSVString(TestGlobals::instance().backendKeyspace, value, handle);
+        }
+        writeLedgerRange(TestGlobals::instance().backendKeyspace, 5619393, 5619442, handle);
+    }
+};
+
+TEST_F(MigrationCassandraManagerLedgerTableTest, MigrateExampleLedgerMigrator)
+{
+    auto constexpr HeaderMigratorName = "ExampleLedgerMigrator";
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(HeaderMigratorName), MigratorStatus::NotMigrated);
+
+    testMigrationManager->runMigration(HeaderMigratorName);
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(HeaderMigratorName), MigratorStatus::Migrated);
+
+    auto const newTableSize =
+        data::synchronous([&](auto ctx) { return testMigrationBackend->fetchLedgerTableSize(ctx); });
+    EXPECT_EQ(newTableSize, LedgerHeaderRawData.size());
+
+    auto const getAccountHash = [this](std::uint32_t seq) {
+        return data::synchronous([&](auto ctx) { return testMigrationBackend->fetchAccountHashViaSequence(seq, ctx); });
+    };
+
+    EXPECT_EQ(
+        getAccountHash(5619393), ripple::uint256("D1DE0F83A6858DF52811E31FE97B8449A4DD55A7D9E0023FE5DC2B335E4C49E8")
+    );
+    EXPECT_EQ(
+        getAccountHash(5619394), ripple::uint256("3FEF485204772F03842AA8757B4631E8F146E17AD9762E0552540A517DD38A24")
+    );
+    EXPECT_EQ(
+        getAccountHash(5619395), ripple::uint256("D0A61C158AD8941868666AD51C4662EEAAA2A141BF0F4435BC22B9BC6783AF65")
+    );
+}
+
+// The test suite for testing migration process for ExampleDropTableMigrator.
+class MigrationCassandraManagerDropTableTest : public MigrationCassandraSimpleTest {};
+
+TEST_F(MigrationCassandraManagerDropTableTest, MigrateDropTableMigrator)
+{
+    auto constexpr DropTableMigratorName = "ExampleDropTableMigrator";
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(DropTableMigratorName), MigratorStatus::NotMigrated);
+
+    auto const beforeDropSize =
+        data::synchronous([&](auto ctx) { return testMigrationBackend->fetchDiffTableSize(ctx); });
+    EXPECT_EQ(beforeDropSize, 0);
+
+    testMigrationManager->runMigration(DropTableMigratorName);
+    EXPECT_EQ(testMigrationManager->getMigratorStatusByName(DropTableMigratorName), MigratorStatus::Migrated);
+
+    auto const newTableSize =
+        data::synchronous([&](auto ctx) { return testMigrationBackend->fetchDiffTableSize(ctx); });
+    EXPECT_EQ(newTableSize, std::nullopt);
+}

tests/integration/migration/cassandra/CassandraMigrationTestBackend.hpp

@@ -0,0 +1,342 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Schema.hpp"
+#include "data/cassandra/SettingsProvider.hpp"
+#include "data/cassandra/Types.hpp"
+#include "migration/cassandra/CassandraMigrationBackend.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <fmt/core.h>
+#include <xrpl/basics/base_uint.h>
+
+#include <cstdint>
+#include <exception>
+#include <optional>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+/**
+ * @brief Test backend for Cassandra migration. The class is mainly to provide an example of how to add the needed
+ * backend for the migrator. It is used in integration tests to provide the backend for the example migrators. In
+ * production, the backend code should be added to CassandraMigrationBackend directly.
+ */
+class CassandraMigrationTestBackend : public migration::cassandra::CassandraMigrationBackend {
+    data::cassandra::SettingsProvider settingsProvider_;
+
+public:
+    /**
+     * @brief Construct a new Cassandra Migration Test Backend object
+     *
+     * @param settingsProvider The settings provider for the Cassandra backend
+     */
+    CassandraMigrationTestBackend(data::cassandra::SettingsProvider settingsProvider)
+        : migration::cassandra::CassandraMigrationBackend(settingsProvider)
+        , settingsProvider_(std::move(settingsProvider))
+
+    {
+        if (auto const res = handle_.executeEach(createTablesSchema()); not res)
+            throw std::runtime_error("Could not create schema: " + res.error());
+    }
+
+    /**
+     * @brief Write a transaction hash and its transaction type to the tx_index_example table. It's used by
+     * ExampleTransactionsMigrator.
+     *
+     * @param hash The transaction hash
+     * @param txType The transaction type
+     */
+    void
+    writeTxIndexExample(std::string const& hash, std::string const& txType)
+    {
+        auto static insertTxIndexExample = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                INSERT INTO {} 
+                       (hash, tx_type)
+                VALUES (?, ?)
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "tx_index_example")
+            ));
+        }();
+        executor_.writeSync(insertTxIndexExample.bind(hash, data::cassandra::Text(txType)));
+    }
+
+    /**
+     * @brief Fetch the transaction type via transaction hash from the tx_index_example table. It's used by
+     * ExampleTransactionsMigrator validation.
+     *
+     * @param hash The transaction hash
+     * @param ctx The boost asio context
+     * @return The transaction type if found, otherwise std::nullopt
+     */
+    std::optional<std::string>
+    fetchTxTypeViaID(std::string const& hash, boost::asio::yield_context ctx)
+    {
+        auto static fetchTxType = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                SELECT tx_type FROM {} WHERE hash = ?
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "tx_index_example")
+            ));
+        }();
+        auto const res = executor_.read(ctx, fetchTxType.bind(hash));
+        if (not res) {
+            return std::nullopt;
+        }
+
+        auto const& result = res.value();
+        if (not result.hasRows()) {
+            return std::nullopt;
+        }
+
+        for (auto const& [txType] : data::cassandra::extract<std::string>(result)) {
+            return txType;
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Fetch the transaction index table size. It's used by ExampleTransactionsMigrator validation.
+     *
+     * @param ctx The boost asio context
+     * @return The size of the transaction index table if found, otherwise std::nullopt
+     */
+    std::optional<std::uint64_t>
+    fetchTxIndexTableSize(boost::asio::yield_context ctx)
+    {
+        auto static insertTxIndexExample = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                SELECT COUNT(*) FROM {}
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "tx_index_example")
+            ));
+        }();
+
+        // This function will be called after table being dropped, catch the exception
+        try {
+            auto const res = executor_.read(ctx, insertTxIndexExample);
+            if (not res) {
+                return std::nullopt;
+            }
+
+            auto const& result = res.value();
+            if (not result.hasRows()) {
+                return std::nullopt;
+            }
+
+            for (auto const& [size] : data::cassandra::extract<std::uint64_t>(result)) {
+                return size;
+            }
+        } catch (std::exception& e) {
+            return std::nullopt;
+        }
+        return std::nullopt;
+    }
+
+    /**
+     *@brief Write the ledger account hash to the ledger_example table. It's used by ExampleLedgerMigrator.
+     *
+     * @param sequence The ledger sequence
+     * @param accountHash The account hash
+     */
+    void
+    writeLedgerAccountHash(std::uint64_t sequence, std::string const& accountHash)
+    {
+        auto static insertLedgerExample = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                INSERT INTO {} 
+                       (sequence, account_hash)
+                VALUES (?, ?)
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "ledger_example")
+            ));
+        }();
+        executor_.writeSync(insertLedgerExample.bind(sequence, accountHash));
+    }
+
+    /**
+     * @brief Fetch the account hash via ledger sequence from the ledger_example table. It's used by
+     * ExampleLedgerMigrator validation.
+     *
+     * @param sequence The ledger sequence
+     * @param ctx The boost asio context
+     * @return The account hash if found, otherwise std::nullopt
+     */
+    std::optional<ripple::uint256>
+    fetchAccountHashViaSequence(std::uint64_t sequence, boost::asio::yield_context ctx)
+    {
+        auto static fetchAccountHash = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                SELECT account_hash FROM {} WHERE sequence = ?
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "ledger_example")
+            ));
+        }();
+        auto const res = executor_.read(ctx, fetchAccountHash.bind(sequence));
+        if (not res) {
+            return std::nullopt;
+        }
+
+        auto const& result = res.value();
+        if (not result.hasRows()) {
+            return std::nullopt;
+        }
+
+        for (auto const& [accountHash] : data::cassandra::extract<ripple::uint256>(result)) {
+            return accountHash;
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Fetch the ledger example table size. It's used by ExampleLedgerMigrator validation.
+     *
+     * @param ctx The boost asio context
+     * @return The size of the ledger example table if found, otherwise std::nullopt
+     */
+    std::optional<std::uint64_t>
+    fetchLedgerTableSize(boost::asio::yield_context ctx)
+    {
+        auto static insertLedgerExample = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                SELECT COUNT(*) FROM {}
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "ledger_example")
+            ));
+        }();
+
+        // This function will be called after table being dropped, catch the exception
+        try {
+            auto const res = executor_.read(ctx, insertLedgerExample);
+            if (not res) {
+                return std::nullopt;
+            }
+
+            auto const& result = res.value();
+            if (not result.hasRows()) {
+                return std::nullopt;
+            }
+
+            for (auto const& [size] : data::cassandra::extract<std::uint64_t>(result)) {
+                return size;
+            }
+        } catch (std::exception& e) {
+            return std::nullopt;
+        }
+        return std::nullopt;
+    }
+
+    /**
+     * @brief Drop the diff table. It's used by ExampleDropTableMigrator.
+     *
+     * @return The result of the operation
+     */
+    auto
+    dropDiffTable()
+    {
+        return handle_.execute(fmt::format(
+            R"(
+            DROP TABLE IF EXISTS {}
+            )",
+            data::cassandra::qualifiedTableName(settingsProvider_, "diff")
+        ));
+    }
+
+    /**
+     * @brief Fetch the diff table size. It's used by ExampleDropTableMigrator validation.
+     *
+     * @param ctx The boost asio context
+     * @return The size of the diff table if found, otherwise std::nullopt
+     */
+    std::optional<std::uint64_t>
+    fetchDiffTableSize(boost::asio::yield_context ctx)
+    {
+        auto static countDiff = [this]() {
+            return handle_.prepare(fmt::format(
+                R"(
+                SELECT COUNT(*) FROM {}
+                )",
+                data::cassandra::qualifiedTableName(settingsProvider_, "diff")
+            ));
+        }();
+
+        // This function will be called after table being dropped, catch the exception
+        try {
+            auto const res = executor_.read(ctx, countDiff);
+            if (not res) {
+                return std::nullopt;
+            }
+
+            auto const& result = res.value();
+            if (not result.hasRows()) {
+                return std::nullopt;
+            }
+
+            for (auto const& [size] : data::cassandra::extract<std::uint64_t>(result)) {
+                return size;
+            }
+        } catch (std::exception& e) {
+            return std::nullopt;
+        }
+        return std::nullopt;
+    }
+
+private:
+    std::vector<data::cassandra::Statement>
+    createTablesSchema()
+    {
+        std::vector<data::cassandra::Statement> statements;
+
+        statements.emplace_back(fmt::format(
+            R"(
+            CREATE TABLE IF NOT EXISTS {}
+                   (      
+                        hash blob,
+                     tx_type text,
+                     PRIMARY KEY (hash) 
+                   ) 
+            )",
+            data::cassandra::qualifiedTableName(settingsProvider_, "tx_index_example")
+        ));
+
+        statements.emplace_back(fmt::format(
+            R"(
+            CREATE TABLE IF NOT EXISTS {}
+                   (      
+                        sequence bigint,
+                    account_hash blob,
+                         PRIMARY KEY (sequence) 
+                   ) 
+            )",
+            data::cassandra::qualifiedTableName(settingsProvider_, "ledger_example")
+        ));
+        return statements;
+    }
+};

tests/integration/migration/cassandra/DBRawData.hpp

@@ -0,0 +1,29 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include <array>
+#include <string>
+
+extern std::array<std::string, 100> TransactionsRawData;
+
+extern std::array<std::string, 100> ObjectsRawData;
+
+extern std::array<std::string, 50> LedgerHeaderRawData;

tests/integration/migration/cassandra/ExampleDropTableMigrator.cpp

@@ -0,0 +1,30 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/ExampleDropTableMigrator.hpp"
+
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+
+void
+ExampleDropTableMigrator::runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const&)
+{
+    backend->dropDiffTable();
+}

tests/integration/migration/cassandra/ExampleDropTableMigrator.hpp

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationTestBackend.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+
+/**
+ * @brief Example migrator for dropping the table. In this example, our migrator will drop the table. The table removal
+ * is not reversible.
+ */
+struct ExampleDropTableMigrator {
+    using Backend = CassandraMigrationTestBackend;
+
+    static constexpr char const* name = "ExampleDropTableMigrator";
+    static constexpr char const* description = "The migrator for dropping the table";
+
+    static void
+    runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& config);
+};

tests/integration/migration/cassandra/ExampleLedgerMigrator.cpp

@@ -0,0 +1,49 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/ExampleLedgerMigrator.hpp"
+
+#include "data/BackendInterface.hpp"
+#include "data/DBHelpers.hpp"
+#include "util/Assert.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <boost/asio/io_context.hpp>
+#include <boost/asio/spawn.hpp>
+
+#include <memory>
+
+void
+ExampleLedgerMigrator::runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const&)
+{
+    auto const range =
+        data::synchronous([&](boost::asio::yield_context yield) { return backend->hardFetchLedgerRange(yield); });
+
+    if (!range.has_value())
+        return;
+
+    data::synchronous([&](boost::asio::yield_context yield) {
+        for (auto seq = range->minSequence; seq <= range->maxSequence; seq++) {
+            auto const ledgerHeader = backend->fetchLedgerBySequence(seq, yield);
+            ASSERT(ledgerHeader.has_value(), "Can not find the sequence: {}", seq);
+
+            backend->writeLedgerAccountHash(seq, uint256ToString(ledgerHeader->accountHash));
+        }
+    });
+}

tests/integration/migration/cassandra/ExampleLedgerMigrator.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationTestBackend.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <memory>
+
+/**
+ * @brief Example migrator for the ledgers table. In this example, we show how to migrate the data from table without
+ * full table scan. We create an index table called "ledger_example" which maintains the map of ledger
+ * sequence to account hash. Because ledger sequence is the partition key of ledgers table, we can just fetch the data
+ * via ledger sequence without full table scan.
+ */
+struct ExampleLedgerMigrator {
+    static constexpr char const* name = "ExampleLedgerMigrator";
+    static constexpr char const* description = "The migrator for ledgers table";
+
+    using Backend = CassandraMigrationTestBackend;
+
+    static void
+    runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& config);
+};

tests/integration/migration/cassandra/ExampleObjectsMigrator.cpp

@@ -0,0 +1,65 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/ExampleObjectsMigrator.hpp"
+
+#include "migration/cassandra/impl/ObjectsAdapter.hpp"
+#include "migration/cassandra/impl/Types.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/LedgerFormats.h>
+#include <xrpl/protocol/STLedgerEntry.h>
+
+#include <atomic>
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <unordered_set>
+
+std::atomic_int64_t ExampleObjectsMigrator::count;
+std::atomic_int64_t ExampleObjectsMigrator::accountCount;
+
+void
+ExampleObjectsMigrator::runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& config)
+{
+    auto const ctxFullScanThreads = config.get<std::uint32_t>("full_scan_threads");
+    auto const jobsFullScan = config.get<std::uint32_t>("full_scan_jobs");
+    auto const cursorPerJobsFullScan = config.get<std::uint32_t>("cursors_per_job");
+
+    std::unordered_set<ripple::uint256> idx;
+    migration::cassandra::impl::ObjectsScanner scaner(
+        {.ctxThreadsNum = ctxFullScanThreads, .jobsNum = jobsFullScan, .cursorsPerJob = cursorPerJobsFullScan},
+        migration::cassandra::impl::ObjectsAdapter(
+            backend,
+            [&](std::uint32_t, std::optional<ripple::SLE> sle) {
+                if (sle.has_value()) {
+                    if (sle->getType() == ripple::ltACCOUNT_ROOT) {
+                        if (!idx.contains(sle->key())) {
+                            ExampleObjectsMigrator::accountCount++;
+                        }
+                    }
+                    idx.insert(sle->key());
+                    ExampleObjectsMigrator::count++;
+                }
+            }
+        )
+    );
+    scaner.wait();
+}

tests/integration/migration/cassandra/ExampleObjectsMigrator.hpp

@@ -0,0 +1,46 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationTestBackend.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <xrpl/protocol/STLedgerEntry.h>
+#include <xrpl/protocol/STObject.h>
+
+#include <atomic>
+#include <memory>
+
+/**
+ * @brief Example migrator for the objects table. In this example, we show how to traverse objects table.
+ * We will count the number of account root in the objects table.
+ */
+struct ExampleObjectsMigrator {
+    using Backend = CassandraMigrationTestBackend;
+
+    static constexpr char const* name = "ExampleObjectsMigrator";
+    static constexpr char const* description = "The migrator for objects table";
+
+    static std::atomic_int64_t count;
+    static std::atomic_int64_t accountCount;
+
+    static void
+    runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& config);
+};

tests/integration/migration/cassandra/ExampleTransactionsMigrator.cpp

@@ -0,0 +1,69 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/ExampleTransactionsMigrator.hpp"
+
+#include "data/DBHelpers.hpp"
+#include "migration/cassandra/impl/TransactionsAdapter.hpp"
+#include "migration/cassandra/impl/Types.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/protocol/STBase.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <unordered_set>
+
+std::uint64_t ExampleTransactionsMigrator::count;
+
+void
+ExampleTransactionsMigrator::runMigration(
+    std::shared_ptr<Backend> const& backend,
+    util::config::ObjectView const& config
+)
+{
+    auto const ctxFullScanThreads = config.get<std::uint32_t>("full_scan_threads");
+    auto const jobsFullScan = config.get<std::uint32_t>("full_scan_jobs");
+    auto const cursorPerJobsFullScan = config.get<std::uint32_t>("cursors_per_job");
+
+    std::unordered_set<std::string> hashSet;
+    std::mutex mtx;  // protect hashSet
+    migration::cassandra::impl::TransactionsScanner scaner(
+        {.ctxThreadsNum = ctxFullScanThreads, .jobsNum = jobsFullScan, .cursorsPerJob = cursorPerJobsFullScan},
+        migration::cassandra::impl::TransactionsAdapter(
+            backend,
+            [&](ripple::STTx const& tx, ripple::TxMeta const&) {
+                {
+                    std::lock_guard<std::mutex> lock(mtx);
+                    hashSet.insert(ripple::to_string(tx.getTransactionID()));
+                }
+                auto const json = tx.getJson(ripple::JsonOptions::none);
+                auto const txType = json["TransactionType"].asString();
+                backend->writeTxIndexExample(uint256ToString(tx.getTransactionID()), txType);
+            }
+        )
+    );
+    scaner.wait();
+    count = hashSet.size();
+}

tests/integration/migration/cassandra/ExampleTransactionsMigrator.hpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "migration/cassandra/CassandraMigrationTestBackend.hpp"
+#include "util/newconfig/ObjectView.hpp"
+
+#include <cstdint>
+#include <memory>
+
+/**
+ * @brief Example migrator for the transactions table. In this example, we show how to traverse the transactions table
+ * and migrate the data to index table. We create an index table for transaction hash to transaction type string.
+ */
+struct ExampleTransactionsMigrator {
+    static constexpr char const* name = "ExampleTransactionsMigrator";
+    static constexpr char const* description = "The migrator for transactions table";
+
+    using Backend = CassandraMigrationTestBackend;
+    static std::uint64_t count;
+
+    static void
+    runMigration(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& config);
+};

tests/integration/util/CassandraDBHelper.cpp

@@ -0,0 +1,72 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+#include "data/cassandra/impl/Result.hpp"
+
+#include <fmt/core.h>
+#include <fmt/format.h>
+
+#include <cstdint>
+#include <string>
+
+data::cassandra::ResultOrError
+writeTxFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler)
+{
+    std::string statement = fmt::format(
+        "INSERT INTO {}.transactions (hash, date, ledger_sequence, metadata, transaction) VALUES ({})", space, record
+    );
+
+    return handler.execute(statement);
+}
+
+data::cassandra::ResultOrError
+writeObjectFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler)
+{
+    std::string statement = fmt::format("INSERT INTO {}.objects (key, sequence, object) VALUES ({})", space, record);
+
+    return handler.execute(statement);
+}
+
+data::cassandra::ResultOrError
+writeLedgerFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler)
+{
+    std::string statement = fmt::format("INSERT INTO {}.ledgers (sequence, header) VALUES ({})", space, record);
+    return handler.execute(statement);
+}
+
+data::cassandra::ResultOrError
+writeLedgerRange(
+    std::string const& space,
+    std::uint32_t minSeq,
+    std::uint32_t maxSeq,
+    data::cassandra::Handle const& handler
+)
+{
+    std::string statement =
+        fmt::format("INSERT INTO {}.ledger_range (sequence, is_latest) VALUES ({},false)", space, minSeq);
+    auto ret = handler.execute(statement);
+
+    if (!ret)
+        return ret;
+
+    statement = fmt::format("INSERT INTO {}.ledger_range (sequence, is_latest) VALUES ({},true)", space, maxSeq);
+    return handler.execute(statement);
+}

tests/integration/util/CassandraDBHelper.hpp

@@ -0,0 +1,76 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2022-2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "data/cassandra/Handle.hpp"
+#include "data/cassandra/Types.hpp"
+
+#include <cstdint>
+#include <string>
+
+/**
+ * @brief Write a transaction to the database from a CSV string.
+ *
+ * @param space The keyspace to write the transaction to.
+ * @param record The CSV string representing the transaction.
+ * @param handler The Cassandra database handler.
+ * @return The result of the operation.
+ */
+data::cassandra::ResultOrError
+writeTxFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler);
+
+/**
+ * @brief Write an object to the database from a CSV string.
+ *
+ * @param space The keyspace to write the object to.
+ * @param record The CSV string representing the object.
+ * @param handler The Cassandra database handler.
+ * @return The result of the operation.
+ */
+data::cassandra::ResultOrError
+writeObjectFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler);
+
+/**
+ * @brief Write a ledger to the database from a CSV string.
+ *
+ * @param space The keyspace to write the ledger to.
+ * @param record The CSV string representing the ledger.
+ * @param handler The Cassandra database handler.
+ * @return The result of the operation.
+ */
+data::cassandra::ResultOrError
+writeLedgerFromCSVString(std::string const& space, std::string const& record, data::cassandra::Handle const& handler);
+
+/**
+ * @brief Write a range of ledgers to the database.
+ *
+ * @param space The keyspace to write the ledgers to.
+ * @param minSeq The minimum ledger sequence.
+ * @param maxSeq The maximum ledger sequence.
+ * @param handler The Cassandra database handler.
+ * @return The result of the operation.
+ */
+data::cassandra::ResultOrError
+writeLedgerRange(
+    std::string const& space,
+    std::uint32_t minSeq,
+    std::uint32_t maxSeq,
+    data::cassandra::Handle const& handler
+);

tests/unit/CMakeLists.txt

@@ -50,6 +50,14 @@ target_sources(
           Main.cpp
           Playground.cpp
           ProfilerTests.cpp
+          # Migration
+          migration/cassandra/FullTableScannerTests.cpp
+          migration/cassandra/SpecTests.cpp
+          migration/MigratorRegisterTests.cpp
+          migration/MigratorStatusTests.cpp
+          migration/MigrationManagerBaseTests.cpp
+          migration/MigrationManagerFactoryTests.cpp
+          migration/SpecTests.cpp
           # RPC
           rpc/APIVersionTests.cpp
           rpc/BaseTests.cpp
@@ -112,6 +120,7 @@ target_sources(
           util/async/AnyStrandTests.cpp
           util/async/AsyncExecutionContextTests.cpp
           util/BatchingTests.cpp
+          util/ConceptsTests.cpp
           util/CoroutineGroupTests.cpp
           util/LedgerUtilsTests.cpp
           # Prometheus support

tests/unit/app/CliArgsTests.cpp

@@ -31,6 +31,7 @@ using namespace app;
 struct CliArgsTests : testing::Test {
     testing::StrictMock<testing::MockFunction<int(CliArgs::Action::Run)>> onRunMock;
     testing::StrictMock<testing::MockFunction<int(CliArgs::Action::Exit)>> onExitMock;
+    testing::StrictMock<testing::MockFunction<int(CliArgs::Action::Migrate)>> onMigrateMock;
 };
 
 TEST_F(CliArgsTests, Parse_NoArgs)
@@ -44,7 +45,9 @@ TEST_F(CliArgsTests, Parse_NoArgs)
         EXPECT_FALSE(run.useNgWebServer);
         return returnCode;
     });
-    EXPECT_EQ(action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction()), returnCode);
+    EXPECT_EQ(
+        action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction(), onMigrateMock.AsStdFunction()), returnCode
+    );
 }
 
 TEST_F(CliArgsTests, Parse_NgWebServer)
@@ -58,7 +61,10 @@ TEST_F(CliArgsTests, Parse_NgWebServer)
             EXPECT_TRUE(run.useNgWebServer);
             return returnCode;
         });
-        EXPECT_EQ(action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction()), returnCode);
+        EXPECT_EQ(
+            action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction(), onMigrateMock.AsStdFunction()),
+            returnCode
+        );
     }
 }
 
@@ -72,7 +78,10 @@ TEST_F(CliArgsTests, Parse_VersionHelp)
         auto const action = CliArgs::parse(argv.size(), const_cast<char const**>(argv.data()));
 
         EXPECT_CALL(onExitMock, Call).WillOnce([](CliArgs::Action::Exit const& exit) { return exit.exitCode; });
-        EXPECT_EQ(action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction()), EXIT_SUCCESS);
+        EXPECT_EQ(
+            action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction(), onMigrateMock.AsStdFunction()),
+            EXIT_SUCCESS
+        );
     }
 }
 
@@ -87,5 +96,7 @@ TEST_F(CliArgsTests, Parse_Config)
         EXPECT_EQ(run.configPath, configPath);
         return returnCode;
     });
-    EXPECT_EQ(action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction()), returnCode);
+    EXPECT_EQ(
+        action.apply(onRunMock.AsStdFunction(), onExitMock.AsStdFunction(), onMigrateMock.AsStdFunction()), returnCode
+    );
 }

tests/unit/migration/MigrationManagerBaseTests.cpp

@@ -0,0 +1,110 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+#include "migration/TestMigrators.hpp"
+#include "migration/impl/MigrationManagerBase.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+#include "util/MockMigrationBackend.hpp"
+#include "util/MockMigrationBackendFixture.hpp"
+#include "util/MockPrometheus.hpp"
+#include "util/newconfig/ConfigConstraints.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <tuple>
+
+using TestMigratorRegister =
+    migration::impl::MigratorsRegister<MockMigrationBackend, SimpleTestMigrator, SimpleTestMigrator2>;
+
+using TestCassandraMigrationManager = migration::impl::MigrationManagerBase<TestMigratorRegister>;
+
+struct MigrationManagerBaseTest : public util::prometheus::WithMockPrometheus, public MockMigrationBackendTestStrict {
+    util::config::ClioConfigDefinition cfg{
+        {"migration.full_scan_threads",
+         util::config::ConfigValue{util::config::ConfigType::Integer}.defaultValue(2).withConstraint(
+             util::config::validateUint32
+         )}
+    };
+    std::shared_ptr<TestCassandraMigrationManager> migrationManager;
+
+    MigrationManagerBaseTest()
+    {
+        auto mockBackendPtr = backend_.operator std::shared_ptr<MockMigrationBackend>();
+        TestMigratorRegister migratorRegister(mockBackendPtr);
+        migrationManager = std::make_shared<TestCassandraMigrationManager>(mockBackendPtr, cfg.getObject("migration"));
+    }
+};
+
+TEST_F(MigrationManagerBaseTest, AllStatus)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator", testing::_)).WillOnce(testing::Return("Migrated"));
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator2", testing::_))
+        .WillOnce(testing::Return("NotMigrated"));
+    auto const status = migrationManager->allMigratorsStatusPairs();
+    EXPECT_EQ(status.size(), 2);
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator", migration::MigratorStatus::Migrated)
+        ) != status.end()
+    );
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator2", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+}
+
+TEST_F(MigrationManagerBaseTest, AllNames)
+{
+    auto const names = migrationManager->allMigratorsNames();
+    EXPECT_EQ(names.size(), 2);
+    EXPECT_EQ(names[0], "SimpleTestMigrator");
+    EXPECT_EQ(names[1], "SimpleTestMigrator2");
+}
+
+TEST_F(MigrationManagerBaseTest, Description)
+{
+    EXPECT_EQ(migrationManager->getMigratorDescriptionByName("unknown"), "No Description");
+    EXPECT_EQ(migrationManager->getMigratorDescriptionByName("SimpleTestMigrator"), "The migrator for version 0 -> 1");
+    EXPECT_EQ(migrationManager->getMigratorDescriptionByName("SimpleTestMigrator2"), "The migrator for version 1 -> 2");
+}
+
+TEST_F(MigrationManagerBaseTest, RunMigration)
+{
+    EXPECT_CALL(*backend_, writeMigratorStatus("SimpleTestMigrator", "Migrated"));
+    migrationManager->runMigration("SimpleTestMigrator");
+}
+
+TEST_F(MigrationManagerBaseTest, getMigratorStatusByName)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator", testing::_)).WillOnce(testing::Return("Migrated"));
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator2", testing::_))
+        .WillOnce(testing::Return("NotMigrated"));
+
+    EXPECT_EQ(migrationManager->getMigratorStatusByName("SimpleTestMigrator"), migration::MigratorStatus::Migrated);
+    EXPECT_EQ(migrationManager->getMigratorStatusByName("SimpleTestMigrator2"), migration::MigratorStatus::NotMigrated);
+}

tests/unit/migration/MigrationManagerFactoryTests.cpp

@@ -0,0 +1,39 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/impl/MigrationManagerFactory.hpp"
+#include "util/LoggerFixtures.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <boost/json/parse.hpp>
+#include <gtest/gtest.h>
+
+struct MigrationManagerFactoryTests : public NoLoggerFixture {};
+
+TEST_F(MigrationManagerFactoryTests, InvalidDBType)
+{
+    util::config::ClioConfigDefinition const configDef{
+        {"database.type", util::config::ConfigValue{util::config::ConfigType::String}.defaultValue("invalid")}
+    };
+    auto const ret = migration::impl::makeMigrationManager(configDef);
+    EXPECT_FALSE(ret);
+    EXPECT_EQ(ret.error(), "Invalid database type");
+}

tests/unit/migration/MigratorRegisterTests.cpp

@@ -0,0 +1,189 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+#include "migration/TestMigrators.hpp"
+#include "migration/impl/MigratorsRegister.hpp"
+#include "util/MockMigrationBackend.hpp"
+#include "util/MockMigrationBackendFixture.hpp"
+#include "util/MockPrometheus.hpp"
+#include "util/newconfig/ConfigConstraints.hpp"
+#include "util/newconfig/ConfigDefinition.hpp"
+#include "util/newconfig/ConfigValue.hpp"
+#include "util/newconfig/Types.hpp"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <optional>
+#include <string>
+#include <tuple>
+
+using EmptyMigratorRegister = migration::impl::MigratorsRegister<MockMigrationBackend>;
+
+namespace {
+util::config::ClioConfigDefinition cfg{
+    {{"migration.full_scan_threads",
+      util::config::ConfigValue{util::config::ConfigType::Integer}.defaultValue(2).withConstraint(
+          util::config::validateUint32
+      )},
+     {"migration.full_scan_jobs",
+      util::config::ConfigValue{util::config::ConfigType::Integer}.defaultValue(4).withConstraint(
+          util::config::validateUint32
+      )},
+     {"migration.cursors_per_job",
+      util::config::ConfigValue{util::config::ConfigType::Integer}.defaultValue(100).withConstraint(
+          util::config::validateUint32
+      )}}
+};
+}  // namespace
+
+struct MigratorRegisterTests : public util::prometheus::WithMockPrometheus, public MockMigrationBackendTest {};
+
+TEST_F(MigratorRegisterTests, EmptyMigratorRegister)
+{
+    EmptyMigratorRegister migratorRegister(backend_);
+    EXPECT_EQ(migratorRegister.getMigratorsStatus().size(), 0);
+    EXPECT_EQ(migratorRegister.getMigratorNames().size(), 0);
+    EXPECT_EQ(migratorRegister.getMigratorStatus("unknown"), migration::MigratorStatus::NotKnown);
+    EXPECT_NO_THROW(migratorRegister.runMigrator("unknown", cfg.getObject("migration")));
+    EXPECT_EQ(migratorRegister.getMigratorDescription("unknown"), "No Description");
+}
+
+using MultipleMigratorRegister =
+    migration::impl::MigratorsRegister<MockMigrationBackend, SimpleTestMigrator, SimpleTestMigrator2>;
+
+struct MultipleMigratorRegisterTests : public util::prometheus::WithMockPrometheus, public MockMigrationBackendTest {
+    std::optional<MultipleMigratorRegister> migratorRegister;
+
+    MultipleMigratorRegisterTests()
+    {
+        migratorRegister.emplace(backend_);
+    }
+};
+
+TEST_F(MultipleMigratorRegisterTests, GetMigratorsStatusWhenError)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus(testing::_, testing::_))
+        .Times(2)
+        .WillRepeatedly(testing::Return(std::nullopt));
+
+    auto const status = migratorRegister->getMigratorsStatus();
+    EXPECT_EQ(status.size(), 2);
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator2", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+}
+
+TEST_F(MultipleMigratorRegisterTests, GetMigratorsStatusWhenReturnInvalidStatus)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus(testing::_, testing::_))
+        .Times(2)
+        .WillRepeatedly(testing::Return("Invalid"));
+
+    auto const status = migratorRegister->getMigratorsStatus();
+    EXPECT_EQ(status.size(), 2);
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator2", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+}
+
+TEST_F(MultipleMigratorRegisterTests, GetMigratorsStatusWhenOneMigrated)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator", testing::_)).WillOnce(testing::Return("Migrated"));
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator2", testing::_))
+        .WillOnce(testing::Return("NotMigrated"));
+
+    auto const status = migratorRegister->getMigratorsStatus();
+    EXPECT_EQ(status.size(), 2);
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator", migration::MigratorStatus::Migrated)
+        ) != status.end()
+    );
+    EXPECT_TRUE(
+        std::find(
+            status.begin(), status.end(), std::make_tuple("SimpleTestMigrator2", migration::MigratorStatus::NotMigrated)
+        ) != status.end()
+    );
+}
+
+TEST_F(MultipleMigratorRegisterTests, GetMigratorStatus)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator", testing::_)).WillOnce(testing::Return("Migrated"));
+    EXPECT_CALL(*backend_, fetchMigratorStatus("SimpleTestMigrator2", testing::_))
+        .WillOnce(testing::Return("NotMigrated"));
+
+    EXPECT_EQ(migratorRegister->getMigratorStatus("unknown"), migration::MigratorStatus::NotKnown);
+    EXPECT_EQ(migratorRegister->getMigratorStatus("SimpleTestMigrator"), migration::MigratorStatus::Migrated);
+    EXPECT_EQ(migratorRegister->getMigratorStatus("SimpleTestMigrator2"), migration::MigratorStatus::NotMigrated);
+}
+
+TEST_F(MultipleMigratorRegisterTests, GetMigratorStatusWhenError)
+{
+    EXPECT_CALL(*backend_, fetchMigratorStatus(testing::_, testing::_))
+        .Times(2)
+        .WillRepeatedly(testing::Return(std::nullopt));
+
+    EXPECT_EQ(migratorRegister->getMigratorStatus("unknown"), migration::MigratorStatus::NotKnown);
+    EXPECT_EQ(migratorRegister->getMigratorStatus("SimpleTestMigrator"), migration::MigratorStatus::NotMigrated);
+    EXPECT_EQ(migratorRegister->getMigratorStatus("SimpleTestMigrator2"), migration::MigratorStatus::NotMigrated);
+}
+
+TEST_F(MultipleMigratorRegisterTests, Names)
+{
+    auto names = migratorRegister->getMigratorNames();
+    EXPECT_EQ(names.size(), 2);
+    EXPECT_TRUE(std::find(names.begin(), names.end(), "SimpleTestMigrator") != names.end());
+    EXPECT_TRUE(std::find(names.begin(), names.end(), "SimpleTestMigrator2") != names.end());
+}
+
+TEST_F(MultipleMigratorRegisterTests, Description)
+{
+    EXPECT_EQ(migratorRegister->getMigratorDescription("unknown"), "No Description");
+    EXPECT_EQ(migratorRegister->getMigratorDescription("SimpleTestMigrator"), "The migrator for version 0 -> 1");
+    EXPECT_EQ(migratorRegister->getMigratorDescription("SimpleTestMigrator2"), "The migrator for version 1 -> 2");
+}
+
+TEST_F(MultipleMigratorRegisterTests, RunUnknownMigrator)
+{
+    EXPECT_CALL(*backend_, writeMigratorStatus(testing::_, testing::_)).Times(0);
+    EXPECT_NO_THROW(migratorRegister->runMigrator("unknown", cfg.getObject("migration")));
+}
+
+TEST_F(MultipleMigratorRegisterTests, MigrateNormalMigrator)
+{
+    EXPECT_CALL(*backend_, writeMigratorStatus("SimpleTestMigrator", "Migrated")).Times(1);
+    EXPECT_NO_THROW(migratorRegister->runMigrator("SimpleTestMigrator", cfg.getObject("migration")));
+}

tests/unit/migration/MigratorStatusTests.cpp

@@ -0,0 +1,51 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/MigratiorStatus.hpp"
+
+#include <gtest/gtest.h>
+
+TEST(MigratiorStatus, ToString)
+{
+    migration::MigratorStatus status(migration::MigratorStatus::Migrated);
+    EXPECT_EQ(status.toString(), "Migrated");
+    status = migration::MigratorStatus(migration::MigratorStatus::NotMigrated);
+    EXPECT_EQ(status.toString(), "NotMigrated");
+    status = migration::MigratorStatus(migration::MigratorStatus::NotKnown);
+    EXPECT_EQ(status.toString(), "NotKnown");
+}
+
+TEST(MigratiorStatus, FromString)
+{
+    EXPECT_EQ(migration::MigratorStatus::fromString("Migrated"), migration::MigratorStatus::Migrated);
+    EXPECT_EQ(migration::MigratorStatus::fromString("NotMigrated"), migration::MigratorStatus::NotMigrated);
+    EXPECT_EQ(migration::MigratorStatus::fromString("NotKnown"), migration::MigratorStatus::NotKnown);
+    EXPECT_EQ(migration::MigratorStatus::fromString("Unknown"), migration::MigratorStatus::NotMigrated);
+}
+
+TEST(MigratiorStatus, Compare)
+{
+    migration::MigratorStatus status1(migration::MigratorStatus::Migrated);
+    migration::MigratorStatus status2(migration::MigratorStatus::Migrated);
+    EXPECT_TRUE(status1 == status2);
+    status2 = migration::MigratorStatus(migration::MigratorStatus::NotMigrated);
+    EXPECT_FALSE(status1 == status2);
+    EXPECT_FALSE(status1 == migration::MigratorStatus::NotMigrated);
+    EXPECT_TRUE(status1 == migration::MigratorStatus::Migrated);
+}

tests/unit/migration/SpecTests.cpp

@@ -0,0 +1,41 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/TestMigrators.hpp"
+#include "migration/impl/Spec.hpp"
+#include "util/MockMigrationBackend.hpp"
+
+#include <gtest/gtest.h>
+
+namespace {
+class Fake {};
+
+}  // namespace
+
+TEST(MigrationSpec, MigratorSpec)
+{
+    static_assert(!migration::impl::MigratorSpec<Fake, MockMigrationBackend>);
+    static_assert(migration::impl::MigratorSpec<SimpleTestMigrator, MockMigrationBackend>);
+}
+
+TEST(MigrationSpec, AllMigratorSpec)
+{
+    static_assert(!migration::impl::AllMigratorSpec<SimpleTestMigrator, SimpleTestMigrator2, Fake>);
+    static_assert(migration::impl::AllMigratorSpec<SimpleTestMigrator2, SimpleTestMigrator>);
+}

tests/unit/migration/cassandra/FullTableScannerTests.cpp

@@ -0,0 +1,111 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/FullTableScanner.hpp"
+#include "util/LoggerFixtures.hpp"
+
+#include <boost/asio/spawn.hpp>
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <functional>
+#include <limits>
+
+namespace {
+
+struct TestScannerAdaper {
+    TestScannerAdaper(
+        testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)>& func
+    )
+        : callback(func) {};
+
+    TestScannerAdaper(TestScannerAdaper const&) = default;
+    TestScannerAdaper(TestScannerAdaper&&) = default;
+
+    std::reference_wrapper<
+        testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)>>
+        callback;
+
+    void
+    readByTokenRange(migration::cassandra::impl::TokenRange const& range, boost::asio::yield_context yield) const
+    {
+        callback.get().Call(range, yield);
+    }
+};
+}  // namespace
+
+struct FullTableScannerTests : public NoLoggerFixture {};
+
+TEST_F(FullTableScannerTests, workerNumZero)
+{
+    testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)> mockCallback;
+    EXPECT_DEATH(
+        migration::cassandra::impl::FullTableScanner<TestScannerAdaper>(
+            {.ctxThreadsNum = 1, .jobsNum = 0, .cursorsPerJob = 100}, TestScannerAdaper(mockCallback)
+        ),
+        "jobsNum for full table scanner must be greater than 0"
+    );
+}
+
+TEST_F(FullTableScannerTests, cursorsPerWorkerZero)
+{
+    testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)> mockCallback;
+    EXPECT_DEATH(
+        migration::cassandra::impl::FullTableScanner<TestScannerAdaper>(
+            {.ctxThreadsNum = 1, .jobsNum = 1, .cursorsPerJob = 0}, TestScannerAdaper(mockCallback)
+        ),
+        "cursorsPerJob for full table scanner must be greater than 0"
+    );
+}
+
+TEST_F(FullTableScannerTests, SingleThreadCtx)
+{
+    testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)> mockCallback;
+    EXPECT_CALL(mockCallback, Call(testing::_, testing::_)).Times(100);
+    auto scanner = migration::cassandra::impl::FullTableScanner<TestScannerAdaper>(
+        {.ctxThreadsNum = 1, .jobsNum = 1, .cursorsPerJob = 100}, TestScannerAdaper(mockCallback)
+    );
+    scanner.wait();
+}
+
+TEST_F(FullTableScannerTests, MultipleThreadCtx)
+{
+    testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)> mockCallback;
+    EXPECT_CALL(mockCallback, Call(testing::_, testing::_)).Times(200);
+    auto scanner = migration::cassandra::impl::FullTableScanner<TestScannerAdaper>(
+        {.ctxThreadsNum = 2, .jobsNum = 2, .cursorsPerJob = 100}, TestScannerAdaper(mockCallback)
+    );
+    scanner.wait();
+}
+
+MATCHER(RangeMinMax, "Matches the range with min and max")
+{
+    return (arg.start == std::numeric_limits<std::int64_t>::min()) &&
+        (arg.end == std::numeric_limits<std::int64_t>::max());
+}
+TEST_F(FullTableScannerTests, RangeSizeIsOne)
+{
+    testing::MockFunction<void(migration::cassandra::impl::TokenRange const&, boost::asio::yield_context)> mockCallback;
+    EXPECT_CALL(mockCallback, Call(RangeMinMax(), testing::_)).Times(1);
+    auto scanner = migration::cassandra::impl::FullTableScanner<TestScannerAdaper>(
+        {.ctxThreadsNum = 2, .jobsNum = 1, .cursorsPerJob = 1}, TestScannerAdaper(mockCallback)
+    );
+    scanner.wait();
+}

tests/unit/migration/cassandra/SpecTests.cpp

@@ -0,0 +1,40 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "migration/cassandra/impl/Spec.hpp"
+
+#include <gtest/gtest.h>
+
+#include <cstdint>
+#include <tuple>
+
+namespace {
+class Empty {};
+
+struct SimpleTestTable {
+    using Row = std::tuple<std::uint32_t, std::uint32_t>;
+    static constexpr char const* PARTITION_KEY = "key";
+    static constexpr char const* TABLE_NAME = "test";
+};
+}  // namespace
+TEST(MigrationSpec, TableSpec)
+{
+    static_assert(!migration::cassandra::impl::TableSpec<Empty>);
+    static_assert(migration::cassandra::impl::TableSpec<SimpleTestTable>);
+}

tests/unit/util/ConceptsTests.cpp

@@ -0,0 +1,55 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#include "util/Concepts.hpp"
+
+#include <gtest/gtest.h>
+
+TEST(ConceptTests, SomeNumberType)
+{
+    static_assert(util::SomeNumberType<int>);
+    static_assert(!util::SomeNumberType<bool>);
+    static_assert(util::SomeNumberType<char>);
+    static_assert(!util::SomeNumberType<int const>);
+}
+
+TEST(ConceptTests, hasNoDuplicates)
+{
+    static_assert(util::hasNoDuplicates(1, 2, 3, 4, 5));
+    static_assert(!util::hasNoDuplicates(1, 2, 3, 4, 5, 5));
+}
+
+struct TestA {
+    static constexpr auto name = "TestA";
+};
+
+struct AnotherA {
+    static constexpr auto name = "TestA";
+};
+
+struct TestB {
+    static constexpr auto name = "TestB";
+};
+
+TEST(ConceptTests, hasNoDuplicateNames)
+{
+    static_assert(util::hasNoDuplicateNames<TestA, TestB>());
+    static_assert(!util::hasNoDuplicateNames<TestA, AnotherA, TestB>());
+    static_assert(!util::hasNoDuplicateNames<TestA, TestB, AnotherA>());
+}