ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2026-04-29 15:37:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Migration framework (#1768)

Browse Source 
This PR implemented the migration framework, which contains the command
line interface to execute migration and helps to migrate data easily.
Please read README.md for more information about this framework.
This commit is contained in:
cyan317
2024-12-17 14:50:51 +00:00
committed by GitHub
parent 15a441b084
commit 8dc7f16ef1
69 changed files with 4395 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
src/migration/impl/MigrationManagerBase.hpp Normal file
View File

@@ -0,0 +1,120 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022-2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "migration/MigrationManagerInterface.hpp"
#include "migration/MigratiorStatus.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <memory>
#include <string>
#include <tuple>
#include <vector>
namespace migration::impl {
/**
* @brief The migration manager implementation for Cassandra. It will run the migration for the Cassandra
* database.
*
* @tparam SupportedMigrators The migrators resgister that contains all the migrators
*/
template <typename SupportedMigrators>
class MigrationManagerBase : public MigrationManagerInterface {
SupportedMigrators migrators_;
// contains only migration related settings
util::config::ObjectView config_;
public:
/**
* @brief Construct a new Cassandra Migration Manager object
*
* @param backend The backend of the Cassandra database
* @param config The configuration of the migration
*/
explicit MigrationManagerBase(
std::shared_ptr<typename SupportedMigrators::BackendType> backend,
util::config::ObjectView const& config
)
: migrators_{backend}, config_{config}
{
}
/**
* @brief Run the the migration according to the given migrator's name
*
* @param name The name of the migrator
*/
void
runMigration(std::string const& name) override
{
migrators_.runMigrator(name, config_);
}
/**
* @brief Get the status of all the migrators
*
* @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
* migrator
*/
std::vector<std::tuple<std::string, MigratorStatus>>
allMigratorsStatusPairs() const override
{
return migrators_.getMigratorsStatus();
}
/**
* @brief Get the status of a migrator by its name
*
* @param name The name of the migrator
* @return The status of the migrator
*/
MigratorStatus
getMigratorStatusByName(std::string const& name) const override
{
return migrators_.getMigratorStatus(name);
}
/**
* @brief Get all registered migrators' names
*
* @return A vector of string, the names of all the migrators
*/
std::vector<std::string>
allMigratorsNames() const override
{
auto const names = migrators_.getMigratorNames();
return std::vector<std::string>{names.begin(), names.end()};
}
/**
* @brief Get the description of a migrator by its name
*
* @param name The name of the migrator
* @return The description of the migrator
*/
std::string
getMigratorDescriptionByName(std::string const& name) const override
{
return migrators_.getMigratorDescription(name);
}
};
} // namespace migration::impl

60
src/migration/impl/MigrationManagerFactory.cpp Normal file
View File

@@ -0,0 +1,60 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022-2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "migration/impl/MigrationManagerFactory.hpp"
#include "data/cassandra/SettingsProvider.hpp"
#include "migration/MigrationManagerInterface.hpp"
#include "migration/cassandra/CassandraMigrationBackend.hpp"
#include "migration/cassandra/CassandraMigrationManager.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include <boost/algorithm/string/predicate.hpp>
#include <memory>
#include <string>
#include <utility>
namespace migration::impl {
std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
makeMigrationManager(util::config::ClioConfigDefinition const& config)
{
static util::Logger const log{"Migration"};
LOG(log.info()) << "Constructing MigrationManager";
auto const type = config.get<std::string>("database.type");
if (not boost::iequals(type, "cassandra")) {
LOG(log.error()) << "Unknown database type to migrate: " << type;
return std::unexpected(std::string("Invalid database type"));
}
auto const cfg = config.getObject("database." + type);
auto migrationCfg = config.getObject("migration");
return std::make_shared<cassandra::CassandraMigrationManager>(
std::make_shared<cassandra::CassandraMigrationBackend>(data::cassandra::SettingsProvider{cfg}),
std::move(migrationCfg)
);
}
} // namespace migration::impl

41
src/migration/impl/MigrationManagerFactory.hpp Normal file
View File

@@ -0,0 +1,41 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022-2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "migration/MigrationManagerInterface.hpp"
#include "util/newconfig/ConfigDefinition.hpp"
#include <expected>
#include <memory>
#include <string>
namespace migration::impl {
/**
* @brief The factory to create a MigrationManagerInferface
*
* @param config The configuration of the migration application, it contains the database connection configuration and
* other migration specific configurations
* @return A shared pointer to the MigrationManagerInterface if the creation was successful, otherwise an error message
*/
std::expected<std::shared_ptr<MigrationManagerInterface>, std::string>
makeMigrationManager(util::config::ClioConfigDefinition const& config);
} // namespace migration::impl

184
src/migration/impl/MigratorsRegister.hpp Normal file
View File

@@ -0,0 +1,184 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022-2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "migration/MigratiorStatus.hpp"
#include "migration/impl/Spec.hpp"
#include "util/Concepts.hpp"
#include "util/log/Logger.hpp"
#include "util/newconfig/ObjectView.hpp"
#include <algorithm>
#include <array>
#include <iterator>
#include <memory>
#include <ranges>
#include <string>
#include <string_view>
#include <tuple>
#include <vector>
namespace migration::impl {
/**
* The concept to check if BackendType is the same as the migrator's required backend type
*/
template <typename BackendType, typename MigratorType>
concept MigrationBackend = requires { requires std::same_as<typename MigratorType::Backend, BackendType>; };
template <typename Backend, typename... MigratorType>
concept BackendMatchAllMigrators = (MigrationBackend<Backend, MigratorType> && ...);
/**
*@brief The register of migrators. It will dispatch the migration to the corresponding migrator. It also
*hold the shared pointer of backend, which is used by the migrators.
*
*@tparam Backend The backend type
*@tparam MigratorType The migrator types. It should be a concept of MigratorSpec and not have duplicate names.
*/
template <typename Backend, typename... MigratorType>
requires AllMigratorSpec<MigratorType...>
class MigratorsRegister {
static_assert(util::hasNoDuplicateNames<MigratorType...>());
util::Logger log_{"Migration"};
std::shared_ptr<Backend> backend_;
template <typename Migrator>
void
callMigration(std::string const& name, util::config::ObjectView const& config)
{
if (name == Migrator::name) {
LOG(log_.info()) << "Running migration: " << name;
Migrator::runMigration(backend_, config);
backend_->writeMigratorStatus(name, MigratorStatus(MigratorStatus::Migrated).toString());
LOG(log_.info()) << "Finished migration: " << name;
}
}
template <typename T>
static constexpr std::string_view
getDescriptionIfMatch(std::string_view targetName)
{
return (T::name == targetName) ? T::description : "";
}
public:
/**
* @brief The backend type which is used by the migrators
*/
using BackendType = Backend;
/**
* @brief Construct a new Migrators Register object
*
* @param backend The backend shared pointer
*/
MigratorsRegister(std::shared_ptr<BackendType> backend) : backend_{std::move(backend)}
{
}
/**
* @brief Run the migration according to the given migrator's name
*
* @param name The migrator's name
* @param config The configuration of the migration
*/
void
runMigrator(std::string const& name, util::config::ObjectView const& config)
requires BackendMatchAllMigrators<BackendType, MigratorType...>
{
(callMigration<MigratorType>(name, config), ...);
}
/**
* @brief Get the status of all the migrators
*
* @return A vector of tuple, the first element is the migrator's name, the second element is the status of the
* migrator
*/
std::vector<std::tuple<std::string, MigratorStatus>>
getMigratorsStatus() const
{
auto const fullList = getMigratorNames();
std::vector<std::tuple<std::string, MigratorStatus>> status;
std::ranges::transform(fullList, std::back_inserter(status), [&](auto const& migratorName) {
auto const migratorNameStr = std::string(migratorName);
return std::make_tuple(migratorNameStr, getMigratorStatus(migratorNameStr));
});
return status;
}
/**
* @brief Get the status of a migrator by its name
*
* @param name The migrator's name to get the status
* @return The status of the migrator
*/
MigratorStatus
getMigratorStatus(std::string const& name) const
{
auto const fullList = getMigratorNames();
if (std::ranges::find(fullList, name) == fullList.end()) {
return MigratorStatus::NotKnown;
}
auto const statusStringOpt =
data::synchronous([&](auto yield) { return backend_->fetchMigratorStatus(name, yield); });
return statusStringOpt ? MigratorStatus::fromString(statusStringOpt.value()) : MigratorStatus::NotMigrated;
}
/**
* @brief Get all registered migrators' names
*
* @return A array of migrator's names
*/
constexpr auto
getMigratorNames() const
{
return std::array<std::string_view, sizeof...(MigratorType)>{MigratorType::name...};
}
/**
* @brief Get the description of a migrator by its name
*
* @param name The migrator's name
* @return The description of the migrator
*/
std::string
getMigratorDescription(std::string const& name) const
{
if constexpr (sizeof...(MigratorType) == 0) {
return "No Description";
} else {
// Fold expression to search through all types
std::string result = ([](std::string const& name) {
return std::string(getDescriptionIfMatch<MigratorType>(name));
}(name) + ...);
return result.empty() ? "No Description" : result;
}
}
};
} // namespace migration::impl

56
src/migration/impl/Spec.hpp Normal file
View File

@@ -0,0 +1,56 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022-2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "util/newconfig/ObjectView.hpp"
#include <boost/asio/spawn.hpp>
#include <memory>
#include <string>
namespace migration::impl {
/**
* @brief The migrator specification concept
*/
template <typename T, typename Backend>
concept MigratorSpec = requires(std::shared_ptr<Backend> const& backend, util::config::ObjectView const& cfg) {
// Check that 'name' exists and is a string
{ T::name } -> std::convertible_to<std::string>;
// Check that 'description' exists and is a string
{ T::description } -> std::convertible_to<std::string>;
// Check that the migrator specifies the backend type it supports
typename T::Backend;
// Check that 'runMigration' exists and is callable
{ T::runMigration(backend, cfg) } -> std::same_as<void>;
};
/**
* @brief used by variadic template to check all migrators are MigratorSpec
*/
template <typename... Types>
concept AllMigratorSpec = (MigratorSpec<Types, typename Types::Backend> && ...);
} // namespace migration::impl