ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 11:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor namespaces part 1 (#817)

Browse Source 
Part 1 of refactoring effort
This commit is contained in:
Peter Chen
2023-08-10 13:05:13 -04:00
committed by GitHub
parent db4046e02a
commit 23442ff1a7
195 changed files with 964 additions and 888 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
CMakeLists.txt
View File

@@ -63,16 +63,16 @@ target_sources(clio PRIVATE
## Main
src/main/impl/Build.cpp
## Backend
src/backend/BackendInterface.cpp
src/backend/LedgerCache.cpp
src/backend/cassandra/impl/Future.cpp
src/backend/cassandra/impl/Cluster.cpp
src/backend/cassandra/impl/Batch.cpp
src/backend/cassandra/impl/Result.cpp
src/backend/cassandra/impl/Tuple.cpp
src/backend/cassandra/impl/SslContext.cpp
src/backend/cassandra/Handle.cpp
src/backend/cassandra/SettingsProvider.cpp
src/data/BackendInterface.cpp
src/data/LedgerCache.cpp
src/data/cassandra/impl/Future.cpp
src/data/cassandra/impl/Cluster.cpp
src/data/cassandra/impl/Batch.cpp
src/data/cassandra/impl/Result.cpp
src/data/cassandra/impl/Tuple.cpp
src/data/cassandra/impl/SslContext.cpp
src/data/cassandra/Handle.cpp
src/data/cassandra/SettingsProvider.cpp
## ETL
src/etl/Source.cpp
src/etl/ProbingSource.cpp
@@ -120,8 +120,8 @@ target_sources(clio PRIVATE
src/rpc/handlers/TransactionEntry.cpp
src/rpc/handlers/Tx.cpp
## Util
src/config/Config.cpp
src/log/Logger.cpp
src/util/config/Config.cpp
src/util/log/Logger.cpp
src/util/Taggable.cpp)
# Clio server
@@ -135,75 +135,75 @@ if(tests)
# Common
unittests/Main.cpp
unittests/Playground.cpp
unittests/Logger.cpp
unittests/Config.cpp
unittests/ProfilerTest.cpp
unittests/JsonUtilTest.cpp
unittests/DOSGuard.cpp
unittests/SubscriptionTest.cpp
unittests/SubscriptionManagerTest.cpp
unittests/LoggerTests.cpp
unittests/ConfigTests.cpp
unittests/ProfilerTests.cpp
unittests/JsonUtilTests.cpp
unittests/DOSGuardTests.cpp
unittests/SubscriptionTests.cpp
unittests/SubscriptionManagerTests.cpp
unittests/util/TestObject.cpp
unittests/util/StringUtils.cpp
# ETL
unittests/etl/ExtractionDataPipeTest.cpp
unittests/etl/ExtractorTest.cpp
unittests/etl/TransformerTest.cpp
unittests/etl/CacheLoaderTest.cpp
unittests/etl/ExtractionDataPipeTests.cpp
unittests/etl/ExtractorTests.cpp
unittests/etl/TransformerTests.cpp
unittests/etl/CacheLoaderTests.cpp
# RPC
unittests/rpc/ErrorTests.cpp
unittests/rpc/BaseTests.cpp
unittests/rpc/RPCHelpersTest.cpp
unittests/rpc/CountersTest.cpp
unittests/rpc/AdminVerificationTest.cpp
unittests/rpc/RPCHelpersTests.cpp
unittests/rpc/CountersTests.cpp
unittests/rpc/AdminVerificationTests.cpp
unittests/rpc/APIVersionTests.cpp
unittests/rpc/ForwardingProxyTests.cpp
unittests/rpc/WorkQueueTest.cpp
unittests/rpc/AmendmentsTest.cpp
unittests/rpc/WorkQueueTests.cpp
unittests/rpc/AmendmentsTests.cpp
## RPC handlers
unittests/rpc/handlers/DefaultProcessorTests.cpp
unittests/rpc/handlers/TestHandlerTests.cpp
unittests/rpc/handlers/AccountCurrenciesTest.cpp
unittests/rpc/handlers/AccountLinesTest.cpp
unittests/rpc/handlers/AccountTxTest.cpp
unittests/rpc/handlers/AccountOffersTest.cpp
unittests/rpc/handlers/AccountInfoTest.cpp
unittests/rpc/handlers/AccountChannelsTest.cpp
unittests/rpc/handlers/AccountNFTsTest.cpp
unittests/rpc/handlers/BookOffersTest.cpp
unittests/rpc/handlers/DepositAuthorizedTest.cpp
unittests/rpc/handlers/GatewayBalancesTest.cpp
unittests/rpc/handlers/TxTest.cpp
unittests/rpc/handlers/TransactionEntryTest.cpp
unittests/rpc/handlers/LedgerEntryTest.cpp
unittests/rpc/handlers/LedgerRangeTest.cpp
unittests/rpc/handlers/NoRippleCheckTest.cpp
unittests/rpc/handlers/ServerInfoTest.cpp
unittests/rpc/handlers/PingTest.cpp
unittests/rpc/handlers/RandomTest.cpp
unittests/rpc/handlers/NFTInfoTest.cpp
unittests/rpc/handlers/NFTBuyOffersTest.cpp
unittests/rpc/handlers/NFTSellOffersTest.cpp
unittests/rpc/handlers/NFTHistoryTest.cpp
unittests/rpc/handlers/SubscribeTest.cpp
unittests/rpc/handlers/UnsubscribeTest.cpp
unittests/rpc/handlers/LedgerDataTest.cpp
unittests/rpc/handlers/AccountObjectsTest.cpp
unittests/rpc/handlers/BookChangesTest.cpp
unittests/rpc/handlers/LedgerTest.cpp
unittests/rpc/handlers/VersionHandlerTest.cpp
unittests/rpc/handlers/AccountCurrenciesTests.cpp
unittests/rpc/handlers/AccountLinesTests.cpp
unittests/rpc/handlers/AccountTxTests.cpp
unittests/rpc/handlers/AccountOffersTests.cpp
unittests/rpc/handlers/AccountInfoTests.cpp
unittests/rpc/handlers/AccountChannelsTests.cpp
unittests/rpc/handlers/AccountNFTsTests.cpp
unittests/rpc/handlers/BookOffersTests.cpp
unittests/rpc/handlers/DepositAuthorizedTests.cpp
unittests/rpc/handlers/GatewayBalancesTests.cpp
unittests/rpc/handlers/TxTests.cpp
unittests/rpc/handlers/TransactionEntryTests.cpp
unittests/rpc/handlers/LedgerEntryTests.cpp
unittests/rpc/handlers/LedgerRangeTests.cpp
unittests/rpc/handlers/NoRippleCheckTests.cpp
unittests/rpc/handlers/ServerInfoTests.cpp
unittests/rpc/handlers/PingTests.cpp
unittests/rpc/handlers/RandomTests.cpp
unittests/rpc/handlers/NFTInfoTests.cpp
unittests/rpc/handlers/NFTBuyOffersTests.cpp
unittests/rpc/handlers/NFTSellOffersTests.cpp
unittests/rpc/handlers/NFTHistoryTests.cpp
unittests/rpc/handlers/SubscribeTests.cpp
unittests/rpc/handlers/UnsubscribeTests.cpp
unittests/rpc/handlers/LedgerDataTests.cpp
unittests/rpc/handlers/AccountObjectsTests.cpp
unittests/rpc/handlers/BookChangesTests.cpp
unittests/rpc/handlers/LedgerTests.cpp
unittests/rpc/handlers/VersionHandlerTests.cpp
# Backend
unittests/backend/BackendFactoryTest.cpp
unittests/backend/cassandra/BaseTests.cpp
unittests/backend/cassandra/BackendTests.cpp
unittests/backend/cassandra/RetryPolicyTests.cpp
unittests/backend/cassandra/SettingsProviderTests.cpp
unittests/backend/cassandra/ExecutionStrategyTests.cpp
unittests/backend/cassandra/AsyncExecutorTests.cpp
unittests/data/BackendFactoryTests.cpp
unittests/data/cassandra/BaseTests.cpp
unittests/data/cassandra/BackendTests.cpp
unittests/data/cassandra/RetryPolicyTests.cpp
unittests/data/cassandra/SettingsProviderTests.cpp
unittests/data/cassandra/ExecutionStrategyTests.cpp
unittests/data/cassandra/AsyncExecutorTests.cpp
# Webserver
unittests/webserver/ServerTest.cpp
unittests/webserver/RPCServerHandlerTest.cpp
unittests/webserver/WhitelistHandlerTest.cpp
unittests/webserver/SweepHandlerTest.cpp)
unittests/webserver/ServerTests.cpp
unittests/webserver/RPCServerHandlerTests.cpp
unittests/webserver/WhitelistHandlerTests.cpp
unittests/webserver/SweepHandlerTests.cpp)
include(CMake/deps/gtest.cmake)

19
src/backend/BackendFactory.h → src/data/BackendFactory.h
View File

@@ -19,18 +19,18 @@
#pragma once
#include <backend/BackendInterface.h>
#include <backend/CassandraBackend.h>
#include <config/Config.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <data/CassandraBackend.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <boost/algorithm/string.hpp>
namespace Backend {
namespace data {
std::shared_ptr<BackendInterface>
make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
make_Backend(boost::asio::io_context& ioc, util::Config const& config)
{
static clio::Logger log{"Backend"};
static util::Logger log{"Backend"};
log.info() << "Constructing BackendInterface";
auto const readOnly = config.valueOr("read_only", false);
@@ -42,8 +42,7 @@ make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
if (boost::iequals(type, "cassandra") or boost::iequals(type, "cassandra-new"))
{
auto cfg = config.section("database." + type);
backend =
std::make_shared<Backend::Cassandra::CassandraBackend>(Backend::Cassandra::SettingsProvider{cfg}, readOnly);
backend = std::make_shared<data::cassandra::CassandraBackend>(data::cassandra::SettingsProvider{cfg}, readOnly);
}
if (!backend)
@@ -59,4 +58,4 @@ make_Backend(boost::asio::io_context& ioc, clio::Config const& config)
log.info() << "Constructed BackendInterface Successfully";
return backend;
}
} // namespace Backend
} // namespace data

12
src/backend/BackendInterface.cpp → src/data/BackendInterface.cpp
View File

@@ -17,20 +17,18 @@
*/
//==============================================================================
#include <backend/BackendInterface.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <util/log/Logger.h>
#include <ripple/protocol/Indexes.h>
#include <ripple/protocol/STLedgerEntry.h>
using namespace clio;
// local to compilation unit loggers
namespace {
clio::Logger gLog{"Backend"};
util::Logger gLog{"Backend"};
} // namespace
namespace Backend {
namespace data {
bool
BackendInterface::finishWrites(std::uint32_t const ledgerSequence)
{
@@ -315,4 +313,4 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
return fees;
}
} // namespace Backend
} // namespace data

18
src/backend/BackendInterface.h → src/data/BackendInterface.h
View File

@@ -19,11 +19,11 @@
#pragma once
#include <backend/DBHelpers.h>
#include <backend/LedgerCache.h>
#include <backend/Types.h>
#include <config/Config.h>
#include <log/Logger.h>
#include <data/DBHelpers.h>
#include <data/LedgerCache.h>
#include <data/Types.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <ripple/protocol/Fees.h>
#include <ripple/protocol/LedgerHeader.h>
@@ -33,7 +33,7 @@
#include <thread>
#include <type_traits>
namespace Backend {
namespace data {
/**
* @brief Throws an error when database read time limit is exceeded.
@@ -63,7 +63,7 @@ template <class F>
auto
retryOnTimeout(F func, size_t waitMs = 500)
{
static clio::Logger log{"Backend"};
static util::Logger log{"Backend"};
while (true)
{
@@ -548,5 +548,5 @@ private:
doFinishWrites() = 0;
};
} // namespace Backend
using BackendInterface = Backend::BackendInterface;
} // namespace data
using BackendInterface = data::BackendInterface;

20
src/backend/CassandraBackend.h → src/data/CassandraBackend.h
View File

@@ -19,21 +19,21 @@
#pragma once
#include <backend/BackendInterface.h>
#include <backend/cassandra/Concepts.h>
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/Schema.h>
#include <backend/cassandra/SettingsProvider.h>
#include <backend/cassandra/impl/ExecutionStrategy.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <data/cassandra/Concepts.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/Schema.h>
#include <data/cassandra/SettingsProvider.h>
#include <data/cassandra/impl/ExecutionStrategy.h>
#include <util/LedgerUtils.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/protocol/LedgerHeader.h>
#include <ripple/protocol/nft.h>
#include <boost/asio/spawn.hpp>
namespace Backend::Cassandra {
namespace data::cassandra {
/**
* @brief Implements @ref BackendInterface for Cassandra/Scylladb
@@ -47,7 +47,7 @@ namespace Backend::Cassandra {
template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategy>
class BasicCassandraBackend : public BackendInterface
{
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
SettingsProviderType settingsProvider_;
Schema<SettingsProviderType> schema_;
@@ -820,4 +820,4 @@ private:
using CassandraBackend = BasicCassandraBackend<SettingsProvider, detail::DefaultExecutionStrategy<>>;
} // namespace Backend::Cassandra
} // namespace data::cassandra

2
src/backend/DBHelpers.h → src/data/DBHelpers.h
View File

@@ -27,7 +27,7 @@
#include <boost/container/flat_set.hpp>
#include <backend/Types.h>
#include <data/Types.h>
/**
* @brief Struct used to keep track of what to write to account_transactions/account_tx tables

6
src/backend/LedgerCache.cpp → src/data/LedgerCache.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include <backend/LedgerCache.h>
#include <data/LedgerCache.h>
namespace Backend {
namespace data {
uint32_t
LedgerCache::latestLedgerSequence() const
@@ -157,4 +157,4 @@ LedgerCache::getSuccessorHitRate() const
return ((float)successorHitCounter_) / successorReqCounter_;
}
} // namespace Backend
} // namespace data

6
src/backend/LedgerCache.h → src/data/LedgerCache.h
View File

@@ -21,14 +21,14 @@
#include <ripple/basics/base_uint.h>
#include <ripple/basics/hardened_hash.h>
#include <backend/Types.h>
#include <data/Types.h>
#include <map>
#include <mutex>
#include <shared_mutex>
#include <utility>
#include <vector>
namespace Backend {
namespace data {
class LedgerCache
{
@@ -95,4 +95,4 @@ public:
getSuccessorHitRate() const;
};
} // namespace Backend
} // namespace data

0
src/backend/README.md → src/data/README.md
View File

4
src/backend/Types.h → src/data/Types.h
View File

@@ -25,7 +25,7 @@
#include <string>
#include <vector>
namespace Backend {
namespace data {
// *** return types
@@ -161,4 +161,4 @@ struct LedgerRange
constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"};
} // namespace Backend
} // namespace data

6
src/backend/cassandra/Concepts.h → src/data/cassandra/Concepts.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/cassandra/Types.h>
#include <data/cassandra/Types.h>
#include <boost/asio/spawn.hpp>
@@ -28,7 +28,7 @@
#include <optional>
#include <string>
namespace Backend::Cassandra {
namespace data::cassandra {
// clang-format off
template <typename T>
@@ -76,4 +76,4 @@ concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraEr
};
// clang-format on
} // namespace Backend::Cassandra
} // namespace data::cassandra

4
src/backend/cassandra/Error.h → src/data/cassandra/Error.h
View File

@@ -23,7 +23,7 @@
#include <string>
namespace Backend::Cassandra {
namespace data::cassandra {
/**
* @brief A simple container for both error message and error code
@@ -96,4 +96,4 @@ public:
}
};
} // namespace Backend::Cassandra
} // namespace data::cassandra

6
src/backend/cassandra/Handle.cpp → src/data/cassandra/Handle.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include <backend/cassandra/Handle.h>
#include <data/cassandra/Handle.h>
namespace Backend::Cassandra {
namespace data::cassandra {
Handle::Handle(Settings clusterSettings) : cluster_{clusterSettings}
{
@@ -152,4 +152,4 @@ Handle::prepare(std::string_view query) const
throw std::runtime_error(rc.error().message());
}
} // namespace Backend::Cassandra
} // namespace data::cassandra

22
src/backend/cassandra/Handle.h → src/data/cassandra/Handle.h
View File

@@ -19,15 +19,15 @@
#pragma once
#include <backend/cassandra/Error.h>
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/Batch.h>
#include <backend/cassandra/impl/Cluster.h>
#include <backend/cassandra/impl/Future.h>
#include <backend/cassandra/impl/ManagedObject.h>
#include <backend/cassandra/impl/Result.h>
#include <backend/cassandra/impl/Session.h>
#include <backend/cassandra/impl/Statement.h>
#include <data/cassandra/Error.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/Batch.h>
#include <data/cassandra/impl/Cluster.h>
#include <data/cassandra/impl/Future.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/Result.h>
#include <data/cassandra/impl/Session.h>
#include <data/cassandra/impl/Statement.h>
#include <util/Expected.h>
#include <cassandra.h>
@@ -37,7 +37,7 @@
#include <iterator>
#include <vector>
namespace Backend::Cassandra {
namespace data::cassandra {
/**
* @brief Represents a handle to the cassandra database cluster
@@ -292,4 +292,4 @@ extract(Handle::ResultType const& result)
return {result};
}
} // namespace Backend::Cassandra
} // namespace data::cassandra

18
src/backend/cassandra/Schema.h → src/data/cassandra/Schema.h
View File

@@ -19,17 +19,17 @@
#pragma once
#include <backend/cassandra/Concepts.h>
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/SettingsProvider.h>
#include <backend/cassandra/Types.h>
#include <config/Config.h>
#include <log/Logger.h>
#include <data/cassandra/Concepts.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/SettingsProvider.h>
#include <data/cassandra/Types.h>
#include <util/Expected.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <fmt/compile.h>
namespace Backend::Cassandra {
namespace data::cassandra {
template <SomeSettingsProvider SettingsProviderType>
[[nodiscard]] std::string inline qualifiedTableName(SettingsProviderType const& provider, std::string_view name)
@@ -48,7 +48,7 @@ class Schema
// Migrations will be ran automatically based on this value.
static constexpr uint16_t version = 1u;
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
std::reference_wrapper<SettingsProviderType const> settingsProvider_;
public:
@@ -664,4 +664,4 @@ private:
std::unique_ptr<Statements> statements_{nullptr};
};
} // namespace Backend::Cassandra
} // namespace data::cassandra

16
src/backend/cassandra/SettingsProvider.cpp → src/data/cassandra/SettingsProvider.cpp
View File

@@ -17,10 +17,10 @@
*/
//==============================================================================
#include <backend/cassandra/SettingsProvider.h>
#include <backend/cassandra/impl/Cluster.h>
#include <backend/cassandra/impl/Statement.h>
#include <config/Config.h>
#include <data/cassandra/SettingsProvider.h>
#include <data/cassandra/impl/Cluster.h>
#include <data/cassandra/impl/Statement.h>
#include <util/config/Config.h>
#include <boost/json.hpp>
@@ -28,7 +28,7 @@
#include <string>
#include <thread>
namespace Backend::Cassandra {
namespace data::cassandra {
namespace detail {
inline Settings::ContactPoints
@@ -39,7 +39,7 @@ tag_invoke(boost::json::value_to_tag<Settings::ContactPoints>, boost::json::valu
"Feed entire Cassandra section to parse "
"Settings::ContactPoints instead");
clio::Config obj{value};
util::Config obj{value};
Settings::ContactPoints out;
out.contactPoints = obj.valueOrThrow<std::string>("contact_points", "`contact_points` must be a string");
@@ -57,7 +57,7 @@ tag_invoke(boost::json::value_to_tag<Settings::SecureConnectionBundle>, boost::j
}
} // namespace detail
SettingsProvider::SettingsProvider(clio::Config const& cfg, uint16_t ttl)
SettingsProvider::SettingsProvider(util::Config const& cfg, uint16_t ttl)
: config_{cfg}
, keyspace_{cfg.valueOr<std::string>("keyspace", "clio")}
, tablePrefix_{cfg.maybeValue<std::string>("table_prefix")}
@@ -148,4 +148,4 @@ SettingsProvider::parseSettings() const
return settings;
}
} // namespace Backend::Cassandra
} // namespace data::cassandra

16
src/backend/cassandra/SettingsProvider.h → src/data/cassandra/SettingsProvider.h
View File

@@ -19,20 +19,20 @@
#pragma once
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/Types.h>
#include <config/Config.h>
#include <log/Logger.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/Types.h>
#include <util/Expected.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
namespace Backend::Cassandra {
namespace data::cassandra {
/**
* @brief Provides settings for @ref CassandraBackend
*/
class SettingsProvider
{
clio::Config config_;
util::Config config_;
std::string keyspace_;
std::optional<std::string> tablePrefix_;
@@ -41,7 +41,7 @@ class SettingsProvider
Settings settings_;
public:
explicit SettingsProvider(clio::Config const& cfg, uint16_t ttl = 0);
explicit SettingsProvider(util::Config const& cfg, uint16_t ttl = 0);
/*! Get the cluster settings */
[[nodiscard]] Settings
@@ -83,4 +83,4 @@ private:
parseSettings() const;
};
} // namespace Backend::Cassandra
} // namespace data::cassandra

4
src/backend/cassandra/Types.h → src/data/cassandra/Types.h
View File

@@ -23,7 +23,7 @@
#include <string>
namespace Backend::Cassandra {
namespace data::cassandra {
namespace detail {
struct Settings;
@@ -64,4 +64,4 @@ using MaybeError = util::Expected<void, CassandraError>;
using ResultOrError = util::Expected<Result, CassandraError>;
using Error = util::Unexpected<CassandraError>;
} // namespace Backend::Cassandra
} // namespace data::cassandra

16
src/backend/cassandra/impl/AsyncExecutor.h → src/data/cassandra/impl/AsyncExecutor.h
View File

@@ -19,19 +19,19 @@
#pragma once
#include <backend/cassandra/Concepts.h>
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/RetryPolicy.h>
#include <log/Logger.h>
#include <data/cassandra/Concepts.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/RetryPolicy.h>
#include <util/Expected.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
#include <functional>
#include <memory>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/**
* @brief A query executor with a changable retry policy
@@ -53,7 +53,7 @@ class AsyncExecutor : public std::enable_shared_from_this<AsyncExecutor<Statemen
using FutureWithCallbackType = typename HandleType::FutureWithCallbackType;
using CallbackType = std::function<void(typename HandleType::ResultOrErrorType)>;
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
StatementType data_;
RetryPolicyType retryPolicy_;
@@ -116,4 +116,4 @@ private:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

10
src/backend/cassandra/impl/Batch.cpp → src/data/cassandra/impl/Batch.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include <backend/cassandra/Error.h>
#include <backend/cassandra/impl/Batch.h>
#include <backend/cassandra/impl/Statement.h>
#include <data/cassandra/Error.h>
#include <data/cassandra/impl/Batch.h>
#include <data/cassandra/impl/Statement.h>
#include <util/Expected.h>
#include <exception>
@@ -29,7 +29,7 @@ namespace {
static constexpr auto batchDeleter = [](CassBatch* ptr) { cass_batch_free(ptr); };
};
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
// todo: use an appropritae value instead of CASS_BATCH_TYPE_LOGGED for
// different use cases
@@ -53,4 +53,4 @@ Batch::add(Statement const& statement)
return {};
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

8
src/backend/cassandra/impl/Batch.h → src/data/cassandra/impl/Batch.h
View File

@@ -19,12 +19,12 @@
#pragma once
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/ManagedObject.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <cassandra.h>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
struct Batch : public ManagedObject<CassBatch>
{
@@ -34,4 +34,4 @@ struct Batch : public ManagedObject<CassBatch>
add(Statement const& statement);
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

10
src/backend/cassandra/impl/Cluster.cpp → src/data/cassandra/impl/Cluster.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include <backend/cassandra/impl/Cluster.h>
#include <backend/cassandra/impl/SslContext.h>
#include <backend/cassandra/impl/Statement.h>
#include <data/cassandra/impl/Cluster.h>
#include <data/cassandra/impl/SslContext.h>
#include <data/cassandra/impl/Statement.h>
#include <util/Expected.h>
#include <fmt/core.h>
@@ -41,7 +41,7 @@ template <class... Ts>
overloadSet(Ts...) -> overloadSet<Ts...>;
}; // namespace
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
Cluster::Cluster(Settings const& settings) : ManagedObject{cass_cluster_new(), clusterDeleter}
{
@@ -221,4 +221,4 @@ Cluster::setupCredentials(Settings const& settings)
cass_cluster_set_credentials(*this, settings.username.value().c_str(), settings.password.value().c_str());
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

10
src/backend/cassandra/impl/Cluster.h → src/data/cassandra/impl/Cluster.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <backend/cassandra/impl/ManagedObject.h>
#include <log/Logger.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <util/log/Logger.h>
#include <cassandra.h>
@@ -31,7 +31,7 @@
#include <thread>
#include <variant>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
struct Settings
{
@@ -86,7 +86,7 @@ struct Settings
class Cluster : public ManagedObject<CassCluster>
{
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
public:
Cluster(Settings const& settings);
@@ -108,4 +108,4 @@ private:
setupCredentials(Settings const& settings);
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

14
src/backend/cassandra/impl/ExecutionStrategy.h → src/data/cassandra/impl/ExecutionStrategy.h
View File

@@ -19,11 +19,11 @@
#pragma once
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/AsyncExecutor.h>
#include <log/Logger.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/AsyncExecutor.h>
#include <util/Expected.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
#include <boost/asio/spawn.hpp>
@@ -36,7 +36,7 @@
#include <optional>
#include <thread>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/**
* @brief Implements async and sync querying against the cassandra DB with
@@ -48,7 +48,7 @@ namespace Backend::Cassandra::detail {
template <typename HandleType = Handle>
class DefaultExecutionStrategy
{
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
std::uint32_t maxWriteRequestsOutstanding_;
std::atomic_uint32_t numWriteRequestsOutstanding_ = 0;
@@ -448,4 +448,4 @@ private:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

10
src/backend/cassandra/impl/Future.cpp → src/data/cassandra/impl/Future.cpp
View File

@@ -17,9 +17,9 @@
*/
//==============================================================================
#include <backend/cassandra/Error.h>
#include <backend/cassandra/impl/Future.h>
#include <backend/cassandra/impl/Result.h>
#include <data/cassandra/Error.h>
#include <data/cassandra/impl/Future.h>
#include <data/cassandra/impl/Result.h>
#include <exception>
#include <vector>
@@ -28,7 +28,7 @@ namespace {
static constexpr auto futureDeleter = [](CassFuture* ptr) { cass_future_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/* implicit */ Future::Future(CassFuture* ptr) : ManagedObject{ptr, futureDeleter}
{
@@ -100,4 +100,4 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
cass_future_set_callback(*this, &invokeHelper, cb_.get());
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

8
src/backend/cassandra/impl/Future.h → src/data/cassandra/impl/Future.h
View File

@@ -19,12 +19,12 @@
#pragma once
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/ManagedObject.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <cassandra.h>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
struct Future : public ManagedObject<CassFuture>
{
@@ -55,4 +55,4 @@ private:
fn_ptr_t cb_;
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

4
src/backend/cassandra/impl/ManagedObject.h → src/data/cassandra/impl/ManagedObject.h
View File

@@ -21,7 +21,7 @@
#include <memory>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
template <typename Managed>
class ManagedObject
@@ -44,4 +44,4 @@ public:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/Result.cpp → src/data/cassandra/impl/Result.cpp
View File

@@ -17,14 +17,14 @@
*/
//==============================================================================
#include <backend/cassandra/impl/Result.h>
#include <data/cassandra/impl/Result.h>
namespace {
static constexpr auto resultDeleter = [](CassResult const* ptr) { cass_result_free(ptr); };
static constexpr auto resultIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/* implicit */ Result::Result(CassResult const* ptr) : ManagedObject{ptr, resultDeleter}
{
@@ -66,4 +66,4 @@ ResultIterator::hasMore() const
return hasMore_;
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

8
src/backend/cassandra/impl/Result.h → src/data/cassandra/impl/Result.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <backend/cassandra/impl/ManagedObject.h>
#include <backend/cassandra/impl/Tuple.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/Tuple.h>
#include <util/Expected.h>
#include <ripple/basics/base_uint.h>
@@ -31,7 +31,7 @@
#include <iterator>
#include <tuple>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
template <typename>
static constexpr bool unsupported_v = false;
@@ -254,4 +254,4 @@ public:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

12
src/backend/cassandra/impl/RetryPolicy.h → src/data/cassandra/impl/RetryPolicy.h
View File

@@ -19,10 +19,10 @@
#pragma once
#include <backend/cassandra/Handle.h>
#include <backend/cassandra/Types.h>
#include <log/Logger.h>
#include <data/cassandra/Handle.h>
#include <data/cassandra/Types.h>
#include <util/Expected.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
@@ -30,14 +30,14 @@
#include <chrono>
#include <cmath>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/**
* @brief A retry policy that employs exponential backoff
*/
class ExponentialBackoffRetryPolicy
{
clio::Logger log_{"Backend"};
util::Logger log_{"Backend"};
boost::asio::steady_timer timer_;
uint32_t attempt_ = 0u;
@@ -91,4 +91,4 @@ public:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/Session.h → src/data/cassandra/impl/Session.h
View File

@@ -19,11 +19,11 @@
#pragma once
#include <backend/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <cassandra.h>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
class Session : public ManagedObject<CassSession>
{
@@ -35,4 +35,4 @@ public:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/SslContext.cpp → src/data/cassandra/impl/SslContext.cpp
View File

@@ -17,13 +17,13 @@
*/
//==============================================================================
#include <backend/cassandra/impl/SslContext.h>
#include <data/cassandra/impl/SslContext.h>
namespace {
static constexpr auto contextDeleter = [](CassSsl* ptr) { cass_ssl_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_new(), contextDeleter}
{
@@ -34,4 +34,4 @@ SslContext::SslContext(std::string const& certificate) : ManagedObject{cass_ssl_
}
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/SslContext.h → src/data/cassandra/impl/SslContext.h
View File

@@ -19,17 +19,17 @@
#pragma once
#include <backend/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <cassandra.h>
#include <string>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
struct SslContext : public ManagedObject<CassSsl>
{
explicit SslContext(std::string const& certificate);
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

10
src/backend/cassandra/impl/Statement.h → src/data/cassandra/impl/Statement.h
View File

@@ -19,9 +19,9 @@
#pragma once
#include <backend/cassandra/Types.h>
#include <backend/cassandra/impl/ManagedObject.h>
#include <backend/cassandra/impl/Tuple.h>
#include <data/cassandra/Types.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/Tuple.h>
#include <util/Expected.h>
#include <ripple/basics/base_uint.h>
@@ -33,7 +33,7 @@
#include <compare>
#include <iterator>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
class Statement : public ManagedObject<CassStatement>
{
@@ -161,4 +161,4 @@ public:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/Tuple.cpp → src/data/cassandra/impl/Tuple.cpp
View File

@@ -17,14 +17,14 @@
*/
//==============================================================================
#include <backend/cassandra/impl/Tuple.h>
#include <data/cassandra/impl/Tuple.h>
namespace {
static constexpr auto tupleDeleter = [](CassTuple* ptr) { cass_tuple_free(ptr); };
static constexpr auto tupleIteratorDeleter = [](CassIterator* ptr) { cass_iterator_free(ptr); };
} // namespace
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
/* implicit */ Tuple::Tuple(CassTuple* ptr) : ManagedObject{ptr, tupleDeleter}
{
@@ -40,4 +40,4 @@ TupleIterator::fromTuple(CassValue const* value)
return {cass_iterator_from_tuple(value)};
}
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

6
src/backend/cassandra/impl/Tuple.h → src/data/cassandra/impl/Tuple.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/cassandra/impl/ManagedObject.h>
#include <data/cassandra/impl/ManagedObject.h>
#include <cassandra.h>
@@ -28,7 +28,7 @@
#include <string_view>
#include <tuple>
namespace Backend::Cassandra::detail {
namespace data::cassandra::detail {
class Tuple : public ManagedObject<CassTuple>
{
@@ -146,4 +146,4 @@ private:
}
};
} // namespace Backend::Cassandra::detail
} // namespace data::cassandra::detail

4
src/etl/ETLService.cpp
View File

@@ -21,8 +21,6 @@
#include <ripple/protocol/LedgerHeader.h>
using namespace clio;
// Database must be populated when this starts
std::optional<uint32_t>
ETLService::runETLPipeline(uint32_t startSequence, uint32_t numExtractors)
@@ -247,7 +245,7 @@ ETLService::doWork()
}
ETLService::ETLService(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,

26
src/etl/ETLService.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <backend/BackendInterface.h>
#include <backend/LedgerCache.h>
#include <data/BackendInterface.h>
#include <data/LedgerCache.h>
#include <etl/LoadBalancer.h>
#include <etl/Source.h>
#include <etl/SystemState.h>
@@ -31,8 +31,8 @@
#include <etl/impl/LedgerLoader.h>
#include <etl/impl/LedgerPublisher.h>
#include <etl/impl/Transformer.h>
#include <log/Logger.h>
#include <subscriptions/SubscriptionManager.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <grpcpp/grpcpp.h>
@@ -63,15 +63,15 @@ class ETLService
using SubscriptionManagerType = SubscriptionManager;
using LoadBalancerType = LoadBalancer;
using NetworkValidatedLedgersType = NetworkValidatedLedgers;
using DataPipeType = clio::detail::ExtractionDataPipe<org::xrpl::rpc::v1::GetLedgerResponse>;
using CacheLoaderType = clio::detail::CacheLoader<Backend::LedgerCache>;
using LedgerFetcherType = clio::detail::LedgerFetcher<LoadBalancerType>;
using ExtractorType = clio::detail::Extractor<DataPipeType, NetworkValidatedLedgersType, LedgerFetcherType>;
using LedgerLoaderType = clio::detail::LedgerLoader<LoadBalancerType, LedgerFetcherType>;
using LedgerPublisherType = clio::detail::LedgerPublisher<SubscriptionManagerType>;
using TransformerType = clio::detail::Transformer<DataPipeType, LedgerLoaderType, LedgerPublisherType>;
using DataPipeType = etl::detail::ExtractionDataPipe<org::xrpl::rpc::v1::GetLedgerResponse>;
using CacheLoaderType = etl::detail::CacheLoader<data::LedgerCache>;
using LedgerFetcherType = etl::detail::LedgerFetcher<LoadBalancerType>;
using ExtractorType = etl::detail::Extractor<DataPipeType, NetworkValidatedLedgersType, LedgerFetcherType>;
using LedgerLoaderType = etl::detail::LedgerLoader<LoadBalancerType, LedgerFetcherType>;
using LedgerPublisherType = etl::detail::LedgerPublisher<SubscriptionManagerType>;
using TransformerType = etl::detail::Transformer<DataPipeType, LedgerLoaderType, LedgerPublisherType>;
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
@@ -104,7 +104,7 @@ public:
* @param ledgers The network validated ledgers datastructure
*/
ETLService(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,
@@ -113,7 +113,7 @@ public:
static std::shared_ptr<ETLService>
make_ETLService(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManagerType> subscriptions,

12
src/etl/LoadBalancer.cpp
View File

@@ -17,14 +17,14 @@
*/
//==============================================================================
#include <backend/DBHelpers.h>
#include <data/DBHelpers.h>
#include <etl/ETLService.h>
#include <etl/NFTHelpers.h>
#include <etl/ProbingSource.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <rpc/RPCHelpers.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/beast/net/IPEndpoint.h>
#include <ripple/protocol/STLedgerEntry.h>
@@ -36,11 +36,11 @@
#include <thread>
using namespace clio;
using namespace util;
std::unique_ptr<Source>
LoadBalancer::make_Source(
clio::Config const& config,
Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -57,7 +57,7 @@ LoadBalancer::make_Source(
std::shared_ptr<LoadBalancer>
LoadBalancer::make_LoadBalancer(
clio::Config const& config,
Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -67,7 +67,7 @@ LoadBalancer::make_LoadBalancer(
}
LoadBalancer::LoadBalancer(
clio::Config const& config,
Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

14
src/etl/LoadBalancer.h
View File

@@ -19,11 +19,11 @@
#pragma once
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <data/BackendInterface.h>
#include <etl/ETLHelpers.h>
#include <log/Logger.h>
#include <subscriptions/SubscriptionManager.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <boost/asio.hpp>
@@ -48,7 +48,7 @@ public:
using OptionalGetLedgerResponseType = std::optional<GetLedgerResponseType>;
private:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::vector<std::unique_ptr<Source>> sources_;
std::uint32_t downloadRanges_ = 16;
@@ -63,7 +63,7 @@ public:
* @param nwvl The network validated ledgers datastructure
*/
LoadBalancer(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -71,7 +71,7 @@ public:
static std::shared_ptr<LoadBalancer>
make_LoadBalancer(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -79,7 +79,7 @@ public:
static std::unique_ptr<Source>
make_Source(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

6
src/etl/NFTHelpers.cpp
View File

@@ -22,9 +22,9 @@
#include <ripple/protocol/TxMeta.h>
#include <vector>
#include <backend/BackendInterface.h>
#include <backend/DBHelpers.h>
#include <backend/Types.h>
#include <data/BackendInterface.h>
#include <data/DBHelpers.h>
#include <data/Types.h>
#include <fmt/core.h>
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>

2
src/etl/NFTHelpers.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/DBHelpers.h>
#include <data/DBHelpers.h>
#include <ripple/protocol/STTx.h>
#include <ripple/protocol/TxMeta.h>

7
src/etl/ProbingSource.cpp
View File

@@ -18,12 +18,9 @@
//==============================================================================
#include <etl/ProbingSource.h>
#include <log/Logger.h>
using namespace clio;
ProbingSource::ProbingSource(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -199,4 +196,4 @@ ProbingSource::make_PlainHooks() noexcept
}
return SourceHooks::Action::STOP;
}};
}
};

8
src/etl/ProbingSource.h
View File

@@ -19,9 +19,9 @@
#pragma once
#include <config/Config.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
#include <boost/beast/core.hpp>
@@ -44,7 +44,7 @@ public:
using GetLedgerResponseType = org::xrpl::rpc::v1::GetLedgerResponse;
private:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::mutex mtx_;
boost::asio::ssl::context sslCtx_;
@@ -65,7 +65,7 @@ public:
* @param sslCtx The SSL context to use; defaults to tlsv12
*/
ProbingSource(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,

5
src/etl/Source.cpp
View File

@@ -17,12 +17,11 @@
*/
//==============================================================================
#include <backend/DBHelpers.h>
#include <data/DBHelpers.h>
#include <etl/ETLService.h>
#include <etl/LoadBalancer.h>
#include <etl/ProbingSource.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <rpc/RPCHelpers.h>
#include <util/Profiler.h>
@@ -35,8 +34,6 @@
#include <thread>
using namespace clio;
static boost::beast::websocket::stream_base::timeout
make_TimeoutOption()
{

28
src/etl/Source.h
View File

@@ -19,14 +19,14 @@
#pragma once
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <data/BackendInterface.h>
#include <etl/ETLHelpers.h>
#include <etl/LoadBalancer.h>
#include <etl/impl/AsyncData.h>
#include <etl/impl/ForwardCache.h>
#include <log/Logger.h>
#include <subscriptions/SubscriptionManager.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <boost/algorithm/string.hpp>
@@ -97,10 +97,10 @@ public:
}
protected:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
private:
friend clio::detail::ForwardCache;
friend etl::detail::ForwardCache;
friend ProbingSource;
virtual std::optional<boost::json::object>
@@ -148,7 +148,7 @@ class SourceImpl : public Source
std::shared_ptr<SubscriptionManager> subscriptions_;
LoadBalancer& balancer_;
clio::detail::ForwardCache forwardCache_;
etl::detail::ForwardCache forwardCache_;
boost::uuids::uuid uuid_;
protected:
@@ -176,7 +176,7 @@ public:
* Primarly used in read-only mode, to monitor when ledgers are validated.
*/
SourceImpl(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioContext,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -494,7 +494,7 @@ public:
grpc::CompletionQueue cq;
void* tag;
bool ok = false;
std::vector<clio::detail::AsyncCallData> calls;
std::vector<etl::detail::AsyncCallData> calls;
auto markers = getMarkers(numMarkers);
for (size_t i = 0; i < markers.size(); ++i)
@@ -521,7 +521,7 @@ public:
while (numFinished < calls.size() && cq.Next(&tag, &ok))
{
assert(tag);
auto ptr = static_cast<clio::detail::AsyncCallData*>(tag);
auto ptr = static_cast<etl::detail::AsyncCallData*>(tag);
if (!ok)
{
@@ -533,7 +533,7 @@ public:
log_.trace() << "Marker prefix = " << ptr->getMarkerPrefix();
auto result = ptr->process(stub_, cq, *backend_, abort, cacheOnly);
if (result != clio::detail::AsyncCallData::CallStatus::MORE)
if (result != etl::detail::AsyncCallData::CallStatus::MORE)
{
++numFinished;
log_.debug() << "Finished a marker. "
@@ -545,7 +545,7 @@ public:
edgeKeys.push_back(ptr->getLastKey());
}
if (result == clio::detail::AsyncCallData::CallStatus::ERRORED)
if (result == etl::detail::AsyncCallData::CallStatus::ERRORED)
abort = true;
if (backend_->cache().size() > progress)
@@ -569,7 +569,7 @@ public:
if (paused_)
return;
if (connected_)
if (isConnected())
hooks_.onDisconnected(ec);
connected_ = false;
@@ -836,7 +836,7 @@ class PlainSource : public SourceImpl<PlainSource>
public:
PlainSource(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
@@ -876,7 +876,7 @@ class SslSource : public SourceImpl<SslSource>
public:
SslSource(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx,
std::shared_ptr<BackendInterface> backend,

10
src/etl/impl/AsyncData.h
View File

@@ -20,16 +20,16 @@
#pragma once
#include <etl/NFTHelpers.h>
#include <log/Logger.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <grpcpp/grpcpp.h>
namespace clio::detail {
namespace etl::detail {
class AsyncCallData
{
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> cur_;
std::unique_ptr<org::xrpl::rpc::v1::GetLedgerDataResponse> next_;
@@ -120,7 +120,7 @@ public:
auto const numObjects = cur_->ledger_objects().objects_size();
log_.debug() << "Writing " << numObjects << " objects";
std::vector<Backend::LedgerObject> cacheUpdates;
std::vector<data::LedgerObject> cacheUpdates;
cacheUpdates.reserve(numObjects);
for (int i = 0; i < numObjects; ++i)
@@ -179,4 +179,4 @@ public:
}
};
} // namespace clio::detail
} // namespace etl::detail

24
src/etl/impl/CacheLoader.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <backend/BackendInterface.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <boost/algorithm/string.hpp>
@@ -34,7 +34,7 @@
#include <mutex>
#include <thread>
namespace clio::detail {
namespace etl::detail {
/**
* @brief Cache loading interface
@@ -44,7 +44,7 @@ class CacheLoader
{
enum class LoadStyle { ASYNC, SYNC, NOT_AT_ALL };
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::reference_wrapper<boost::asio::io_context> ioContext_;
std::shared_ptr<BackendInterface> backend_;
@@ -73,7 +73,7 @@ class CacheLoader
public:
CacheLoader(
clio::Config const& config,
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> const& backend,
CacheType& ledgerCache)
@@ -300,13 +300,13 @@ private:
auto const& state = response.at("state").as_array();
std::vector<Backend::LedgerObject> objects;
std::vector<data::LedgerObject> objects;
objects.reserve(state.size());
for (auto const& ledgerObject : state)
{
auto const& obj = ledgerObject.as_object();
Backend::LedgerObject stateObject = {};
data::LedgerObject stateObject = {};
if (!stateObject.key.parseHex(obj.at("index").as_string().c_str()))
{
@@ -337,14 +337,14 @@ private:
void
loadCacheFromDb(uint32_t seq)
{
std::vector<Backend::LedgerObject> diff;
std::vector<data::LedgerObject> diff;
std::vector<std::optional<ripple::uint256>> cursors;
auto append = [](auto&& a, auto&& b) { a.insert(std::end(a), std::begin(b), std::end(b)); };
for (size_t i = 0; i < numCacheDiffs_; ++i)
{
append(diff, Backend::synchronousAndRetryOnTimeout([&](auto yield) {
append(diff, data::synchronousAndRetryOnTimeout([&](auto yield) {
return backend_->fetchLedgerDiff(seq - i, yield);
}));
}
@@ -387,12 +387,12 @@ private:
[this, seq, start, end, numRemaining, startTime, markers](boost::asio::yield_context yield) {
std::optional<ripple::uint256> cursor = start;
std::string cursorStr =
cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(Backend::firstKey);
cursor.has_value() ? ripple::strHex(cursor.value()) : ripple::strHex(data::firstKey);
log_.debug() << "Starting a cursor: " << cursorStr << " markers = " << *markers;
while (not stopping_)
{
auto res = Backend::retryOnTimeout([this, seq, &cursor, &yield]() {
auto res = data::retryOnTimeout([this, seq, &cursor, &yield]() {
return backend_->fetchLedgerPage(cursor, seq, cachePageFetchSize_, false, yield);
});
@@ -431,4 +431,4 @@ private:
}
};
} // namespace clio::detail
} // namespace etl::detail

8
src/etl/impl/ExtractionDataPipe.h
View File

@@ -20,12 +20,12 @@
#pragma once
#include <etl/ETLHelpers.h>
#include <log/Logger.h>
#include <util/log/Logger.h>
#include <memory>
#include <vector>
namespace clio::detail {
namespace etl::detail {
/**
* @brief A collection of thread safe async queues used by Extractor and Transformer to communicate
@@ -40,7 +40,7 @@ public:
constexpr static auto TOTAL_MAX_IN_QUEUE = 1000u;
private:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
uint32_t stride_;
uint32_t startSequence_;
@@ -131,4 +131,4 @@ private:
}
};
} // namespace clio::detail
} // namespace etl::detail

10
src/etl/impl/Extractor.h
View File

@@ -20,8 +20,8 @@
#pragma once
#include <etl/SystemState.h>
#include <log/Logger.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/beast/core/CurrentThreadName.h>
@@ -29,7 +29,7 @@
#include <mutex>
#include <thread>
namespace clio::detail {
namespace etl::detail {
/**
* @brief Extractor thread that is fetching GRPC data and enqueue it on the DataPipeType
@@ -37,7 +37,7 @@ namespace clio::detail {
template <typename DataPipeType, typename NetworkValidatedLedgersType, typename LedgerFetcherType>
class Extractor
{
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::reference_wrapper<DataPipeType> pipe_;
std::shared_ptr<NetworkValidatedLedgersType> networkValidatedLedgers_;
@@ -90,7 +90,7 @@ private:
while (!shouldFinish(currentSequence) && networkValidatedLedgers_->waitUntilValidatedByNetwork(currentSequence))
{
auto [fetchResponse, time] = util::timed<std::chrono::duration<double>>(
auto [fetchResponse, time] = ::util::timed<std::chrono::duration<double>>(
[this, currentSequence]() { return ledgerFetcher_.get().fetchDataAndDiff(currentSequence); });
totalTime += time;
@@ -137,4 +137,4 @@ private:
}
};
} // namespace clio::detail
} // namespace etl::detail

4
src/etl/impl/ForwardCache.cpp
View File

@@ -24,7 +24,7 @@
#include <boost/asio/spawn.hpp>
#include <boost/json.hpp>
namespace clio::detail {
namespace etl::detail {
void
ForwardCache::freshen()
@@ -80,4 +80,4 @@ ForwardCache::get(boost::json::object const& request) const
return {latestForwarded_.at(*command)};
}
} // namespace clio::detail
} // namespace etl::detail

16
src/etl/impl/ForwardCache.h
View File

@@ -19,10 +19,10 @@
#pragma once
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <data/BackendInterface.h>
#include <etl/ETLHelpers.h>
#include <log/Logger.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
#include <boost/json.hpp>
@@ -33,7 +33,7 @@
class Source;
namespace clio::detail {
namespace etl::detail {
/**
* @brief Cache for rippled responses
@@ -42,19 +42,19 @@ class ForwardCache
{
using ResponseType = std::optional<boost::json::object>;
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
mutable std::shared_mutex mtx_;
std::unordered_map<std::string, ResponseType> latestForwarded_;
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
Source const& source_;
::Source const& source_;
std::uint32_t duration_ = 10;
void
clear();
public:
ForwardCache(clio::Config const& config, boost::asio::io_context& ioc, Source const& source)
ForwardCache(util::Config const& config, boost::asio::io_context& ioc, Source const& source)
: strand_(boost::asio::make_strand(ioc)), source_(source)
{
if (config.contains("cache"))
@@ -79,4 +79,4 @@ public:
get(boost::json::object const& command) const;
};
} // namespace clio::detail
} // namespace etl::detail

10
src/etl/impl/LedgerFetcher.h
View File

@@ -19,16 +19,16 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <util/log/Logger.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
#include <grpcpp/grpcpp.h>
#include <optional>
namespace clio::detail {
namespace etl::detail {
/**
* @brief GRPC Ledger data fetcher
@@ -40,7 +40,7 @@ public:
using OptionalGetLedgerResponseType = typename LoadBalancerType::OptionalGetLedgerResponseType;
private:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
@@ -98,4 +98,4 @@ public:
}
};
} // namespace clio::detail
} // namespace etl::detail

100
src/etl/impl/LedgerLoader.h
View File

@@ -19,13 +19,13 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <etl/NFTHelpers.h>
#include <etl/SystemState.h>
#include <etl/impl/LedgerFetcher.h>
#include <log/Logger.h>
#include <util/LedgerUtils.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/beast/core/CurrentThreadName.h>
@@ -38,7 +38,7 @@ struct FormattedTransactionsData
std::vector<NFTsData> nfTokensData;
};
namespace clio::detail {
namespace etl::detail {
/**
* @brief Loads ledger data into the DB
@@ -52,7 +52,7 @@ public:
using RawLedgerObjectType = typename LoadBalancerType::RawLedgerObjectType;
private:
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<LoadBalancerType> loadBalancer_;
@@ -157,11 +157,11 @@ public:
if (!ledgerData)
return {};
ripple::LedgerHeader lgrInfo = util::deserializeHeader(ripple::makeSlice(ledgerData->ledger_header()));
ripple::LedgerHeader lgrInfo = ::util::deserializeHeader(ripple::makeSlice(ledgerData->ledger_header()));
log_.debug() << "Deserialized ledger header. " << util::toString(lgrInfo);
log_.debug() << "Deserialized ledger header. " << ::util::toString(lgrInfo);
auto timeDiff = util::timed<std::chrono::duration<double>>([this, sequence, &lgrInfo, &ledgerData]() {
auto timeDiff = ::util::timed<std::chrono::duration<double>>([this, sequence, &lgrInfo, &ledgerData]() {
backend_->startWrites();
log_.debug() << "Started writes";
@@ -184,51 +184,53 @@ public:
size_t numWrites = 0;
backend_->cache().setFull();
auto seconds = util::timed<std::chrono::seconds>([this, edgeKeys = &edgeKeys, sequence, &numWrites]() {
for (auto& key : *edgeKeys)
{
log_.debug() << "Writing edge key = " << ripple::strHex(key);
auto succ = backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
if (succ)
backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
}
ripple::uint256 prev = Backend::firstKey;
while (auto cur = backend_->cache().getSuccessor(prev, sequence))
{
assert(cur);
if (prev == Backend::firstKey)
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
if (isBookDir(cur->key, cur->blob))
auto seconds =
::util::timed<std::chrono::seconds>([this, edgeKeys = &edgeKeys, sequence, &numWrites]() {
for (auto& key : *edgeKeys)
{
auto base = getBookBase(cur->key);
// make sure the base is not an actual object
if (!backend_->cache().get(cur->key, sequence))
{
auto succ = backend_->cache().getSuccessor(base, sequence);
assert(succ);
if (succ->key == cur->key)
{
log_.debug() << "Writing book successor = " << ripple::strHex(base) << " - "
<< ripple::strHex(cur->key);
backend_->writeSuccessor(
uint256ToString(base), sequence, uint256ToString(cur->key));
}
}
++numWrites;
log_.debug() << "Writing edge key = " << ripple::strHex(key);
auto succ =
backend_->cache().getSuccessor(*ripple::uint256::fromVoidChecked(key), sequence);
if (succ)
backend_->writeSuccessor(std::move(key), sequence, uint256ToString(succ->key));
}
prev = std::move(cur->key);
if (numWrites % 100000 == 0 && numWrites != 0)
log_.info() << "Wrote " << numWrites << " book successors";
}
ripple::uint256 prev = data::firstKey;
while (auto cur = backend_->cache().getSuccessor(prev, sequence))
{
assert(cur);
if (prev == data::firstKey)
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(cur->key));
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(Backend::lastKey));
++numWrites;
});
if (isBookDir(cur->key, cur->blob))
{
auto base = getBookBase(cur->key);
// make sure the base is not an actual object
if (!backend_->cache().get(cur->key, sequence))
{
auto succ = backend_->cache().getSuccessor(base, sequence);
assert(succ);
if (succ->key == cur->key)
{
log_.debug() << "Writing book successor = " << ripple::strHex(base) << " - "
<< ripple::strHex(cur->key);
backend_->writeSuccessor(
uint256ToString(base), sequence, uint256ToString(cur->key));
}
}
++numWrites;
}
prev = std::move(cur->key);
if (numWrites % 100000 == 0 && numWrites != 0)
log_.info() << "Wrote " << numWrites << " book successors";
}
backend_->writeSuccessor(uint256ToString(prev), sequence, uint256ToString(data::lastKey));
++numWrites;
});
log_.info() << "Looping through cache and submitting all writes took " << seconds
<< " seconds. numWrites = " << std::to_string(numWrites);
@@ -251,4 +253,4 @@ public:
}
};
} // namespace clio::detail
} // namespace etl::detail

18
src/etl/impl/LedgerPublisher.h
View File

@@ -19,17 +19,17 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <etl/SystemState.h>
#include <log/Logger.h>
#include <util/LedgerUtils.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/protocol/LedgerHeader.h>
#include <chrono>
namespace clio::detail {
namespace etl::detail {
/**
* @brief Publishes ledgers in a synchronized fashion.
@@ -45,7 +45,7 @@ namespace clio::detail {
template <typename SubscriptionManagerType>
class LedgerPublisher
{
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
boost::asio::strand<boost::asio::io_context::executor_type> publishStrand_;
@@ -113,7 +113,7 @@ public:
}
else
{
auto lgr = Backend::synchronousAndRetryOnTimeout(
auto lgr = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerBySequence(ledgerSequence, yield); });
assert(lgr);
@@ -142,7 +142,7 @@ public:
{
log_.info() << "Updating cache";
std::vector<Backend::LedgerObject> diff = Backend::synchronousAndRetryOnTimeout(
std::vector<data::LedgerObject> diff = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchLedgerDiff(lgrInfo.seq, yield); });
backend_->cache().update(diff, lgrInfo.seq); // todo: inject cache to update, don't use backend cache
@@ -156,10 +156,10 @@ public:
// TODO: this probably should be a strategy
if (age < 600)
{
std::optional<ripple::Fees> fees = Backend::synchronousAndRetryOnTimeout(
std::optional<ripple::Fees> fees = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchFees(lgrInfo.seq, yield); });
std::vector<Backend::TransactionAndMetadata> transactions = Backend::synchronousAndRetryOnTimeout(
std::vector<data::TransactionAndMetadata> transactions = data::synchronousAndRetryOnTimeout(
[&](auto yield) { return backend_->fetchAllTransactionsInLedger(lgrInfo.seq, yield); });
auto ledgerRange = backend_->fetchLedgerRange();
@@ -252,4 +252,4 @@ private:
}
};
} // namespace clio::detail
} // namespace etl::detail

34
src/etl/impl/Transformer.h
View File

@@ -19,12 +19,12 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <etl/SystemState.h>
#include <etl/impl/LedgerLoader.h>
#include <log/Logger.h>
#include <util/LedgerUtils.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/beast/core/CurrentThreadName.h>
#include <ripple/proto/org/xrpl/rpc/v1/xrp_ledger.grpc.pb.h>
@@ -34,7 +34,7 @@
#include <memory>
#include <thread>
namespace clio::detail {
namespace etl::detail {
/*
* TODO:
@@ -53,7 +53,7 @@ class Transformer
using GetLedgerResponseType = typename LedgerLoaderType::GetLedgerResponseType;
using RawLedgerObjectType = typename LedgerLoaderType::RawLedgerObjectType;
clio::Logger log_{"ETL"};
util::Logger log_{"ETL"};
std::reference_wrapper<DataPipeType> pipe_;
std::shared_ptr<BackendInterface> backend_;
@@ -167,9 +167,9 @@ private:
buildNextLedger(GetLedgerResponseType& rawData)
{
log_.debug() << "Beginning ledger update";
ripple::LedgerHeader lgrInfo = util::deserializeHeader(ripple::makeSlice(rawData.ledger_header()));
ripple::LedgerHeader lgrInfo = ::util::deserializeHeader(ripple::makeSlice(rawData.ledger_header()));
log_.debug() << "Deserialized ledger header. " << util::toString(lgrInfo);
log_.debug() << "Deserialized ledger header. " << ::util::toString(lgrInfo);
backend_->startWrites();
backend_->writeLedger(lgrInfo, std::move(*rawData.mutable_ledger_header()));
@@ -202,10 +202,10 @@ private:
backend_->writeNFTTransactions(std::move(insertTxResultOp->nfTokenTxData));
auto [success, duration] =
util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });
::util::timed<std::chrono::duration<double>>([&]() { return backend_->finishWrites(lgrInfo.seq); });
log_.debug() << "Finished writes. Total time: " << std::to_string(duration);
log_.debug() << "Finished ledger update: " << util::toString(lgrInfo);
log_.debug() << "Finished ledger update: " << ::util::toString(lgrInfo);
return {lgrInfo, success};
}
@@ -219,7 +219,7 @@ private:
void
updateCache(ripple::LedgerHeader const& lgrInfo, GetLedgerResponseType& rawData)
{
std::vector<Backend::LedgerObject> cacheUpdates;
std::vector<data::LedgerObject> cacheUpdates;
cacheUpdates.reserve(rawData.ledger_objects().objects_size());
// TODO change these to unordered_set
@@ -297,11 +297,11 @@ private:
auto lb = backend_->cache().getPredecessor(obj.key, lgrInfo.seq);
if (!lb)
lb = {Backend::firstKey, {}};
lb = {data::firstKey, {}};
auto ub = backend_->cache().getSuccessor(obj.key, lgrInfo.seq);
if (!ub)
ub = {Backend::lastKey, {}};
ub = {data::lastKey, {}};
if (obj.blob.size() == 0)
{
@@ -332,10 +332,10 @@ private:
}
else
{
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(Backend::lastKey));
backend_->writeSuccessor(uint256ToString(base), lgrInfo.seq, uint256ToString(data::lastKey));
log_.debug() << "Updating book successor " << ripple::strHex(base) << " - "
<< ripple::strHex(Backend::lastKey);
<< ripple::strHex(data::lastKey);
}
}
}
@@ -359,7 +359,7 @@ private:
{
auto firstBook = std::move(*obj.mutable_first_book());
if (!firstBook.size())
firstBook = uint256ToString(Backend::lastKey);
firstBook = uint256ToString(data::lastKey);
log_.debug() << "writing book successor " << ripple::strHex(obj.book_base()) << " - "
<< ripple::strHex(firstBook);
@@ -372,10 +372,10 @@ private:
{
std::string* predPtr = obj.mutable_predecessor();
if (!predPtr->size())
*predPtr = uint256ToString(Backend::firstKey);
*predPtr = uint256ToString(data::firstKey);
std::string* succPtr = obj.mutable_successor();
if (!succPtr->size())
*succPtr = uint256ToString(Backend::lastKey);
*succPtr = uint256ToString(data::lastKey);
if (obj.mod_type() == RawLedgerObjectType::DELETED)
{
@@ -424,4 +424,4 @@ private:
}
};
} // namespace clio::detail
} // namespace etl::detail

17
src/main/Main.cpp
View File

@@ -25,12 +25,12 @@
#undef GRPC_ASAN_ENABLED
#endif
#include <backend/BackendFactory.h>
#include <config/Config.h>
#include <data/BackendFactory.h>
#include <etl/ETLService.h>
#include <rpc/Counters.h>
#include <rpc/RPCEngine.h>
#include <rpc/common/impl/HandlerProvider.h>
#include <util/config/Config.h>
#include <webserver/RPCServerHandler.h>
#include <webserver/Server.h>
@@ -47,8 +47,9 @@
#include <thread>
#include <vector>
using namespace clio;
using namespace util;
using namespace boost::asio;
namespace po = boost::program_options;
/**
@@ -178,12 +179,12 @@ try
io_context ioc{threads};
// Rate limiter, to prevent abuse
auto sweepHandler = IntervalSweepHandler{config, ioc};
auto whitelistHandler = WhitelistHandler{config};
auto dosGuard = DOSGuard{config, whitelistHandler, sweepHandler};
auto sweepHandler = web::IntervalSweepHandler{config, ioc};
auto whitelistHandler = web::WhitelistHandler{config};
auto dosGuard = web::DOSGuard{config, whitelistHandler, sweepHandler};
// Interface to the database
auto backend = Backend::make_Backend(ioc, config);
auto backend = data::make_Backend(ioc, config);
// Manages clients subscribed to streams
auto subscriptions = SubscriptionManager::make_SubscriptionManager(config, backend);
@@ -212,7 +213,7 @@ try
std::make_shared<RPCServerHandler<RPC::RPCEngine, ETLService>>(config, backend, rpcEngine, etl, subscriptions);
auto ctx = parseCerts(config);
auto const ctxRef = ctx ? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()} : std::nullopt;
auto const httpServer = Server::make_HttpServer(config, ioc, ctxRef, dosGuard, handler);
auto const httpServer = web::make_HttpServer(config, ioc, ctxRef, dosGuard, handler);
// Blocks until stopped.
// When stopped, shared_ptrs fall out of scope

10
src/rpc/BookChangesHelper.h
View File

@@ -53,7 +53,7 @@ public:
* @return std::vector<BookChange> Book changes
*/
[[nodiscard]] static std::vector<BookChange>
compute(std::vector<Backend::TransactionAndMetadata> const& transactions)
compute(std::vector<data::TransactionAndMetadata> const& transactions)
{
return HandlerImpl{}(transactions);
}
@@ -66,7 +66,7 @@ private:
public:
[[nodiscard]] std::vector<BookChange>
operator()(std::vector<Backend::TransactionAndMetadata> const& transactions)
operator()(std::vector<data::TransactionAndMetadata> const& transactions)
{
for (auto const& tx : transactions)
handleBookChange(tx);
@@ -176,7 +176,7 @@ private:
}
void
handleBookChange(Backend::TransactionAndMetadata const& blob)
handleBookChange(data::TransactionAndMetadata const& blob)
{
auto const [tx, meta] = RPC::deserializeTxPlusMeta(blob);
if (!tx || !meta || !tx->isFieldPresent(ripple::sfTransactionType))
@@ -229,8 +229,6 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookChange const
}
[[nodiscard]] boost::json::object const
computeBookChanges(
ripple::LedgerHeader const& lgrInfo,
std::vector<Backend::TransactionAndMetadata> const& transactions);
computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::TransactionAndMetadata> const& transactions);
} // namespace RPC

24
src/rpc/Factories.cpp
View File

@@ -21,21 +21,21 @@
#include <rpc/common/Types.h>
using namespace std;
using namespace clio;
using namespace util;
namespace RPC {
util::Expected<Web::Context, Status>
util::Expected<web::Context, Status>
make_WsContext(
boost::asio::yield_context yc,
boost::json::object const& request,
shared_ptr<Server::ConnectionBase> const& session,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,
shared_ptr<web::ConnectionBase> const& session,
TagDecoratorFactory const& tagFactory,
data::LedgerRange const& range,
string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser)
{
using Error = util::Unexpected<Status>;
using Error = Unexpected<Status>;
boost::json::value commandValue = nullptr;
if (!request.contains("command") && request.contains("method"))
@@ -51,19 +51,19 @@ make_WsContext(
return Error{{ClioError::rpcINVALID_API_VERSION, apiVersion.error()}};
string command = commandValue.as_string().c_str();
return Web::Context(yc, command, *apiVersion, request, session, tagFactory, range, clientIp);
return web::Context(yc, command, *apiVersion, request, session, tagFactory, range, clientIp);
}
util::Expected<Web::Context, Status>
Expected<web::Context, Status>
make_HttpContext(
boost::asio::yield_context yc,
boost::json::object const& request,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,
TagDecoratorFactory const& tagFactory,
data::LedgerRange const& range,
string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser)
{
using Error = util::Unexpected<Status>;
using Error = Unexpected<Status>;
if (!request.contains("method"))
return Error{{ClioError::rpcCOMMAND_IS_MISSING}};
@@ -91,7 +91,7 @@ make_HttpContext(
if (!apiVersion)
return Error{{ClioError::rpcINVALID_API_VERSION, apiVersion.error()}};
return Web::Context(yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp);
return web::Context(yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp);
}
} // namespace RPC

12
src/rpc/Factories.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/Errors.h>
#include <rpc/common/APIVersion.h>
#include <util/Expected.h>
@@ -41,22 +41,22 @@
*/
namespace RPC {
util::Expected<Web::Context, Status>
util::Expected<web::Context, Status>
make_WsContext(
boost::asio::yield_context yc,
boost::json::object const& request,
std::shared_ptr<Server::ConnectionBase> const& session,
std::shared_ptr<web::ConnectionBase> const& session,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,
data::LedgerRange const& range,
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser);
util::Expected<Web::Context, Status>
util::Expected<web::Context, Status>
make_HttpContext(
boost::asio::yield_context yc,
boost::json::object const& request,
util::TagDecoratorFactory const& tagFactory,
Backend::LedgerRange const& range,
data::LedgerRange const& range,
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser);

22
src/rpc/RPCEngine.h
View File

@@ -19,10 +19,8 @@
#pragma once
#include <backend/BackendInterface.h>
#include <config/Config.h>
#include <data/BackendInterface.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <rpc/Counters.h>
#include <rpc/Errors.h>
#include <rpc/RPCHelpers.h>
@@ -31,6 +29,8 @@
#include <rpc/common/impl/AdminVerificationStrategy.h>
#include <rpc/common/impl/ForwardingProxy.h>
#include <util/Taggable.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <webserver/Context.h>
#include <webserver/DOSGuard.h>
@@ -55,13 +55,13 @@ namespace RPC {
template <typename AdminVerificationStrategyType>
class RPCEngineBase
{
clio::Logger perfLog_{"Performance"};
clio::Logger log_{"RPC"};
util::Logger perfLog_{"Performance"};
util::Logger log_{"RPC"};
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<SubscriptionManager> subscriptions_;
std::shared_ptr<LoadBalancer> balancer_;
std::reference_wrapper<clio::DOSGuard const> dosGuard_;
std::reference_wrapper<web::DOSGuard const> dosGuard_;
std::reference_wrapper<WorkQueue> workQueue_;
std::reference_wrapper<Counters> counters_;
@@ -76,7 +76,7 @@ public:
std::shared_ptr<SubscriptionManager> const& subscriptions,
std::shared_ptr<LoadBalancer> const& balancer,
std::shared_ptr<ETLService> const& etl,
clio::DOSGuard const& dosGuard,
web::DOSGuard const& dosGuard,
WorkQueue& workQueue,
Counters& counters,
std::shared_ptr<HandlerProvider const> const& handlerProvider)
@@ -93,12 +93,12 @@ public:
static std::shared_ptr<RPCEngineBase>
make_RPCEngine(
clio::Config const& config,
util::Config const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<SubscriptionManager> const& subscriptions,
std::shared_ptr<LoadBalancer> const& balancer,
std::shared_ptr<ETLService> const& etl,
clio::DOSGuard const& dosGuard,
web::DOSGuard const& dosGuard,
WorkQueue& workQueue,
Counters& counters,
std::shared_ptr<HandlerProvider const> const& handlerProvider)
@@ -112,7 +112,7 @@ public:
* @param ctx The @ref Context of the request
*/
Result
buildResponse(Web::Context const& ctx)
buildResponse(web::Context const& ctx)
{
if (forwardingProxy_.shouldForward(ctx))
return forwardingProxy_.forward(ctx);
@@ -149,7 +149,7 @@ public:
return Status{v.error()};
}
}
catch (Backend::DatabaseTimeout const& t)
catch (data::DatabaseTimeout const& t)
{
log_.error() << "Database timeout";
notifyTooBusy();

28
src/rpc/RPCHelpers.cpp
View File

@@ -17,12 +17,12 @@
*/
//==============================================================================
#include <backend/BackendInterface.h>
#include <backend/DBHelpers.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <data/DBHelpers.h>
#include <rpc/Errors.h>
#include <rpc/RPCHelpers.h>
#include <util/Profiler.h>
#include <util/log/Logger.h>
#include <ripple/basics/StringUtilities.h>
#include <ripple/protocol/NFTSyntheticSerializer.h>
@@ -30,11 +30,9 @@
#include <boost/algorithm/string.hpp>
#include <boost/format.hpp>
using namespace clio;
// local to compilation unit loggers
namespace {
clio::Logger gLog{"RPC"};
util::Logger gLog{"RPC"};
} // namespace
namespace RPC {
@@ -144,7 +142,7 @@ accountFromStringStrict(std::string const& account)
return {};
}
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs)
{
try
{
@@ -173,7 +171,7 @@ deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
}
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_t seq)
deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs, std::uint32_t seq)
{
auto [tx, meta] = deserializeTxPlusMeta(blobs);
@@ -191,7 +189,7 @@ toJson(ripple::STBase const& obj)
}
std::pair<boost::json::object, boost::json::object>
toExpandedJson(Backend::TransactionAndMetadata const& blobs, NFTokenjson nftEnabled)
toExpandedJson(data::TransactionAndMetadata const& blobs, NFTokenjson nftEnabled)
{
auto [txn, meta] = deserializeTxPlusMeta(blobs, blobs.ledgerSequence);
auto txnJson = toJson(*txn);
@@ -300,7 +298,7 @@ parseStringAsUInt(std::string const& value)
}
std::variant<Status, ripple::LedgerHeader>
ledgerInfoFromRequest(std::shared_ptr<Backend::BackendInterface const> const& backend, Web::Context const& ctx)
ledgerInfoFromRequest(std::shared_ptr<data::BackendInterface const> const& backend, web::Context const& ctx)
{
auto hashValue = ctx.params.contains("ledger_hash") ? ctx.params.at("ledger_hash") : nullptr;
@@ -677,10 +675,10 @@ traverseOwnedNodes(
std::shared_ptr<ripple::SLE const>
read(
std::shared_ptr<Backend::BackendInterface const> const& backend,
std::shared_ptr<data::BackendInterface const> const& backend,
ripple::Keylet const& keylet,
ripple::LedgerHeader const& lgrInfo,
Web::Context const& context)
web::Context const& context)
{
if (auto const blob = backend->fetchLedgerObject(keylet.key, lgrInfo.seq, context.yield); blob)
{
@@ -1025,10 +1023,10 @@ transferRate(
boost::json::array
postProcessOrderBook(
std::vector<Backend::LedgerObject> const& offers,
std::vector<data::LedgerObject> const& offers,
ripple::Book const& book,
ripple::AccountID const& takerID,
Backend::BackendInterface const& backend,
data::BackendInterface const& backend,
std::uint32_t const ledgerSequence,
boost::asio::yield_context yield)
{
@@ -1327,7 +1325,7 @@ getNFTID(boost::json::object const& request)
bool
isAmendmentEnabled(
std::shared_ptr<Backend::BackendInterface const> const& backend,
std::shared_ptr<data::BackendInterface const> const& backend,
boost::asio::yield_context yield,
uint32_t seq,
ripple::uint256 amendmentId)

24
src/rpc/RPCHelpers.h
View File

@@ -23,7 +23,7 @@
* This file contains a variety of utility functions used when executing the handlers.
*/
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/Amendments.h>
#include <rpc/JS.h>
#include <rpc/common/Types.h>
@@ -55,14 +55,14 @@ parseAccountCursor(std::optional<std::string> jsonCursor);
// TODO this function should probably be in a different file and namespace
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs);
deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs);
// TODO this function should probably be in a different file and namespace
std::pair<std::shared_ptr<ripple::STTx const>, std::shared_ptr<ripple::TxMeta const>>
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_t seq);
deserializeTxPlusMeta(data::TransactionAndMetadata const& blobs, std::uint32_t seq);
std::pair<boost::json::object, boost::json::object>
toExpandedJson(Backend::TransactionAndMetadata const& blobs, NFTokenjson includeNFTIDs = NFTokenjson::DISABLE);
toExpandedJson(data::TransactionAndMetadata const& blobs, NFTokenjson includeNFTIDs = NFTokenjson::DISABLE);
bool
insertDeliveredAmount(
@@ -95,7 +95,7 @@ generatePubLedgerMessage(
std::uint32_t txnCount);
std::variant<Status, ripple::LedgerHeader>
ledgerInfoFromRequest(std::shared_ptr<Backend::BackendInterface const> const& backend, Web::Context const& ctx);
ledgerInfoFromRequest(std::shared_ptr<data::BackendInterface const> const& backend, web::Context const& ctx);
std::variant<Status, ripple::LedgerHeader>
getLedgerInfoFromHashOrSeq(
@@ -131,10 +131,10 @@ traverseOwnedNodes(
std::shared_ptr<ripple::SLE const>
read(
std::shared_ptr<Backend::BackendInterface const> const& backend,
std::shared_ptr<data::BackendInterface const> const& backend,
ripple::Keylet const& keylet,
ripple::LedgerHeader const& lgrInfo,
Web::Context const& context);
web::Context const& context);
std::variant<Status, std::pair<ripple::PublicKey, ripple::SecretKey>>
keypairFromRequst(boost::json::object const& request);
@@ -195,10 +195,10 @@ xrpLiquid(
boost::json::array
postProcessOrderBook(
std::vector<Backend::LedgerObject> const& offers,
std::vector<data::LedgerObject> const& offers,
ripple::Book const& book,
ripple::AccountID const& takerID,
Backend::BackendInterface const& backend,
data::BackendInterface const& backend,
std::uint32_t ledgerSequence,
boost::asio::yield_context yield);
@@ -219,18 +219,18 @@ getNFTID(boost::json::object const& request);
bool
isAmendmentEnabled(
std::shared_ptr<Backend::BackendInterface const> const& backend,
std::shared_ptr<data::BackendInterface const> const& backend,
boost::asio::yield_context yield,
uint32_t seq,
ripple::uint256 amendmentId);
template <class T>
void
logDuration(Web::Context const& ctx, T const& dur)
logDuration(web::Context const& ctx, T const& dur)
{
using boost::json::serialize;
static clio::Logger log{"RPC"};
static util::Logger log{"RPC"};
auto const millis = std::chrono::duration_cast<std::chrono::milliseconds>(dur).count();
auto const seconds = std::chrono::duration_cast<std::chrono::seconds>(dur).count();
auto const msg = fmt::format(

10
src/rpc/WorkQueue.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <config/Config.h>
#include <log/Logger.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <boost/asio.hpp>
#include <boost/asio/spawn.hpp>
@@ -41,7 +41,7 @@ class WorkQueue
std::atomic_uint64_t curSize_ = 0;
uint32_t maxSize_ = std::numeric_limits<uint32_t>::max();
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
boost::asio::thread_pool ioc_;
public:
@@ -49,9 +49,9 @@ public:
~WorkQueue();
static WorkQueue
make_WorkQueue(clio::Config const& config)
make_WorkQueue(util::Config const& config)
{
static clio::Logger log{"RPC"};
static util::Logger log{"RPC"};
auto const serverConfig = config.section("server");
auto const numThreads = config.valueOr<uint32_t>("workers", std::thread::hardware_concurrency());
auto const maxQueueSize = serverConfig.valueOr<uint32_t>("max_queue_size", 0); // 0 is no limit

4
src/rpc/common/Types.h
View File

@@ -28,7 +28,7 @@
#include <boost/json/value.hpp>
#include <boost/json/value_from.hpp>
namespace Server {
namespace web {
struct ConnectionBase;
}
@@ -73,7 +73,7 @@ struct VoidOutput
struct Context
{
boost::asio::yield_context yield;
std::shared_ptr<Server::ConnectionBase> session;
std::shared_ptr<web::ConnectionBase> session;
bool isAdmin = false;
std::string clientIp;
uint32_t apiVersion = 0u; // invalid by default

4
src/rpc/common/impl/APIVersionParser.cpp
View File

@@ -17,8 +17,8 @@
*/
//==============================================================================
#include <log/Logger.h>
#include <rpc/common/impl/APIVersionParser.h>
#include <util/log/Logger.h>
#include <fmt/core.h>
@@ -26,7 +26,7 @@ using namespace std;
namespace RPC::detail {
ProductionAPIVersionParser::ProductionAPIVersionParser(clio::Config const& config)
ProductionAPIVersionParser::ProductionAPIVersionParser(util::Config const& config)
: ProductionAPIVersionParser(
config.valueOr("default", API_VERSION_DEFAULT),
config.valueOr("min", API_VERSION_MIN),

8
src/rpc/common/impl/APIVersionParser.h
View File

@@ -19,10 +19,10 @@
#pragma once
#include <config/Config.h>
#include <log/Logger.h>
#include <rpc/common/APIVersion.h>
#include <util/Expected.h>
#include <util/config/Config.h>
#include <util/log/Logger.h>
#include <algorithm>
#include <string>
@@ -31,7 +31,7 @@ namespace RPC::detail {
class ProductionAPIVersionParser : public APIVersionParser
{
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
uint32_t defaultVersion_;
uint32_t minVersion_;
@@ -70,7 +70,7 @@ public:
<< "; default = " << defaultVersion_ << "]";
}
ProductionAPIVersionParser(clio::Config const& config);
ProductionAPIVersionParser(util::Config const& config);
util::Expected<uint32_t, std::string>
parse(boost::json::object const& request) const override;

8
src/rpc/common/impl/ForwardingProxy.h
View File

@@ -21,10 +21,10 @@
#include <etl/LoadBalancer.h>
#include <etl/Source.h>
#include <log/Logger.h>
#include <rpc/Counters.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>
#include <util/log/Logger.h>
#include <webserver/Context.h>
#include <memory>
@@ -35,7 +35,7 @@ namespace RPC::detail {
template <typename LoadBalancerType, typename CountersType, typename HandlerProviderType>
class ForwardingProxy
{
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
std::shared_ptr<LoadBalancerType> balancer_;
std::reference_wrapper<CountersType> counters_;
@@ -51,7 +51,7 @@ public:
}
bool
shouldForward(Web::Context const& ctx) const
shouldForward(web::Context const& ctx) const
{
if (ctx.method == "subscribe" || ctx.method == "unsubscribe")
return false;
@@ -91,7 +91,7 @@ public:
}
Result
forward(Web::Context const& ctx)
forward(web::Context const& ctx)
{
auto toForward = ctx.params;
toForward["command"] = ctx.method;

2
src/rpc/common/impl/HandlerProvider.cpp
View File

@@ -56,7 +56,7 @@
namespace RPC::detail {
ProductionHandlerProvider::ProductionHandlerProvider(
clio::Config const& config,
util::Config const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<SubscriptionManager> const& subscriptionManager,
std::shared_ptr<LoadBalancer> const& balancer,

4
src/rpc/common/impl/HandlerProvider.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/common/AnyHandler.h>
#include <rpc/common/Types.h>
#include <subscriptions/SubscriptionManager.h>
@@ -50,7 +50,7 @@ class ProductionHandlerProvider final : public HandlerProvider
public:
ProductionHandlerProvider(
clio::Config const& config,
util::Config const& config,
std::shared_ptr<BackendInterface> const& backend,
std::shared_ptr<SubscriptionManager> const& subscriptionManager,
std::shared_ptr<LoadBalancer> const& balancer,

2
src/rpc/handlers/AccountChannels.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/JS.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/AccountCurrencies.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/AccountInfo.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/AccountLines.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>

2
src/rpc/handlers/AccountNFTs.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/AccountObjects.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/AccountOffers.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>

2
src/rpc/handlers/AccountTx.cpp
View File

@@ -64,7 +64,7 @@ AccountTxHandler::process(AccountTxHandler::Input input, Context const& ctx) con
maxIndex = minIndex = std::get<ripple::LedgerHeader>(lgrInfoOrStatus).seq;
}
std::optional<Backend::TransactionsCursor> cursor;
std::optional<data::TransactionsCursor> cursor;
// if marker exists
if (input.marker)

6
src/rpc/handlers/AccountTx.h
View File

@@ -19,13 +19,13 @@
#pragma once
#include <backend/BackendInterface.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
#include <util/log/Logger.h>
namespace RPC {
@@ -36,7 +36,7 @@ namespace RPC {
*/
class AccountTxHandler
{
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
std::shared_ptr<BackendInterface> sharedPtrBackend_;
public:

4
src/rpc/handlers/BookChanges.cpp
View File

@@ -79,9 +79,7 @@ tag_invoke(boost::json::value_to_tag<BookChangesHandler::Input>, boost::json::va
}
[[nodiscard]] boost::json::object const
computeBookChanges(
ripple::LedgerHeader const& lgrInfo,
std::vector<Backend::TransactionAndMetadata> const& transactions)
computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::TransactionAndMetadata> const& transactions)
{
using boost::json::value_from;

2
src/rpc/handlers/BookChanges.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/BookChangesHelper.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/BookOffers.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/DepositAuthorized.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/JS.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>

2
src/rpc/handlers/GatewayBalances.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/Ledger.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>

2
src/rpc/handlers/LedgerData.cpp
View File

@@ -108,7 +108,7 @@ LedgerDataHandler::process(Input input, Context const& ctx) const
output.ledgerIndex = lgrInfo.seq;
auto const start = std::chrono::system_clock::now();
std::vector<Backend::LedgerObject> results;
std::vector<data::LedgerObject> results;
if (input.diffMarker)
{

4
src/rpc/handlers/LedgerData.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>
@@ -40,7 +40,7 @@ class LedgerDataHandler
{
// dependencies
std::shared_ptr<BackendInterface> sharedPtrBackend_;
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
static const std::unordered_map<std::string, ripple::LedgerEntryType> TYPES_MAP;

2
src/rpc/handlers/LedgerEntry.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>

4
src/rpc/handlers/LedgerRange.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/common/Types.h>
#include <boost/json.hpp>
@@ -40,7 +40,7 @@ class LedgerRangeHandler
public:
struct Output
{
Backend::LedgerRange range;
data::LedgerRange range;
};
using Result = HandlerReturnType<Output>;

2
src/rpc/handlers/NFTBuyOffers.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/handlers/NFTOffersCommon.h>
namespace RPC {

2
src/rpc/handlers/NFTHistory.cpp
View File

@@ -66,7 +66,7 @@ NFTHistoryHandler::process(NFTHistoryHandler::Input input, Context const& ctx) c
maxIndex = minIndex = std::get<ripple::LedgerHeader>(lgrInfoOrStatus).seq;
}
std::optional<Backend::TransactionsCursor> cursor;
std::optional<data::TransactionsCursor> cursor;
// if marker exists
if (input.marker)

6
src/rpc/handlers/NFTHistory.h
View File

@@ -19,13 +19,13 @@
#pragma once
#include <backend/BackendInterface.h>
#include <log/Logger.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
#include <util/log/Logger.h>
namespace RPC {
@@ -36,7 +36,7 @@ namespace RPC {
*/
class NFTHistoryHandler
{
clio::Logger log_{"RPC"};
util::Logger log_{"RPC"};
std::shared_ptr<BackendInterface> sharedPtrBackend_;
public:

2
src/rpc/handlers/NFTInfo.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>

2
src/rpc/handlers/NFTOffersCommon.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Modifiers.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/NFTSellOffers.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/handlers/NFTOffersCommon.h>
namespace RPC {

2
src/rpc/handlers/NoRippleCheck.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Modifiers.h>

4
src/rpc/handlers/ServerInfo.h
View File

@@ -19,8 +19,8 @@
#pragma once
#include <backend/BackendInterface.h>
#include <backend/DBHelpers.h>
#include <data/BackendInterface.h>
#include <data/DBHelpers.h>
#include <main/Build.h>
#include <rpc/Errors.h>
#include <rpc/RPCHelpers.h>

15
src/rpc/handlers/Subscribe.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
@@ -143,7 +143,7 @@ private:
subscribeToStreams(
boost::asio::yield_context yield,
std::vector<std::string> const& streams,
std::shared_ptr<Server::ConnectionBase> const& session) const
std::shared_ptr<web::ConnectionBase> const& session) const
{
auto response = boost::json::object{};
@@ -167,9 +167,8 @@ private:
}
void
subscribeToAccounts(
std::vector<std::string> const& accounts,
std::shared_ptr<Server::ConnectionBase> const& session) const
subscribeToAccounts(std::vector<std::string> const& accounts, std::shared_ptr<web::ConnectionBase> const& session)
const
{
for (auto const& account : accounts)
{
@@ -181,7 +180,7 @@ private:
void
subscribeToAccountsProposed(
std::vector<std::string> const& accounts,
std::shared_ptr<Server::ConnectionBase> const& session) const
std::shared_ptr<web::ConnectionBase> const& session) const
{
for (auto const& account : accounts)
{
@@ -193,13 +192,13 @@ private:
void
subscribeToBooks(
std::vector<OrderBook> const& books,
std::shared_ptr<Server::ConnectionBase> const& session,
std::shared_ptr<web::ConnectionBase> const& session,
boost::asio::yield_context yield,
Output& output) const
{
static auto constexpr fetchLimit = 200;
std::optional<Backend::LedgerRange> rng;
std::optional<data::LedgerRange> rng;
for (auto const& internalBook : books)
{

2
src/rpc/handlers/TransactionEntry.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/MetaProcessors.h>
#include <rpc/common/Types.h>

2
src/rpc/handlers/Tx.h
View File

@@ -19,7 +19,7 @@
#pragma once
#include <backend/BackendInterface.h>
#include <data/BackendInterface.h>
#include <rpc/RPCHelpers.h>
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>

Some files were not shown because too many files have changed in this diff Show More