ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-04 11:55:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update doxygen comments (#818)

Browse Source 
Fixes #421
This commit is contained in:
Alex Kremer
2023-08-11 21:32:32 +01:00
committed by GitHub
parent c20b14494a
commit 547cb340bd
206 changed files with 3004 additions and 1937 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
CMake/Docs.cmake Normal file
View File

@@ -0,0 +1,11 @@
find_package(Doxygen REQUIRED)
set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile)
set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
add_custom_target(docs
COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Generating API documentation with Doxygen"
VERBATIM)

13
CMakeLists.txt
View File

@@ -6,6 +6,7 @@ project(clio)
# ==================================================== #
option(verbose "Verbose build" FALSE)
option(tests "Build tests" FALSE)
option(docs "Generate doxygen docs" FALSE)
option(coverage "Build test coverage report" FALSE)
option(packaging "Create distribution packages" FALSE)
# ==================================================== #
@@ -222,15 +223,21 @@ if(tests)
target_include_directories(${TEST_TARGET} PRIVATE unittests)
target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest)
# Generate `clio_test-ccov` if coverage is enabled
# Note: use `make clio_test-ccov` to generate report
# Generate `clio_tests-ccov` if coverage is enabled
# Note: use `make clio_tests-ccov` to generate report
if(coverage)
include(CMake/Coverage.cmake)
add_coverage(${TEST_TARGET})
endif()
endif()
# Generate `docs` target for doxygen documentation if enabled
# Note: use `make docs` to generate the documentation
if(docs)
include(CMake/Docs.cmake)
endif()
include(CMake/install/install.cmake)
if(packaging)
include(CMake/packaging.cmake) # This file exists only in build runner
include(CMake/packaging.cmake) # This file exists only in build runner
endif()

17
Doxyfile
View File

@@ -1,3 +1,16 @@
PROJECT_NAME = "Clio"
INPUT = src
RECURSIVE = YES
INPUT = ../src ../unittests
EXCLUDE_PATTERNS = *Test*.cpp *Test*.h
RECURSIVE = YES
HAVE_DOT = YES
QUIET = YES
WARNINGS = NO
WARN_NO_PARAMDOC = NO
WARN_IF_INCOMPLETE_DOC = NO
WARN_IF_UNDOCUMENTED = NO
GENERATE_LATEX = NO
GENERATE_HTML = YES
SORT_MEMBERS_CTORS_1ST = YES

14
README.md
View File

@@ -1,15 +1,16 @@
# Clio
Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. Validated
historical ledger and transaction data are stored in a more space-efficient format,
Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC.
Validated historical ledger and transaction data are stored in a more space-efficient format,
using up to 4 times less space than rippled. Clio can be configured to store data in Apache Cassandra or ScyllaDB,
allowing for scalable read throughput. Multiple Clio nodes can share
access to the same dataset, allowing for a highly available cluster of Clio nodes,
without the need for redundant data storage or computation.
allowing for scalable read throughput. Multiple Clio nodes can share access to the same dataset,
allowing for a highly available cluster of Clio nodes, without the need for redundant data storage or computation.
Clio offers the full rippled API, with the caveat that Clio by default only returns validated data.
This means that `ledger_index` defaults to `validated` instead of `current` for all requests.
Other non-validated data is also not returned, such as information about queued transactions.
For requests that require access to the p2p network, such as `fee` or `submit`, Clio automatically forwards the request to a rippled node and propagates the response back to the client. To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to rippled.
For requests that require access to the p2p network, such as `fee` or `submit`, Clio automatically forwards the request to a rippled node and propagates the response back to the client.
To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to rippled.
Clio does not connect to the peer-to-peer network. Instead, Clio extracts data from a group of specified rippled nodes. Running Clio requires access to at least one rippled node
from which data can be extracted. The rippled node does not need to be running on the same machine as Clio.
@@ -25,7 +26,6 @@ It is written in C++20 and therefore requires a modern compiler.
## Prerequisites
### Minimum Requirements
- [Python 3.7](https://www.python.org/downloads/)

13
conanfile.py
View File

@@ -12,9 +12,10 @@ class Clio(ConanFile):
options = {
'fPIC': [True, False],
'verbose': [True, False],
'tests': [True, False], # build unit tests
'tests': [True, False], # build unit tests; create `clio_tests` binary
'docs': [True, False], # doxygen API docs; create custom target 'docs'
'packaging': [True, False], # create distribution packages
'coverage': [True, False], # build for test coverage report
'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov`
}
requires = [
@@ -22,7 +23,6 @@ class Clio(ConanFile):
'cassandra-cpp-driver/2.16.2',
'fmt/10.0.0',
'grpc/1.50.1',
'gtest/1.13.0',
'openssl/1.1.1u',
'xrpl/1.12.0-b2',
]
@@ -33,6 +33,7 @@ class Clio(ConanFile):
'tests': False,
'packaging': False,
'coverage': False,
'docs': False,
'xrpl/*:tests': False,
'cassandra-cpp-driver/*:shared': False,
@@ -52,6 +53,10 @@ class Clio(ConanFile):
'CMakeLists.txt', 'CMake/*', 'src/*'
)
def requirements(self):
if self.options.tests:
self.requires('gtest/1.13.0')
def configure(self):
if self.settings.compiler == 'apple-clang':
self.options['boost'].visibility = 'global'
@@ -68,6 +73,8 @@ class Clio(ConanFile):
tc.variables['verbose'] = self.options.verbose
tc.variables['tests'] = self.options.tests
tc.variables['coverage'] = self.options.coverage
tc.variables['docs'] = self.options.docs
tc.variables['packaging'] = self.options.packaging
tc.generate()
def build(self):

8
src/data/BackendFactory.h
View File

@@ -27,6 +27,14 @@
#include <boost/algorithm/string.hpp>
namespace data {
/**
* @brief A factory function that creates the backend based on a config.
*
* @param ioc The boost::asio::io_context to use
* @param config The clio config to use
* @return A shared_ptr<BackendInterface> with the selected implementation
*/
std::shared_ptr<BackendInterface>
make_Backend(boost::asio::io_context& ioc, util::Config const& config)
{

40
src/data/BackendInterface.cpp
View File

@@ -48,22 +48,6 @@ BackendInterface::writeLedgerObject(std::string&& key, std::uint32_t const seq,
doWriteLedgerObject(std::move(key), seq, std::move(blob));
}
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const
{
while (true)
{
try
{
return hardFetchLedgerRange(yield);
}
catch (DatabaseTimeout& t)
{
;
}
}
}
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRangeNoThrow() const
{
@@ -238,6 +222,30 @@ BackendInterface::fetchBookOffers(
return page;
}
std::optional<LedgerRange>
BackendInterface::hardFetchLedgerRange() const
{
return synchronous([this](auto yield) { return hardFetchLedgerRange(yield); });
}
std::optional<LedgerRange>
BackendInterface::fetchLedgerRange() const
{
std::shared_lock lck(rngMtx_);
return range;
}
void
BackendInterface::updateRange(uint32_t newMax)
{
std::scoped_lock lck(rngMtx_);
assert(!range || newMax >= range->maxSequence);
if (!range)
range = {newMax, newMax};
else
range->maxSequence = newMax;
}
LedgerPage
BackendInterface::fetchLedgerPage(
std::optional<ripple::uint256> const& cursor,

395
src/data/BackendInterface.h
View File

@@ -36,10 +36,7 @@
namespace data {
/**
* @brief Throws an error when database read time limit is exceeded.
*
* This class is throws an error when read time limit is exceeded but
* is also paired with a separate class to retry the connection.
* @brief Represents a database timeout error.
*/
class DatabaseTimeout : public std::exception
{
@@ -52,16 +49,16 @@ public:
};
/**
* @brief Separate class that reattempts connection after time limit.
* @brief A helper function that catches DatabaseTimout exceptions and retries indefinitely.
*
* @tparam F Represents a class of handlers for Cassandra database.
* @param func Instance of Cassandra database handler class.
* @param waitMs Is the arbitrary time limit of 500ms.
* @return auto
* @tparam FnType The type of function object to execute
* @param func The function object to execute
* @param waitMs Delay between retry attempts
* @return auto The same as the return type of func
*/
template <class F>
template <class FnType>
auto
retryOnTimeout(F func, size_t waitMs = 500)
retryOnTimeout(FnType func, size_t waitMs = 500)
{
static util::Logger log{"Backend"};
@@ -71,7 +68,7 @@ retryOnTimeout(F func, size_t waitMs = 500)
{
return func();
}
catch (DatabaseTimeout& t)
catch (DatabaseTimeout const&)
{
log.error() << "Database request timed out. Sleeping and retrying ... ";
std::this_thread::sleep_for(std::chrono::milliseconds(waitMs));
@@ -80,80 +77,64 @@ retryOnTimeout(F func, size_t waitMs = 500)
}
/**
* @brief Passes in serialized handlers in an asynchronous fashion.
* @brief Synchronously executes the given function object inside a coroutine.
*
* Note that the synchronous auto passes handlers critical to supporting
* the Clio backend. The coroutine types are checked if same/different.
*
* @tparam F Represents a class of handlers for Cassandra database.
* @param f R-value instance of Cassandra handler class.
* @return auto
* @tparam FnType The type of function object to execute
* @param func The function object to execute
* @return auto The same as the return type of func
*/
template <class F>
template <class FnType>
auto
synchronous(F&& f)
synchronous(FnType&& func)
{
boost::asio::io_context ctx;
using R = typename boost::result_of<F(boost::asio::yield_context)>::type;
using R = typename boost::result_of<FnType(boost::asio::yield_context)>::type;
if constexpr (!std::is_same<R, void>::value)
{
R res;
boost::asio::spawn(ctx, [&f, &res](boost::asio::yield_context yield) { res = f(yield); });
boost::asio::spawn(ctx, [&func, &res](auto yield) { res = func(yield); });
ctx.run();
return res;
}
else
{
boost::asio::spawn(ctx, [&f](boost::asio::yield_context yield) { f(yield); });
boost::asio::spawn(ctx, [&func](auto yield) { func(yield); });
ctx.run();
}
}
/**
* @brief Reestablishes synchronous connection on timeout.
* @brief Synchronously execute the given function object and retry until no DatabaseTimeout is thrown.
*
* @tparam Represents a class of handlers for Cassandra database.
* @param f R-value instance of Cassandra database handler class.
* @return auto
* @tparam FnType The type of function object to execute
* @param func The function object to execute
* @return auto The same as the return type of func
*/
template <class F>
template <class FnType>
auto
synchronousAndRetryOnTimeout(F&& f)
synchronousAndRetryOnTimeout(FnType&& func)
{
return retryOnTimeout([&]() { return synchronous(f); });
return retryOnTimeout([&]() { return synchronous(func); });
}
/*! @brief Handles ledger and transaction backend data. */
/**
* @brief The interface to the database used by Clio.
*/
class BackendInterface
{
/**
* @brief Shared mutexes and a cache for the interface.
*
* rngMutex is a shared mutex. Shared mutexes prevent shared data
* from being accessed by multiple threads and has two levels of
* access: shared and exclusive.
*/
protected:
mutable std::shared_mutex rngMtx_;
std::optional<LedgerRange> range;
LedgerCache cache_;
/**
* @brief Public read methods
*
* All of these reads methods can throw DatabaseTimeout. When writing
* code in an RPC handler, this exception does not need to be caught:
* when an RPC results in a timeout, an error is returned to the client.
*/
public:
BackendInterface() = default;
virtual ~BackendInterface() = default;
// TODO: Remove this hack. Cache should not be exposed thru BackendInterface
/**
* @brief Cache that holds states of the ledger
* @return Immutable cache
*/
LedgerCache const&
@@ -163,7 +144,6 @@ public:
}
/**
* @brief Cache that holds states of the ledger
* @return Mutable cache
*/
LedgerCache&
@@ -172,62 +152,67 @@ public:
return cache_;
}
/*! @brief Fetches a specific ledger by sequence number. */
/**
* @brief Fetches a specific ledger by sequence number.
*
* @param sequence The sequence number to fetch for
* @param yield The coroutine context
* @return The ripple::LedgerHeader if found; nullopt otherwise
*/
virtual std::optional<ripple::LedgerHeader>
fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const = 0;
/*! @brief Fetches a specific ledger by hash. */
/**
* @brief Fetches a specific ledger by hash.
*
* @param hash The hash to fetch for
* @param yield The coroutine context
* @return The ripple::LedgerHeader if found; nullopt otherwise
*/
virtual std::optional<ripple::LedgerHeader>
fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
/*! @brief Fetches the latest ledger sequence. */
/**
* @brief Fetches the latest ledger sequence.
*
* @param yield The coroutine context
* @return Latest sequence wrapped in an optional if found; nullopt otherwise
*/
virtual std::optional<std::uint32_t>
fetchLatestLedgerSequence(boost::asio::yield_context yield) const = 0;
/*! @brief Fetches the current ledger range while locking that process */
/**
* @brief Fetch the current ledger range.
*
* @return The current ledger range if populated; nullopt otherwise
*/
std::optional<LedgerRange>
fetchLedgerRange() const
{
std::shared_lock lck(rngMtx_);
return range;
}
fetchLedgerRange() const;
/**
* @brief Updates the range of sequences to be tracked.
* @brief Updates the range of sequences that are stored in the DB.
*
* Function that continues updating the range sliding window or creates
* a new sliding window once the maxSequence limit has been reached.
*
* @param newMax Unsigned 32-bit integer representing new max of range.
* @param newMax The new maximum sequence available
*/
void
updateRange(uint32_t newMax)
{
std::scoped_lock lck(rngMtx_);
assert(!range || newMax >= range->maxSequence);
if (!range)
range = {newMax, newMax};
else
range->maxSequence = newMax;
}
updateRange(uint32_t newMax);
/**
* @brief Returns the fees for specific transactions.
* @brief Fetch the fees from a specific ledger sequence.
*
* @param seq Unsigned 32-bit integer reprsenting sequence.
* @param yield The currently executing coroutine.
* @return std::optional<ripple::Fees>
* @param seq The sequence to fetch for
* @param yield The coroutine context
* @return ripple::Fees if fees are found; nullopt otherwise
*/
std::optional<ripple::Fees>
fetchFees(std::uint32_t const seq, boost::asio::yield_context yield) const;
/*! @brief TRANSACTION METHODS */
/**
* @brief Fetches a specific transaction.
*
* @param hash Unsigned 256-bit integer representing hash.
* @param yield The currently executing coroutine.
* @return std::optional<TransactionAndMetadata>
* @param hash The hash of the transaction to fetch
* @param yield The coroutine context
* @return TransactionAndMetadata if transaction is found; nullopt otherwise
*/
virtual std::optional<TransactionAndMetadata>
fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0;
@@ -235,24 +220,22 @@ public:
/**
* @brief Fetches multiple transactions.
*
* @param hashes Unsigned integer value representing a hash.
* @param yield The currently executing coroutine.
* @return std::vector<TransactionAndMetadata>
* @param hashes A vector of hashes to fetch transactions for
* @param yield The coroutine context
* @return A vector of TransactionAndMetadata matching the given hashes
*/
virtual std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes, boost::asio::yield_context yield) const = 0;
/**
* @brief Fetches all transactions for a specific account
* @brief Fetches all transactions for a specific account.
*
* @param account A specific XRPL Account, speciifed by unique type
* accountID.
* @param limit Paging limit for how many transactions can be returned per
* page.
* @param forward Boolean whether paging happens forwards or backwards.
* @param cursor Important metadata returned every time paging occurs.
* @param yield Currently executing coroutine.
* @return TransactionsAndCursor
* @param account The account to fetch transactions for
* @param limit The maximum number of transactions per result page
* @param forward Whether to fetch the page forwards or backwards from the given cursor
* @param cursor The cursor to resume fetching from
* @param yield The coroutine context
* @return Results and a cursor to resume from
*/
virtual TransactionsAndCursor
fetchAccountTransactions(
@@ -265,10 +248,9 @@ public:
/**
* @brief Fetches all transactions from a specific ledger.
*
* @param ledgerSequence Unsigned 32-bit integer for latest total
* transactions.
* @param yield Currently executing coroutine.
* @return std::vector<TransactionAndMetadata>
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return Results as a vector of TransactionAndMetadata
*/
virtual std::vector<TransactionAndMetadata>
fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
@@ -276,21 +258,20 @@ public:
/**
* @brief Fetches all transaction hashes from a specific ledger.
*
* @param ledgerSequence Standard unsigned integer.
* @param yield Currently executing coroutine.
* @return std::vector<ripple::uint256>
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return Hashes as ripple::uint256 in a vector
*/
virtual std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
/*! @brief NFT methods */
/**
* @brief Fetches a specific NFT
* @brief Fetches a specific NFT.
*
* @param tokenID Unsigned 256-bit integer.
* @param ledgerSequence Standard unsigned integer.
* @param yield Currently executing coroutine.
* @return std::optional<NFT>
* @param tokenID The ID of the NFT
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return NFT object on success; nullopt otherwise
*/
virtual std::optional<NFT>
fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
@@ -299,12 +280,12 @@ public:
/**
* @brief Fetches all transactions for a specific NFT.
*
* @param tokenID Unsigned 256-bit integer.
* @param limit Paging limit as to how many transactions return per page.
* @param forward Boolean whether paging happens forwards or backwards.
* @param cursorIn Represents transaction number and ledger sequence.
* @param yield Currently executing coroutine is passed in as input.
* @return TransactionsAndCursor
* @param tokenID The ID of the NFT
* @param limit The maximum number of transactions per result page
* @param forward Whether to fetch the page forwards or backwards from the given cursor
* @param cursorIn The cursor to resume fetching from
* @param yield The coroutine context
* @return Results and a cursor to resume from
*/
virtual TransactionsAndCursor
fetchNFTTransactions(
@@ -314,25 +295,30 @@ public:
std::optional<TransactionsCursor> const& cursorIn,
boost::asio::yield_context yield) const = 0;
/*! @brief STATE DATA METHODS */
/**
* @brief Fetches a specific ledger object: vector of unsigned chars
* @brief Fetches a specific ledger object.
*
* @param key Unsigned 256-bit integer.
* @param sequence Unsigned 32-bit integer.
* @param yield Currently executing coroutine.
* @return std::optional<Blob>
* Currently the real fetch happens in doFetchLedgerObject and fetchLedgerObject attempts to fetch from Cache first
* and only calls out to the real DB if a cache miss ocurred.
*
* @param key The key of the object
* @param sequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return The object as a Blob on success; nullopt otherwise
*/
std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield) const;
/**
* @brief Fetches all ledger objects: a vector of vectors of unsigned chars.
* @brief Fetches all ledger objects by their keys.
*
* @param keys Unsigned 256-bit integer.
* @param sequence Unsigned 32-bit integer.
* @param yield Currently executing coroutine.
* @return std::vector<Blob>
* Currently the real fetch happens in doFetchLedgerObjects and fetchLedgerObjects attempts to fetch from Cache
* first and only calls out to the real DB for each of the keys that was not found in the cache.
*
* @param keys A vector with the keys of the objects to fetch
* @param sequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return A vector of ledger objects as Blobs
*/
std::vector<Blob>
fetchLedgerObjects(
@@ -340,12 +326,26 @@ public:
std::uint32_t const sequence,
boost::asio::yield_context yield) const;
/*! @brief Virtual function version of fetchLedgerObject */
/**
* @brief The database-specific implementation for fetching a ledger object.
*
* @param key The key to fetch for
* @param sequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return The object as a Blob on success; nullopt otherwise
*/
virtual std::optional<Blob>
doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield)
const = 0;
/*! @brief Virtual function version of fetchLedgerObjects */
/**
* @brief The database-specific implementation for fetching ledger objects.
*
* @param keys The keys to fetch for
* @param sequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return A vector of Blobs representing each fetched object
*/
virtual std::vector<Blob>
doFetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
@@ -353,14 +353,11 @@ public:
boost::asio::yield_context yield) const = 0;
/**
* @brief Returns the difference between ledgers: vector of objects
* @brief Returns the difference between ledgers.
*
* Objects are made of a key value, vector of unsigned chars (blob),
* and a boolean detailing whether keys and blob match.
*
* @param ledgerSequence Standard unsigned integer.
* @param yield Currently executing coroutine.
* @return std::vector<LedgerObject>
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return A vector of LedgerObject representing the diff
*/
virtual std::vector<LedgerObject>
fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0;
@@ -368,12 +365,12 @@ public:
/**
* @brief Fetches a page of ledger objects, ordered by key/index.
*
* @param cursor Important metadata returned every time paging occurs.
* @param ledgerSequence Standard unsigned integer.
* @param limit Paging limit as to how many transactions returned per page.
* @param outOfOrder Boolean on whether ledger page is out of order.
* @param yield Currently executing coroutine.
* @return LedgerPage
* @param cursor The cursor to resume fetching from
* @param ledgerSequence The ledger sequence to fetch for
* @param limit The maximum number of transactions per result page
* @param outOfOrder If set to true max available sequence is used instead of ledgerSequence
* @param yield The coroutine context
* @return The ledger page
*/
LedgerPage
fetchLedgerPage(
@@ -383,16 +380,40 @@ public:
bool outOfOrder,
boost::asio::yield_context yield) const;
/*! @brief Fetches successor object from key/index. */
/**
* @brief Fetches the successor object.
*
* @param key The key to fetch for
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return The sucessor on success; nullopt otherwise
*/
std::optional<LedgerObject>
fetchSuccessorObject(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const;
/*! @brief Fetches successor key from key/index. */
/**
* @brief Fetches the successor key.
*
* Thea real fetch happens in doFetchSuccessorKey. This function will attempt to lookup the successor in the cache
* first and only if it's not found in the cache will it fetch from the actual DB.
*
* @param key The key to fetch for
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return The sucessor key on success; nullopt otherwise
*/
std::optional<ripple::uint256>
fetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const;
/*! @brief Virtual function version of fetchSuccessorKey. */
/**
* @brief Database-specific implementation of fetching the successor key
*
* @param key The key to fetch for
* @param ledgerSequence The ledger sequence to fetch for
* @param yield The coroutine context
* @return The sucessor on success; nullopt otherwise
*/
virtual std::optional<ripple::uint256>
doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield)
const = 0;
@@ -401,11 +422,10 @@ public:
* @brief Fetches book offers.
*
* @param book Unsigned 256-bit integer.
* @param ledgerSequence Standard unsigned integer.
* @param ledgerSequence The ledger sequence to fetch for
* @param limit Pagaing limit as to how many transactions returned per page.
* @param cursor Important metadata returned every time paging occurs.
* @param yield Currently executing coroutine.
* @return BookOffersPage
* @param yield The coroutine context
* @return The book offers page
*/
BookOffersPage
fetchBookOffers(
@@ -415,31 +435,30 @@ public:
boost::asio::yield_context yield) const;
/**
* @brief Returns a ledger range
* @brief Synchronously fetches the ledger range from DB.
*
* Ledger range is a struct of min and max sequence numbers). Due to
* the use of [&], which denotes a special case of a lambda expression
* where values found outside the scope are passed by reference, wrt the
* currently executing coroutine.
* This function just wraps hardFetchLedgerRange(boost::asio::yield_context) using synchronous(FnType&&).
*
* @return std::optional<LedgerRange>
* @return The ledger range if available; nullopt otherwise
*/
std::optional<LedgerRange>
hardFetchLedgerRange() const
{
return synchronous([&](boost::asio::yield_context yield) { return hardFetchLedgerRange(yield); });
}
hardFetchLedgerRange() const;
/*! @brief Virtual function equivalent of hardFetchLedgerRange. */
/**
* @brief Fetches the ledger range from DB.
*
* @return The ledger range if available; nullopt otherwise
*/
virtual std::optional<LedgerRange>
hardFetchLedgerRange(boost::asio::yield_context yield) const = 0;
/*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
/**
* @brief Fetches the ledger range from DB retrying until no DatabaseTimeout is thrown.
*
* @return The ledger range if available; nullopt otherwise
*/
std::optional<LedgerRange>
hardFetchLedgerRangeNoThrow() const;
/*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */
std::optional<LedgerRange>
hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const;
/**
* @brief Writes to a specific ledger.
@@ -453,11 +472,9 @@ public:
/**
* @brief Writes a new ledger object.
*
* The key and blob are r-value references and do NOT have memory addresses.
*
* @param key String represented as an r-value.
* @param seq Unsigned integer representing a sequence.
* @param blob r-value vector of unsigned characters (blob).
* @param key The key to write the ledger object under
* @param seq The ledger sequence to write for
* @param blob The data to write
*/
virtual void
writeLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob);
@@ -465,11 +482,11 @@ public:
/**
* @brief Writes a new transaction.
*
* @param hash r-value reference. No memory address.
* @param seq Unsigned 32-bit integer.
* @param date Unsigned 32-bit integer.
* @param transaction r-value reference. No memory address.
* @param metadata r-value refrence. No memory address.
* @param hash The hash of the transaction
* @param seq The ledger sequence to write for
* @param date The timestamp of the entry
* @param transaction The transaction data to write
* @param metadata The metadata to write
*/
virtual void
writeTransaction(
@@ -480,9 +497,9 @@ public:
std::string&& metadata) = 0;
/**
* @brief Write a new NFT.
* @brief Writes NFTs to the database.
*
* @param data Passed in as an r-value reference.
* @param data A vector of NFTsData objects representing the NFTs
*/
virtual void
writeNFTs(std::vector<NFTsData>&& data) = 0;
@@ -490,15 +507,15 @@ public:
/**
* @brief Write a new set of account transactions.
*
* @param data Passed in as an r-value reference.
* @param data A vector of AccountTransactionsData objects representing the account transactions
*/
virtual void
writeAccountTransactions(std::vector<AccountTransactionsData>&& data) = 0;
/**
* @brief Write a new transaction for a specific NFT.
* @brief Write NFTs transactions.
*
* @param data Passed in as an r-value reference.
* @param data A vector of NFTTransactionsData objects
*/
virtual void
writeNFTTransactions(std::vector<NFTTransactionsData>&& data) = 0;
@@ -506,41 +523,39 @@ public:
/**
* @brief Write a new successor.
*
* @param key Passed in as an r-value reference.
* @param seq Unsigned 32-bit integer.
* @param successor Passed in as an r-value reference.
* @param key Key of the object that the passed successor will be the successor for
* @param seq The ledger sequence to write for
* @param successor The successor data to write
*/
virtual void
writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) = 0;
/*! @brief Tells database we will write data for a specific ledger. */
/**
* @brief Starts a write transaction with the DB. No-op for cassandra.
*
* Note: Can potentially be deprecated and removed.
*/
virtual void
startWrites() const = 0;
/**
* @brief Tells database we finished writing all data for a specific ledger.
*
* TODO: change the return value to represent different results:
* Committed, write conflict, errored, successful but not committed
* Uses doFinishWrites to synchronize with the pending writes.
*
* @param ledgerSequence Const unsigned 32-bit integer on ledger sequence.
* @return true
* @return false
* @param ledgerSequence The ledger sequence to finish writing for
* @return true on success; false otherwise
*/
bool
finishWrites(std::uint32_t const ledgerSequence);
/**
* @return true if database is overwhelmed; false otherwise
*/
virtual bool
isTooBusy() const = 0;
private:
/**
* @brief Private helper method to write ledger object
*
* @param key r-value string representing key.
* @param seq Unsigned 32-bit integer representing sequence.
* @param blob r-value vector of unsigned chars.
*/
virtual void
doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) = 0;

18
src/data/CassandraBackend.h
View File

@@ -36,15 +36,14 @@
namespace data::cassandra {
/**
* @brief Implements @ref BackendInterface for Cassandra/Scylladb
* @brief Implements @ref BackendInterface for Cassandra/ScyllaDB.
*
* Note: this is a safer and more correct rewrite of the original implementation
* of the backend. We deliberately did not change the interface for now so that
* other parts such as ETL do not have to change at all.
* Eventually we should change the interface so that it does not have to know
* about yield_context.
* Note: This is a safer and more correct rewrite of the original implementation of the backend.
*
* @tparam SettingsProviderType The settings provider type to use
* @tparam ExecutionStrategyType The execution strategy type to use
*/
template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategy>
template <SomeSettingsProvider SettingsProviderType, SomeExecutionStrategy ExecutionStrategyType>
class BasicCassandraBackend : public BackendInterface
{
util::Logger log_{"Backend"};
@@ -54,7 +53,7 @@ class BasicCassandraBackend : public BackendInterface
Handle handle_;
// have to be mutable because BackendInterface constness :(
mutable ExecutionStrategy executor_;
mutable ExecutionStrategyType executor_;
std::atomic_uint32_t ledgerSequence_ = 0u;
@@ -62,7 +61,8 @@ public:
/**
* @brief Create a new cassandra/scylla backend instance.
*
* @param settingsProvider
* @param settingsProvider The settings provider to use
* @param readOnly Whether the database should be in readonly mode
*/
BasicCassandraBackend(SettingsProviderType settingsProvider, bool readOnly)
: settingsProvider_{std::move(settingsProvider)}

63
src/data/DBHelpers.h
View File

@@ -17,6 +17,7 @@
*/
//==============================================================================
/** @file */
#pragma once
#include <ripple/basics/Log.h>
@@ -30,7 +31,7 @@
#include <data/Types.h>
/**
* @brief Struct used to keep track of what to write to account_transactions/account_tx tables
* @brief Struct used to keep track of what to write to account_transactions/account_tx tables.
*/
struct AccountTransactionsData
{
@@ -51,7 +52,7 @@ struct AccountTransactionsData
};
/**
* @brief Represents a link from a tx to an NFT that was targeted/modified/created by it
* @brief Represents a link from a tx to an NFT that was targeted/modified/created by it.
*
* Gets written to nf_token_transactions table and the like.
*/
@@ -138,6 +139,12 @@ struct NFTsData
}
};
/**
* @brief Check whether the supplied object is an offer.
*
* @param object The object to check
* @return true if the object is an offer; false otherwise
*/
template <class T>
inline bool
isOffer(T const& object)
@@ -146,19 +153,28 @@ isOffer(T const& object)
return offer_bytes == 0x006f;
}
/**
* @brief Check whether the supplied hex represents an offer object.
*
* @param object The object to check
* @return true if the object is an offer; false otherwise
*/
template <class T>
inline bool
isOfferHex(T const& object)
{
auto blob = ripple::strUnHex(4, object.begin(), object.begin() + 4);
if (blob)
{
short offer_bytes = ((*blob)[1] << 8) | (*blob)[2];
return offer_bytes == 0x006f;
}
return isOffer(*blob);
return false;
}
/**
* @brief Check whether the supplied object is a dir node.
*
* @param object The object to check
* @return true if the object is a dir node; false otherwise
*/
template <class T>
inline bool
isDirNode(T const& object)
@@ -167,6 +183,13 @@ isDirNode(T const& object)
return spaceKey == 0x0064;
}
/**
* @brief Check whether the supplied object is a book dir.
*
* @param key The key into the object
* @param object The object to check
* @return true if the object is a book dir; false otherwise
*/
template <class T, class R>
inline bool
isBookDir(T const& key, R const& object)
@@ -178,6 +201,12 @@ isBookDir(T const& key, R const& object)
return !sle[~ripple::sfOwner].has_value();
}
/**
* @brief Get the book out of an offer object.
*
* @param offer The offer to get the book for
* @return Book as ripple::uint256
*/
template <class T>
inline ripple::uint256
getBook(T const& offer)
@@ -185,26 +214,40 @@ getBook(T const& offer)
ripple::SerialIter it{offer.data(), offer.size()};
ripple::SLE sle{it, {}};
ripple::uint256 book = sle.getFieldH256(ripple::sfBookDirectory);
return book;
}
/**
* @brief Get the book base.
*
* @param key The key to get the book base out of
* @return Book base as ripple::uint256
*/
template <class T>
inline ripple::uint256
getBookBase(T const& key)
{
assert(key.size() == ripple::uint256::size());
ripple::uint256 ret;
for (size_t i = 0; i < 24; ++i)
{
ret.data()[i] = key.data()[i];
}
return ret;
}
/**
* @brief Stringify a ripple::uint256.
*
* @param input The input value
* @return The input value as a string
*/
inline std::string
uint256ToString(ripple::uint256 const& uint)
uint256ToString(ripple::uint256 const& input)
{
return {reinterpret_cast<const char*>(uint.data()), uint.size()};
return {reinterpret_cast<const char*>(input.data()), input.size()};
}
/** @brief The ripple epoch start timestamp. Midnight on 1st January 2000. */
static constexpr std::uint32_t rippleEpochStart = 946684800;

64
src/data/LedgerCache.h
View File

@@ -30,6 +30,9 @@
namespace data {
/**
* @brief Cache for an entire ledger.
*/
class LedgerCache
{
struct CacheEntry
@@ -57,40 +60,93 @@ class LedgerCache
std::unordered_set<ripple::uint256, ripple::hardened_hash<>> deletes_;
public:
// Update the cache with new ledger objects set isBackground to true when writing old data from a background thread
/**
* @brief Update the cache with new ledger objects.
*
* @param blobs The ledger objects to update cache with
* @param seq The sequence to update cache for
* @param isBackground Should be set to true when writing old data from a background thread
*/
void
update(std::vector<LedgerObject> const& blobs, uint32_t seq, bool isBackground = false);
/**
* @brief Fetch a cached object by its key and sequence number.
*
* @param key The key to fetch for
* @param seq The sequence to fetch for
* @return If found in cache, will return the cached Blob; otherwise nullopt is returned
*/
std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const;
// always returns empty optional if isFull() is false
/**
* @brief Gets a cached successor.
*
* Note: This function always returns std::nullopt when @ref isFull() returns false.
*
* @param key The key to fetch for
* @param seq The sequence to fetch for
* @return If found in cache, will return the cached successor; otherwise nullopt is returned
*/
std::optional<LedgerObject>
getSuccessor(ripple::uint256 const& key, uint32_t seq) const;
// always returns empty optional if isFull() is false
/**
* @brief Gets a cached predcessor.
*
* Note: This function always returns std::nullopt when @ref isFull() returns false.
*
* @param key The key to fetch for
* @param seq The sequence to fetch for
* @return If found in cache, will return the cached predcessor; otherwise nullopt is returned
*/
std::optional<LedgerObject>
getPredecessor(ripple::uint256 const& key, uint32_t seq) const;
/**
* @brief Disables the cache.
*/
void
setDisabled();
/**
* @brief Sets the full flag to true.
*
* This is used when cache loaded in its entirety at startup of the application. This can be either loaded from DB,
* populated together with initial ledger download (on first run) or downloaded from a peer node (specified in
* config).
*/
void
setFull();
/**
* @return The latest ledger sequence for which cache is available.
*/
uint32_t
latestLedgerSequence() const;
// whether the cache has all data for the most recent ledger
/**
* @return true if the cache has all data for the most recent ledger; false otherwise
*/
bool
isFull() const;
/**
* @return The total size of the cache.
*/
size_t
size() const;
/**
* @return A number representing the success rate of hitting an object in the cache versus missing it.
*/
float
getObjectHitRate() const;
/**
* @return A number representing the success rate of hitting a successor in the cache versus missing it.
*/
float
getSuccessorHitRate() const;
};

3
src/data/README.md
View File

@@ -1,4 +1,5 @@
# Clio Backend
# Backend
## Background
The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. As of right now, Cassandra and ScyllaDB are the only supported databases that are production-ready. Support for database types can be easily extended by creating new implementations which implements the virtual methods of `BackendInterface.h`. Then, use the Factory Object Design Pattern to simply add logic statements to `BackendFactory.h` that return the new database interface for a specific `type` in Clio's configuration file.

37
src/data/Types.h
View File

@@ -21,20 +21,23 @@
#include <ripple/basics/base_uint.h>
#include <ripple/protocol/AccountID.h>
#include <optional>
#include <string>
#include <vector>
namespace data {
// *** return types
using Blob = std::vector<unsigned char>;
/**
* @brief Represents an object in the ledger.
*/
struct LedgerObject
{
ripple::uint256 key;
Blob blob;
bool
operator==(const LedgerObject& other) const
{
@@ -42,16 +45,27 @@ struct LedgerObject
}
};
/**
* @brief Represents a page of LedgerObjects.
*/
struct LedgerPage
{
std::vector<LedgerObject> objects;
std::optional<ripple::uint256> cursor;
};
/**
* @brief Represents a page of book offer objects.
*/
struct BookOffersPage
{
std::vector<LedgerObject> offers;
std::optional<ripple::uint256> cursor;
};
/**
* @brief Represents a transaction and its metadata bundled together.
*/
struct TransactionAndMetadata
{
Blob transaction;
@@ -85,6 +99,9 @@ struct TransactionAndMetadata
}
};
/**
* @brief Represents a cursor into the transactions table.
*/
struct TransactionsCursor
{
std::uint32_t ledgerSequence = 0;
@@ -114,12 +131,18 @@ struct TransactionsCursor
}
};
/**
* @brief Represests a bundle of transactions with metadata and a cursor to the next page.
*/
struct TransactionsAndCursor
{
std::vector<TransactionAndMetadata> txns;
std::optional<TransactionsCursor> cursor;
};
/**
* @brief Represents a NFToken.
*/
struct NFT
{
ripple::uint256 tokenID;
@@ -143,9 +166,8 @@ struct NFT
{
}
// clearly two tokens are the same if they have the same ID, but this
// struct stores the state of a given token at a given ledger sequence, so
// we also need to compare with ledgerSequence
// clearly two tokens are the same if they have the same ID, but this struct stores the state of a given token at a
// given ledger sequence, so we also need to compare with ledgerSequence.
bool
operator==(NFT const& other) const
{
@@ -153,12 +175,17 @@ struct NFT
}
};
/**
* @brief Stores a range of sequences as a min and max pair.
*/
struct LedgerRange
{
std::uint32_t minSequence = 0;
std::uint32_t maxSequence = 0;
};
constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"};
constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"};
constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"};
} // namespace data

9
src/data/cassandra/Concepts.h
View File

@@ -30,6 +30,9 @@
namespace data::cassandra {
/**
* @brief The requirements of a settings provider.
*/
// clang-format off
template <typename T>
concept SomeSettingsProvider = requires(T a) {
@@ -41,6 +44,9 @@ concept SomeSettingsProvider = requires(T a) {
};
// clang-format on
/**
* @brief The requirements of an execution strategy.
*/
// clang-format off
template <typename T>
concept SomeExecutionStrategy = requires(
@@ -66,6 +72,9 @@ concept SomeExecutionStrategy = requires(
};
// clang-format on
/**
* @brief The requirements of a retry policy.
*/
// clang-format off
template <typename T>
concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) {

14
src/data/cassandra/Error.h
View File

@@ -26,7 +26,7 @@
namespace data::cassandra {
/**
* @brief A simple container for both error message and error code
* @brief A simple container for both error message and error code.
*/
class CassandraError
{
@@ -67,18 +67,27 @@ public:
return os;
}
/**
* @return The final error message as a std::string
*/
std::string
message() const
{
return message_;
}
/**
* @return The error code
*/
uint32_t
code() const
{
return code_;
}
/**
* @return true if the wrapped error is considered a timeout; false otherwise
*/
bool
isTimeout() const
{
@@ -89,6 +98,9 @@ public:
return false;
}
/**
* @return true if the wrapped error is an invalid query; false otherwise
*/
bool
isInvalidQuery() const
{

131
src/data/cassandra/Handle.h
View File

@@ -57,28 +57,31 @@ public:
using ResultType = Result;
/**
* @brief Construct a new handle from a @ref Settings object
* @brief Construct a new handle from a @ref detail::Settings object.
*
* @param clusterSettings The settings to use
*/
explicit Handle(Settings clusterSettings = Settings::defaultSettings());
/**
* @brief Construct a new handle with default settings and only by setting
* the contact points
* @brief Construct a new handle with default settings and only by setting the contact points.
*
* @param contactPoints The contact points to use instead of settings
*/
explicit Handle(std::string_view contactPoints);
/**
* @brief Disconnects gracefully if possible
* @brief Disconnects gracefully if possible.
*/
~Handle();
/**
* @brief Move is supported
* @brief Move is supported.
*/
Handle(Handle&&) = default;
/**
* @brief Connect to the cluster asynchronously
* @brief Connect to the cluster asynchronously.
*
* @return A future
*/
@@ -86,31 +89,37 @@ public:
asyncConnect() const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncConnect() const for how this works.
*
* @return Possibly an error
*/
[[nodiscard]] MaybeErrorType
connect() const;
/**
* @brief Connect to the the specified keyspace asynchronously
* @brief Connect to the the specified keyspace asynchronously.
*
* @param keyspace The keyspace to use
* @return A future
*/
[[nodiscard]] FutureType
asyncConnect(std::string_view keyspace) const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncConnect(std::string_view) const for how this works.
*
* @param keyspace The keyspace to use
* @return Possibly an error
*/
[[nodiscard]] MaybeErrorType
connect(std::string_view keyspace) const;
/**
* @brief Disconnect from the cluster asynchronously
* @brief Disconnect from the cluster asynchronously.
*
* @return A future
*/
@@ -118,32 +127,40 @@ public:
asyncDisconnect() const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncDisconnect() const for how this works.
*
* @return Possibly an error
*/
[[maybe_unused]] MaybeErrorType
disconnect() const;
/**
* @brief Reconnect to the the specified keyspace asynchronously
* @brief Reconnect to the the specified keyspace asynchronously.
*
* @param keyspace The keyspace to use
* @return A future
*/
[[nodiscard]] FutureType
asyncReconnect(std::string_view keyspace) const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncReconnect(std::string_view) const for how this works.
*
* @param keyspace The keyspace to use
* @return Possibly an error
*/
[[nodiscard]] MaybeErrorType
reconnect(std::string_view keyspace) const;
/**
* @brief Execute a simple query with optional args asynchronously
* @brief Execute a simple query with optional args asynchronously.
*
* @param query The query to execute
* @param args The arguments to bind for execution
* @return A future
*/
template <typename... Args>
@@ -155,10 +172,13 @@ public:
}
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncExecute(std::string_view, Args&&...) const for how this
* works.
* See asyncExecute(std::string_view, Args&&...) const for how this works.
*
* @param query The query to execute
* @param args The arguments to bind for execution
* @return The result or an error
*/
template <typename... Args>
[[maybe_unused]] ResultOrErrorType
@@ -168,30 +188,34 @@ public:
}
/**
* @brief Execute each of the statements asynchronously
* @brief Execute each of the statements asynchronously.
*
* Batched version is not always the right option. Especially since it only
* supports INSERT, UPDATE and DELETE statements.
* This can be used as an alternative when statements need to execute in
* bulk.
* Batched version is not always the right option.
* Especially since it only supports INSERT, UPDATE and DELETE statements.
* This can be used as an alternative when statements need to execute in bulk.
*
* @param statements The statements to execute
* @return A vector of future objects
*/
[[nodiscard]] std::vector<FutureType>
asyncExecuteEach(std::vector<StatementType> const& statements) const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncExecuteEach(std::vector<StatementType> const&) const for
* how this works.
* See @ref asyncExecuteEach(std::vector<StatementType> const&) const for how this works.
*
* @param statements The statements to execute
* @return Possibly an error
*/
[[maybe_unused]] MaybeErrorType
executeEach(std::vector<StatementType> const& statements) const;
/**
* @brief Execute a prepared statement with optional args asynchronously
* @brief Execute a prepared statement with optional args asynchronously.
*
* @param statement The prepared statement to execute
* @param args The arguments to bind for execution
* @return A future
*/
template <typename... Args>
@@ -203,10 +227,13 @@ public:
}
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncExecute(std::vector<StatementType> const&, Args&&...) const
* for how this works.
* See asyncExecute(std::vector<StatementType> const&, Args&&...) const for how this works.
*
* @param statement The prepared statement to bind and execute
* @param args The arguments to bind for execution
* @return The result or an error
*/
template <typename... Args>
[[maybe_unused]] ResultOrErrorType
@@ -216,61 +243,70 @@ public:
}
/**
* @brief Execute one (bound or simple) statements asynchronously
* @brief Execute one (bound or simple) statements asynchronously.
*
* @param statement The statement to execute
* @return A future
*/
[[nodiscard]] FutureType
asyncExecute(StatementType const& statement) const;
/**
* @brief Execute one (bound or simple) statements asynchronously with a
* callback
* @brief Execute one (bound or simple) statements asynchronously with a callback.
*
* @param statement The statement to execute
* @param cb The callback to execute when data is ready
* @return A future that holds onto the callback provided
*/
[[nodiscard]] FutureWithCallbackType
asyncExecute(StatementType const& statement, std::function<void(ResultOrErrorType)>&& cb) const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncExecute(StatementType const&) const for how this
* works.
* See @ref asyncExecute(StatementType const&) const for how this works.
*
* @param statement The statement to execute
* @return The result or an error
*/
[[maybe_unused]] ResultOrErrorType
execute(StatementType const& statement) const;
/**
* @brief Execute a batch of (bound or simple) statements asynchronously
* @brief Execute a batch of (bound or simple) statements asynchronously.
*
* @param statements The statements to execute
* @return A future
*/
[[nodiscard]] FutureType
asyncExecute(std::vector<StatementType> const& statements) const;
/**
* @brief Synchonous version of the above
* @brief Synchonous version of the above.
*
* See @ref asyncExecute(std::vector<StatementType> const&) const for how
* this works.
* See @ref asyncExecute(std::vector<StatementType> const&) const for how this works.
*
* @param statements The statements to execute
* @return Possibly an error
*/
[[maybe_unused]] MaybeErrorType
execute(std::vector<StatementType> const& statements) const;
/**
* @brief Execute a batch of (bound or simple) statements asynchronously
* with a completion callback
* @brief Execute a batch of (bound or simple) statements asynchronously with a completion callback.
*
* @param statements The statements to execute
* @param cb The callback to execute when data is ready
* @return A future that holds onto the callback provided
*/
[[nodiscard]] FutureWithCallbackType
asyncExecute(std::vector<StatementType> const& statements, std::function<void(ResultOrErrorType)>&& cb) const;
/**
* @brief Prepare a statement
* @brief Prepare a statement.
*
* @return A @ref PreparedStatementType
* @param query
* @return A prepared statement
* @throws std::runtime_error with underlying error description on failure
*/
[[nodiscard]] PreparedStatementType
@@ -278,12 +314,13 @@ public:
};
/**
* @brief Extracts the results into series of std::tuple<Types...> by creating a
* simple wrapper with an STL input iterator inside.
* @brief Extracts the results into series of std::tuple<Types...> by creating a simple wrapper with an STL input
* iterator inside.
*
* You can call .begin() and .end() in order to iterate as usual.
* This also means that you can use it in a range-based for or with some
* algorithms.
* This also means that you can use it in a range-based for or with some algorithms.
*
* @param result The result to iterate
*/
template <typename... Types>
[[nodiscard]] detail::ResultExtractor<Types...>

13
src/data/cassandra/Schema.h
View File

@@ -38,16 +38,11 @@ template <SomeSettingsProvider SettingsProviderType>
}
/**
* @brief Manages the DB schema and provides access to prepared statements
* @brief Manages the DB schema and provides access to prepared statements.
*/
template <SomeSettingsProvider SettingsProviderType>
class Schema
{
// Current schema version.
// Update this everytime you update the schema.
// Migrations will be ran automatically based on this value.
static constexpr uint16_t version = 1u;
util::Logger log_{"Backend"};
std::reference_wrapper<SettingsProviderType const> settingsProvider_;
@@ -261,7 +256,7 @@ public:
}();
/**
* @brief Prepared statements holder
* @brief Prepared statements holder.
*/
class Statements
{
@@ -641,7 +636,7 @@ public:
};
/**
* @brief Recreates the prepared statements
* @brief Recreates the prepared statements.
*/
void
prepareStatements(Handle const& handle)
@@ -652,7 +647,7 @@ public:
}
/**
* @brief Provides access to statements
* @brief Provides access to statements.
*/
std::unique_ptr<Statements> const&
operator->() const

28
src/data/cassandra/SettingsProvider.h
View File

@@ -28,7 +28,7 @@
namespace data::cassandra {
/**
* @brief Provides settings for @ref CassandraBackend
* @brief Provides settings for @ref BasicCassandraBackend.
*/
class SettingsProvider
{
@@ -41,34 +41,50 @@ class SettingsProvider
Settings settings_;
public:
/**
* @brief Create a settings provider from the specified config.
*
* @param cfg The config of Clio to use
* @param ttl Time to live setting
*/
explicit SettingsProvider(util::Config const& cfg, uint16_t ttl = 0);
/*! Get the cluster settings */
/**
* @return The cluster settings
*/
[[nodiscard]] Settings
getSettings() const;
/*! Get the specified keyspace */
/**
* @return The specified keyspace
*/
[[nodiscard]] inline std::string
getKeyspace() const
{
return keyspace_;
}
/*! Get an optional table prefix to use in all queries */
/**
* @return The optional table prefix to use in all queries
*/
[[nodiscard]] inline std::optional<std::string>
getTablePrefix() const
{
return tablePrefix_;
}
/*! Get the replication factor */
/**
* @return The replication factor
*/
[[nodiscard]] inline uint16_t
getReplicationFactor() const
{
return replicationFactor_;
}
/*! Get the default time to live to use in all `create` queries */
/**
* @return The default time to live to use in all `create` queries
*/
[[nodiscard]] inline uint16_t
getTtl() const
{

3
src/data/cassandra/impl/Batch.cpp
View File

@@ -31,8 +31,7 @@ static constexpr auto batchDeleter = [](CassBatch* ptr) { cass_batch_free(ptr);
namespace data::cassandra::detail {
// todo: use an appropritae value instead of CASS_BATCH_TYPE_LOGGED for
// different use cases
// TODO: Use an appropriate value instead of CASS_BATCH_TYPE_LOGGED for different use cases
Batch::Batch(std::vector<Statement> const& statements)
: ManagedObject{cass_batch_new(CASS_BATCH_TYPE_LOGGED), batchDeleter}
{

58
src/data/cassandra/impl/Cluster.h
View File

@@ -33,42 +33,97 @@
namespace data::cassandra::detail {
// TODO: move Settings to public interface, not detail
/**
* @brief Bundles all cassandra settings in one place.
*/
struct Settings
{
/**
* @brief Represents the configuration of contact points for cassandra.
*/
struct ContactPoints
{
std::string contactPoints = "127.0.0.1"; // defaults to localhost
std::optional<uint16_t> port;
};
/**
* @brief Represents the configuration of a secure connection bundle.
*/
struct SecureConnectionBundle
{
std::string bundle; // no meaningful default
};
/** @brief Enables or disables cassandra driver logger */
bool enableLog = false;
/** @brief Connect timeout specified in milliseconds */
std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{10000};
/** @brief Request timeout specified in milliseconds */
std::chrono::milliseconds requestTimeout = std::chrono::milliseconds{0}; // no timeout at all
/** @brief Connection information; either ContactPoints or SecureConnectionBundle */
std::variant<ContactPoints, SecureConnectionBundle> connectionInfo = ContactPoints{};
/** @brief The number of threads for the driver to pool */
uint32_t threads = std::thread::hardware_concurrency();
/** @brief The maximum number of outstanding write requests at any given moment */
uint32_t maxWriteRequestsOutstanding = 10'000;
/** @brief The maximum number of outstanding read requests at any given moment */
uint32_t maxReadRequestsOutstanding = 100'000;
/** @brief The maximum number of connections per host */
uint32_t maxConnectionsPerHost = 2u;
/** @brief The number of connection per host to always have active */
uint32_t coreConnectionsPerHost = 2u;
/** @brief The maximum concurrent requests per connection; new connections will be created when reached */
uint32_t maxConcurrentRequestsThreshold =
(maxWriteRequestsOutstanding + maxReadRequestsOutstanding) / coreConnectionsPerHost;
/** @brief Size of the event queue */
std::optional<uint32_t> queueSizeEvent;
/** @brief Size of the IO queue */
std::optional<uint32_t> queueSizeIO;
/** @brief High watermark for bytes written */
std::optional<uint32_t> writeBytesHighWatermark;
/** @brief Low watermark for bytes written */
std::optional<uint32_t> writeBytesLowWatermark;
/** @brief High watermark for pending requests */
std::optional<uint32_t> pendingRequestsHighWatermark;
/** @brief Low watermark for pending requests */
std::optional<uint32_t> pendingRequestsLowWatermark;
/** @brief Maximum number of requests per flush */
std::optional<uint32_t> maxRequestsPerFlush;
/** @brief Maximum number of connections that will be created concurrently */
std::optional<uint32_t> maxConcurrentCreation;
/** @brief SSL certificate */
std::optional<std::string> certificate; // ssl context
/** @brief Username/login */
std::optional<std::string> username;
/** @brief Password to match the `username` */
std::optional<std::string> password;
/**
* @brief Creates a new Settings object as a copy of the current one with overridden contact points.
*/
Settings
withContactPoints(std::string_view contactPoints)
{
@@ -77,6 +132,9 @@ struct Settings
return tmp;
}
/**
* @brief Returns the default settings.
*/
static Settings
defaultSettings()
{

30
src/data/cassandra/impl/ExecutionStrategy.h
View File

@@ -38,12 +38,13 @@
namespace data::cassandra::detail {
// TODO: this could probably be also moved out of detail and into the main cassandra namespace.
/**
* @brief Implements async and sync querying against the cassandra DB with
* support for throttling.
* @brief Implements async and sync querying against the cassandra DB with support for throttling.
*
* Note: A lot of the code that uses yield is repeated below. This is ok for now
* because we are hopefully going to be getting rid of it entirely later on.
* Note: A lot of the code that uses yield is repeated below.
* This is ok for now because we are hopefully going to be getting rid of it entirely later on.
*/
template <typename HandleType = Handle>
class DefaultExecutionStrategy
@@ -77,6 +78,10 @@ public:
using ResultType = typename HandleType::ResultType;
using CompletionTokenType = boost::asio::yield_context;
/**
* @param settings The settings to use
* @param handle A handle to the cassandra database
*/
DefaultExecutionStrategy(Settings const& settings, HandleType const& handle)
: maxWriteRequestsOutstanding_{settings.maxWriteRequestsOutstanding}
, maxReadRequestsOutstanding_{settings.maxReadRequestsOutstanding}
@@ -96,7 +101,7 @@ public:
}
/**
* @brief Wait for all async writes to finish before unblocking
* @brief Wait for all async writes to finish before unblocking.
*/
void
sync()
@@ -107,6 +112,9 @@ public:
log_.debug() << "Sync done.";
}
/**
* @return true if outstanding read requests allowance is exhausted; false otherwise
*/
bool
isTooBusy() const
{
@@ -114,7 +122,7 @@ public:
}
/**
* @brief Blocking query execution used for writing data
* @brief Blocking query execution used for writing data.
*
* Retries forever sleeping for 5 milliseconds between attempts.
*/
@@ -136,7 +144,7 @@ public:
}
/**
* @brief Blocking query execution used for writing data
* @brief Blocking query execution used for writing data.
*
* Retries forever sleeping for 5 milliseconds between attempts.
*/
@@ -148,11 +156,11 @@ public:
}
/**
* @brief Non-blocking query execution used for writing data
* @brief Non-blocking query execution used for writing data.
*
* Retries forever with retry policy specified by @ref AsyncExecutor
*
* @param prepradeStatement Statement to prepare and execute
* @param preparedStatement Statement to prepare and execute
* @param args Args to bind to the prepared statement
* @throw DatabaseTimeout on timeout
*/
@@ -169,7 +177,7 @@ public:
}
/**
* @brief Non-blocking batched query execution used for writing data
* @brief Non-blocking batched query execution used for writing data.
*
* Retries forever with retry policy specified by @ref AsyncExecutor.
*
@@ -195,7 +203,7 @@ public:
* Retries forever until successful or throws an exception on timeout.
*
* @param token Completion token (yield_context)
* @param prepradeStatement Statement to prepare and execute
* @param preparedStatement Statement to prepare and execute
* @param args Args to bind to the prepared statement
* @throw DatabaseTimeout on timeout
* @return ResultType or error wrapped in Expected

7
src/data/cassandra/impl/Future.cpp
View File

@@ -73,7 +73,7 @@ void
invokeHelper(CassFuture* ptr, void* cbPtr)
{
// Note: can't use Future{ptr}.get() because double free will occur :/
auto* cb = static_cast<FutureWithCallback::fn_t*>(cbPtr);
auto* cb = static_cast<FutureWithCallback::FnType*>(cbPtr);
if (auto const rc = cass_future_error_code(ptr); rc)
{
auto const errMsg = [&ptr](std::string const& label) {
@@ -90,9 +90,8 @@ invokeHelper(CassFuture* ptr, void* cbPtr)
}
}
// TODO: cb_ can be deleted before cassandra-driver calls it if the user fails to hold onto the future object
/* implicit */ FutureWithCallback::FutureWithCallback(CassFuture* ptr, fn_t&& cb)
: Future{ptr}, cb_{std::make_unique<fn_t>(std::move(cb))}
/* implicit */ FutureWithCallback::FutureWithCallback(CassFuture* ptr, FnType&& cb)
: Future{ptr}, cb_{std::make_unique<FnType>(std::move(cb))}
{
// Instead of passing `this` as the userdata void*, we pass the address of
// the callback itself which will survive std::move of the

10
src/data/cassandra/impl/Future.h
View File

@@ -43,16 +43,16 @@ invokeHelper(CassFuture* ptr, void* self);
class FutureWithCallback : public Future
{
public:
using fn_t = std::function<void(ResultOrError)>;
using fn_ptr_t = std::unique_ptr<fn_t>;
using FnType = std::function<void(ResultOrError)>;
using FnPtrType = std::unique_ptr<FnType>;
/* implicit */ FutureWithCallback(CassFuture* ptr, fn_t&& cb);
/* implicit */ FutureWithCallback(CassFuture* ptr, FnType&& cb);
FutureWithCallback(FutureWithCallback const&) = delete;
FutureWithCallback(FutureWithCallback&&) = default;
private:
/*! Wrapped in a unique_ptr so it can survive std::move :/ */
fn_ptr_t cb_;
/** Wrapped in a unique_ptr so it can survive std::move :/ */
FnPtrType cb_;
};
} // namespace data::cassandra::detail

26
src/data/cassandra/impl/Result.h
View File

@@ -51,11 +51,11 @@ extractColumn(CassRow const* row, std::size_t idx)
}
};
using decayed_t = std::decay_t<Type>;
using uint_tuple_t = std::tuple<uint32_t, uint32_t>;
using uchar_vector_t = std::vector<unsigned char>;
using DecayedType = std::decay_t<Type>;
using UintTupleType = std::tuple<uint32_t, uint32_t>;
using UCharVectorType = std::vector<unsigned char>;
if constexpr (std::is_same_v<decayed_t, ripple::uint256>)
if constexpr (std::is_same_v<DecayedType, ripple::uint256>)
{
cass_byte_t const* buf;
std::size_t bufSize;
@@ -63,7 +63,7 @@ extractColumn(CassRow const* row, std::size_t idx)
throwErrorIfNeeded(rc, "Extract ripple::uint256");
output = ripple::uint256::fromVoid(buf);
}
else if constexpr (std::is_same_v<decayed_t, ripple::AccountID>)
else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>)
{
cass_byte_t const* buf;
std::size_t bufSize;
@@ -71,20 +71,20 @@ extractColumn(CassRow const* row, std::size_t idx)
throwErrorIfNeeded(rc, "Extract ripple::AccountID");
output = ripple::AccountID::fromVoid(buf);
}
else if constexpr (std::is_same_v<decayed_t, uchar_vector_t>)
else if constexpr (std::is_same_v<DecayedType, UCharVectorType>)
{
cass_byte_t const* buf;
std::size_t bufSize;
auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize);
throwErrorIfNeeded(rc, "Extract vector<unsigned char>");
output = uchar_vector_t{buf, buf + bufSize};
output = UCharVectorType{buf, buf + bufSize};
}
else if constexpr (std::is_same_v<decayed_t, uint_tuple_t>)
else if constexpr (std::is_same_v<DecayedType, UintTupleType>)
{
auto const* tuple = cass_row_get_column(row, idx);
output = TupleIterator::fromTuple(tuple).extract<uint32_t, uint32_t>();
}
else if constexpr (std::is_convertible_v<decayed_t, std::string>)
else if constexpr (std::is_convertible_v<DecayedType, std::string>)
{
char const* value;
std::size_t len;
@@ -92,7 +92,7 @@ extractColumn(CassRow const* row, std::size_t idx)
throwErrorIfNeeded(rc, "Extract string");
output = std::string{value, len};
}
else if constexpr (std::is_same_v<decayed_t, bool>)
else if constexpr (std::is_same_v<DecayedType, bool>)
{
cass_bool_t flag;
auto const rc = cass_value_get_bool(cass_row_get_column(row, idx), &flag);
@@ -100,17 +100,17 @@ extractColumn(CassRow const* row, std::size_t idx)
output = flag ? true : false;
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<decayed_t, int64_t>)
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
int64_t out;
auto const rc = cass_value_get_int64(cass_row_get_column(row, idx), &out);
throwErrorIfNeeded(rc, "Extract int64");
output = static_cast<decayed_t>(out);
output = static_cast<DecayedType>(out);
}
else
{
// type not supported for extraction
static_assert(unsupported_v<decayed_t>);
static_assert(unsupported_v<DecayedType>);
}
return output;

48
src/data/cassandra/impl/Statement.h
View File

@@ -44,7 +44,7 @@ class Statement : public ManagedObject<CassStatement>
public:
/**
* @brief Construct a new statement with optionally provided arguments
* @brief Construct a new statement with optionally provided arguments.
*
* Note: it's up to the user to make sure the bound parameters match
* the format of the query (e.g. amount of '?' matches count of args).
@@ -66,6 +66,11 @@ public:
Statement(Statement&&) = default;
/**
* @brief Binds the given arguments to the statement.
*
* @param args Arguments to bind
*/
template <typename... Args>
void
bind(Args&&... args) const
@@ -74,6 +79,12 @@ public:
(this->bindAt<Args>(idx++, std::forward<Args>(args)), ...);
}
/**
* @brief Binds an argument to a specific index.
*
* @param idx The index of the argument
* @param value The value to bind it to
*/
template <typename Type>
void
bindAt(std::size_t const idx, Type&& value) const
@@ -88,48 +99,48 @@ public:
return cass_statement_bind_bytes(*this, idx, static_cast<cass_byte_t const*>(data), size);
};
using decayed_t = std::decay_t<Type>;
using uchar_vec_t = std::vector<unsigned char>;
using uint_tuple_t = std::tuple<uint32_t, uint32_t>;
using DecayedType = std::decay_t<Type>;
using UCharVectorType = std::vector<unsigned char>;
using UintTupleType = std::tuple<uint32_t, uint32_t>;
if constexpr (std::is_same_v<decayed_t, ripple::uint256>)
if constexpr (std::is_same_v<DecayedType, ripple::uint256>)
{
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind ripple::uint256");
}
else if constexpr (std::is_same_v<decayed_t, ripple::AccountID>)
else if constexpr (std::is_same_v<DecayedType, ripple::AccountID>)
{
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind ripple::AccountID");
}
else if constexpr (std::is_same_v<decayed_t, uchar_vec_t>)
else if constexpr (std::is_same_v<DecayedType, UCharVectorType>)
{
auto const rc = bindBytes(value.data(), value.size());
throwErrorIfNeeded(rc, "Bind vector<unsigned char>");
}
else if constexpr (std::is_convertible_v<decayed_t, std::string>)
else if constexpr (std::is_convertible_v<DecayedType, std::string>)
{
// reinterpret_cast is needed here :'(
auto const rc = bindBytes(reinterpret_cast<unsigned char const*>(value.data()), value.size());
throwErrorIfNeeded(rc, "Bind string (as bytes)");
}
else if constexpr (std::is_same_v<decayed_t, uint_tuple_t>)
else if constexpr (std::is_same_v<DecayedType, UintTupleType>)
{
auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::move(value)});
throwErrorIfNeeded(rc, "Bind tuple<uint32, uint32>");
}
else if constexpr (std::is_same_v<decayed_t, bool>)
else if constexpr (std::is_same_v<DecayedType, bool>)
{
auto const rc = cass_statement_bind_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
else if constexpr (std::is_same_v<decayed_t, Limit>)
else if constexpr (std::is_same_v<DecayedType, Limit>)
{
auto const rc = cass_statement_bind_int32(*this, idx, value.limit);
throwErrorIfNeeded(rc, "Bind limit (int32)");
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<decayed_t, int64_t>)
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
auto const rc = cass_statement_bind_int64(*this, idx, value);
throwErrorIfNeeded(rc, "Bind int64");
@@ -137,11 +148,16 @@ public:
else
{
// type not supported for binding
static_assert(unsupported_v<decayed_t>);
static_assert(unsupported_v<DecayedType>);
}
}
};
/**
* @brief Represents a prepared statement on the DB side.
*
* This is used to produce Statement objects that can be executed.
*/
class PreparedStatement : public ManagedObject<CassPrepared const>
{
static constexpr auto deleter = [](CassPrepared const* ptr) { cass_prepared_free(ptr); };
@@ -151,6 +167,12 @@ public:
{
}
/**
* @brief Bind the given arguments and produce a ready to execute Statement.
*
* @param args The arguments to bind
* @return A bound and ready to execute Statement object
*/
template <typename... Args>
Statement
bind(Args&&... args) const

16
src/data/cassandra/impl/Tuple.h
View File

@@ -68,15 +68,15 @@ public:
}
};
using decayed_t = std::decay_t<Type>;
using DecayedType = std::decay_t<Type>;
if constexpr (std::is_same_v<decayed_t, bool>)
if constexpr (std::is_same_v<DecayedType, bool>)
{
auto const rc = cass_tuple_set_bool(*this, idx, value ? cass_true : cass_false);
throwErrorIfNeeded(rc, "Bind bool");
}
// clio only uses bigint (int64_t) so we convert any incoming type
else if constexpr (std::is_convertible_v<decayed_t, int64_t>)
else if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
auto const rc = cass_tuple_set_int64(*this, idx, value);
throwErrorIfNeeded(rc, "Bind int64");
@@ -84,7 +84,7 @@ public:
else
{
// type not supported for binding
static_assert(unsupported_v<decayed_t>);
static_assert(unsupported_v<DecayedType>);
}
}
};
@@ -126,20 +126,20 @@ private:
}
};
using decayed_t = std::decay_t<Type>;
using DecayedType = std::decay_t<Type>;
// clio only uses bigint (int64_t) so we convert any incoming type
if constexpr (std::is_convertible_v<decayed_t, int64_t>)
if constexpr (std::is_convertible_v<DecayedType, int64_t>)
{
int64_t out;
auto const rc = cass_value_get_int64(cass_iterator_get_value(*this), &out);
throwErrorIfNeeded(rc, "Extract int64 from tuple");
output = static_cast<decayed_t>(out);
output = static_cast<DecayedType>(out);
}
else
{
// type not supported for extraction
static_assert(unsupported_v<decayed_t>);
static_assert(unsupported_v<DecayedType>);
}
return output;

42
src/etl/ETLHelpers.h
View File

@@ -17,6 +17,7 @@
*/
//==============================================================================
/** @file */
#pragma once
#include <ripple/basics/base_uint.h>
@@ -44,6 +45,9 @@ class NetworkValidatedLedgers
std::condition_variable cv_;
public:
/**
* @brief A factory function for NetworkValidatedLedgers.
*/
static std::shared_ptr<NetworkValidatedLedgers>
make_ValidatedLedgers()
{
@@ -51,9 +55,9 @@ public:
}
/**
* @brief Notify the datastructure that idx has been validated by the network
* @brief Notify the datastructure that idx has been validated by the network.
*
* @param idx sequence validated by network
* @param idx Sequence validated by network
*/
void
push(uint32_t idx)
@@ -69,7 +73,7 @@ public:
*
* If no ledgers are known to have been validated, this function waits until the next ledger is validated
*
* @return sequence of most recently validated ledger. empty optional if the datastructure has been stopped
* @return Sequence of most recently validated ledger. empty optional if the datastructure has been stopped
*/
std::optional<uint32_t>
getMostRecent()
@@ -80,9 +84,9 @@ public:
}
/**
* @brief Waits for the sequence to be validated by the network
* @brief Waits for the sequence to be validated by the network.
*
* @param sequence to wait for
* @param sequence The sequence to wait for
* @return true if sequence was validated, false otherwise a return value of false means the datastructure has been
* stopped
*/
@@ -101,7 +105,7 @@ public:
// TODO: does the note make sense? lockfree queues provide the same blocking behaviour just without mutex, don't they?
/**
* @brief Generic thread-safe queue with a max capacity
* @brief Generic thread-safe queue with a max capacity.
*
* @note (original note) We can't use a lockfree queue here, since we need the ability to wait for an element to be
* added or removed from the queue. These waits are blocking calls.
@@ -117,21 +121,21 @@ class ThreadSafeQueue
public:
/**
* @brief Create an instance of the queue
* @brief Create an instance of the queue.
*
* @param maxSize maximum size of the queue. Calls that would cause the queue to exceed this size will block until
* free space is available
* free space is available.
*/
ThreadSafeQueue(uint32_t maxSize) : maxSize_(maxSize)
{
}
/**
* @brief Push element onto the queue
* @brief Push element onto the queue.
*
* Note: This method will block until free space is available
* Note: This method will block until free space is available.
*
* @param elt element to push onto queue
* @param elt Element to push onto queue
*/
void
push(T const& elt)
@@ -143,11 +147,11 @@ public:
}
/**
* @brief Push element onto the queue
* @brief Push element onto the queue.
*
* Note: This method will block until free space is available
*
* @param elt element to push onto queue. elt is moved from
* @param elt Element to push onto queue. Ownership is transferred
*/
void
push(T&& elt)
@@ -159,11 +163,11 @@ public:
}
/**
* @brief Pop element from the queue
* @brief Pop element from the queue.
*
* Note: Will block until queue is non-empty
* Note: Will block until queue is non-empty.
*
* @return element popped from queue
* @return Element popped from queue
*/
T
pop()
@@ -179,9 +183,9 @@ public:
}
/**
* @brief Attempt to pop an element
* @brief Attempt to pop an element.
*
* @return element popped from queue or empty optional if queue was empty
* @return Element popped from queue or empty optional if queue was empty
*/
std::optional<T>
tryPop()
@@ -201,7 +205,7 @@ public:
/**
* @brief Parititions the uint256 keyspace into numMarkers partitions, each of equal size.
*
* @param numMarkers total markers to partition for
* @param numMarkers Total markers to partition for
*/
inline std::vector<ripple::uint256>
getMarkers(size_t numMarkers)

31
src/etl/ETLService.h
View File

@@ -46,6 +46,9 @@ namespace feed {
class SubscriptionManager;
}
/**
* @brief This namespace contains everything to do with the ETL and ETL sources.
*/
namespace etl {
/**
@@ -98,7 +101,7 @@ class ETLService
public:
/**
* @brief Create an instance of ETLService
* @brief Create an instance of ETLService.
*
* @param config The configuration to use
* @param ioc io context to run on
@@ -115,6 +118,18 @@ public:
std::shared_ptr<LoadBalancerType> balancer,
std::shared_ptr<NetworkValidatedLedgersType> ledgers);
/**
* @brief A factory function to spawn new ETLService instances.
*
* Creates and runs the ETL service.
*
* @param config The configuration to use
* @param ioc io context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param balancer Load balancer to use
* @param ledgers The network validated ledgers datastructure
*/
static std::shared_ptr<ETLService>
make_ETLService(
util::Config const& config,
@@ -131,7 +146,7 @@ public:
}
/**
* @brief Stops components and joins worker thread
* @brief Stops components and joins worker thread.
*/
~ETLService()
{
@@ -148,7 +163,7 @@ public:
}
/**
* @brief Get time passed since last ledger close, in seconds
* @brief Get time passed since last ledger close, in seconds.
*/
std::uint32_t
lastCloseAgeSeconds() const
@@ -229,7 +244,7 @@ private:
}
/**
* @brief Get the number of markers to use during the initial ledger download
* @brief Get the number of markers to use during the initial ledger download.
*
* This is equivelent to the degree of parallelism during the initial ledger download.
*
@@ -242,19 +257,19 @@ private:
}
/**
* @brief Start all components to run ETL service
* @brief Start all components to run ETL service.
*/
void
run();
/**
* @brief Spawn the worker thread and start monitoring
* @brief Spawn the worker thread and start monitoring.
*/
void
doWork();
/**
* @brief Sets amendment blocked flag
* @brief Sets amendment blocked flag.
*/
void
setAmendmentBlocked()
@@ -262,4 +277,4 @@ private:
state_.isAmendmentBlocked = true;
}
};
} // namespace etl
} // namespace etl

21
src/etl/LoadBalancer.cpp
View File

@@ -43,15 +43,13 @@ namespace etl {
std::unique_ptr<Source>
LoadBalancer::make_Source(
Config const& config,
boost::asio::io_context& ioContext,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer)
{
auto src =
std::make_unique<ProbingSource>(config, ioContext, backend, subscriptions, networkValidatedLedgers, balancer);
auto src = std::make_unique<ProbingSource>(config, ioc, backend, subscriptions, validatedLedgers, balancer);
src->run();
return src;
@@ -70,10 +68,10 @@ LoadBalancer::make_LoadBalancer(
LoadBalancer::LoadBalancer(
Config const& config,
boost::asio::io_context& ioContext,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl)
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers)
{
if (auto value = config.maybeValue<uint32_t>("num_markers"); value)
downloadRanges_ = std::clamp(*value, 1u, 256u);
@@ -82,13 +80,18 @@ LoadBalancer::LoadBalancer(
for (auto const& entry : config.array("etl_sources"))
{
std::unique_ptr<Source> source = make_Source(entry, ioContext, backend, subscriptions, nwvl, *this);
std::unique_ptr<Source> source = make_Source(entry, ioc, backend, subscriptions, validatedLedgers, *this);
sources_.push_back(std::move(source));
log_.info() << "Added etl source - " << sources_.back()->toString();
}
}
LoadBalancer::~LoadBalancer()
{
sources_.clear();
}
std::pair<std::vector<std::string>, bool>
LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly)
{
@@ -235,4 +238,4 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence)
}
return true;
}
} // namespace etl
} // namespace etl

75
src/etl/LoadBalancer.h
View File

@@ -33,14 +33,15 @@ namespace etl {
class Source;
class ProbingSource;
} // namespace etl
namespace feed {
class SubscriptionManager;
}
} // namespace feed
namespace etl {
/**
* @brief This class is used to manage connections to transaction processing processes
* @brief This class is used to manage connections to transaction processing processes.
*
* This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of
* which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also
@@ -56,25 +57,34 @@ public:
private:
util::Logger log_{"ETL"};
std::vector<std::unique_ptr<Source>> sources_;
std::uint32_t downloadRanges_ = 16;
std::uint32_t downloadRanges_ = 16; /*< The number of markers to use when downloading intial ledger */
public:
/**
* @brief Create an instance of the load balancer
* @brief Create an instance of the load balancer.
*
* @param config The configuration to use
* @param ioContext io context to run on
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param nwvl The network validated ledgers datastructure
* @param validatedLedgers The network validated ledgers datastructure
*/
LoadBalancer(
util::Config const& config,
boost::asio::io_context& ioContext,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl);
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers);
/**
* @brief A factory function for the load balancer.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
*/
static std::shared_ptr<LoadBalancer>
make_LoadBalancer(
util::Config const& config,
@@ -83,37 +93,46 @@ public:
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers);
/**
* @brief A factory function for the ETL source.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param balancer The load balancer
*/
static std::unique_ptr<Source>
make_Source(
util::Config const& config,
boost::asio::io_context& ioContext,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer);
~LoadBalancer()
{
sources_.clear();
}
~LoadBalancer();
/**
* @brief Load the initial ledger, writing data to the queue
* @brief Load the initial ledger, writing data to the queue.
*
* @param sequence sequence of ledger to download
* @param sequence Sequence of ledger to download
* @param cacheOnly Whether to only write to cache and not to the DB; defaults to false
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, bool cacheOnly = false);
/**
* @brief Fetch data for a specific ledger
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
*
* @param ledgerSequence sequence of ledger to fetch data for
* @param getObjects if true, fetch diff between specified ledger and previous
* @return the extracted data, if extraction was successful. If the ledger was found in the database or the server
* @param ledgerSequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one
* @param getObjectNeighbors Whether to request object neighbors
* @return The extracted data, if extraction was successful. If the ledger was found in the database or the server
* is shutting down, the optional will be empty
*/
OptionalGetLedgerResponseType
@@ -133,16 +152,18 @@ public:
shouldPropagateTxnStream(Source* in) const;
/**
* @return JSON representation of the state of this load balancer
* @return JSON representation of the state of this load balancer.
*/
boost::json::value
toJson() const;
/**
* @brief Forward a JSON RPC request to a randomly selected rippled node
* @brief Forward a JSON RPC request to a randomly selected rippled node.
*
* @param request JSON-RPC request
* @return response received from rippled node
* @param request JSON-RPC request to forward
* @param clientIp The IP address of the peer
* @param yield The coroutine context
* @return Response received from rippled node as JSON object on success; nullopt on failure
*/
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
@@ -150,13 +171,13 @@ public:
private:
/**
* @brief Execute a function on a randomly selected source
* @brief Execute a function on a randomly selected source.
*
* @note f is a function that takes an Source as an argument and returns a bool.
* Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another
* randomly chosen Source is used. The process repeats until f returns true.
*
* @param f function to execute. This function takes the ETL source as an argument, and returns a bool.
* @param f Function to execute. This function takes the ETL source as an argument, and returns a bool
* @param ledgerSequence f is executed for each Source that has this ledger
* @return true if f was eventually executed successfully. false if the ledger was found in the database or the
* server is shutting down
@@ -165,4 +186,4 @@ private:
bool
execute(Func f, uint32_t ledgerSequence);
};
} // namespace etl
} // namespace etl

14
src/etl/NFTHelpers.h
View File

@@ -17,6 +17,7 @@
*/
//==============================================================================
/** @file */
#pragma once
#include <data/DBHelpers.h>
@@ -27,13 +28,22 @@
namespace etl {
/**
* @brief Pull NFT data from TX via ETLService
* @brief Pull NFT data from TX via ETLService.
*
* @param txMeta Transaction metadata
* @param sttx The transaction
* @return NFT transactions data as a pair of transactions and optional NFTsData
*/
std::pair<std::vector<NFTTransactionsData>, std::optional<NFTsData>>
getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx);
/**
* @brief Pull NFT data from ledger object via loadInitialLedger
* @brief Pull NFT data from ledger object via loadInitialLedger.
*
* @param seq The ledger sequence to pull for
* @param key The owner key
* @param blob Object data as blob
* @return The NFT data as a vector
*/
std::vector<NFTsData>
getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob);

8
src/etl/ProbingSource.cpp
View File

@@ -105,19 +105,19 @@ ProbingSource::token() const
}
std::pair<std::vector<std::string>, bool>
ProbingSource::loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly)
ProbingSource::loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly)
{
if (!currentSrc_)
return {{}, false};
return currentSrc_->loadInitialLedger(ledgerSequence, numMarkers, cacheOnly);
return currentSrc_->loadInitialLedger(sequence, numMarkers, cacheOnly);
}
std::pair<grpc::Status, ProbingSource::GetLedgerResponseType>
ProbingSource::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors)
ProbingSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors)
{
if (!currentSrc_)
return {};
return currentSrc_->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors);
return currentSrc_->fetchLedger(sequence, getObjects, getObjectNeighbors);
}
std::optional<boost::json::object>

6
src/etl/ProbingSource.h
View File

@@ -56,7 +56,7 @@ private:
public:
/**
* @brief Create an instance of the probing source
* @brief Create an instance of the probing source.
*
* @param config The configuration to use
* @param ioc io context to run on
@@ -99,10 +99,10 @@ public:
toString() const override;
std::pair<std::vector<std::string>, bool>
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override;
loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) override;
std::pair<grpc::Status, GetLedgerResponseType>
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override;
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) override;
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)

2
src/etl/README.md
View File

@@ -1,3 +1,5 @@
# ETL subsystem
A single clio node has one or more ETL sources, specified in the config
file. clio will subscribe to the `ledgers` stream of each of the ETL
sources. This stream sends a message whenever a new ledger is validated. Upon

453
src/etl/Source.h
View File

@@ -53,47 +53,98 @@ class SubscriptionManager;
namespace etl {
/**
* @brief Base class for all ETL sources
* @brief Base class for all ETL sources.
*
* Note: Since sources below are implemented via CRTP, it sort of makes no sense to have a virtual base class.
* We should consider using a vector of ProbingSources instead of vector of unique ptrs to this virtual base.
*/
class Source
{
public:
/** @return true if source is connected; false otherwise */
virtual bool
isConnected() const = 0;
/** @return JSON representation of the source */
virtual boost::json::object
toJson() const = 0;
/** @brief Runs the source */
virtual void
run() = 0;
/** @brief Request to pause the source (i.e. disconnect and do nothing) */
virtual void
pause() = 0;
/** @brief Reconnect and resume this source */
virtual void
resume() = 0;
/** @return String representation of the source (for debug) */
virtual std::string
toString() const = 0;
/**
* @brief Check if ledger is known by this source.
*
* @param sequence The ledger sequence to check
* @return true if ledger is in the range of this source; false otherwise
*/
virtual bool
hasLedger(uint32_t sequence) const = 0;
/**
* @brief Fetch data for a specific ledger.
*
* This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger
* is found in the database, or the server is shutting down.
*
* @param sequence Sequence of the ledger to fetch
* @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true
* @param getObjectNeighbors Whether to request object neighbors; defaults to false
* @return A std::pair of the response status and the response itself
*/
virtual std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) = 0;
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0;
/**
* @brief Download a ledger in full.
*
* @param sequence Sequence of the ledger to download
* @param numMarkers Number of markers to generate for async calls
* @param cacheOnly Only insert into cache, not the DB; defaults to false
* @return A std::pair of the data and a bool indicating whether the download was successfull
*/
virtual std::pair<std::vector<std::string>, bool>
loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0;
/**
* @brief Forward a request to rippled.
*
* @param request The request to forward
* @param clientIp IP of the client forwarding this request
* @param yield The coroutine context
* @return Response wrapped in an optional on success; nullopt otherwise
*/
virtual std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const = 0;
/**
* @return A token that uniquely identifies this source instance.
*/
virtual boost::uuids::uuid
token() const = 0;
virtual ~Source() = default;
/**
* @brief Comparison is done via comparing tokens provided by the token() function.
*
* @param other The other source to compare to
* @return true if sources are equal; false otherwise
*/
bool
operator==(Source const& other) const
{
@@ -115,7 +166,7 @@ private:
};
/**
* @brief Hooks for source events such as connects and disconnects
* @brief Hooks for source events such as connects and disconnects.
*/
struct SourceHooks
{
@@ -126,7 +177,9 @@ struct SourceHooks
};
/**
* @brief Base implementation of shared source logic (using CRTP)
* @brief Base implementation of shared source logic.
*
* @tparam Derived The derived class for CRTP
*/
template <class Derived>
class SourceImpl : public Source
@@ -174,25 +227,30 @@ protected:
public:
/**
* @brief Create ETL source without gRPC endpoint
* @brief Create the base portion of ETL source.
*
* Fetch ledger and load initial ledger will fail for this source.
* Primarly used in read-only mode, to monitor when ledgers are validated.
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param balancer Load balancer to use
* @param hooks Hooks to use for connect/disconnect events
*/
SourceImpl(
util::Config const& config,
boost::asio::io_context& ioContext,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> networkValidatedLedgers,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
: networkValidatedLedgers_(networkValidatedLedgers)
: networkValidatedLedgers_(validatedLedgers)
, backend_(backend)
, subscriptions_(subscriptions)
, balancer_(balancer)
, forwardCache_(config, ioContext, *this)
, strand_(boost::asio::make_strand(ioContext))
, forwardCache_(config, ioc, *this)
, strand_(boost::asio::make_strand(ioc))
, timer_(strand_)
, resolver_(strand_)
, hooks_(hooks)
@@ -241,20 +299,6 @@ public:
return uuid_;
}
std::chrono::system_clock::time_point
getLastMsgTime() const
{
std::lock_guard lck(lastMsgTimeMtx_);
return lastMsgTime_;
}
void
setLastMsgTime()
{
std::lock_guard lck(lastMsgTimeMtx_);
lastMsgTime_ = std::chrono::system_clock::now();
}
std::optional<boost::json::object>
requestFromRippled(
boost::json::object const& request,
@@ -337,10 +381,6 @@ public:
}
}
/**
* @param sequence ledger sequence to check for
* @return true if this source has the desired ledger
*/
bool
hasLedger(uint32_t sequence) const override
{
@@ -362,64 +402,8 @@ public:
return false;
}
/**
* @brief Process the validated range received on the ledgers stream. set the appropriate member variable
*
* @param range validated range received on ledgers stream
*/
void
setValidatedRange(std::string const& range)
{
std::vector<std::pair<uint32_t, uint32_t>> pairs;
std::vector<std::string> ranges;
boost::split(ranges, range, boost::is_any_of(","));
for (auto& pair : ranges)
{
std::vector<std::string> minAndMax;
boost::split(minAndMax, pair, boost::is_any_of("-"));
if (minAndMax.size() == 1)
{
uint32_t sequence = std::stoll(minAndMax[0]);
pairs.push_back(std::make_pair(sequence, sequence));
}
else
{
assert(minAndMax.size() == 2);
uint32_t min = std::stoll(minAndMax[0]);
uint32_t max = std::stoll(minAndMax[1]);
pairs.push_back(std::make_pair(min, max));
}
}
std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; });
// we only hold the lock here, to avoid blocking while string processing
std::lock_guard lck(mtx_);
validatedLedgers_ = std::move(pairs);
validatedLedgersRaw_ = range;
}
/**
* @return the validated range of this source
* @note this is only used by server_info
*/
std::string
getValidatedRange() const
{
std::lock_guard lck(mtx_);
return validatedLedgersRaw_;
}
/**
* @brief Fetch the specified ledger
*
* @param ledgerSequence sequence of the ledger to fetch @getObjects whether to get the account state diff between
* this ledger and the prior one
* @return the extracted data and the result status
*/
std::pair<grpc::Status, org::xrpl::rpc::v1::GetLedgerResponse>
fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override
fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) override
{
org::xrpl::rpc::v1::GetLedgerResponse response;
if (!stub_)
@@ -429,7 +413,7 @@ public:
org::xrpl::rpc::v1::GetLedgerRequest request;
grpc::ClientContext context;
request.mutable_ledger()->set_sequence(ledgerSequence);
request.mutable_ledger()->set_sequence(sequence);
request.set_transactions(true);
request.set_expand(true);
request.set_get_objects(getObjects);
@@ -448,9 +432,6 @@ public:
return {status, std::move(response)};
}
/**
* @brief Produces a human-readable string with info about the source
*/
std::string
toString() const override
{
@@ -458,10 +439,6 @@ public:
", grpc port: " + grpcPort_ + "}";
}
/**
* @brief Produces stats for this source in a json object
* @return json object with stats
*/
boost::json::object
toJson() const override
{
@@ -482,15 +459,8 @@ public:
return res;
}
/**
* @brief Download a ledger in full
*
* @param ledgerSequence sequence of the ledger to download
* @param writeQueue queue to push downloaded ledger objects
* @return true if the download was successful
*/
std::pair<std::vector<std::string>, bool>
loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override
loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) override
{
if (!stub_)
return {{}, false};
@@ -508,10 +478,10 @@ public:
if (i + 1 < markers.size())
nextMarker = markers[i + 1];
calls.emplace_back(ledgerSequence, markers[i], nextMarker);
calls.emplace_back(sequence, markers[i], nextMarker);
}
log_.debug() << "Starting data download for ledger " << ledgerSequence << ". Using source = " << toString();
log_.debug() << "Starting data download for ledger " << sequence << ". Using source = " << toString();
for (auto& c : calls)
c.call(stub_, cq);
@@ -564,60 +534,19 @@ public:
return {std::move(edgeKeys), !abort};
}
/**
* @brief Attempt to reconnect to the ETL source
*/
void
reconnect(boost::beast::error_code ec)
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const override
{
if (paused_)
return;
if (isConnected())
hooks_.onDisconnected(ec);
connected_ = false;
readBuffer_ = {};
// These are somewhat normal errors. operation_aborted occurs on shutdown,
// when the timer is cancelled. connection_refused will occur repeatedly
std::string err = ec.message();
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
if (auto resp = forwardCache_.get(request); resp)
{
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
// ERR_PACK /* crypto/err/err.h */
char buf[128];
::ERR_error_string_n(ec.value(), buf, sizeof(buf));
err += buf;
log_.error() << err;
log_.debug() << "request hit forwardCache";
return resp;
}
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
log_.error() << "error code = " << ec << " - " << toString();
}
else
{
log_.warn() << "error code = " << ec << " - " << toString();
}
// exponentially increasing timeouts, with a max of 30 seconds
size_t waitTime = std::min(pow(2, numFailures_), 30.0);
numFailures_++;
timer_.expires_after(boost::asio::chrono::seconds(waitTime));
timer_.async_wait([this](auto ec) {
bool startAgain = (ec != boost::asio::error::operation_aborted);
derived().close(startAgain);
});
return requestFromRippled(request, clientIp, yield);
}
/**
* @brief Pause the source effectively stopping it from trying to reconnect
*/
void
pause() override
{
@@ -625,9 +554,6 @@ public:
derived().close(false);
}
/**
* @brief Resume the source allowing it to reconnect again
*/
void
resume() override
{
@@ -636,7 +562,10 @@ public:
}
/**
* @brief Callback for resolving the server host
* @brief Callback for resolving the server host.
*
* @param ec The error code
* @param results Result of the resolve operation
*/
void
onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results)
@@ -656,7 +585,9 @@ public:
}
/**
* @brief Callback for handshake with the server
* @brief Callback for handshake with the server.
*
* @param ec The error code
*/
void
onHandshake(boost::beast::error_code ec)
@@ -692,10 +623,13 @@ public:
}
/**
* @brief Callback for writing data
* @brief Callback for writing data.
*
* @param ec The error code
* @param size Amount of bytes written
*/
void
onWrite(boost::beast::error_code ec, size_t size)
onWrite(boost::beast::error_code ec, [[maybe_unused]] size_t size)
{
if (ec)
reconnect(ec);
@@ -704,7 +638,10 @@ public:
}
/**
* @brief Callback for data available to read
* @brief Callback for data available to read.
*
* @param ec The error code
* @param size Amount of bytes read
*/
void
onRead(boost::beast::error_code ec, size_t size)
@@ -721,8 +658,10 @@ public:
}
/**
* @brief Handle the most recently received message
* @return true if the message was handled successfully. false on error
* @brief Handle the most recently received message.
*
* @param size Amount of bytes available in the read buffer
* @return true if the message was handled successfully; false otherwise
*/
bool
handleMessage(size_t size)
@@ -802,23 +741,6 @@ public:
}
}
/**
* @brief Forward a request to rippled
* @return response wrapped in an optional on success; nullopt otherwise
*/
std::optional<boost::json::object>
forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield)
const override
{
if (auto resp = forwardCache_.get(request); resp)
{
log_.debug() << "request hit forwardCache";
return resp;
}
return requestFromRippled(request, clientIp, yield);
}
protected:
Derived&
derived()
@@ -831,47 +753,172 @@ protected:
{
resolver_.async_resolve(ip_, wsPort_, [this](auto ec, auto results) { onResolve(ec, results); });
}
void
reconnect(boost::beast::error_code ec)
{
if (paused_)
return;
if (isConnected())
hooks_.onDisconnected(ec);
connected_ = false;
readBuffer_ = {};
// These are somewhat normal errors. operation_aborted occurs on shutdown,
// when the timer is cancelled. connection_refused will occur repeatedly
std::string err = ec.message();
// if we cannot connect to the transaction processing process
if (ec.category() == boost::asio::error::get_ssl_category())
{
err = std::string(" (") + boost::lexical_cast<std::string>(ERR_GET_LIB(ec.value())) + "," +
boost::lexical_cast<std::string>(ERR_GET_REASON(ec.value())) + ") ";
// ERR_PACK /* crypto/err/err.h */
char buf[128];
::ERR_error_string_n(ec.value(), buf, sizeof(buf));
err += buf;
log_.error() << err;
}
if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused)
{
log_.error() << "error code = " << ec << " - " << toString();
}
else
{
log_.warn() << "error code = " << ec << " - " << toString();
}
// exponentially increasing timeouts, with a max of 30 seconds
size_t waitTime = std::min(pow(2, numFailures_), 30.0);
numFailures_++;
timer_.expires_after(boost::asio::chrono::seconds(waitTime));
timer_.async_wait([this](auto ec) {
bool startAgain = (ec != boost::asio::error::operation_aborted);
derived().close(startAgain);
});
}
private:
void
setLastMsgTime()
{
std::lock_guard lck(lastMsgTimeMtx_);
lastMsgTime_ = std::chrono::system_clock::now();
}
std::chrono::system_clock::time_point
getLastMsgTime() const
{
std::lock_guard lck(lastMsgTimeMtx_);
return lastMsgTime_;
}
void
setValidatedRange(std::string const& range)
{
std::vector<std::pair<uint32_t, uint32_t>> pairs;
std::vector<std::string> ranges;
boost::split(ranges, range, boost::is_any_of(","));
for (auto& pair : ranges)
{
std::vector<std::string> minAndMax;
boost::split(minAndMax, pair, boost::is_any_of("-"));
if (minAndMax.size() == 1)
{
uint32_t sequence = std::stoll(minAndMax[0]);
pairs.push_back(std::make_pair(sequence, sequence));
}
else
{
assert(minAndMax.size() == 2);
uint32_t min = std::stoll(minAndMax[0]);
uint32_t max = std::stoll(minAndMax[1]);
pairs.push_back(std::make_pair(min, max));
}
}
std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; });
// we only hold the lock here, to avoid blocking while string processing
std::lock_guard lck(mtx_);
validatedLedgers_ = std::move(pairs);
validatedLedgersRaw_ = range;
}
std::string
getValidatedRange() const
{
std::lock_guard lck(mtx_);
return validatedLedgersRaw_;
}
};
/**
* @brief Implementation of a source that uses a regular, non-secure websocket connection.
*/
class PlainSource : public SourceImpl<PlainSource>
{
using StreamType = boost::beast::websocket::stream<boost::beast::tcp_stream>;
std::unique_ptr<StreamType> ws_;
public:
/**
* @brief Create a non-secure ETL source.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param balancer Load balancer to use
* @param hooks Hooks to use for connect/disconnect events
*/
PlainSource(
util::Config const& config,
boost::asio::io_context& ioc,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
: SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
: SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks))
, ws_(std::make_unique<StreamType>(strand_))
{
}
/**
* @brief Callback for connection to the server
* @brief Callback for connection to the server.
*
* @param ec The error code
* @param endpoint The resolved endpoint
*/
void
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/**
* @brief Close the websocket
* @param startAgain whether to reconnect
* @brief Close the websocket.
*
* @param startAgain Whether to automatically reconnect
*/
void
close(bool startAgain);
boost::beast::websocket::stream<boost::beast::tcp_stream>&
/** @return The underlying TCP stream */
StreamType&
ws()
{
return *ws_;
}
};
/**
* @brief Implementation of a source that uses a secure websocket connection.
*/
class SslSource : public SourceImpl<SslSource>
{
using StreamType = boost::beast::websocket::stream<boost::beast::ssl_stream<boost::beast::tcp_stream>>;
@@ -879,44 +926,64 @@ class SslSource : public SourceImpl<SslSource>
std::unique_ptr<StreamType> ws_;
public:
/**
* @brief Create a secure ETL source.
*
* @param config The configuration to use
* @param ioc The io_context to run on
* @param sslCtx The SSL context if any
* @param backend BackendInterface implementation
* @param subscriptions Subscription manager
* @param validatedLedgers The network validated ledgers datastructure
* @param balancer Load balancer to use
* @param hooks Hooks to use for connect/disconnect events
*/
SslSource(
util::Config const& config,
boost::asio::io_context& ioc,
std::optional<std::reference_wrapper<boost::asio::ssl::context>> sslCtx,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<feed::SubscriptionManager> subscriptions,
std::shared_ptr<NetworkValidatedLedgers> nwvl,
std::shared_ptr<NetworkValidatedLedgers> validatedLedgers,
LoadBalancer& balancer,
SourceHooks hooks)
: SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks))
: SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks))
, sslCtx_(sslCtx)
, ws_(std::make_unique<StreamType>(strand_, *sslCtx_))
{
}
/**
* @brief Callback for connection to the server
* @brief Callback for connection to the server.
*
* @param ec The error code
* @param endpoint The resolved endpoint
*/
void
onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/**
* @brief Callback for SSL handshake completion
* @brief Callback for SSL handshake completion.
*
* @param ec The error code
* @param endpoint The resolved endpoint
*/
void
onSslHandshake(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint);
/**
* @brief Close the websocket
* @param startAgain whether to reconnect
* @brief Close the websocket.
*
* @param startAgain Whether to automatically reconnect
*/
void
close(bool startAgain);
/** @return The underlying SSL stream */
StreamType&
ws()
{
return *ws_;
}
};
} // namespace etl
} // namespace etl

31
src/etl/SystemState.h
View File

@@ -23,36 +23,23 @@
namespace etl {
/**
* @brief Represents the state of the ETL subsystem.
*/
struct SystemState
{
/**
* @brief Whether the process is in strict read-only mode
* @brief Whether the process is in strict read-only mode.
*
* In strict read-only mode, the process will never attempt to become the ETL writer, and will only publish ledgers
* as they are written to the database.
*/
bool isReadOnly = false;
/**
* @brief Whether the process is writing to the database.
*
* Used by server_info
*/
std::atomic_bool isWriting = false;
/**
* @brief Whether the software is stopping
*/
std::atomic_bool isStopping = false;
/**
* @brief Whether a write conflict was detected
*/
std::atomic_bool writeConflict = false;
/**
* @brief Whether we detected an amendment block
*/
std::atomic_bool isAmendmentBlocked = false;
std::atomic_bool isWriting = false; /**< @brief Whether the process is writing to the database. */
std::atomic_bool isStopping = false; /**< @brief Whether the software is stopping. */
std::atomic_bool writeConflict = false; /**< @brief Whether a write conflict was detected. */
std::atomic_bool isAmendmentBlocked = false; /**< @brief Whether we detected an amendment block. */
};
} // namespace etl

2
src/etl/impl/ForwardCache.cpp
View File

@@ -70,7 +70,7 @@ ForwardCache::get(boost::json::object const& request) const
if (!command)
return {};
if (RPC::specifiesCurrentOrClosedLedger(request))
if (rpc::specifiesCurrentOrClosedLedger(request))
return {};
std::shared_lock lk(mtx_);

12
src/etl/impl/LedgerFetcher.h
View File

@@ -64,11 +64,11 @@ public:
* @return ledger header and transaction+metadata blobs; empty optional if the server is shutting down
*/
OptionalGetLedgerResponseType
fetchData(uint32_t seq)
fetchData(uint32_t sequence)
{
log_.debug() << "Attempting to fetch ledger with sequence = " << seq;
log_.debug() << "Attempting to fetch ledger with sequence = " << sequence;
auto response = loadBalancer_->fetchLedger(seq, false, false);
auto response = loadBalancer_->fetchLedger(sequence, false, false);
if (response)
log_.trace() << "GetLedger reply = " << response->DebugString();
return response;
@@ -85,12 +85,12 @@ public:
* this ledger and the parent; Empty optional if the server is shutting down
*/
OptionalGetLedgerResponseType
fetchDataAndDiff(uint32_t seq)
fetchDataAndDiff(uint32_t sequence)
{
log_.debug() << "Attempting to fetch ledger with sequence = " << seq;
log_.debug() << "Attempting to fetch ledger with sequence = " << sequence;
auto response = loadBalancer_->fetchLedger(
seq, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= seq);
sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence);
if (response)
log_.trace() << "GetLedger reply = " << response->DebugString();

3
src/etl/impl/LedgerLoader.h
View File

@@ -31,6 +31,9 @@
#include <memory>
/**
* @brief Account transactions, NFT transactions and NFT data bundled togeher.
*/
struct FormattedTransactionsData
{
std::vector<AccountTransactionsData> accountTxData;

29
src/etl/impl/Transformer.h
View File

@@ -45,7 +45,7 @@ namespace etl::detail {
*/
/**
* @brief Transformer thread that prepares new ledger out of raw data from GRPC
* @brief Transformer thread that prepares new ledger out of raw data from GRPC.
*/
template <typename DataPipeType, typename LedgerLoaderType, typename LedgerPublisherType>
class Transformer
@@ -66,7 +66,7 @@ class Transformer
public:
/**
* @brief Create an instance of the transformer
* @brief Create an instance of the transformer.
*
* This spawns a new thread that reads from the data pipe and writes ledgers to the DB using LedgerLoader and
* LedgerPublisher.
@@ -89,7 +89,7 @@ public:
}
/**
* @brief Joins the transformer thread
* @brief Joins the transformer thread.
*/
~Transformer()
{
@@ -98,7 +98,7 @@ public:
}
/**
* @brief Block calling thread until transformer thread exits
* @brief Block calling thread until transformer thread exits.
*/
void
waitTillFinished()
@@ -155,13 +155,12 @@ private:
}
}
// TODO update this documentation
/**
* @brief Build the next ledger using the previous ledger and the extracted data.
* @note rawData should be data that corresponds to the ledger immediately following the previous seq.
*
* @param rawData data extracted from an ETL source
* @return the newly built ledger and data to write to the database
* @param rawData Data extracted from an ETL source
* @return The newly built ledger and data to write to the database
*/
std::pair<ripple::LedgerHeader, bool>
buildNextLedger(GetLedgerResponseType& rawData)
@@ -342,7 +341,7 @@ private:
}
/**
* @brief Write successors info into DB
* @brief Write successors info into DB.
*
* @param lgrInfo Ledger info
* @param rawData Ledger data from GRPC
@@ -399,24 +398,38 @@ private:
}
}
/** @return true if the transformer is stopping; false otherwise */
bool
isStopping() const
{
return state_.get().isStopping;
}
/** @return true if there was a write conflict; false otherwise */
bool
hasWriteConflict() const
{
return state_.get().writeConflict;
}
/**
* @brief Sets the write conflict flag.
*
* @param conflict The value to set
*/
void
setWriteConflict(bool conflict)
{
state_.get().writeConflict = conflict;
}
/**
* @brief Sets the amendment blocked flag.
*
* Being amendment blocked means that Clio was compiled with libxrpl that does not yet support some field that
* arrived from rippled and therefore can't extract the ledger diff. When this happens, Clio can't proceed with ETL
* and should log this error and only handle RPC requests.
*/
void
setAmendmentBlocked()
{

59
src/feed/Message.h
View File

@@ -1,59 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2022, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include <string>
namespace feed {
// This class should only be constructed once, then it can
// be read from in parallel by many websocket senders
class Message
{
std::string message_;
public:
Message() = delete;
Message(std::string&& message) : message_(std::move(message))
{
}
Message(Message const&) = delete;
Message(Message&&) = delete;
Message&
operator=(Message const&) = delete;
Message&
operator=(Message&&) = delete;
~Message() = default;
char*
data()
{
return message_.data();
}
std::size_t
size()
{
return message_.size();
}
};
} // namespace feed

22
src/feed/SubscriptionManager.cpp
View File

@@ -55,9 +55,9 @@ getLedgerPubMessage(
pubMsg["ledger_hash"] = to_string(lgrInfo.hash);
pubMsg["ledger_time"] = lgrInfo.closeTime.time_since_epoch().count();
pubMsg["fee_base"] = RPC::toBoostJson(fees.base.jsonClipped());
pubMsg["reserve_base"] = RPC::toBoostJson(fees.reserve.jsonClipped());
pubMsg["reserve_inc"] = RPC::toBoostJson(fees.increment.jsonClipped());
pubMsg["fee_base"] = rpc::toBoostJson(fees.base.jsonClipped());
pubMsg["reserve_base"] = rpc::toBoostJson(fees.reserve.jsonClipped());
pubMsg["reserve_inc"] = rpc::toBoostJson(fees.increment.jsonClipped());
pubMsg["validated_ledgers"] = ledgerRange;
pubMsg["txn_count"] = txnCount;
@@ -159,11 +159,11 @@ SubscriptionManager::pubLedger(
void
SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, ripple::LedgerHeader const& lgrInfo)
{
auto [tx, meta] = RPC::deserializeTxPlusMeta(blobs, lgrInfo.seq);
auto [tx, meta] = rpc::deserializeTxPlusMeta(blobs, lgrInfo.seq);
boost::json::object pubObj;
pubObj["transaction"] = RPC::toJson(*tx);
pubObj["meta"] = RPC::toJson(*meta);
RPC::insertDeliveredAmount(pubObj["meta"].as_object(), tx, meta, blobs.date);
pubObj["transaction"] = rpc::toJson(*tx);
pubObj["meta"] = rpc::toJson(*meta);
rpc::insertDeliveredAmount(pubObj["meta"].as_object(), tx, meta, blobs.date);
pubObj["type"] = "transaction";
pubObj["validated"] = true;
pubObj["status"] = "closed";
@@ -187,7 +187,7 @@ SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, r
ripple::STAmount ownerFunds;
auto fetchFundsSynchronous = [&]() {
data::synchronous([&](boost::asio::yield_context yield) {
ownerFunds = RPC::accountFunds(*backend_, lgrInfo.seq, amount, account, yield);
ownerFunds = rpc::accountFunds(*backend_, lgrInfo.seq, amount, account, yield);
});
};
@@ -248,7 +248,7 @@ SubscriptionManager::pubBookChanges(
ripple::LedgerHeader const& lgrInfo,
std::vector<data::TransactionAndMetadata> const& transactions)
{
auto const json = RPC::computeBookChanges(lgrInfo, transactions);
auto const json = rpc::computeBookChanges(lgrInfo, transactions);
auto const bookChangesMsg = std::make_shared<std::string>(boost::json::serialize(json));
bookChangesSubscribers_.publish(bookChangesMsg);
}
@@ -260,7 +260,7 @@ SubscriptionManager::forwardProposedTransaction(boost::json::object const& respo
txProposedSubscribers_.publish(pubMsg);
auto transaction = response.at("transaction").as_object();
auto accounts = RPC::getAccountsFromTransaction(transaction);
auto accounts = rpc::getAccountsFromTransaction(transaction);
for (ripple::AccountID const& account : accounts)
accountProposedSubscribers_.publish(pubMsg, account);
@@ -367,4 +367,4 @@ SubscriptionManager::cleanup(SessionPtrType session)
cleanupFuncs_.erase(session);
}
} // namespace feed
} // namespace feed

480
src/feed/SubscriptionManager.h
View File

@@ -28,85 +28,20 @@
#include <memory>
/**
* @brief This namespace deals with subscriptions.
*/
namespace feed {
using SessionPtrType = std::shared_ptr<web::ConnectionBase>;
class Subscription
{
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
std::unordered_set<SessionPtrType> subscribers_ = {};
std::atomic_uint64_t subCount_ = 0;
public:
Subscription() = delete;
Subscription(Subscription&) = delete;
Subscription(Subscription&&) = delete;
explicit Subscription(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
~Subscription() = default;
void
subscribe(SessionPtrType const& session);
void
unsubscribe(SessionPtrType const& session);
void
publish(std::shared_ptr<std::string> const& message);
std::uint64_t
count() const
{
return subCount_.load();
}
bool
empty() const
{
return count() == 0;
}
};
template <class Key>
class SubscriptionMap
{
using subscribers = std::set<SessionPtrType>;
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
std::unordered_map<Key, subscribers> subscribers_ = {};
std::atomic_uint64_t subCount_ = 0;
public:
SubscriptionMap() = delete;
SubscriptionMap(SubscriptionMap&) = delete;
SubscriptionMap(SubscriptionMap&&) = delete;
explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
~SubscriptionMap() = default;
void
subscribe(SessionPtrType const& session, Key const& key);
void
unsubscribe(SessionPtrType const& session, Key const& key);
void
publish(std::shared_ptr<std::string> const& message, Key const& key);
std::uint64_t
count() const
{
return subCount_.load();
}
};
/**
* @brief Sends a message to subscribers.
*
* @param message The message to send
* @param subscribers The subscription stream to send the message to
* @param counter The subscription counter to decrement if session is detected as dead
*/
template <class T>
inline void
sendToSubscribers(std::shared_ptr<std::string> const& message, T& subscribers, std::atomic_uint64_t& counter)
@@ -127,6 +62,13 @@ sendToSubscribers(std::shared_ptr<std::string> const& message, T& subscribers, s
}
}
/**
* @brief Adds a session to the subscription stream.
*
* @param session The session to add
* @param subscribers The stream to subscribe to
* @param counter The counter representing the current total subscribers
*/
template <class T>
inline void
addSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter)
@@ -138,6 +80,13 @@ addSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter
}
}
/**
* @brief Removes a session from the subscription stream.
*
* @param session The session to remove
* @param subscribers The stream to unsubscribe from
* @param counter The counter representing the current total subscribers
*/
template <class T>
inline void
removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter)
@@ -149,47 +98,170 @@ removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& coun
}
}
template <class Key>
void
SubscriptionMap<Key>::subscribe(SessionPtrType const& session, Key const& account)
/**
* @brief Represents a subscription stream.
*/
class Subscription
{
boost::asio::post(strand_, [this, session, account]() { addSession(session, subscribers_[account], subCount_); });
}
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
std::unordered_set<SessionPtrType> subscribers_ = {};
std::atomic_uint64_t subCount_ = 0;
public:
Subscription() = delete;
Subscription(Subscription&) = delete;
Subscription(Subscription&&) = delete;
/**
* @brief Create a new subscription stream.
*
* @param ioc The io_context to run on
*/
explicit Subscription(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
~Subscription() = default;
/**
* @brief Adds the given session to the subscribers set.
*
* @param session The session to add
*/
void
subscribe(SessionPtrType const& session);
/**
* @brief Removes the given session from the subscribers set.
*
* @param session The session to remove
*/
void
unsubscribe(SessionPtrType const& session);
/**
* @brief Sends the given message to all subscribers.
*
* @param message The message to send
*/
void
publish(std::shared_ptr<std::string> const& message);
/**
* @return Total subscriber count on this stream.
*/
std::uint64_t
count() const
{
return subCount_.load();
}
/**
* @return true if the stream currently has no subscribers; false otherwise
*/
bool
empty() const
{
return count() == 0;
}
};
/**
* @brief Represents a collection of subscriptions where each stream is mapped to a key.
*/
template <class Key>
void
SubscriptionMap<Key>::unsubscribe(SessionPtrType const& session, Key const& account)
class SubscriptionMap
{
boost::asio::post(strand_, [this, account, session]() {
if (!subscribers_.contains(account))
return;
using SubscribersType = std::set<SessionPtrType>;
if (!subscribers_[account].contains(session))
return;
boost::asio::strand<boost::asio::io_context::executor_type> strand_;
std::unordered_map<Key, SubscribersType> subscribers_ = {};
std::atomic_uint64_t subCount_ = 0;
--subCount_;
public:
SubscriptionMap() = delete;
SubscriptionMap(SubscriptionMap&) = delete;
SubscriptionMap(SubscriptionMap&&) = delete;
subscribers_[account].erase(session);
/**
* @brief Create a new subscription map.
*
* @param ioc The io_context to run on
*/
explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc))
{
}
if (subscribers_[account].size() == 0)
{
subscribers_.erase(account);
}
});
}
~SubscriptionMap() = default;
template <class Key>
void
SubscriptionMap<Key>::publish(std::shared_ptr<std::string> const& message, Key const& account)
{
boost::asio::post(strand_, [this, account, message]() {
if (!subscribers_.contains(account))
return;
/**
* @brief Subscribe to a specific stream by its key.
*
* @param session The session to add
* @param key The key for the subscription to subscribe to
*/
void
subscribe(SessionPtrType const& session, Key const& key)
{
boost::asio::post(strand_, [this, session, key]() { addSession(session, subscribers_[key], subCount_); });
}
sendToSubscribers(message, subscribers_[account], subCount_);
});
}
/**
* @brief Unsubscribe from a specific stream by its key.
*
* @param session The session to remove
* @param key The key for the subscription to unsubscribe from
*/
void
unsubscribe(SessionPtrType const& session, Key const& key)
{
boost::asio::post(strand_, [this, key, session]() {
if (!subscribers_.contains(key))
return;
if (!subscribers_[key].contains(session))
return;
--subCount_;
subscribers_[key].erase(session);
if (subscribers_[key].size() == 0)
{
subscribers_.erase(key);
}
});
}
/**
* @brief Sends the given message to all subscribers.
*
* @param message The message to send
* @param key The key for the subscription to send the message to
*/
void
publish(std::shared_ptr<std::string> const& message, Key const& key)
{
boost::asio::post(strand_, [this, key, message]() {
if (!subscribers_.contains(key))
return;
sendToSubscribers(message, subscribers_[key], subCount_);
});
}
/**
* @return Total subscriber count on all streams in the collection.
*/
std::uint64_t
count() const
{
return subCount_.load();
}
};
/**
* @brief Manages subscriptions.
*/
class SubscriptionManager
{
util::Logger log_{"Subscriptions"};
@@ -212,14 +284,26 @@ class SubscriptionManager
std::shared_ptr<data::BackendInterface const> backend_;
public:
/**
* @brief A factory function that creates a new subscription manager configured from the config provided.
*
* @param config The configuration to use
* @param backend The backend to use
*/
static std::shared_ptr<SubscriptionManager>
make_SubscriptionManager(util::Config const& config, std::shared_ptr<data::BackendInterface const> const& b)
make_SubscriptionManager(util::Config const& config, std::shared_ptr<data::BackendInterface const> const& backend)
{
auto numThreads = config.valueOr<uint64_t>("subscription_workers", 1);
return std::make_shared<SubscriptionManager>(numThreads, b);
return std::make_shared<SubscriptionManager>(numThreads, backend);
}
SubscriptionManager(std::uint64_t numThreads, std::shared_ptr<data::BackendInterface const> const& b)
/**
* @brief Creates a new instance of the subscription manager.
*
* @param numThreads The number of worker threads to manage subscriptions
* @param backend The backend to use
*/
SubscriptionManager(std::uint64_t numThreads, std::shared_ptr<data::BackendInterface const> const& backend)
: ledgerSubscribers_(ioc_)
, txSubscribers_(ioc_)
, txProposedSubscribers_(ioc_)
@@ -229,7 +313,7 @@ public:
, accountSubscribers_(ioc_)
, accountProposedSubscribers_(ioc_)
, bookSubscribers_(ioc_)
, backend_(b)
, backend_(backend)
{
work_.emplace(ioc_);
@@ -243,6 +327,7 @@ public:
workers_.emplace_back([this] { ioc_.run(); });
}
/** @brief Stops the worker threads of the subscription manager. */
~SubscriptionManager()
{
work_.reset();
@@ -252,9 +337,24 @@ public:
worker.join();
}
/**
* @brief Subscribe to the ledger stream.
*
* @param yield The coroutine context
* @param session The session to subscribe to the stream
* @return JSON object representing the first message to be sent to the new subscriber
*/
boost::json::object
subLedger(boost::asio::yield_context yield, SessionPtrType session);
/**
* @brief Publish to the ledger stream.
*
* @param lgrInfo The ledger header to serialize
* @param fees The fees to serialize
* @param ledgerRange The ledger range this message applies to
* @param txnCount The total number of transactions to serialize
*/
void
pubLedger(
ripple::LedgerHeader const& lgrInfo,
@@ -262,97 +362,216 @@ public:
std::string const& ledgerRange,
std::uint32_t txnCount);
/**
* @brief Publish to the book changes stream.
*
* @param lgrInfo The ledger header to serialize
* @param transactions The transactions to serialize
*/
void
pubBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::TransactionAndMetadata> const& transactions);
/**
* @brief Unsubscribe from the ledger stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubLedger(SessionPtrType session);
/**
* @brief Subscribe to the transactions stream.
*
* @param session The session to subscribe to the stream
*/
void
subTransactions(SessionPtrType session);
/**
* @brief Unsubscribe from the transactions stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubTransactions(SessionPtrType session);
/**
* @brief Publish to the book changes stream.
*
* @param blobs The transactions to serialize
* @param lgrInfo The ledger header to serialize
*/
void
pubTransaction(data::TransactionAndMetadata const& blobs, ripple::LedgerHeader const& lgrInfo);
/**
* @brief Subscribe to the account changes stream.
*
* @param account The account to monitor changes for
* @param session The session to subscribe to the stream
*/
void
subAccount(ripple::AccountID const& account, SessionPtrType const& session);
/**
* @brief Unsubscribe from the account changes stream.
*
* @param account The account the stream is for
* @param session The session to unsubscribe from the stream
*/
void
unsubAccount(ripple::AccountID const& account, SessionPtrType const& session);
/**
* @brief Subscribe to a specific book changes stream.
*
* @param book The book to monitor changes for
* @param session The session to subscribe to the stream
*/
void
subBook(ripple::Book const& book, SessionPtrType session);
/**
* @brief Unsubscribe from the specific book changes stream.
*
* @param book The book to stop monitoring changes for
* @param session The session to unsubscribe from the stream
*/
void
unsubBook(ripple::Book const& book, SessionPtrType session);
/**
* @brief Subscribe to the book changes stream.
*
* @param session The session to subscribe to the stream
*/
void
subBookChanges(SessionPtrType session);
/**
* @brief Unsubscribe from the book changes stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubBookChanges(SessionPtrType session);
/**
* @brief Subscribe to the manifest stream.
*
* @param session The session to subscribe to the stream
*/
void
subManifest(SessionPtrType session);
/**
* @brief Unsubscribe from the manifest stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubManifest(SessionPtrType session);
/**
* @brief Subscribe to the validation stream.
*
* @param session The session to subscribe to the stream
*/
void
subValidation(SessionPtrType session);
/**
* @brief Unsubscribe from the validation stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubValidation(SessionPtrType session);
/**
* @brief Publish proposed transactions and proposed accounts from a JSON response.
*
* @param response The JSON response to use
*/
void
forwardProposedTransaction(boost::json::object const& response);
/**
* @brief Publish manifest updates from a JSON response.
*
* @param response The JSON response to use
*/
void
forwardManifest(boost::json::object const& response);
/**
* @brief Publish validation updates from a JSON response.
*
* @param response The JSON response to use
*/
void
forwardValidation(boost::json::object const& response);
/**
* @brief Subscribe to the proposed account stream.
*
* @param account The account to monitor
* @param session The session to subscribe to the stream
*/
void
subProposedAccount(ripple::AccountID const& account, SessionPtrType session);
/**
* @brief Unsubscribe from the proposed account stream.
*
* @param account The account the stream is for
* @param session The session to unsubscribe from the stream
*/
void
unsubProposedAccount(ripple::AccountID const& account, SessionPtrType session);
/**
* @brief Subscribe to the processed transactions stream.
*
* @param session The session to subscribe to the stream
*/
void
subProposedTransactions(SessionPtrType session);
/**
* @brief Unsubscribe from the proposed transactions stream.
*
* @param session The session to unsubscribe from the stream
*/
void
unsubProposedTransactions(SessionPtrType session);
/** @brief Clenup the session on removal. */
void
cleanup(SessionPtrType session);
/**
* @brief Generate a JSON report on the current state of the subscriptions.
*
* @return The report as a JSON object
*/
boost::json::object
report() const
{
boost::json::object counts = {};
counts["ledger"] = ledgerSubscribers_.count();
counts["transactions"] = txSubscribers_.count();
counts["transactions_proposed"] = txProposedSubscribers_.count();
counts["manifests"] = manifestSubscribers_.count();
counts["validations"] = validationsSubscribers_.count();
counts["account"] = accountSubscribers_.count();
counts["accounts_proposed"] = accountProposedSubscribers_.count();
counts["books"] = bookSubscribers_.count();
counts["book_changes"] = bookChangesSubscribers_.count();
return counts;
return {
{"ledger", ledgerSubscribers_.count()},
{"transactions", txSubscribers_.count()},
{"transactions_proposed", txProposedSubscribers_.count()},
{"manifests", manifestSubscribers_.count()},
{"validations", validationsSubscribers_.count()},
{"account", accountSubscribers_.count()},
{"accounts_proposed", accountProposedSubscribers_.count()},
{"books", bookSubscribers_.count()},
{"book_changes", bookChangesSubscribers_.count()},
};
}
private:
void
sendAll(std::string const& pubMsg, std::unordered_set<SessionPtrType>& subs);
using CleanupFunction = std::function<void(SessionPtrType const)>;
void
@@ -362,14 +581,11 @@ private:
void
subscribeHelper(SessionPtrType const& session, Key const& k, SubscriptionMap<Key>& subs, CleanupFunction&& func);
/**
* This is how we chose to cleanup subscriptions that have been closed.
* Each time we add a subscriber, we add the opposite lambda that
* unsubscribes that subscriber when cleanup is called with the session that
* closed.
*/
// This is how we chose to cleanup subscriptions that have been closed.
// Each time we add a subscriber, we add the opposite lambda that unsubscribes that subscriber when cleanup is
// called with the session that closed.
std::mutex cleanupMtx_;
std::unordered_map<SessionPtrType, std::vector<CleanupFunction>> cleanupFuncs_ = {};
};
} // namespace feed
} // namespace feed

10
src/main/Main.cpp
View File

@@ -201,15 +201,15 @@ try
// ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes
auto etl = etl::ETLService::make_ETLService(config, ioc, backend, subscriptions, balancer, ledgers);
auto workQueue = WorkQueue::make_WorkQueue(config);
auto counters = RPC::Counters::make_Counters(workQueue);
auto const handlerProvider = std::make_shared<RPC::detail::ProductionHandlerProvider const>(
auto workQueue = rpc::WorkQueue::make_WorkQueue(config);
auto counters = rpc::Counters::make_Counters(workQueue);
auto const handlerProvider = std::make_shared<rpc::detail::ProductionHandlerProvider const>(
config, backend, subscriptions, balancer, etl, counters);
auto const rpcEngine = RPC::RPCEngine::make_RPCEngine(
auto const rpcEngine = rpc::RPCEngine::make_RPCEngine(
config, backend, subscriptions, balancer, etl, dosGuard, workQueue, counters, handlerProvider);
// init the web server
auto handler = std::make_shared<RPCServerHandler<RPC::RPCEngine, etl::ETLService>>(
auto handler = std::make_shared<web::RPCServerHandler<rpc::RPCEngine, etl::ETLService>>(
config, backend, rpcEngine, etl, subscriptions);
auto ctx = parseCerts(config);
auto const ctxRef = ctx ? std::optional<std::reference_wrapper<ssl::context>>{ctx.value()} : std::nullopt;

43
src/main/Mainpage.h Normal file
View File

@@ -0,0 +1,43 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2023, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
/**
* @mainpage Clio API server
*
* @section intro Introduction
*
* Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC.
*
* Validated historical ledger and transaction data are stored in a more space-efficient format, using up to 4 times
* less space than rippled.
*
* Clio can be configured to store data in Apache Cassandra or ScyllaDB, allowing for scalable read throughput.
* Multiple Clio nodes can share access to the same dataset, allowing for a highly available cluster of Clio nodes,
* without the need for redundant data storage or computation.
*
* You can read more general information about Clio and its subsystems from the `Related Pages` section.
*
* @section Develop
*
* As you prepare to develop code for Clio, please be sure you are aware of our current
* <A HREF="https://github.com/XRPLF/clio/blob/develop/CONTRIBUTING.md">Contribution guidelines</A>.
*
* Read `rpc/README.md` carefully to know more about writing your own handlers for
* Clio.
*/

11
src/rpc/Amendments.h
View File

@@ -23,12 +23,19 @@
#include <ripple/protocol/digest.h>
namespace RPC {
namespace rpc {
#define REGISTER_AMENDMENT(name) inline static const ripple::uint256 name = GetAmendmentId(#name);
/**
* @brief Represents a list of amendments in the XRPL.
*/
struct Amendments
{
/**
* @param name The name of the amendment
* @return The corresponding amendment Id
*/
static ripple::uint256 const
GetAmendmentId(std::string_view const name)
{
@@ -38,4 +45,4 @@ struct Amendments
REGISTER_AMENDMENT(DisallowIncoming)
REGISTER_AMENDMENT(Clawback)
};
} // namespace RPC
} // namespace rpc

19
src/rpc/BookChangesHelper.h
View File

@@ -17,13 +17,14 @@
*/
//==============================================================================
/** @file */
#pragma once
#include <rpc/RPCHelpers.h>
#include <set>
namespace RPC {
namespace rpc {
/**
* @brief Represents an entry in the book_changes' changes array.
@@ -178,7 +179,7 @@ private:
void
handleBookChange(data::TransactionAndMetadata const& blob)
{
auto const [tx, meta] = RPC::deserializeTxPlusMeta(blob);
auto const [tx, meta] = rpc::deserializeTxPlusMeta(blob);
if (!tx || !meta || !tx->isFieldPresent(ripple::sfTransactionType))
return;
@@ -205,6 +206,12 @@ private:
};
};
/**
* @brief Implementation of value_from for BookChange type.
*
* @param jv The JSON value to populate
* @param change The BookChange to serialize
*/
inline void
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookChange const& change)
{
@@ -228,7 +235,13 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookChange const
};
}
/**
* @brief Computes all book changes for the given ledger header and transactions.
*
* @param lgrInfo The ledger header
* @param transactions The vector of transactions with heir metadata
*/
[[nodiscard]] boost::json::object const
computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::TransactionAndMetadata> const& transactions);
} // namespace RPC
} // namespace rpc

4
src/rpc/Counters.cpp
View File

@@ -21,7 +21,7 @@
#include <rpc/JS.h>
#include <rpc/RPCHelpers.h>
namespace RPC {
namespace rpc {
void
Counters::rpcFailed(std::string const& method)
@@ -137,4 +137,4 @@ Counters::report() const
return obj;
}
} // namespace RPC
} // namespace rpc

53
src/rpc/Counters.h
View File

@@ -28,10 +28,16 @@
#include <string>
#include <unordered_map>
namespace RPC {
namespace rpc {
/**
* @brief Holds information about successful, failed, forwarded, etc. RPC handler calls.
*/
class Counters
{
/**
* @brief All counters the system keeps track of for each RPC method.
*/
struct MethodInfo
{
std::uint64_t started = 0u;
@@ -57,49 +63,92 @@ class Counters
std::chrono::time_point<std::chrono::system_clock> startupTime_;
public:
/**
* @brief Creates a new counters instance that operates on the given WorkQueue.
*
* @param wq The work queue to operate on
*/
Counters(WorkQueue const& wq) : workQueue_(std::cref(wq)), startupTime_{std::chrono::system_clock::now()} {};
/**
* @brief A factory function that creates a new counters instance.
*
* @param wq The work queue to operate on
* @return The new instance
*/
static Counters
make_Counters(WorkQueue const& wq)
{
return Counters{wq};
}
/**
* @brief Increments the failed count for a particular RPC method.
*
* @param method The method to increment the count for
*/
void
rpcFailed(std::string const& method);
/**
* @brief Increments the errored count for a particular RPC method.
*
* @param method The method to increment the count for
*/
void
rpcErrored(std::string const& method);
/**
* @brief Increments the completed count for a particular RPC method.
*
* @param method The method to increment the count for
*/
void
rpcComplete(std::string const& method, std::chrono::microseconds const& rpcDuration);
/**
* @brief Increments the forwarded count for a particular RPC method.
*
* @param method The method to increment the count for
*/
void
rpcForwarded(std::string const& method);
/**
* @brief Increments the failed to forward count for a particular RPC method.
*
* @param method The method to increment the count for
*/
void
rpcFailedToForward(std::string const& method);
/** @brief Increments the global too busy counter. */
void
onTooBusy();
/** @brief Increments the global not ready counter. */
void
onNotReady();
/** @brief Increments the global bad syntax counter. */
void
onBadSyntax();
/** @brief Increments the global unknown command/method counter. */
void
onUnknownCommand();
/** @brief Increments the global internal error counter. */
void
onInternalError();
/** @return Uptime of this instance in seconds. */
std::chrono::seconds
uptime() const;
/** @return A JSON report with current state of all counters for every method. */
boost::json::object
report() const;
};
} // namespace RPC
} // namespace rpc

4
src/rpc/Errors.cpp
View File

@@ -36,7 +36,7 @@ template <class... Ts>
overloadSet(Ts...) -> overloadSet<Ts...>;
} // namespace
namespace RPC {
namespace rpc {
WarningInfo const&
getWarningInfo(WarningCode code)
@@ -152,4 +152,4 @@ makeError(Status const& status)
return res;
}
} // namespace RPC
} // namespace rpc

81
src/rpc/Errors.h
View File

@@ -17,6 +17,7 @@
*/
//==============================================================================
/** @file */
#pragma once
#include <ripple/protocol/ErrorCodes.h>
@@ -28,11 +29,9 @@
#include <string_view>
#include <variant>
namespace RPC {
namespace rpc {
/**
* @brief Custom clio RPC Errors.
*/
/** @brief Custom clio RPC Errors. */
enum class ClioError {
// normal clio errors start with 5000
rpcMALFORMED_CURRENCY = 5000,
@@ -51,9 +50,7 @@ enum class ClioError {
rpcPARAMS_UNPARSEABLE = 6004,
};
/**
* @brief Holds info about a particular @ref ClioError.
*/
/** @brief Holds info about a particular @ref ClioError. */
struct ClioErrorInfo
{
ClioError const code;
@@ -61,9 +58,7 @@ struct ClioErrorInfo
std::string_view const message;
};
/**
* @brief Clio uses compatible Rippled error codes for most RPC errors.
*/
/** @brief Clio uses compatible Rippled error codes for most RPC errors. */
using RippledError = ripple::error_code_i;
/**
@@ -74,9 +69,7 @@ using RippledError = ripple::error_code_i;
*/
using CombinedError = std::variant<RippledError, ClioError>;
/**
* @brief A status returned from any RPC handler.
*/
/** @brief A status returned from any RPC handler. */
struct Status
{
CombinedError code = RippledError::rpcSUCCESS;
@@ -102,9 +95,7 @@ struct Status
{
}
/**
* @brief Returns true if the Status is *not* OK.
*/
/** @brief Returns true if the Status is *not* OK. */
operator bool() const
{
if (auto err = std::get_if<RippledError>(&code))
@@ -114,10 +105,10 @@ struct Status
}
/**
* @brief Returns true if the Status contains the desired @ref RippledError
* @brief Returns true if the @ref rpc::Status contains the desired @ref rpc::RippledError
*
* @param other The RippledError to match
* @return bool true if status matches given error; false otherwise
* @param other The @ref rpc::RippledError to match
* @return true if status matches given error; false otherwise
*/
bool
operator==(RippledError other) const
@@ -132,7 +123,7 @@ struct Status
* @brief Returns true if the Status contains the desired @ref ClioError
*
* @param other The RippledError to match
* @return bool true if status matches given error; false otherwise
* @return true if status matches given error; false otherwise
*/
bool
operator==(ClioError other) const
@@ -144,14 +135,10 @@ struct Status
}
};
/**
* @brief Warning codes that can be returned by clio.
*/
/** @brief Warning codes that can be returned by clio. */
enum WarningCode { warnUNKNOWN = -1, warnRPC_CLIO = 2001, warnRPC_OUTDATED = 2002, warnRPC_RATE_LIMIT = 2003 };
/**
* @brief Holds information about a clio warning.
*/
/** @brief Holds information about a clio warning. */
struct WarningInfo
{
constexpr WarningInfo() = default;
@@ -163,9 +150,7 @@ struct WarningInfo
std::string_view const message = "unknown warning";
};
/**
* @brief Invalid parameters error.
*/
/** @brief Invalid parameters error. */
class InvalidParamsError : public std::exception
{
std::string msg;
@@ -182,9 +167,7 @@ public:
}
};
/**
* @brief Account not found error.
*/
/** @brief Account not found error. */
class AccountNotFoundError : public std::exception
{
std::string account;
@@ -201,16 +184,14 @@ public:
}
};
/**
* @brief A globally available @ref Status that represents a successful state
*/
/** @brief A globally available @ref rpc::Status that represents a successful state. */
static Status OK;
/**
* @brief Get the warning info object from a warning code.
*
* @param code The warning code
* @return WarningInfo const& A reference to the static warning info
* @return A reference to the static warning info
*/
WarningInfo const&
getWarningInfo(WarningCode code);
@@ -219,34 +200,34 @@ getWarningInfo(WarningCode code);
* @brief Get the error info object from an clio-specific error code.
*
* @param code The error code
* @return ClioErrorInfo const& A reference to the static error info
* @return A reference to the static error info
*/
ClioErrorInfo const&
getErrorInfo(ClioError code);
/**
* @brief Generate JSON from a warning code.
* @brief Generate JSON from a @ref rpc::WarningCode.
*
* @param code The @ref WarningCode
* @return boost::json::object The JSON output
* @param code The warning code
* @return The JSON output
*/
boost::json::object
makeWarning(WarningCode code);
/**
* @brief Generate JSON from a @ref Status.
* @brief Generate JSON from a @ref rpc::Status.
*
* @param status The @ref Status
* @return boost::json::object The JSON output
* @param status The status object
* @return The JSON output
*/
boost::json::object
makeError(Status const& status);
/**
* @brief Generate JSON from a @ref RippledError.
* @brief Generate JSON from a @ref rpc::RippledError.
*
* @param status The rippled @ref RippledError
* @return boost::json::object The JSON output
* @param err The rippled error
* @return The JSON output
*/
boost::json::object
makeError(
@@ -255,10 +236,10 @@ makeError(
std::optional<std::string_view> customMessage = std::nullopt);
/**
* @brief Generate JSON from a @ref ClioError.
* @brief Generate JSON from a @ref rpc::ClioError.
*
* @param status The clio's custom @ref ClioError
* @return boost::json::object The JSON output
* @param err The clio's custom error
* @return The JSON output
*/
boost::json::object
makeError(
@@ -266,4 +247,4 @@ makeError(
std::optional<std::string_view> customError = std::nullopt,
std::optional<std::string_view> customMessage = std::nullopt);
} // namespace RPC
} // namespace rpc

4
src/rpc/Factories.cpp
View File

@@ -23,7 +23,7 @@
using namespace std;
using namespace util;
namespace RPC {
namespace rpc {
util::Expected<web::Context, Status>
make_WsContext(
@@ -94,4 +94,4 @@ make_HttpContext(
return web::Context(yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp);
}
} // namespace RPC
} // namespace rpc

29
src/rpc/Factories.h
View File

@@ -36,11 +36,22 @@
* This file contains various classes necessary for executing RPC handlers.
* Context gives the handlers access to various other parts of the application Status is used to report errors.
* And lastly, there are various functions for making Contexts, Statuses and serializing Status to JSON.
* This file is meant to contain any class or function that code outside of the rpc folder needs to use. For helper
* functions or classes used within the rpc folder, use RPCHelpers.h.
* This file is meant to contain any class or function that code outside of the rpc folder needs to use.
* For helper functions or classes used within the rpc folder, use RPCHelpers.h.
*/
namespace RPC {
namespace rpc {
/**
* @brief A factory function that creates a Websocket context.
*
* @param yc The coroutine context
* @param request The request as JSON object
* @param session The connection
* @param tagFactory A factory that provides tags to track requests
* @param range The ledger range that is available at request time
* @param clientIp The IP address of the connected client
* @param apiVersionParser A parser that is used to parse out the "api_version" field
*/
util::Expected<web::Context, Status>
make_WsContext(
boost::asio::yield_context yc,
@@ -51,6 +62,16 @@ make_WsContext(
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser);
/**
* @brief A factory function that creates a HTTP context.
*
* @param yc The coroutine context
* @param request The request as JSON object
* @param tagFactory A factory that provides tags to track requests
* @param range The ledger range that is available at request time
* @param clientIp The IP address of the connected client
* @param apiVersionParser A parser that is used to parse out the "api_version" field
*/
util::Expected<web::Context, Status>
make_HttpContext(
boost::asio::yield_context yc,
@@ -60,4 +81,4 @@ make_HttpContext(
std::string const& clientIp,
std::reference_wrapper<APIVersionParser const> apiVersionParser);
} // namespace RPC
} // namespace rpc

5
src/rpc/JS.h
View File

@@ -21,9 +21,8 @@
#include <ripple/protocol/jss.h>
// Useful macro for borrowing from ripple::jss
// static strings. (J)son (S)trings
/** @brief Helper macro for borrowing from ripple::jss static (J)son (S)trings. */
#define JS(x) ripple::jss::x.c_str()
// Access (SF)ield name (S)trings
/** @brief Provides access to (SF)ield name (S)trings. */
#define SFS(x) ripple::x.jsonName.c_str()

19
src/rpc/README.md
View File

@@ -1,29 +1,22 @@
# Clio RPC subsystem
# RPC subsystem
## Background
The RPC subsystem is where the common framework for handling incoming JSON requests is implemented.
Currently the NextGen RPC framework is a work in progress and the handlers are not yet implemented using the new common framework classes.
## Integration plan
- Implement base framework - **done**
- Migrate handlers one by one, making them injectable, adding unit-tests - **in progress**
- Integrate all new handlers into clio in one go
- Cover the rest with unit-tests
- Release first time with new subsystem active
## Components
See `common` subfolder.
- **AnyHandler**: The type-erased wrapper that allows for storing different handlers in one map/vector.
- **RpcSpec/FieldSpec**: The RPC specification classes, used to specify how incoming JSON is to be validated before it's parsed and passed on to individual handler implementations.
- **Validators**: A bunch of supported validators that can be specified as requirements for each **`FieldSpec`** to make up the final **`RpcSpec`** of any given RPC handler.
- **Validators/Modifiers**: A bunch of supported validators and modifiers that can be specified as requirements for each **`FieldSpec`** to make up the final **`RpcSpec`** of any given RPC handler.
## Implementing a (NextGen) handler
## Implementing a handler
See `unittests/rpc` for exmaples.
Handlers need to fulfil the requirements specified by the **`Handler`** concept (see `rpc/common/Concepts.h`):
Handlers need to fulfil the requirements specified by the **`SomeHandler`** concept (see `rpc/common/Concepts.h`):
- Expose types:
* `Input` - The POD struct which acts as input for the handler
* `Output` - The POD struct which acts as output of a valid handler invocation
- Have a `spec(uint32_t)` member function returning a const reference to an **`RpcSpec`** describing the JSON input.
- Have a `spec(uint32_t)` member function returning a const reference to an **`RpcSpec`** describing the JSON input for the specified API version.
- Have a `process(Input)` member function that operates on `Input` POD and returns `HandlerReturnType<Output>`
- Implement `value_from` and `value_to` support using `tag_invoke` as per `boost::json` documentation for these functions.

43
src/rpc/RPCEngine.h
View File

@@ -52,10 +52,13 @@ class LoadBalancer;
class ETLService;
} // namespace etl
namespace RPC {
/**
* @brief This namespace contains all the RPC logic and handlers.
*/
namespace rpc {
/**
* @brief The RPC engine that ties all RPC-related functionality together
* @brief The RPC engine that ties all RPC-related functionality together.
*/
template <typename AdminVerificationStrategyType>
class RPCEngineBase
@@ -113,8 +116,10 @@ public:
}
/**
* @brief Main request processor routine
* @brief Main request processor routine.
*
* @param ctx The @ref Context of the request
* @return A result which can be an error status or a valid JSON response
*/
Result
buildResponse(web::Context const& ctx)
@@ -171,22 +176,24 @@ public:
}
/**
* @brief Used to schedule request processing onto the work queue
* @brief Used to schedule request processing onto the work queue.
*
* @tparam FnType The type of function
* @param func The lambda to execute when this request is handled
* @param ip The ip address for which this request is being executed
*/
template <typename Fn>
template <typename FnType>
bool
post(Fn&& func, std::string const& ip)
post(FnType&& func, std::string const& ip)
{
return workQueue_.get().postCoro(std::forward<Fn>(func), dosGuard_.get().isWhiteListed(ip));
return workQueue_.get().postCoro(std::forward<FnType>(func), dosGuard_.get().isWhiteListed(ip));
}
/**
* @brief Notify the system that specified method was executed
* @brief Notify the system that specified method was executed.
*
* @param method
* @param duration The time it took to execute the method specified in
* microseconds
* @param duration The time it took to execute the method specified in microseconds
*/
void
notifyComplete(std::string const& method, std::chrono::microseconds const& duration)
@@ -196,7 +203,7 @@ public:
}
/**
* @brief Notify the system that specified method failed to execute due to a recoverable user error
* @brief Notify the system that specified method failed to execute due to a recoverable user error.
*
* Used for errors based on user input, not actual failures of the db or clio itself.
*
@@ -211,7 +218,7 @@ public:
}
/**
* @brief Notify the system that specified method failed due to some unrecoverable error
* @brief Notify the system that specified method failed due to some unrecoverable error.
*
* Used for erors such as database timeout, internal errors, etc.
*
@@ -225,7 +232,7 @@ public:
}
/**
* @brief Notify the system that the RPC system is too busy to handle an incoming request
* @brief Notify the system that the RPC system is too busy to handle an incoming request.
*/
void
notifyTooBusy()
@@ -234,7 +241,7 @@ public:
}
/**
* @brief Notify the system that the RPC system was not ready to handle an incoming request
* @brief Notify the system that the RPC system was not ready to handle an incoming request.
*
* This happens when the backend is not yet have a ledger range
*/
@@ -245,7 +252,7 @@ public:
}
/**
* @brief Notify the system that the incoming request did not specify the RPC method/command
* @brief Notify the system that the incoming request did not specify the RPC method/command.
*/
void
notifyBadSyntax()
@@ -254,7 +261,7 @@ public:
}
/**
* @brief Notify the system that the incoming request specified an unknown/unsupported method/command
* @brief Notify the system that the incoming request specified an unknown/unsupported method/command.
*/
void
notifyUnknownCommand()
@@ -263,7 +270,7 @@ public:
}
/**
* @brief Notify the system that the incoming request lead to an internal error (unrecoverable)
* @brief Notify the system that the incoming request lead to an internal error (unrecoverable).
*/
void
notifyInternalError()
@@ -281,4 +288,4 @@ private:
using RPCEngine = RPCEngineBase<detail::IPAdminVerificationStrategy>;
} // namespace RPC
} // namespace rpc

4
src/rpc/RPCHelpers.cpp
View File

@@ -35,7 +35,7 @@ namespace {
util::Logger gLog{"RPC"};
} // namespace
namespace RPC {
namespace rpc {
std::optional<AccountCursor>
parseAccountCursor(std::optional<std::string> jsonCursor)
@@ -1340,4 +1340,4 @@ isAmendmentEnabled(
return std::find(listAmendments.begin(), listAmendments.end(), amendmentId) != listAmendments.end();
}
} // namespace RPC
} // namespace rpc

5
src/rpc/RPCHelpers.h
View File

@@ -17,6 +17,7 @@
*/
//==============================================================================
/** @file */
#pragma once
/*
@@ -37,7 +38,7 @@
#include <fmt/core.h>
namespace RPC {
namespace rpc {
enum class NFTokenjson { ENABLE, DISABLE };
@@ -246,4 +247,4 @@ logDuration(web::Context const& ctx, T const& dur)
log.info() << ctx.tag() << msg;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/WorkQueue.cpp
View File

@@ -19,6 +19,8 @@
#include <rpc/WorkQueue.h>
namespace rpc {
WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize) : ioc_{numWorkers}
{
if (maxSize != 0)
@@ -29,3 +31,5 @@ WorkQueue::~WorkQueue()
{
ioc_.join();
}
} // namespace rpc

42
src/rpc/WorkQueue.h
View File

@@ -32,6 +32,11 @@
#include <shared_mutex>
#include <thread>
namespace rpc {
/**
* @brief An asynchronous, thread-safe queue for RPC requests.
*/
class WorkQueue
{
// these are cumulative for the lifetime of the process
@@ -45,9 +50,20 @@ class WorkQueue
boost::asio::thread_pool ioc_;
public:
/**
* @brief Create an we instance of the work queue.
*
* @param numWorkers The amount of threads to spawn in the pool
* @param maxSize The maximum capacity of the queue; 0 means unlimited
*/
WorkQueue(std::uint32_t numWorkers, uint32_t maxSize = 0);
~WorkQueue();
/**
* @brief A factory function that creates the work queue based on a config.
*
* @param config The Clio config to use
*/
static WorkQueue
make_WorkQueue(util::Config const& config)
{
@@ -60,9 +76,19 @@ public:
return WorkQueue{numThreads, maxQueueSize};
}
template <typename F>
/**
* @brief Submit a job to the work queue.
*
* The job will be rejected if isWhiteListed is set to false and the current size of the queue reached capacity.
*
* @tparam FnType The function object type
* @param func The function object to queue as a job
* @param isWhiteListed Whether the queue capacity applies to this job
* @return true if the job was successfully queued; false otherwise
*/
template <typename FnType>
bool
postCoro(F&& f, bool isWhiteListed)
postCoro(FnType&& func, bool isWhiteListed)
{
if (curSize_ >= maxSize_ && !isWhiteListed)
{
@@ -75,7 +101,8 @@ public:
// Each time we enqueue a job, we want to post a symmetrical job that will dequeue and run the job at the front
// of the job queue.
boost::asio::spawn(
ioc_, [this, f = std::forward<F>(f), start = std::chrono::system_clock::now()](auto yield) mutable {
ioc_,
[this, func = std::forward<FnType>(func), start = std::chrono::system_clock::now()](auto yield) mutable {
auto const run = std::chrono::system_clock::now();
auto const wait = std::chrono::duration_cast<std::chrono::microseconds>(run - start).count();
@@ -83,13 +110,18 @@ public:
durationUs_ += wait;
log_.info() << "WorkQueue wait time = " << wait << " queue size = " << curSize_;
f(yield);
func(yield);
--curSize_;
});
return true;
}
/**
* @brief Generate a report of the work queue state.
*
* @return The report as a JSON object.
*/
boost::json::object
report() const
{
@@ -103,3 +135,5 @@ public:
return obj;
}
};
} // namespace rpc

10
src/rpc/common/APIVersion.h
View File

@@ -26,7 +26,7 @@
#include <string>
namespace RPC {
namespace rpc {
/**
* @brief Default API version to use if no version is specified by clients
@@ -54,7 +54,13 @@ class APIVersionParser
public:
virtual ~APIVersionParser() = default;
/**
* @brief Extracts API version information from a JSON object.
*
* @param request A JSON object representing the request
* @return The specified API version if contained in the JSON object; error string otherwise
*/
util::Expected<uint32_t, std::string> virtual parse(boost::json::object const& request) const = 0;
};
} // namespace RPC
} // namespace rpc

8
src/rpc/common/AnyHandler.h
View File

@@ -23,7 +23,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/impl/Processors.h>
namespace RPC {
namespace rpc {
/**
* @brief A type-erased Handler that can contain any (NextGen) RPC handler class
@@ -40,9 +40,9 @@ public:
*
* @tparam HandlerType The real type of wrapped handler class
* @tparam ProcessingStrategy A strategy that implements how processing of JSON is to be done
* @param handler The handler to wrap. Required to fulfil the @ref Handler concept.
* @param handler The handler to wrap. Required to fulfil the @ref rpc::SomeHandler concept.
*/
template <Handler HandlerType, typename ProcessingStrategy = detail::DefaultProcessor<HandlerType>>
template <SomeHandler HandlerType, typename ProcessingStrategy = detail::DefaultProcessor<HandlerType>>
/* implicit */ AnyHandler(HandlerType&& handler)
: pimpl_{std::make_unique<Model<HandlerType, ProcessingStrategy>>(std::forward<HandlerType>(handler))}
{
@@ -117,4 +117,4 @@ private:
std::unique_ptr<Concept> pimpl_;
};
} // namespace RPC
} // namespace rpc

60
src/rpc/common/Concepts.h
View File

@@ -26,65 +26,87 @@
#include <string>
namespace RPC {
namespace rpc {
struct RpcSpec;
/**
* @brief A concept that specifies what a requirement used with @ref FieldSpec
* must provide
* @brief Specifies what a requirement used with @ref rpc::FieldSpec must provide.
*/
// clang-format off
template <typename T>
concept Requirement = requires(T a, boost::json::value lval) {
concept SomeRequirement = requires(T a, boost::json::value lval) {
{ a.verify(lval, std::string{}) } -> std::same_as<MaybeError>;
};
// clang-format on
/**
* @brief Specifies what a modifier used with @ref rpc::FieldSpec must provide.
*/
// clang-format off
template <typename T>
concept Modifier = requires(T a, boost::json::value lval) {
concept SomeModifier = requires(T a, boost::json::value lval) {
{ a.modify(lval, std::string{}) } -> std::same_as<MaybeError>;
};
// clang-format on
/**
* @brief The requirements of a processor to be used with @ref rpc::FieldSpec.
*/
template <typename T>
concept Processor = (Requirement<T> or Modifier<T>);
concept SomeProcessor = (SomeRequirement<T> or SomeModifier<T>);
/**
* @brief A concept that specifies what a Handler type must provide
*
* Note that value_from and value_to should be implemented using tag_invoke
* as per boost::json documentation for these functions.
* @brief A process function that expects both some Input and a Context.
*/
// clang-format off
template <typename T>
concept ContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) {
concept SomeContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) {
{ a.process(in, ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
};
// clang-format on
/**
* @brief A process function that expects no Input but does take a Context.
*/
// clang-format off
template <typename T>
concept ContextProcessWithoutInput = requires(T a, typename T::Output out, Context const& ctx) {
concept SomeContextProcessWithoutInput = requires(T a, typename T::Output out, Context const& ctx) {
{ a.process(ctx) } -> std::same_as<HandlerReturnType<decltype(out)>>;
};
// clang-format on
/**
* @brief Specifies what a Handler with Input must provide.
*/
// clang-format off
template <typename T>
concept HandlerWithInput = requires(T a, uint32_t version) {
concept SomeHandlerWithInput = requires(T a, uint32_t version) {
{ a.spec(version) } -> std::same_as<RpcSpecConstRef>;
}
and ContextProcessWithInput<T>
and SomeContextProcessWithInput<T>
and boost::json::has_value_to<typename T::Input>::value;
// clang-format on
/**
* @brief Specifies what a Handler without Input must provide.
*/
// clang-format off
template <typename T>
concept HandlerWithoutInput = ContextProcessWithoutInput<T>;
concept SomeHandlerWithoutInput = SomeContextProcessWithoutInput<T>;
// clang-format on
/**
* @brief Specifies what a Handler type must provide.
*/
// clang-format off
template <typename T>
concept Handler =
concept SomeHandler =
(
HandlerWithInput<T> or
HandlerWithoutInput<T>
SomeHandlerWithInput<T> or
SomeHandlerWithoutInput<T>
)
and boost::json::has_value_from<typename T::Output>::value;
// clang-format on
} // namespace RPC
} // namespace rpc

4
src/rpc/common/MetaProcessors.cpp
View File

@@ -24,7 +24,7 @@
#include <string_view>
namespace RPC::meta {
namespace rpc::meta {
[[nodiscard]] MaybeError
Section::verify(boost::json::value& value, std::string_view key) const
@@ -68,4 +68,4 @@ ValidateArrayAt::verify(boost::json::value& value, std::string_view key) const
return {};
}
} // namespace RPC::meta
} // namespace rpc::meta

47
src/rpc/common/MetaProcessors.h
View File

@@ -26,10 +26,10 @@
#include <fmt/core.h>
namespace RPC::meta {
namespace rpc::meta {
/**
* @brief A meta-processor that acts as a spec for a sub-object/section
* @brief A meta-processor that acts as a spec for a sub-object/section.
*/
class Section final
{
@@ -37,7 +37,7 @@ class Section final
public:
/**
* @brief Construct new section validator from a list of specs
* @brief Construct new section validator from a list of specs.
*
* @param specs List of specs @ref FieldSpec
*/
@@ -46,17 +46,18 @@ public:
}
/**
* @brief Verify that the JSON value representing the section is valid according to the given specs
* @brief Verify that the JSON value representing the section is valid according to the given specs.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the section from the outer object
* @return Possibly an error
*/
[[nodiscard]] MaybeError
verify(boost::json::value& value, std::string_view key) const;
};
/**
* @brief A meta-processor that specifies a list of specs to run against the object at the given index in the array
* @brief A meta-processor that specifies a list of specs to run against the object at the given index in the array.
*/
class ValidateArrayAt final
{
@@ -65,7 +66,7 @@ class ValidateArrayAt final
public:
/**
* @brief Constructs a processor that validates the specified element of a JSON array
* @brief Constructs a processor that validates the specified element of a JSON array.
*
* @param idx The index inside the array to validate
* @param specs The specifications to validate against
@@ -75,10 +76,11 @@ public:
}
/**
* @brief Verify that the JSON array element at given index is valid according the stored specs
* @brief Verify that the JSON array element at given index is valid according the stored specs.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the array from the outer object
* @return Possibly an error
*/
[[nodiscard]] MaybeError
verify(boost::json::value& value, std::string_view key) const;
@@ -86,17 +88,17 @@ public:
/**
* @brief A meta-processor that specifies a list of requirements to run against when the type matches the template
* parameter
* parameter.
*/
template <typename Type>
class IfType final
{
public:
/**
* @brief Constructs a validator that validates the specs if the type matches
* @brief Constructs a validator that validates the specs if the type matches.
* @param requirements The requirements to validate against
*/
template <Requirement... Requirements>
template <SomeRequirement... Requirements>
IfType(Requirements&&... requirements)
{
processor_ = [... r = std::forward<Requirements>(requirements)](
@@ -122,11 +124,11 @@ public:
}
/**
* @brief Verify that the element is valid
* according the stored requirements when type matches
* @brief Verify that the element is valid according to the stored requirements when type matches.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the element from the outer object
* @return Possibly an error
*/
[[nodiscard]] MaybeError
verify(boost::json::value& value, std::string_view key) const
@@ -134,7 +136,7 @@ public:
if (not value.is_object() or not value.as_object().contains(key.data()))
return {}; // ignore. field does not exist, let 'required' fail instead
if (not RPC::validation::checkType<Type>(value.as_object().at(key.data())))
if (not rpc::validation::checkType<Type>(value.as_object().at(key.data())))
return {}; // ignore if type does not match
return processor_(value, key);
@@ -145,23 +147,30 @@ private:
};
/**
* @brief A meta-processor that wraps another validator and produces a custom error in case the wrapped validator fails
* @brief A meta-processor that wraps a validator and produces a custom error in case the wrapped validator fails.
*/
template <typename Requirement>
template <typename SomeRequirement>
class WithCustomError final
{
Requirement requirement;
SomeRequirement requirement;
Status error;
public:
/**
* @brief Constructs a validator that calls the given validator `req` and returns a custom error `err` in case `req`
* fails
* fails.
*/
WithCustomError(Requirement req, Status err) : requirement{std::move(req)}, error{err}
WithCustomError(SomeRequirement req, Status err) : requirement{std::move(req)}, error{err}
{
}
/**
* @brief Runs the stored validator and produces a custom error if the wrapped validator fails.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the element from the outer object
* @return Possibly an error
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
{
@@ -172,4 +181,4 @@ public:
}
};
} // namespace RPC::meta
} // namespace rpc::meta

7
src/rpc/common/Modifiers.h
View File

@@ -23,7 +23,7 @@
#include <rpc/common/Specs.h>
#include <rpc/common/Types.h>
namespace RPC::modifiers {
namespace rpc::modifiers {
/**
* @brief Clamp value between min and max.
@@ -36,7 +36,7 @@ class Clamp final
public:
/**
* @brief Construct the modifier storing min and max values
* @brief Construct the modifier storing min and max values.
*
* @param min
* @param max
@@ -50,6 +50,7 @@ public:
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the modified value from the outer object
* @return Possibly an error
*/
[[nodiscard]] MaybeError
modify(boost::json::value& value, std::string_view key) const
@@ -67,4 +68,4 @@ public:
}
};
} // namespace RPC::modifiers
} // namespace rpc::modifiers

4
src/rpc/common/Specs.cpp
View File

@@ -21,7 +21,7 @@
#include <boost/json/value.hpp>
namespace RPC {
namespace rpc {
[[nodiscard]] MaybeError
FieldSpec::process(boost::json::value& value) const
@@ -39,4 +39,4 @@ RpcSpec::process(boost::json::value& value) const
return {};
}
} // namespace RPC
} // namespace rpc

22
src/rpc/common/Specs.h
View File

@@ -26,28 +26,28 @@
#include <string>
#include <vector>
namespace RPC {
namespace rpc {
/**
* @brief Represents a Specification for one field of an RPC command
* @brief Represents a Specification for one field of an RPC command.
*/
struct FieldSpec final
{
/**
* @brief Construct a field specification out of a set of processors
* @brief Construct a field specification out of a set of processors.
*
* @tparam Processors The types of processors @ref Processor
* @tparam Processors The types of processors
* @param key The key in a JSON object that the field validates
* @param processors The processors, each of them have to fulfil the @ref Processor concept
* @param processors The processors, each of them have to fulfil the @ref rpc::SomeProcessor concept
*/
template <Processor... Processors>
template <SomeProcessor... Processors>
FieldSpec(std::string const& key, Processors&&... processors)
: processor_{detail::makeFieldProcessor<Processors...>(key, std::forward<Processors>(processors)...)}
{
}
/**
* @brief Processos the passed JSON value using the stored processors
* @brief Processos the passed JSON value using the stored processors.
*
* @param value The JSON value to validate and/or modify
* @return Nothing on success; @ref Status on error
@@ -60,7 +60,7 @@ private:
};
/**
* @brief Represents a Specification of an entire RPC command
* @brief Represents a Specification of an entire RPC command.
*
* Note: this should really be all constexpr and handlers would expose
* static constexpr RpcSpec spec instead. Maybe some day in the future.
@@ -68,7 +68,7 @@ private:
struct RpcSpec final
{
/**
* @brief Construct a full RPC request specification
* @brief Construct a full RPC request specification.
*
* @param fields The fields of the RPC specification @ref FieldSpec
*/
@@ -77,7 +77,7 @@ struct RpcSpec final
}
/**
* @brief Processos the passed JSON value using the stored field specs
* @brief Processos the passed JSON value using the stored field specs.
*
* @param value The JSON value to validate and/or modify
* @return Nothing on success; @ref Status on error
@@ -89,4 +89,4 @@ private:
std::vector<FieldSpec> fields_;
};
} // namespace RPC
} // namespace rpc

30
src/rpc/common/Types.h
View File

@@ -36,9 +36,12 @@ namespace feed {
class SubscriptionManager;
}
namespace RPC {
namespace rpc {
class Counters;
struct RpcSpec;
struct FieldSpec;
class AnyHandler;
/**
* @brief Return type used for Validators that can return error but don't have
@@ -62,15 +65,21 @@ using HandlerReturnType = util::Expected<OutputType, Status>;
*/
using ReturnType = util::Expected<boost::json::value, Status>;
struct RpcSpec;
struct FieldSpec;
/**
* @brief An alias for a const reference to @ref RpcSpec.
*/
using RpcSpecConstRef = RpcSpec const&;
/**
* @brief An empty type used as Output for handlers than don't actually produce output.
*/
struct VoidOutput
{
};
/**
* @brief Context of an RPC call.
*/
struct Context
{
boost::asio::yield_context yield;
@@ -80,8 +89,14 @@ struct Context
uint32_t apiVersion = 0u; // invalid by default
};
/**
* @brief Result type used to return responses or error statuses to the Webserver subsystem.
*/
using Result = std::variant<Status, boost::json::object>;
/**
* @brief A cursor object used to traverse nodes owned by an account.
*/
struct AccountCursor
{
ripple::uint256 index;
@@ -100,8 +115,9 @@ struct AccountCursor
}
};
class AnyHandler;
/**
* @brief Interface for the provider of RPC handlers.
*/
class HandlerProvider
{
public:
@@ -123,4 +139,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, VoidOutput const
jv = boost::json::object{};
}
} // namespace RPC
} // namespace rpc

4
src/rpc/common/Validators.cpp
View File

@@ -26,7 +26,7 @@
#include <charconv>
#include <string_view>
namespace RPC::validation {
namespace rpc::validation {
[[nodiscard]] MaybeError
Required::verify(boost::json::value const& value, std::string_view key) const
@@ -202,4 +202,4 @@ CustomValidator SubscribeAccountsValidator =
return MaybeError{};
}};
} // namespace RPC::validation
} // namespace rpc::validation

171
src/rpc/common/Validators.h
View File

@@ -25,10 +25,10 @@
#include <fmt/core.h>
namespace RPC::validation {
namespace rpc::validation {
/**
* @brief Check that the type is the same as what was expected
* @brief Check that the type is the same as what was expected.
*
* @tparam Expected The expected type that value should be convertible to
* @param value The json value to check the type of
@@ -79,7 +79,7 @@ template <typename Expected>
}
/**
* @brief A validator that simply requires a field to be present
* @brief A validator that simply requires a field to be present.
*/
struct Required final
{
@@ -88,15 +88,16 @@ struct Required final
};
/**
* @brief A validator that forbids a field to be present
* If there is a value provided, it will forbid the field only when the value equals
* If there is no value provided, it will forbid the field when the field shows up
* @brief A validator that forbids a field to be present.
*
* If there is a value provided, it will forbid the field only when the value equals.
* If there is no value provided, it will forbid the field when the field shows up.
*/
template <typename... T>
class NotSupported;
/**
* @brief A specialized NotSupported validator that forbids a field to be present when the value equals the given value
* @brief A specialized NotSupported validator that forbids a field to be present when the value equals the given value.
*/
template <typename T>
class NotSupported<T> final
@@ -104,6 +105,22 @@ class NotSupported<T> final
T value_;
public:
/**
* @brief Constructs a new NotSupported validator.
*
* @param val The value to store and verify against
*/
NotSupported(T val) : value_(val)
{
}
/**
* @brief Verify whether the field is supported or not.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcNOT_SUPPORTED` if the value matched; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
{
@@ -118,19 +135,22 @@ public:
}
return {};
}
NotSupported(T val) : value_(val)
{
}
};
/**
* @brief A specialized NotSupported validator that forbids a field to be present
* @brief A specialized NotSupported validator that forbids a field to be present.
*/
template <>
class NotSupported<> final
{
public:
/**
* @brief Verify whether the field is supported or not.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcNOT_SUPPORTED` if the field is found; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
{
@@ -141,22 +161,24 @@ public:
}
};
// deduction guide to avoid having to specify the template arguments
/**
* @brief Deduction guide to avoid having to specify the template arguments.
*/
template <typename... T>
NotSupported(T&&... t) -> NotSupported<T...>;
/**
* @brief Validates that the type of the value is one of the given types
* @brief Validates that the type of the value is one of the given types.
*/
template <typename... Types>
struct Type final
{
/**
* @brief Verify that the JSON value is (one) of specified type(s)
* @brief Verify that the JSON value is (one) of specified type(s).
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -175,7 +197,7 @@ struct Type final
};
/**
* @brief Validate that value is between specified min and max
* @brief Validate that value is between specified min and max.
*/
template <typename Type>
class Between final
@@ -185,7 +207,7 @@ class Between final
public:
/**
* @brief Construct the validator storing min and max values
* @brief Construct the validator storing min and max values.
*
* @param min
* @param max
@@ -195,11 +217,11 @@ public:
}
/**
* @brief Verify that the JSON value is within a certain range
* @brief Verify that the JSON value is within a certain range.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -221,7 +243,7 @@ public:
};
/**
* @brief Validate that value is equal or greater than the specified min
* @brief Validate that value is equal or greater than the specified min.
*/
template <typename Type>
class Min final
@@ -230,7 +252,7 @@ class Min final
public:
/**
* @brief Construct the validator storing min value
* @brief Construct the validator storing min value.
*
* @param min
*/
@@ -242,8 +264,8 @@ public:
* @brief Verify that the JSON value is not smaller than min
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -263,7 +285,7 @@ public:
};
/**
* @brief Validate that value is not greater than max
* @brief Validate that value is not greater than max.
*/
template <typename Type>
class Max final
@@ -272,7 +294,7 @@ class Max final
public:
/**
* @brief Construct the validator storing max value
* @brief Construct the validator storing max value.
*
* @param max
*/
@@ -281,10 +303,11 @@ public:
}
/**
* @brief Verify that the JSON value is not greater than max
* @brief Verify that the JSON value is not greater than max.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -304,7 +327,7 @@ public:
};
/**
* @brief Validates that the value is equal to the one passed in
* @brief Validates that the value is equal to the one passed in.
*/
template <typename Type>
class EqualTo final
@@ -313,7 +336,7 @@ class EqualTo final
public:
/**
* @brief Construct the validator with stored original value
* @brief Construct the validator with stored original value.
*
* @param original The original value to store
*/
@@ -322,11 +345,11 @@ public:
}
/**
* @brief Verify that the JSON value is equal to the stored original
* @brief Verify that the JSON value is equal to the stored original.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -345,13 +368,12 @@ public:
};
/**
* @brief Deduction guide to help disambiguate what it means to EqualTo a
* "string" without specifying the type.
* @brief Deduction guide to help disambiguate what it means to EqualTo a "string" without specifying the type.
*/
EqualTo(char const*)->EqualTo<std::string>;
/**
* @brief Validates that the value is one of the values passed in
* @brief Validates that the value is one of the values passed in.
*/
template <typename Type>
class OneOf final
@@ -360,7 +382,7 @@ class OneOf final
public:
/**
* @brief Construct the validator with stored options of initializer list
* @brief Construct the validator with stored options of initializer list.
*
* @param options The list of allowed options
*/
@@ -369,7 +391,7 @@ public:
}
/**
* @brief Construct the validator with stored options of other container
* @brief Construct the validator with stored options of other container.
*
* @param begin,end the range to copy the elements from
*/
@@ -379,11 +401,11 @@ public:
}
/**
* @brief Verify that the JSON value is one of the stored options
* @brief Verify that the JSON value is one of the stored options.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const
@@ -402,13 +424,12 @@ public:
};
/**
* @brief Deduction guide to help disambiguate what it means to OneOf a
* few "strings" without specifying the type.
* @brief Deduction guide to help disambiguate what it means to OneOf a few "strings" without specifying the type.
*/
OneOf(std::initializer_list<char const*>)->OneOf<std::string>;
/**
* @brief A meta-validator that allows to specify a custom validation function
* @brief A meta-validator that allows to specify a custom validation function.
*/
class CustomValidator final
{
@@ -416,7 +437,7 @@ class CustomValidator final
public:
/**
* @brief Constructs a custom validator from any supported callable
* @brief Constructs a custom validator from any supported callable.
*
* @tparam Fn The type of callable
* @param fn The callable/function object
@@ -427,79 +448,83 @@ public:
}
/**
* @brief Verify that the JSON value is valid according to the custom
* validation function stored
* @brief Verify that the JSON value is valid according to the custom validation function stored.
*
* @param value The JSON value representing the outer object
* @param key The key used to retrieve the tested value from the outer
* object
* @param key The key used to retrieve the tested value from the outer object
* @return Any compatible user-provided error if validation failed; otherwise no error is returned
*/
[[nodiscard]] MaybeError
verify(boost::json::value const& value, std::string_view key) const;
};
/**
* @brief Helper function to check if sv is an uint32 number or not
* @brief Helper function to check if input value is an uint32 number or not.
*
* @param sv The input value as a string_view
* @return true if the string can be converted to a uint32; false otherwise
*/
[[nodiscard]] bool
checkIsU32Numeric(std::string_view sv);
/**
* @brief Provide a common used validator for ledger index
* LedgerIndex must be a string or int
* If the specified LedgerIndex is a string, it's value must be either
* @brief Provides a commonly used validator for ledger index.
*
* LedgerIndex must be a string or an int. If the specified LedgerIndex is a string, its value must be either
* "validated" or a valid integer value represented as a string.
*/
extern CustomValidator LedgerIndexValidator;
/**
* @brief Provide a common used validator for account
* Account must be a string and the converted public key is valid
* @brief Provides a commonly used validator for accounts.
*
* Account must be a string and the converted public key is valid.
*/
extern CustomValidator AccountValidator;
/**
* @brief Provide a common used validator for account
* Account must be a string and can convert to base58
* @brief Provides a commonly used validator for accounts.
*
* Account must be a string and can convert to base58.
*/
extern CustomValidator AccountBase58Validator;
/**
* @brief Provide a common used validator for marker
* Marker is composed of a comma separated index and start hint. The
* former will be read as hex, and the latter can cast to uint64.
* @brief Provides a commonly used validator for markers.
*
* A marker is composed of a comma-separated index and a start hint.
* The former will be read as hex, and the latter can be cast to uint64.
*/
extern CustomValidator AccountMarkerValidator;
/**
* @brief Provide a common used validator for uint256 hex string
* It must be a string and hex
* Transaction index, ledger hash all use this validator
* @brief Provides a commonly used validator for uint256 hex string.
*
* It must be a string and also a decodable hex.
* Transaction index, ledger hash all use this validator.
*/
extern CustomValidator Uint256HexStringValidator;
/**
* @brief Provide a common used validator for currency
* including standard currency code and token code
* @brief Provides a commonly used validator for currency, including standard currency code and token code.
*/
extern CustomValidator CurrencyValidator;
/**
* @brief Provide a common used validator for issuer type
* It must be a hex string or base58 string
* @brief Provides a commonly used validator for issuer type.
*
* It must be a hex string or base58 string.
*/
extern CustomValidator IssuerValidator;
/**
* @brief Provide a validator for validating valid streams used in
* subscribe/unsubscribe
* @brief Provides a validator for validating streams used in subscribe/unsubscribe.
*/
extern CustomValidator SubscribeStreamValidator;
/**
* @brief Provide a validator for validating valid accounts used in
* subscribe/unsubscribe
* @brief Provides a validator for validating accounts used in subscribe/unsubscribe.
*/
extern CustomValidator SubscribeAccountsValidator;
} // namespace RPC::validation
} // namespace rpc::validation

4
src/rpc/common/impl/APIVersionParser.cpp
View File

@@ -24,7 +24,7 @@
using namespace std;
namespace RPC::detail {
namespace rpc::detail {
ProductionAPIVersionParser::ProductionAPIVersionParser(util::Config const& config)
: ProductionAPIVersionParser(
@@ -58,4 +58,4 @@ ProductionAPIVersionParser::parse(boost::json::object const& request) const
return defaultVersion_;
}
} // namespace RPC::detail
} // namespace rpc::detail

4
src/rpc/common/impl/APIVersionParser.h
View File

@@ -27,7 +27,7 @@
#include <algorithm>
#include <string>
namespace RPC::detail {
namespace rpc::detail {
class ProductionAPIVersionParser : public APIVersionParser
{
@@ -94,4 +94,4 @@ public:
}
};
} // namespace RPC::detail
} // namespace rpc::detail

4
src/rpc/common/impl/AdminVerificationStrategy.h
View File

@@ -21,7 +21,7 @@
#include <string_view>
namespace RPC::detail {
namespace rpc::detail {
class IPAdminVerificationStrategy final
{
@@ -39,4 +39,4 @@ public:
}
};
} // namespace RPC::detail
} // namespace rpc::detail

10
src/rpc/common/impl/Factories.h
View File

@@ -26,12 +26,12 @@
#include <optional>
namespace RPC::detail {
namespace rpc::detail {
template <typename>
static constexpr bool unsupported_v = false;
template <Processor... Processors>
template <SomeProcessor... Processors>
[[nodiscard]] auto
makeFieldProcessor(std::string const& key, Processors&&... procs)
{
@@ -45,10 +45,10 @@ makeFieldProcessor(std::string const& key, Processors&&... procs)
if (firstFailure)
return; // already failed earlier - skip
if constexpr (Requirement<decltype(*req)>) {
if constexpr (SomeRequirement<decltype(*req)>) {
if (auto const res = req->verify(j, key); not res)
firstFailure = res.error();
} else if constexpr (Modifier<decltype(*req)>) {
} else if constexpr (SomeModifier<decltype(*req)>) {
if (auto const res = req->modify(j, key); not res)
firstFailure = res.error();
} else {
@@ -65,4 +65,4 @@ makeFieldProcessor(std::string const& key, Processors&&... procs)
};
}
} // namespace RPC::detail
} // namespace rpc::detail

4
src/rpc/common/impl/ForwardingProxy.h
View File

@@ -30,7 +30,7 @@
#include <memory>
#include <string>
namespace RPC::detail {
namespace rpc::detail {
template <typename LoadBalancerType, typename CountersType, typename HandlerProviderType>
class ForwardingProxy
@@ -148,4 +148,4 @@ private:
}
};
} // namespace RPC::detail
} // namespace rpc::detail

4
src/rpc/common/impl/HandlerProvider.cpp
View File

@@ -53,7 +53,7 @@
#include <rpc/handlers/Unsubscribe.h>
#include <rpc/handlers/VersionHandler.h>
namespace RPC::detail {
namespace rpc::detail {
ProductionHandlerProvider::ProductionHandlerProvider(
util::Config const& config,
@@ -117,4 +117,4 @@ ProductionHandlerProvider::isClioOnly(std::string const& command) const
return handlerMap_.contains(command) && handlerMap_.at(command).isClioOnly;
}
} // namespace RPC::detail
} // namespace rpc::detail

6
src/rpc/common/impl/HandlerProvider.h
View File

@@ -32,14 +32,14 @@ namespace etl {
class ETLService;
class LoadBalancer;
} // namespace etl
namespace RPC {
namespace rpc {
class Counters;
}
namespace feed {
class SubscriptionManager;
}
namespace RPC::detail {
namespace rpc::detail {
class ProductionHandlerProvider final : public HandlerProvider
{
@@ -70,4 +70,4 @@ public:
isClioOnly(std::string const& command) const override;
};
} // namespace RPC::detail
} // namespace rpc::detail

13
src/rpc/common/impl/Processors.h
View File

@@ -23,12 +23,12 @@
#include <rpc/common/Concepts.h>
#include <rpc/common/Types.h>
namespace RPC::detail {
namespace rpc::detail {
template <typename>
static constexpr bool unsupported_handler_v = false;
template <Handler HandlerType>
template <SomeHandler HandlerType>
struct DefaultProcessor final
{
[[nodiscard]] ReturnType
@@ -36,7 +36,7 @@ struct DefaultProcessor final
{
using boost::json::value_from;
using boost::json::value_to;
if constexpr (HandlerWithInput<HandlerType>)
if constexpr (SomeHandlerWithInput<HandlerType>)
{
// first we run validation against specified API version
auto const spec = handler.spec(ctx.apiVersion);
@@ -54,7 +54,7 @@ struct DefaultProcessor final
else
return value_from(ret.value());
}
else if constexpr (HandlerWithoutInput<HandlerType>)
else if constexpr (SomeHandlerWithoutInput<HandlerType>)
{
// no input to pass, ignore the value
if (auto const ret = handler.process(ctx); not ret)
@@ -64,11 +64,10 @@ struct DefaultProcessor final
}
else
{
// when concept HandlerWithInput and HandlerWithoutInput not cover
// all Handler case
// when concept SomeHandlerWithInput and SomeHandlerWithoutInput not cover all Handler case
static_assert(unsupported_handler_v<HandlerType>);
}
}
};
} // namespace RPC::detail
} // namespace rpc::detail

4
src/rpc/handlers/AccountChannels.cpp
View File

@@ -20,7 +20,7 @@
#include <rpc/RPCHelpers.h>
#include <rpc/handlers/AccountChannels.h>
namespace RPC {
namespace rpc {
void
AccountChannelsHandler::addChannel(std::vector<ChannelResponse>& jsonChannels, ripple::SLE const& channelSle) const
@@ -188,4 +188,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountChannelsH
jv = std::move(obj);
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountChannels.h
View File

@@ -27,7 +27,7 @@
#include <vector>
namespace RPC {
namespace rpc {
/**
* @brief The account_channels method returns information about an account's Payment Channels. This includes only
@@ -126,4 +126,4 @@ private:
friend void
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, ChannelResponse const& channel);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountCurrencies.cpp
View File

@@ -19,7 +19,7 @@
#include <rpc/handlers/AccountCurrencies.h>
namespace RPC {
namespace rpc {
AccountCurrenciesHandler::Result
AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context const& ctx) const
{
@@ -114,4 +114,4 @@ tag_invoke(boost::json::value_to_tag<AccountCurrenciesHandler::Input>, boost::js
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountCurrencies.h
View File

@@ -27,7 +27,7 @@
#include <set>
namespace RPC {
namespace rpc {
/**
* @brief The account_currencies command retrieves a list of currencies that an account can send or receive,
@@ -87,4 +87,4 @@ private:
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

8
src/rpc/handlers/AccountInfo.cpp
View File

@@ -21,7 +21,7 @@
#include <ripple/protocol/ErrorCodes.h>
namespace RPC {
namespace rpc {
AccountInfoHandler::Result
AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx) const
{
@@ -51,10 +51,10 @@ AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx)
return Error{Status{RippledError::rpcDB_DESERIALIZATION}};
auto const isDisallowIncomingEnabled =
RPC::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, RPC::Amendments::DisallowIncoming);
rpc::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, rpc::Amendments::DisallowIncoming);
auto const isClawbackEnabled =
RPC::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, RPC::Amendments::Clawback);
rpc::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, rpc::Amendments::Clawback);
// Return SignerList(s) if that is requested.
if (input.signerLists)
@@ -173,4 +173,4 @@ tag_invoke(boost::json::value_to_tag<AccountInfoHandler::Input>, boost::json::va
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountInfo.h
View File

@@ -25,7 +25,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
namespace RPC {
namespace rpc {
/**
* @brief The account_info command retrieves information about an account, its activity, and its XRP balance.
@@ -106,4 +106,4 @@ private:
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountLines.cpp
View File

@@ -19,7 +19,7 @@
#include <rpc/handlers/AccountLines.h>
namespace RPC {
namespace rpc {
void
AccountLinesHandler::addLine(
@@ -236,4 +236,4 @@ tag_invoke(
jv = std::move(obj);
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountLines.h
View File

@@ -28,7 +28,7 @@
#include <vector>
namespace RPC {
namespace rpc {
/**
* @brief The account_lines method returns information about an account's trust lines, which contain balances in all
@@ -135,4 +135,4 @@ private:
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, LineResponse const& line);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountNFTs.cpp
View File

@@ -22,7 +22,7 @@
#include <ripple/protocol/Serializer.h>
#include <ripple/protocol/nft.h>
namespace RPC {
namespace rpc {
AccountNFTsHandler::Result
AccountNFTsHandler::process(AccountNFTsHandler::Input input, Context const& ctx) const
@@ -146,4 +146,4 @@ tag_invoke(boost::json::value_to_tag<AccountNFTsHandler::Input>, boost::json::va
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountNFTs.h
View File

@@ -25,7 +25,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
namespace RPC {
namespace rpc {
/**
* @brief The account_nfts method returns a list of NFToken objects for the specified account.
@@ -96,4 +96,4 @@ private:
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountObjects.cpp
View File

@@ -19,7 +19,7 @@
#include <rpc/handlers/AccountObjects.h>
namespace RPC {
namespace rpc {
std::unordered_map<std::string, ripple::LedgerEntryType> const AccountObjectsHandler::TYPESMAP{
{"state", ripple::ltRIPPLE_STATE},
@@ -168,4 +168,4 @@ tag_invoke(boost::json::value_to_tag<AccountObjectsHandler::Input>, boost::json:
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountObjects.h
View File

@@ -27,7 +27,7 @@
#include <set>
namespace RPC {
namespace rpc {
/**
* @brief The account_objects command returns the raw ledger format for all objects owned by an account.
@@ -122,4 +122,4 @@ private:
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountOffers.cpp
View File

@@ -19,7 +19,7 @@
#include <rpc/handlers/AccountOffers.h>
namespace RPC {
namespace rpc {
void
AccountOffersHandler::addOffer(std::vector<Offer>& offers, ripple::SLE const& offerSle) const
@@ -157,4 +157,4 @@ tag_invoke(boost::json::value_to_tag<AccountOffersHandler::Input>, boost::json::
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountOffers.h
View File

@@ -26,7 +26,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
namespace RPC {
namespace rpc {
/**
* @brief The account_offers method retrieves a list of offers made by a given account.
@@ -111,4 +111,4 @@ private:
friend void
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Offer const& offer);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountTx.cpp
View File

@@ -20,7 +20,7 @@
#include <rpc/handlers/AccountTx.h>
#include <util/Profiler.h>
namespace RPC {
namespace rpc {
// TODO: this is currently very similar to nft_history but its own copy for time
// being. we should aim to reuse common logic in some way in the future.
@@ -211,4 +211,4 @@ tag_invoke(boost::json::value_to_tag<AccountTxHandler::Input>, boost::json::valu
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/AccountTx.h
View File

@@ -27,7 +27,7 @@
#include <rpc/common/Validators.h>
#include <util/log/Logger.h>
namespace RPC {
namespace rpc {
/**
* @brief The account_tx method retrieves a list of transactions that involved the specified account.
@@ -127,4 +127,4 @@ private:
friend void
tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Marker const& marker);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/BookChanges.cpp
View File

@@ -19,7 +19,7 @@
#include <rpc/handlers/BookChanges.h>
namespace RPC {
namespace rpc {
BookChangesHandler::Result
BookChangesHandler::process(BookChangesHandler::Input input, Context const& ctx) const
@@ -92,4 +92,4 @@ computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector<data::Transa
};
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/BookChanges.h
View File

@@ -25,7 +25,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
namespace RPC {
namespace rpc {
/**
* @brief BookChangesHandler returns the order book changes for a given ledger.
@@ -81,4 +81,4 @@ private:
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/BookOffers.cpp
View File

@@ -20,7 +20,7 @@
#include <rpc/RPCHelpers.h>
#include <rpc/handlers/BookOffers.h>
namespace RPC {
namespace rpc {
BookOffersHandler::Result
BookOffersHandler::process(Input input, Context const& ctx) const
@@ -98,4 +98,4 @@ tag_invoke(boost::json::value_to_tag<BookOffersHandler::Input>, boost::json::val
return input;
}
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/BookOffers.h
View File

@@ -25,7 +25,7 @@
#include <rpc/common/Types.h>
#include <rpc/common/Validators.h>
namespace RPC {
namespace rpc {
/**
* @brief The book_offers method retrieves a list of Offers between two currencies, also known as an order book.
@@ -117,4 +117,4 @@ private:
friend Input
tag_invoke(boost::json::value_to_tag<Input>, boost::json::value const& jv);
};
} // namespace RPC
} // namespace rpc

4
src/rpc/handlers/DepositAuthorized.cpp
View File

@@ -20,7 +20,7 @@
#include <rpc/RPCHelpers.h>
#include <rpc/handlers/DepositAuthorized.h>
namespace RPC {
namespace rpc {
DepositAuthorizedHandler::Result
DepositAuthorizedHandler::process(DepositAuthorizedHandler::Input input, Context const& ctx) const
@@ -112,4 +112,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, DepositAuthorize
};
}
} // namespace RPC
} // namespace rpc

Some files were not shown because too many files have changed in this diff Show More