From 547cb340bd3967a3e721aa00353cdbd911208a34 Mon Sep 17 00:00:00 2001 From: Alex Kremer Date: Fri, 11 Aug 2023 21:32:32 +0100 Subject: [PATCH] Update doxygen comments (#818) Fixes #421 --- CMake/Docs.cmake | 11 + CMakeLists.txt | 13 +- Doxyfile | 17 +- README.md | 14 +- conanfile.py | 13 +- src/data/BackendFactory.h | 8 + src/data/BackendInterface.cpp | 40 +- src/data/BackendInterface.h | 395 +++++++------- src/data/CassandraBackend.h | 18 +- src/data/DBHelpers.h | 63 ++- src/data/LedgerCache.h | 64 ++- src/data/README.md | 3 +- src/data/Types.h | 37 +- src/data/cassandra/Concepts.h | 9 + src/data/cassandra/Error.h | 14 +- src/data/cassandra/Handle.h | 131 +++-- src/data/cassandra/Schema.h | 13 +- src/data/cassandra/SettingsProvider.h | 28 +- src/data/cassandra/impl/Batch.cpp | 3 +- src/data/cassandra/impl/Cluster.h | 58 +++ src/data/cassandra/impl/ExecutionStrategy.h | 30 +- src/data/cassandra/impl/Future.cpp | 7 +- src/data/cassandra/impl/Future.h | 10 +- src/data/cassandra/impl/Result.h | 26 +- src/data/cassandra/impl/Statement.h | 48 +- src/data/cassandra/impl/Tuple.h | 16 +- src/etl/ETLHelpers.h | 42 +- src/etl/ETLService.h | 31 +- src/etl/LoadBalancer.cpp | 21 +- src/etl/LoadBalancer.h | 75 ++- src/etl/NFTHelpers.h | 14 +- src/etl/ProbingSource.cpp | 8 +- src/etl/ProbingSource.h | 6 +- src/etl/README.md | 2 + src/etl/Source.h | 453 ++++++++++------- src/etl/SystemState.h | 31 +- src/etl/impl/ForwardCache.cpp | 2 +- src/etl/impl/LedgerFetcher.h | 12 +- src/etl/impl/LedgerLoader.h | 3 + src/etl/impl/Transformer.h | 29 +- src/feed/Message.h | 59 --- src/feed/SubscriptionManager.cpp | 22 +- src/feed/SubscriptionManager.h | 480 +++++++++++++----- src/main/Main.cpp | 10 +- src/main/Mainpage.h | 43 ++ src/rpc/Amendments.h | 11 +- src/rpc/BookChangesHelper.h | 19 +- src/rpc/Counters.cpp | 4 +- src/rpc/Counters.h | 53 +- src/rpc/Errors.cpp | 4 +- src/rpc/Errors.h | 81 ++- src/rpc/Factories.cpp | 4 +- src/rpc/Factories.h | 29 +- src/rpc/JS.h | 5 +- src/rpc/README.md | 19 +- src/rpc/RPCEngine.h | 43 +- src/rpc/RPCHelpers.cpp | 4 +- src/rpc/RPCHelpers.h | 5 +- src/rpc/WorkQueue.cpp | 4 + src/rpc/WorkQueue.h | 42 +- src/rpc/common/APIVersion.h | 10 +- src/rpc/common/AnyHandler.h | 8 +- src/rpc/common/Concepts.h | 60 ++- src/rpc/common/MetaProcessors.cpp | 4 +- src/rpc/common/MetaProcessors.h | 47 +- src/rpc/common/Modifiers.h | 7 +- src/rpc/common/Specs.cpp | 4 +- src/rpc/common/Specs.h | 22 +- src/rpc/common/Types.h | 30 +- src/rpc/common/Validators.cpp | 4 +- src/rpc/common/Validators.h | 171 ++++--- src/rpc/common/impl/APIVersionParser.cpp | 4 +- src/rpc/common/impl/APIVersionParser.h | 4 +- .../common/impl/AdminVerificationStrategy.h | 4 +- src/rpc/common/impl/Factories.h | 10 +- src/rpc/common/impl/ForwardingProxy.h | 4 +- src/rpc/common/impl/HandlerProvider.cpp | 4 +- src/rpc/common/impl/HandlerProvider.h | 6 +- src/rpc/common/impl/Processors.h | 13 +- src/rpc/handlers/AccountChannels.cpp | 4 +- src/rpc/handlers/AccountChannels.h | 4 +- src/rpc/handlers/AccountCurrencies.cpp | 4 +- src/rpc/handlers/AccountCurrencies.h | 4 +- src/rpc/handlers/AccountInfo.cpp | 8 +- src/rpc/handlers/AccountInfo.h | 4 +- src/rpc/handlers/AccountLines.cpp | 4 +- src/rpc/handlers/AccountLines.h | 4 +- src/rpc/handlers/AccountNFTs.cpp | 4 +- src/rpc/handlers/AccountNFTs.h | 4 +- src/rpc/handlers/AccountObjects.cpp | 4 +- src/rpc/handlers/AccountObjects.h | 4 +- src/rpc/handlers/AccountOffers.cpp | 4 +- src/rpc/handlers/AccountOffers.h | 4 +- src/rpc/handlers/AccountTx.cpp | 4 +- src/rpc/handlers/AccountTx.h | 4 +- src/rpc/handlers/BookChanges.cpp | 4 +- src/rpc/handlers/BookChanges.h | 4 +- src/rpc/handlers/BookOffers.cpp | 4 +- src/rpc/handlers/BookOffers.h | 4 +- src/rpc/handlers/DepositAuthorized.cpp | 4 +- src/rpc/handlers/DepositAuthorized.h | 4 +- src/rpc/handlers/GatewayBalances.cpp | 4 +- src/rpc/handlers/GatewayBalances.h | 4 +- src/rpc/handlers/Ledger.cpp | 6 +- src/rpc/handlers/Ledger.h | 4 +- src/rpc/handlers/LedgerData.cpp | 4 +- src/rpc/handlers/LedgerData.h | 10 +- src/rpc/handlers/LedgerEntry.cpp | 4 +- src/rpc/handlers/LedgerEntry.h | 4 +- src/rpc/handlers/LedgerRange.cpp | 4 +- src/rpc/handlers/LedgerRange.h | 4 +- src/rpc/handlers/NFTBuyOffers.cpp | 4 +- src/rpc/handlers/NFTBuyOffers.h | 4 +- src/rpc/handlers/NFTHistory.cpp | 4 +- src/rpc/handlers/NFTHistory.h | 4 +- src/rpc/handlers/NFTInfo.cpp | 6 +- src/rpc/handlers/NFTInfo.h | 4 +- src/rpc/handlers/NFTOffersCommon.cpp | 6 +- src/rpc/handlers/NFTOffersCommon.h | 4 +- src/rpc/handlers/NFTSellOffers.cpp | 4 +- src/rpc/handlers/NFTSellOffers.h | 4 +- src/rpc/handlers/NoRippleCheck.cpp | 4 +- src/rpc/handlers/NoRippleCheck.h | 4 +- src/rpc/handlers/Ping.h | 4 +- src/rpc/handlers/Random.cpp | 4 +- src/rpc/handlers/Random.h | 4 +- src/rpc/handlers/ServerInfo.h | 8 +- src/rpc/handlers/Subscribe.h | 4 +- src/rpc/handlers/TransactionEntry.cpp | 4 +- src/rpc/handlers/TransactionEntry.h | 4 +- src/rpc/handlers/Tx.cpp | 4 +- src/rpc/handlers/Tx.h | 4 +- src/rpc/handlers/Unsubscribe.h | 4 +- src/rpc/handlers/VersionHandler.h | 8 +- src/util/JsonUtils.h | 10 + src/util/LedgerUtils.h | 27 +- src/util/Profiler.h | 20 +- src/util/Taggable.cpp | 6 +- src/util/Taggable.h | 84 +-- src/util/config/Config.cpp | 34 +- src/util/config/Config.h | 66 +-- src/util/log/Logger.cpp | 16 +- src/util/log/Logger.h | 84 +-- src/web/Context.h | 15 + src/web/DOSGuard.h | 30 +- src/web/HttpSession.h | 35 +- src/web/PlainWsSession.h | 81 ++- src/web/README.md | 9 +- src/web/RPCServerHandler.h | 58 ++- src/web/Server.h | 125 +++-- src/web/SslHttpSession.h | 56 +- src/web/SslWsSession.h | 83 +-- src/web/WhitelistHandler.h | 8 +- src/web/impl/ErrorHandling.h | 42 +- src/web/impl/HttpBase.h | 20 +- src/web/impl/WsBase.h | 26 +- src/web/interface/Concepts.h | 4 +- src/web/interface/ConnectionBase.h | 29 +- unittests/README.md | 4 +- unittests/SubscriptionTests.cpp | 8 - unittests/data/cassandra/BackendTests.cpp | 2 +- unittests/rpc/APIVersionTests.cpp | 2 +- unittests/rpc/AdminVerificationTests.cpp | 2 +- unittests/rpc/AmendmentsTests.cpp | 2 +- unittests/rpc/BaseTests.cpp | 14 +- unittests/rpc/CountersTests.cpp | 2 +- unittests/rpc/ErrorTests.cpp | 2 +- unittests/rpc/ForwardingProxyTests.cpp | 4 +- unittests/rpc/RPCHelpersTests.cpp | 2 +- unittests/rpc/WorkQueueTests.cpp | 1 + .../rpc/handlers/AccountChannelsTests.cpp | 36 +- .../rpc/handlers/AccountCurrenciesTests.cpp | 10 +- unittests/rpc/handlers/AccountInfoTests.cpp | 20 +- unittests/rpc/handlers/AccountLinesTests.cpp | 40 +- unittests/rpc/handlers/AccountNFTsTests.cpp | 12 +- .../rpc/handlers/AccountObjectsTests.cpp | 16 +- unittests/rpc/handlers/AccountOffersTests.cpp | 14 +- unittests/rpc/handlers/AccountTxTests.cpp | 8 +- unittests/rpc/handlers/BookChangesTests.cpp | 10 +- unittests/rpc/handlers/BookOffersTests.cpp | 18 +- .../rpc/handlers/DefaultProcessorTests.cpp | 10 +- .../rpc/handlers/DepositAuthorizedTests.cpp | 14 +- .../rpc/handlers/GatewayBalancesTests.cpp | 14 +- unittests/rpc/handlers/LedgerDataTests.cpp | 12 +- unittests/rpc/handlers/LedgerEntryTests.cpp | 18 +- unittests/rpc/handlers/LedgerRangeTests.cpp | 2 +- unittests/rpc/handlers/LedgerTests.cpp | 10 +- unittests/rpc/handlers/NFTBuyOffersTests.cpp | 34 +- unittests/rpc/handlers/NFTHistoryTests.cpp | 8 +- unittests/rpc/handlers/NFTInfoTests.cpp | 24 +- unittests/rpc/handlers/NFTSellOffersTests.cpp | 34 +- unittests/rpc/handlers/NoRippleCheckTests.cpp | 12 +- unittests/rpc/handlers/PingTests.cpp | 2 +- unittests/rpc/handlers/RandomTests.cpp | 2 +- unittests/rpc/handlers/ServerInfoTests.cpp | 14 +- unittests/rpc/handlers/SubscribeTests.cpp | 12 +- unittests/rpc/handlers/TestHandlerTests.cpp | 8 +- .../rpc/handlers/TransactionEntryTests.cpp | 14 +- unittests/rpc/handlers/TxTests.cpp | 12 +- unittests/rpc/handlers/UnsubscribeTests.cpp | 16 +- .../rpc/handlers/VersionHandlerTests.cpp | 2 +- unittests/rpc/handlers/impl/FakesAndMocks.h | 36 +- unittests/util/MockHandlerProvider.h | 4 +- unittests/util/MockRPCEngine.h | 4 +- unittests/util/StringUtils.cpp | 2 +- unittests/web/RPCServerHandlerTests.cpp | 5 +- 206 files changed, 3004 insertions(+), 1937 deletions(-) create mode 100644 CMake/Docs.cmake delete mode 100644 src/feed/Message.h create mode 100644 src/main/Mainpage.h diff --git a/CMake/Docs.cmake b/CMake/Docs.cmake new file mode 100644 index 00000000..4f2fbb00 --- /dev/null +++ b/CMake/Docs.cmake @@ -0,0 +1,11 @@ +find_package(Doxygen REQUIRED) + +set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile) +set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile) + +configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY) +add_custom_target(docs + COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Generating API documentation with Doxygen" + VERBATIM) diff --git a/CMakeLists.txt b/CMakeLists.txt index 92a62c34..8a375316 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,6 +6,7 @@ project(clio) # ==================================================== # option(verbose "Verbose build" FALSE) option(tests "Build tests" FALSE) +option(docs "Generate doxygen docs" FALSE) option(coverage "Build test coverage report" FALSE) option(packaging "Create distribution packages" FALSE) # ==================================================== # @@ -222,15 +223,21 @@ if(tests) target_include_directories(${TEST_TARGET} PRIVATE unittests) target_link_libraries(${TEST_TARGET} PUBLIC clio gtest::gtest) - # Generate `clio_test-ccov` if coverage is enabled - # Note: use `make clio_test-ccov` to generate report + # Generate `clio_tests-ccov` if coverage is enabled + # Note: use `make clio_tests-ccov` to generate report if(coverage) include(CMake/Coverage.cmake) add_coverage(${TEST_TARGET}) endif() endif() +# Generate `docs` target for doxygen documentation if enabled +# Note: use `make docs` to generate the documentation +if(docs) + include(CMake/Docs.cmake) +endif() + include(CMake/install/install.cmake) if(packaging) - include(CMake/packaging.cmake) # This file exists only in build runner + include(CMake/packaging.cmake) # This file exists only in build runner endif() diff --git a/Doxyfile b/Doxyfile index 1da44b70..9cfee3c0 100644 --- a/Doxyfile +++ b/Doxyfile @@ -1,3 +1,16 @@ PROJECT_NAME = "Clio" -INPUT = src -RECURSIVE = YES \ No newline at end of file +INPUT = ../src ../unittests +EXCLUDE_PATTERNS = *Test*.cpp *Test*.h +RECURSIVE = YES +HAVE_DOT = YES + +QUIET = YES +WARNINGS = NO +WARN_NO_PARAMDOC = NO +WARN_IF_INCOMPLETE_DOC = NO +WARN_IF_UNDOCUMENTED = NO + +GENERATE_LATEX = NO +GENERATE_HTML = YES + +SORT_MEMBERS_CTORS_1ST = YES diff --git a/README.md b/README.md index d9ffe9a9..1f2f28a7 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,16 @@ # Clio -Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. Validated -historical ledger and transaction data are stored in a more space-efficient format, + +Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. +Validated historical ledger and transaction data are stored in a more space-efficient format, using up to 4 times less space than rippled. Clio can be configured to store data in Apache Cassandra or ScyllaDB, -allowing for scalable read throughput. Multiple Clio nodes can share -access to the same dataset, allowing for a highly available cluster of Clio nodes, -without the need for redundant data storage or computation. +allowing for scalable read throughput. Multiple Clio nodes can share access to the same dataset, +allowing for a highly available cluster of Clio nodes, without the need for redundant data storage or computation. Clio offers the full rippled API, with the caveat that Clio by default only returns validated data. This means that `ledger_index` defaults to `validated` instead of `current` for all requests. Other non-validated data is also not returned, such as information about queued transactions. -For requests that require access to the p2p network, such as `fee` or `submit`, Clio automatically forwards the request to a rippled node and propagates the response back to the client. To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to rippled. +For requests that require access to the p2p network, such as `fee` or `submit`, Clio automatically forwards the request to a rippled node and propagates the response back to the client. +To access non-validated data for *any* request, simply add `ledger_index: "current"` to the request, and Clio will forward the request to rippled. Clio does not connect to the peer-to-peer network. Instead, Clio extracts data from a group of specified rippled nodes. Running Clio requires access to at least one rippled node from which data can be extracted. The rippled node does not need to be running on the same machine as Clio. @@ -25,7 +26,6 @@ It is written in C++20 and therefore requires a modern compiler. ## Prerequisites - ### Minimum Requirements - [Python 3.7](https://www.python.org/downloads/) diff --git a/conanfile.py b/conanfile.py index 4c7b6812..6a1b8c39 100644 --- a/conanfile.py +++ b/conanfile.py @@ -12,9 +12,10 @@ class Clio(ConanFile): options = { 'fPIC': [True, False], 'verbose': [True, False], - 'tests': [True, False], # build unit tests + 'tests': [True, False], # build unit tests; create `clio_tests` binary + 'docs': [True, False], # doxygen API docs; create custom target 'docs' 'packaging': [True, False], # create distribution packages - 'coverage': [True, False], # build for test coverage report + 'coverage': [True, False], # build for test coverage report; create custom target `clio_tests-ccov` } requires = [ @@ -22,7 +23,6 @@ class Clio(ConanFile): 'cassandra-cpp-driver/2.16.2', 'fmt/10.0.0', 'grpc/1.50.1', - 'gtest/1.13.0', 'openssl/1.1.1u', 'xrpl/1.12.0-b2', ] @@ -33,6 +33,7 @@ class Clio(ConanFile): 'tests': False, 'packaging': False, 'coverage': False, + 'docs': False, 'xrpl/*:tests': False, 'cassandra-cpp-driver/*:shared': False, @@ -52,6 +53,10 @@ class Clio(ConanFile): 'CMakeLists.txt', 'CMake/*', 'src/*' ) + def requirements(self): + if self.options.tests: + self.requires('gtest/1.13.0') + def configure(self): if self.settings.compiler == 'apple-clang': self.options['boost'].visibility = 'global' @@ -68,6 +73,8 @@ class Clio(ConanFile): tc.variables['verbose'] = self.options.verbose tc.variables['tests'] = self.options.tests tc.variables['coverage'] = self.options.coverage + tc.variables['docs'] = self.options.docs + tc.variables['packaging'] = self.options.packaging tc.generate() def build(self): diff --git a/src/data/BackendFactory.h b/src/data/BackendFactory.h index e5628d4b..4aa28bc7 100644 --- a/src/data/BackendFactory.h +++ b/src/data/BackendFactory.h @@ -27,6 +27,14 @@ #include namespace data { + +/** + * @brief A factory function that creates the backend based on a config. + * + * @param ioc The boost::asio::io_context to use + * @param config The clio config to use + * @return A shared_ptr with the selected implementation + */ std::shared_ptr make_Backend(boost::asio::io_context& ioc, util::Config const& config) { diff --git a/src/data/BackendInterface.cpp b/src/data/BackendInterface.cpp index 7aa9e211..0b59183c 100644 --- a/src/data/BackendInterface.cpp +++ b/src/data/BackendInterface.cpp @@ -48,22 +48,6 @@ BackendInterface::writeLedgerObject(std::string&& key, std::uint32_t const seq, doWriteLedgerObject(std::move(key), seq, std::move(blob)); } -std::optional -BackendInterface::hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const -{ - while (true) - { - try - { - return hardFetchLedgerRange(yield); - } - catch (DatabaseTimeout& t) - { - ; - } - } -} - std::optional BackendInterface::hardFetchLedgerRangeNoThrow() const { @@ -238,6 +222,30 @@ BackendInterface::fetchBookOffers( return page; } +std::optional +BackendInterface::hardFetchLedgerRange() const +{ + return synchronous([this](auto yield) { return hardFetchLedgerRange(yield); }); +} + +std::optional +BackendInterface::fetchLedgerRange() const +{ + std::shared_lock lck(rngMtx_); + return range; +} + +void +BackendInterface::updateRange(uint32_t newMax) +{ + std::scoped_lock lck(rngMtx_); + assert(!range || newMax >= range->maxSequence); + if (!range) + range = {newMax, newMax}; + else + range->maxSequence = newMax; +} + LedgerPage BackendInterface::fetchLedgerPage( std::optional const& cursor, diff --git a/src/data/BackendInterface.h b/src/data/BackendInterface.h index 8ba91f01..ac218a3a 100644 --- a/src/data/BackendInterface.h +++ b/src/data/BackendInterface.h @@ -36,10 +36,7 @@ namespace data { /** - * @brief Throws an error when database read time limit is exceeded. - * - * This class is throws an error when read time limit is exceeded but - * is also paired with a separate class to retry the connection. + * @brief Represents a database timeout error. */ class DatabaseTimeout : public std::exception { @@ -52,16 +49,16 @@ public: }; /** - * @brief Separate class that reattempts connection after time limit. + * @brief A helper function that catches DatabaseTimout exceptions and retries indefinitely. * - * @tparam F Represents a class of handlers for Cassandra database. - * @param func Instance of Cassandra database handler class. - * @param waitMs Is the arbitrary time limit of 500ms. - * @return auto + * @tparam FnType The type of function object to execute + * @param func The function object to execute + * @param waitMs Delay between retry attempts + * @return auto The same as the return type of func */ -template +template auto -retryOnTimeout(F func, size_t waitMs = 500) +retryOnTimeout(FnType func, size_t waitMs = 500) { static util::Logger log{"Backend"}; @@ -71,7 +68,7 @@ retryOnTimeout(F func, size_t waitMs = 500) { return func(); } - catch (DatabaseTimeout& t) + catch (DatabaseTimeout const&) { log.error() << "Database request timed out. Sleeping and retrying ... "; std::this_thread::sleep_for(std::chrono::milliseconds(waitMs)); @@ -80,80 +77,64 @@ retryOnTimeout(F func, size_t waitMs = 500) } /** - * @brief Passes in serialized handlers in an asynchronous fashion. + * @brief Synchronously executes the given function object inside a coroutine. * - * Note that the synchronous auto passes handlers critical to supporting - * the Clio backend. The coroutine types are checked if same/different. - * - * @tparam F Represents a class of handlers for Cassandra database. - * @param f R-value instance of Cassandra handler class. - * @return auto + * @tparam FnType The type of function object to execute + * @param func The function object to execute + * @return auto The same as the return type of func */ -template +template auto -synchronous(F&& f) +synchronous(FnType&& func) { boost::asio::io_context ctx; - using R = typename boost::result_of::type; + using R = typename boost::result_of::type; if constexpr (!std::is_same::value) { R res; - boost::asio::spawn(ctx, [&f, &res](boost::asio::yield_context yield) { res = f(yield); }); + boost::asio::spawn(ctx, [&func, &res](auto yield) { res = func(yield); }); ctx.run(); return res; } else { - boost::asio::spawn(ctx, [&f](boost::asio::yield_context yield) { f(yield); }); + boost::asio::spawn(ctx, [&func](auto yield) { func(yield); }); ctx.run(); } } /** - * @brief Reestablishes synchronous connection on timeout. + * @brief Synchronously execute the given function object and retry until no DatabaseTimeout is thrown. * - * @tparam Represents a class of handlers for Cassandra database. - * @param f R-value instance of Cassandra database handler class. - * @return auto + * @tparam FnType The type of function object to execute + * @param func The function object to execute + * @return auto The same as the return type of func */ -template +template auto -synchronousAndRetryOnTimeout(F&& f) +synchronousAndRetryOnTimeout(FnType&& func) { - return retryOnTimeout([&]() { return synchronous(f); }); + return retryOnTimeout([&]() { return synchronous(func); }); } -/*! @brief Handles ledger and transaction backend data. */ +/** + * @brief The interface to the database used by Clio. + */ class BackendInterface { - /** - * @brief Shared mutexes and a cache for the interface. - * - * rngMutex is a shared mutex. Shared mutexes prevent shared data - * from being accessed by multiple threads and has two levels of - * access: shared and exclusive. - */ protected: mutable std::shared_mutex rngMtx_; std::optional range; LedgerCache cache_; - /** - * @brief Public read methods - * - * All of these reads methods can throw DatabaseTimeout. When writing - * code in an RPC handler, this exception does not need to be caught: - * when an RPC results in a timeout, an error is returned to the client. - */ - public: BackendInterface() = default; virtual ~BackendInterface() = default; + // TODO: Remove this hack. Cache should not be exposed thru BackendInterface /** - * @brief Cache that holds states of the ledger * @return Immutable cache */ LedgerCache const& @@ -163,7 +144,6 @@ public: } /** - * @brief Cache that holds states of the ledger * @return Mutable cache */ LedgerCache& @@ -172,62 +152,67 @@ public: return cache_; } - /*! @brief Fetches a specific ledger by sequence number. */ + /** + * @brief Fetches a specific ledger by sequence number. + * + * @param sequence The sequence number to fetch for + * @param yield The coroutine context + * @return The ripple::LedgerHeader if found; nullopt otherwise + */ virtual std::optional fetchLedgerBySequence(std::uint32_t const sequence, boost::asio::yield_context yield) const = 0; - /*! @brief Fetches a specific ledger by hash. */ + /** + * @brief Fetches a specific ledger by hash. + * + * @param hash The hash to fetch for + * @param yield The coroutine context + * @return The ripple::LedgerHeader if found; nullopt otherwise + */ virtual std::optional fetchLedgerByHash(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0; - /*! @brief Fetches the latest ledger sequence. */ + /** + * @brief Fetches the latest ledger sequence. + * + * @param yield The coroutine context + * @return Latest sequence wrapped in an optional if found; nullopt otherwise + */ virtual std::optional fetchLatestLedgerSequence(boost::asio::yield_context yield) const = 0; - /*! @brief Fetches the current ledger range while locking that process */ + /** + * @brief Fetch the current ledger range. + * + * @return The current ledger range if populated; nullopt otherwise + */ std::optional - fetchLedgerRange() const - { - std::shared_lock lck(rngMtx_); - return range; - } + fetchLedgerRange() const; /** - * @brief Updates the range of sequences to be tracked. + * @brief Updates the range of sequences that are stored in the DB. * - * Function that continues updating the range sliding window or creates - * a new sliding window once the maxSequence limit has been reached. - * - * @param newMax Unsigned 32-bit integer representing new max of range. + * @param newMax The new maximum sequence available */ void - updateRange(uint32_t newMax) - { - std::scoped_lock lck(rngMtx_); - assert(!range || newMax >= range->maxSequence); - if (!range) - range = {newMax, newMax}; - else - range->maxSequence = newMax; - } + updateRange(uint32_t newMax); /** - * @brief Returns the fees for specific transactions. + * @brief Fetch the fees from a specific ledger sequence. * - * @param seq Unsigned 32-bit integer reprsenting sequence. - * @param yield The currently executing coroutine. - * @return std::optional + * @param seq The sequence to fetch for + * @param yield The coroutine context + * @return ripple::Fees if fees are found; nullopt otherwise */ std::optional fetchFees(std::uint32_t const seq, boost::asio::yield_context yield) const; - /*! @brief TRANSACTION METHODS */ /** * @brief Fetches a specific transaction. * - * @param hash Unsigned 256-bit integer representing hash. - * @param yield The currently executing coroutine. - * @return std::optional + * @param hash The hash of the transaction to fetch + * @param yield The coroutine context + * @return TransactionAndMetadata if transaction is found; nullopt otherwise */ virtual std::optional fetchTransaction(ripple::uint256 const& hash, boost::asio::yield_context yield) const = 0; @@ -235,24 +220,22 @@ public: /** * @brief Fetches multiple transactions. * - * @param hashes Unsigned integer value representing a hash. - * @param yield The currently executing coroutine. - * @return std::vector + * @param hashes A vector of hashes to fetch transactions for + * @param yield The coroutine context + * @return A vector of TransactionAndMetadata matching the given hashes */ virtual std::vector fetchTransactions(std::vector const& hashes, boost::asio::yield_context yield) const = 0; /** - * @brief Fetches all transactions for a specific account + * @brief Fetches all transactions for a specific account. * - * @param account A specific XRPL Account, speciifed by unique type - * accountID. - * @param limit Paging limit for how many transactions can be returned per - * page. - * @param forward Boolean whether paging happens forwards or backwards. - * @param cursor Important metadata returned every time paging occurs. - * @param yield Currently executing coroutine. - * @return TransactionsAndCursor + * @param account The account to fetch transactions for + * @param limit The maximum number of transactions per result page + * @param forward Whether to fetch the page forwards or backwards from the given cursor + * @param cursor The cursor to resume fetching from + * @param yield The coroutine context + * @return Results and a cursor to resume from */ virtual TransactionsAndCursor fetchAccountTransactions( @@ -265,10 +248,9 @@ public: /** * @brief Fetches all transactions from a specific ledger. * - * @param ledgerSequence Unsigned 32-bit integer for latest total - * transactions. - * @param yield Currently executing coroutine. - * @return std::vector + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return Results as a vector of TransactionAndMetadata */ virtual std::vector fetchAllTransactionsInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0; @@ -276,21 +258,20 @@ public: /** * @brief Fetches all transaction hashes from a specific ledger. * - * @param ledgerSequence Standard unsigned integer. - * @param yield Currently executing coroutine. - * @return std::vector + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return Hashes as ripple::uint256 in a vector */ virtual std::vector fetchAllTransactionHashesInLedger(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0; - /*! @brief NFT methods */ /** - * @brief Fetches a specific NFT + * @brief Fetches a specific NFT. * - * @param tokenID Unsigned 256-bit integer. - * @param ledgerSequence Standard unsigned integer. - * @param yield Currently executing coroutine. - * @return std::optional + * @param tokenID The ID of the NFT + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return NFT object on success; nullopt otherwise */ virtual std::optional fetchNFT(ripple::uint256 const& tokenID, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) @@ -299,12 +280,12 @@ public: /** * @brief Fetches all transactions for a specific NFT. * - * @param tokenID Unsigned 256-bit integer. - * @param limit Paging limit as to how many transactions return per page. - * @param forward Boolean whether paging happens forwards or backwards. - * @param cursorIn Represents transaction number and ledger sequence. - * @param yield Currently executing coroutine is passed in as input. - * @return TransactionsAndCursor + * @param tokenID The ID of the NFT + * @param limit The maximum number of transactions per result page + * @param forward Whether to fetch the page forwards or backwards from the given cursor + * @param cursorIn The cursor to resume fetching from + * @param yield The coroutine context + * @return Results and a cursor to resume from */ virtual TransactionsAndCursor fetchNFTTransactions( @@ -314,25 +295,30 @@ public: std::optional const& cursorIn, boost::asio::yield_context yield) const = 0; - /*! @brief STATE DATA METHODS */ /** - * @brief Fetches a specific ledger object: vector of unsigned chars + * @brief Fetches a specific ledger object. * - * @param key Unsigned 256-bit integer. - * @param sequence Unsigned 32-bit integer. - * @param yield Currently executing coroutine. - * @return std::optional + * Currently the real fetch happens in doFetchLedgerObject and fetchLedgerObject attempts to fetch from Cache first + * and only calls out to the real DB if a cache miss ocurred. + * + * @param key The key of the object + * @param sequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return The object as a Blob on success; nullopt otherwise */ std::optional fetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield) const; /** - * @brief Fetches all ledger objects: a vector of vectors of unsigned chars. + * @brief Fetches all ledger objects by their keys. * - * @param keys Unsigned 256-bit integer. - * @param sequence Unsigned 32-bit integer. - * @param yield Currently executing coroutine. - * @return std::vector + * Currently the real fetch happens in doFetchLedgerObjects and fetchLedgerObjects attempts to fetch from Cache + * first and only calls out to the real DB for each of the keys that was not found in the cache. + * + * @param keys A vector with the keys of the objects to fetch + * @param sequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return A vector of ledger objects as Blobs */ std::vector fetchLedgerObjects( @@ -340,12 +326,26 @@ public: std::uint32_t const sequence, boost::asio::yield_context yield) const; - /*! @brief Virtual function version of fetchLedgerObject */ + /** + * @brief The database-specific implementation for fetching a ledger object. + * + * @param key The key to fetch for + * @param sequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return The object as a Blob on success; nullopt otherwise + */ virtual std::optional doFetchLedgerObject(ripple::uint256 const& key, std::uint32_t const sequence, boost::asio::yield_context yield) const = 0; - /*! @brief Virtual function version of fetchLedgerObjects */ + /** + * @brief The database-specific implementation for fetching ledger objects. + * + * @param keys The keys to fetch for + * @param sequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return A vector of Blobs representing each fetched object + */ virtual std::vector doFetchLedgerObjects( std::vector const& keys, @@ -353,14 +353,11 @@ public: boost::asio::yield_context yield) const = 0; /** - * @brief Returns the difference between ledgers: vector of objects + * @brief Returns the difference between ledgers. * - * Objects are made of a key value, vector of unsigned chars (blob), - * and a boolean detailing whether keys and blob match. - * - * @param ledgerSequence Standard unsigned integer. - * @param yield Currently executing coroutine. - * @return std::vector + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return A vector of LedgerObject representing the diff */ virtual std::vector fetchLedgerDiff(std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0; @@ -368,12 +365,12 @@ public: /** * @brief Fetches a page of ledger objects, ordered by key/index. * - * @param cursor Important metadata returned every time paging occurs. - * @param ledgerSequence Standard unsigned integer. - * @param limit Paging limit as to how many transactions returned per page. - * @param outOfOrder Boolean on whether ledger page is out of order. - * @param yield Currently executing coroutine. - * @return LedgerPage + * @param cursor The cursor to resume fetching from + * @param ledgerSequence The ledger sequence to fetch for + * @param limit The maximum number of transactions per result page + * @param outOfOrder If set to true max available sequence is used instead of ledgerSequence + * @param yield The coroutine context + * @return The ledger page */ LedgerPage fetchLedgerPage( @@ -383,16 +380,40 @@ public: bool outOfOrder, boost::asio::yield_context yield) const; - /*! @brief Fetches successor object from key/index. */ + /** + * @brief Fetches the successor object. + * + * @param key The key to fetch for + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return The sucessor on success; nullopt otherwise + */ std::optional fetchSuccessorObject(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const; - /*! @brief Fetches successor key from key/index. */ + /** + * @brief Fetches the successor key. + * + * Thea real fetch happens in doFetchSuccessorKey. This function will attempt to lookup the successor in the cache + * first and only if it's not found in the cache will it fetch from the actual DB. + * + * @param key The key to fetch for + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return The sucessor key on success; nullopt otherwise + */ std::optional fetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const; - /*! @brief Virtual function version of fetchSuccessorKey. */ + /** + * @brief Database-specific implementation of fetching the successor key + * + * @param key The key to fetch for + * @param ledgerSequence The ledger sequence to fetch for + * @param yield The coroutine context + * @return The sucessor on success; nullopt otherwise + */ virtual std::optional doFetchSuccessorKey(ripple::uint256 key, std::uint32_t const ledgerSequence, boost::asio::yield_context yield) const = 0; @@ -401,11 +422,10 @@ public: * @brief Fetches book offers. * * @param book Unsigned 256-bit integer. - * @param ledgerSequence Standard unsigned integer. + * @param ledgerSequence The ledger sequence to fetch for * @param limit Pagaing limit as to how many transactions returned per page. - * @param cursor Important metadata returned every time paging occurs. - * @param yield Currently executing coroutine. - * @return BookOffersPage + * @param yield The coroutine context + * @return The book offers page */ BookOffersPage fetchBookOffers( @@ -415,31 +435,30 @@ public: boost::asio::yield_context yield) const; /** - * @brief Returns a ledger range + * @brief Synchronously fetches the ledger range from DB. * - * Ledger range is a struct of min and max sequence numbers). Due to - * the use of [&], which denotes a special case of a lambda expression - * where values found outside the scope are passed by reference, wrt the - * currently executing coroutine. + * This function just wraps hardFetchLedgerRange(boost::asio::yield_context) using synchronous(FnType&&). * - * @return std::optional + * @return The ledger range if available; nullopt otherwise */ std::optional - hardFetchLedgerRange() const - { - return synchronous([&](boost::asio::yield_context yield) { return hardFetchLedgerRange(yield); }); - } + hardFetchLedgerRange() const; - /*! @brief Virtual function equivalent of hardFetchLedgerRange. */ + /** + * @brief Fetches the ledger range from DB. + * + * @return The ledger range if available; nullopt otherwise + */ virtual std::optional hardFetchLedgerRange(boost::asio::yield_context yield) const = 0; - /*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */ + /** + * @brief Fetches the ledger range from DB retrying until no DatabaseTimeout is thrown. + * + * @return The ledger range if available; nullopt otherwise + */ std::optional hardFetchLedgerRangeNoThrow() const; - /*! @brief Fetches ledger range but doesn't throw timeout. Use with care. */ - std::optional - hardFetchLedgerRangeNoThrow(boost::asio::yield_context yield) const; /** * @brief Writes to a specific ledger. @@ -453,11 +472,9 @@ public: /** * @brief Writes a new ledger object. * - * The key and blob are r-value references and do NOT have memory addresses. - * - * @param key String represented as an r-value. - * @param seq Unsigned integer representing a sequence. - * @param blob r-value vector of unsigned characters (blob). + * @param key The key to write the ledger object under + * @param seq The ledger sequence to write for + * @param blob The data to write */ virtual void writeLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob); @@ -465,11 +482,11 @@ public: /** * @brief Writes a new transaction. * - * @param hash r-value reference. No memory address. - * @param seq Unsigned 32-bit integer. - * @param date Unsigned 32-bit integer. - * @param transaction r-value reference. No memory address. - * @param metadata r-value refrence. No memory address. + * @param hash The hash of the transaction + * @param seq The ledger sequence to write for + * @param date The timestamp of the entry + * @param transaction The transaction data to write + * @param metadata The metadata to write */ virtual void writeTransaction( @@ -480,9 +497,9 @@ public: std::string&& metadata) = 0; /** - * @brief Write a new NFT. + * @brief Writes NFTs to the database. * - * @param data Passed in as an r-value reference. + * @param data A vector of NFTsData objects representing the NFTs */ virtual void writeNFTs(std::vector&& data) = 0; @@ -490,15 +507,15 @@ public: /** * @brief Write a new set of account transactions. * - * @param data Passed in as an r-value reference. + * @param data A vector of AccountTransactionsData objects representing the account transactions */ virtual void writeAccountTransactions(std::vector&& data) = 0; /** - * @brief Write a new transaction for a specific NFT. + * @brief Write NFTs transactions. * - * @param data Passed in as an r-value reference. + * @param data A vector of NFTTransactionsData objects */ virtual void writeNFTTransactions(std::vector&& data) = 0; @@ -506,41 +523,39 @@ public: /** * @brief Write a new successor. * - * @param key Passed in as an r-value reference. - * @param seq Unsigned 32-bit integer. - * @param successor Passed in as an r-value reference. + * @param key Key of the object that the passed successor will be the successor for + * @param seq The ledger sequence to write for + * @param successor The successor data to write */ virtual void writeSuccessor(std::string&& key, std::uint32_t const seq, std::string&& successor) = 0; - /*! @brief Tells database we will write data for a specific ledger. */ + /** + * @brief Starts a write transaction with the DB. No-op for cassandra. + * + * Note: Can potentially be deprecated and removed. + */ virtual void startWrites() const = 0; /** * @brief Tells database we finished writing all data for a specific ledger. * - * TODO: change the return value to represent different results: - * Committed, write conflict, errored, successful but not committed + * Uses doFinishWrites to synchronize with the pending writes. * - * @param ledgerSequence Const unsigned 32-bit integer on ledger sequence. - * @return true - * @return false + * @param ledgerSequence The ledger sequence to finish writing for + * @return true on success; false otherwise */ bool finishWrites(std::uint32_t const ledgerSequence); + /** + * @return true if database is overwhelmed; false otherwise + */ virtual bool isTooBusy() const = 0; private: - /** - * @brief Private helper method to write ledger object - * - * @param key r-value string representing key. - * @param seq Unsigned 32-bit integer representing sequence. - * @param blob r-value vector of unsigned chars. - */ virtual void doWriteLedgerObject(std::string&& key, std::uint32_t const seq, std::string&& blob) = 0; diff --git a/src/data/CassandraBackend.h b/src/data/CassandraBackend.h index cb144589..6e8ffa49 100644 --- a/src/data/CassandraBackend.h +++ b/src/data/CassandraBackend.h @@ -36,15 +36,14 @@ namespace data::cassandra { /** - * @brief Implements @ref BackendInterface for Cassandra/Scylladb + * @brief Implements @ref BackendInterface for Cassandra/ScyllaDB. * - * Note: this is a safer and more correct rewrite of the original implementation - * of the backend. We deliberately did not change the interface for now so that - * other parts such as ETL do not have to change at all. - * Eventually we should change the interface so that it does not have to know - * about yield_context. + * Note: This is a safer and more correct rewrite of the original implementation of the backend. + * + * @tparam SettingsProviderType The settings provider type to use + * @tparam ExecutionStrategyType The execution strategy type to use */ -template +template class BasicCassandraBackend : public BackendInterface { util::Logger log_{"Backend"}; @@ -54,7 +53,7 @@ class BasicCassandraBackend : public BackendInterface Handle handle_; // have to be mutable because BackendInterface constness :( - mutable ExecutionStrategy executor_; + mutable ExecutionStrategyType executor_; std::atomic_uint32_t ledgerSequence_ = 0u; @@ -62,7 +61,8 @@ public: /** * @brief Create a new cassandra/scylla backend instance. * - * @param settingsProvider + * @param settingsProvider The settings provider to use + * @param readOnly Whether the database should be in readonly mode */ BasicCassandraBackend(SettingsProviderType settingsProvider, bool readOnly) : settingsProvider_{std::move(settingsProvider)} diff --git a/src/data/DBHelpers.h b/src/data/DBHelpers.h index 6df7711c..85139b24 100644 --- a/src/data/DBHelpers.h +++ b/src/data/DBHelpers.h @@ -17,6 +17,7 @@ */ //============================================================================== +/** @file */ #pragma once #include @@ -30,7 +31,7 @@ #include /** - * @brief Struct used to keep track of what to write to account_transactions/account_tx tables + * @brief Struct used to keep track of what to write to account_transactions/account_tx tables. */ struct AccountTransactionsData { @@ -51,7 +52,7 @@ struct AccountTransactionsData }; /** - * @brief Represents a link from a tx to an NFT that was targeted/modified/created by it + * @brief Represents a link from a tx to an NFT that was targeted/modified/created by it. * * Gets written to nf_token_transactions table and the like. */ @@ -138,6 +139,12 @@ struct NFTsData } }; +/** + * @brief Check whether the supplied object is an offer. + * + * @param object The object to check + * @return true if the object is an offer; false otherwise + */ template inline bool isOffer(T const& object) @@ -146,19 +153,28 @@ isOffer(T const& object) return offer_bytes == 0x006f; } +/** + * @brief Check whether the supplied hex represents an offer object. + * + * @param object The object to check + * @return true if the object is an offer; false otherwise + */ template inline bool isOfferHex(T const& object) { auto blob = ripple::strUnHex(4, object.begin(), object.begin() + 4); if (blob) - { - short offer_bytes = ((*blob)[1] << 8) | (*blob)[2]; - return offer_bytes == 0x006f; - } + return isOffer(*blob); return false; } +/** + * @brief Check whether the supplied object is a dir node. + * + * @param object The object to check + * @return true if the object is a dir node; false otherwise + */ template inline bool isDirNode(T const& object) @@ -167,6 +183,13 @@ isDirNode(T const& object) return spaceKey == 0x0064; } +/** + * @brief Check whether the supplied object is a book dir. + * + * @param key The key into the object + * @param object The object to check + * @return true if the object is a book dir; false otherwise + */ template inline bool isBookDir(T const& key, R const& object) @@ -178,6 +201,12 @@ isBookDir(T const& key, R const& object) return !sle[~ripple::sfOwner].has_value(); } +/** + * @brief Get the book out of an offer object. + * + * @param offer The offer to get the book for + * @return Book as ripple::uint256 + */ template inline ripple::uint256 getBook(T const& offer) @@ -185,26 +214,40 @@ getBook(T const& offer) ripple::SerialIter it{offer.data(), offer.size()}; ripple::SLE sle{it, {}}; ripple::uint256 book = sle.getFieldH256(ripple::sfBookDirectory); + return book; } +/** + * @brief Get the book base. + * + * @param key The key to get the book base out of + * @return Book base as ripple::uint256 + */ template inline ripple::uint256 getBookBase(T const& key) { assert(key.size() == ripple::uint256::size()); + ripple::uint256 ret; for (size_t i = 0; i < 24; ++i) - { ret.data()[i] = key.data()[i]; - } + return ret; } +/** + * @brief Stringify a ripple::uint256. + * + * @param input The input value + * @return The input value as a string + */ inline std::string -uint256ToString(ripple::uint256 const& uint) +uint256ToString(ripple::uint256 const& input) { - return {reinterpret_cast(uint.data()), uint.size()}; + return {reinterpret_cast(input.data()), input.size()}; } +/** @brief The ripple epoch start timestamp. Midnight on 1st January 2000. */ static constexpr std::uint32_t rippleEpochStart = 946684800; diff --git a/src/data/LedgerCache.h b/src/data/LedgerCache.h index 577ff580..8f20662f 100644 --- a/src/data/LedgerCache.h +++ b/src/data/LedgerCache.h @@ -30,6 +30,9 @@ namespace data { +/** + * @brief Cache for an entire ledger. + */ class LedgerCache { struct CacheEntry @@ -57,40 +60,93 @@ class LedgerCache std::unordered_set> deletes_; public: - // Update the cache with new ledger objects set isBackground to true when writing old data from a background thread + /** + * @brief Update the cache with new ledger objects. + * + * @param blobs The ledger objects to update cache with + * @param seq The sequence to update cache for + * @param isBackground Should be set to true when writing old data from a background thread + */ void update(std::vector const& blobs, uint32_t seq, bool isBackground = false); + /** + * @brief Fetch a cached object by its key and sequence number. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in cache, will return the cached Blob; otherwise nullopt is returned + */ std::optional get(ripple::uint256 const& key, uint32_t seq) const; - // always returns empty optional if isFull() is false + /** + * @brief Gets a cached successor. + * + * Note: This function always returns std::nullopt when @ref isFull() returns false. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in cache, will return the cached successor; otherwise nullopt is returned + */ std::optional getSuccessor(ripple::uint256 const& key, uint32_t seq) const; - // always returns empty optional if isFull() is false + /** + * @brief Gets a cached predcessor. + * + * Note: This function always returns std::nullopt when @ref isFull() returns false. + * + * @param key The key to fetch for + * @param seq The sequence to fetch for + * @return If found in cache, will return the cached predcessor; otherwise nullopt is returned + */ std::optional getPredecessor(ripple::uint256 const& key, uint32_t seq) const; + /** + * @brief Disables the cache. + */ void setDisabled(); + /** + * @brief Sets the full flag to true. + * + * This is used when cache loaded in its entirety at startup of the application. This can be either loaded from DB, + * populated together with initial ledger download (on first run) or downloaded from a peer node (specified in + * config). + */ void setFull(); + /** + * @return The latest ledger sequence for which cache is available. + */ uint32_t latestLedgerSequence() const; - // whether the cache has all data for the most recent ledger + /** + * @return true if the cache has all data for the most recent ledger; false otherwise + */ bool isFull() const; + /** + * @return The total size of the cache. + */ size_t size() const; + /** + * @return A number representing the success rate of hitting an object in the cache versus missing it. + */ float getObjectHitRate() const; + /** + * @return A number representing the success rate of hitting a successor in the cache versus missing it. + */ float getSuccessorHitRate() const; }; diff --git a/src/data/README.md b/src/data/README.md index fb227549..8f86b14e 100644 --- a/src/data/README.md +++ b/src/data/README.md @@ -1,4 +1,5 @@ -# Clio Backend +# Backend + ## Background The backend of Clio is responsible for handling the proper reading and writing of past ledger data from and to a given database. As of right now, Cassandra and ScyllaDB are the only supported databases that are production-ready. Support for database types can be easily extended by creating new implementations which implements the virtual methods of `BackendInterface.h`. Then, use the Factory Object Design Pattern to simply add logic statements to `BackendFactory.h` that return the new database interface for a specific `type` in Clio's configuration file. diff --git a/src/data/Types.h b/src/data/Types.h index 925e03c4..5d973dc6 100644 --- a/src/data/Types.h +++ b/src/data/Types.h @@ -21,20 +21,23 @@ #include #include + #include #include #include namespace data { -// *** return types - using Blob = std::vector; +/** + * @brief Represents an object in the ledger. + */ struct LedgerObject { ripple::uint256 key; Blob blob; + bool operator==(const LedgerObject& other) const { @@ -42,16 +45,27 @@ struct LedgerObject } }; +/** + * @brief Represents a page of LedgerObjects. + */ struct LedgerPage { std::vector objects; std::optional cursor; }; + +/** + * @brief Represents a page of book offer objects. + */ struct BookOffersPage { std::vector offers; std::optional cursor; }; + +/** + * @brief Represents a transaction and its metadata bundled together. + */ struct TransactionAndMetadata { Blob transaction; @@ -85,6 +99,9 @@ struct TransactionAndMetadata } }; +/** + * @brief Represents a cursor into the transactions table. + */ struct TransactionsCursor { std::uint32_t ledgerSequence = 0; @@ -114,12 +131,18 @@ struct TransactionsCursor } }; +/** + * @brief Represests a bundle of transactions with metadata and a cursor to the next page. + */ struct TransactionsAndCursor { std::vector txns; std::optional cursor; }; +/** + * @brief Represents a NFToken. + */ struct NFT { ripple::uint256 tokenID; @@ -143,9 +166,8 @@ struct NFT { } - // clearly two tokens are the same if they have the same ID, but this - // struct stores the state of a given token at a given ledger sequence, so - // we also need to compare with ledgerSequence + // clearly two tokens are the same if they have the same ID, but this struct stores the state of a given token at a + // given ledger sequence, so we also need to compare with ledgerSequence. bool operator==(NFT const& other) const { @@ -153,12 +175,17 @@ struct NFT } }; +/** + * @brief Stores a range of sequences as a min and max pair. + */ struct LedgerRange { std::uint32_t minSequence = 0; std::uint32_t maxSequence = 0; }; + constexpr ripple::uint256 firstKey{"0000000000000000000000000000000000000000000000000000000000000000"}; constexpr ripple::uint256 lastKey{"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"}; constexpr ripple::uint256 hi192{"0000000000000000000000000000000000000000000000001111111111111111"}; + } // namespace data diff --git a/src/data/cassandra/Concepts.h b/src/data/cassandra/Concepts.h index 137dd95d..ef312d2c 100644 --- a/src/data/cassandra/Concepts.h +++ b/src/data/cassandra/Concepts.h @@ -30,6 +30,9 @@ namespace data::cassandra { +/** + * @brief The requirements of a settings provider. + */ // clang-format off template concept SomeSettingsProvider = requires(T a) { @@ -41,6 +44,9 @@ concept SomeSettingsProvider = requires(T a) { }; // clang-format on +/** + * @brief The requirements of an execution strategy. + */ // clang-format off template concept SomeExecutionStrategy = requires( @@ -66,6 +72,9 @@ concept SomeExecutionStrategy = requires( }; // clang-format on +/** + * @brief The requirements of a retry policy. + */ // clang-format off template concept SomeRetryPolicy = requires(T a, boost::asio::io_context ioc, CassandraError err, uint32_t attempt) { diff --git a/src/data/cassandra/Error.h b/src/data/cassandra/Error.h index 8d844168..5a070f71 100644 --- a/src/data/cassandra/Error.h +++ b/src/data/cassandra/Error.h @@ -26,7 +26,7 @@ namespace data::cassandra { /** - * @brief A simple container for both error message and error code + * @brief A simple container for both error message and error code. */ class CassandraError { @@ -67,18 +67,27 @@ public: return os; } + /** + * @return The final error message as a std::string + */ std::string message() const { return message_; } + /** + * @return The error code + */ uint32_t code() const { return code_; } + /** + * @return true if the wrapped error is considered a timeout; false otherwise + */ bool isTimeout() const { @@ -89,6 +98,9 @@ public: return false; } + /** + * @return true if the wrapped error is an invalid query; false otherwise + */ bool isInvalidQuery() const { diff --git a/src/data/cassandra/Handle.h b/src/data/cassandra/Handle.h index 675fb14e..37b4056a 100644 --- a/src/data/cassandra/Handle.h +++ b/src/data/cassandra/Handle.h @@ -57,28 +57,31 @@ public: using ResultType = Result; /** - * @brief Construct a new handle from a @ref Settings object + * @brief Construct a new handle from a @ref detail::Settings object. + * + * @param clusterSettings The settings to use */ explicit Handle(Settings clusterSettings = Settings::defaultSettings()); /** - * @brief Construct a new handle with default settings and only by setting - * the contact points + * @brief Construct a new handle with default settings and only by setting the contact points. + * + * @param contactPoints The contact points to use instead of settings */ explicit Handle(std::string_view contactPoints); /** - * @brief Disconnects gracefully if possible + * @brief Disconnects gracefully if possible. */ ~Handle(); /** - * @brief Move is supported + * @brief Move is supported. */ Handle(Handle&&) = default; /** - * @brief Connect to the cluster asynchronously + * @brief Connect to the cluster asynchronously. * * @return A future */ @@ -86,31 +89,37 @@ public: asyncConnect() const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * * See @ref asyncConnect() const for how this works. + * + * @return Possibly an error */ [[nodiscard]] MaybeErrorType connect() const; /** - * @brief Connect to the the specified keyspace asynchronously + * @brief Connect to the the specified keyspace asynchronously. * + * @param keyspace The keyspace to use * @return A future */ [[nodiscard]] FutureType asyncConnect(std::string_view keyspace) const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * * See @ref asyncConnect(std::string_view) const for how this works. + * + * @param keyspace The keyspace to use + * @return Possibly an error */ [[nodiscard]] MaybeErrorType connect(std::string_view keyspace) const; /** - * @brief Disconnect from the cluster asynchronously + * @brief Disconnect from the cluster asynchronously. * * @return A future */ @@ -118,32 +127,40 @@ public: asyncDisconnect() const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * * See @ref asyncDisconnect() const for how this works. + * + * @return Possibly an error */ [[maybe_unused]] MaybeErrorType disconnect() const; /** - * @brief Reconnect to the the specified keyspace asynchronously + * @brief Reconnect to the the specified keyspace asynchronously. * + * @param keyspace The keyspace to use * @return A future */ [[nodiscard]] FutureType asyncReconnect(std::string_view keyspace) const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * * See @ref asyncReconnect(std::string_view) const for how this works. + * + * @param keyspace The keyspace to use + * @return Possibly an error */ [[nodiscard]] MaybeErrorType reconnect(std::string_view keyspace) const; /** - * @brief Execute a simple query with optional args asynchronously + * @brief Execute a simple query with optional args asynchronously. * + * @param query The query to execute + * @param args The arguments to bind for execution * @return A future */ template @@ -155,10 +172,13 @@ public: } /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * - * See @ref asyncExecute(std::string_view, Args&&...) const for how this - * works. + * See asyncExecute(std::string_view, Args&&...) const for how this works. + * + * @param query The query to execute + * @param args The arguments to bind for execution + * @return The result or an error */ template [[maybe_unused]] ResultOrErrorType @@ -168,30 +188,34 @@ public: } /** - * @brief Execute each of the statements asynchronously + * @brief Execute each of the statements asynchronously. * - * Batched version is not always the right option. Especially since it only - * supports INSERT, UPDATE and DELETE statements. - * This can be used as an alternative when statements need to execute in - * bulk. + * Batched version is not always the right option. + * Especially since it only supports INSERT, UPDATE and DELETE statements. + * This can be used as an alternative when statements need to execute in bulk. * + * @param statements The statements to execute * @return A vector of future objects */ [[nodiscard]] std::vector asyncExecuteEach(std::vector const& statements) const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * - * See @ref asyncExecuteEach(std::vector const&) const for - * how this works. + * See @ref asyncExecuteEach(std::vector const&) const for how this works. + * + * @param statements The statements to execute + * @return Possibly an error */ [[maybe_unused]] MaybeErrorType executeEach(std::vector const& statements) const; /** - * @brief Execute a prepared statement with optional args asynchronously + * @brief Execute a prepared statement with optional args asynchronously. * + * @param statement The prepared statement to execute + * @param args The arguments to bind for execution * @return A future */ template @@ -203,10 +227,13 @@ public: } /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * - * See @ref asyncExecute(std::vector const&, Args&&...) const - * for how this works. + * See asyncExecute(std::vector const&, Args&&...) const for how this works. + * + * @param statement The prepared statement to bind and execute + * @param args The arguments to bind for execution + * @return The result or an error */ template [[maybe_unused]] ResultOrErrorType @@ -216,61 +243,70 @@ public: } /** - * @brief Execute one (bound or simple) statements asynchronously + * @brief Execute one (bound or simple) statements asynchronously. * + * @param statement The statement to execute * @return A future */ [[nodiscard]] FutureType asyncExecute(StatementType const& statement) const; /** - * @brief Execute one (bound or simple) statements asynchronously with a - * callback + * @brief Execute one (bound or simple) statements asynchronously with a callback. * + * @param statement The statement to execute + * @param cb The callback to execute when data is ready * @return A future that holds onto the callback provided */ [[nodiscard]] FutureWithCallbackType asyncExecute(StatementType const& statement, std::function&& cb) const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * - * See @ref asyncExecute(StatementType const&) const for how this - * works. + * See @ref asyncExecute(StatementType const&) const for how this works. + * + * @param statement The statement to execute + * @return The result or an error */ [[maybe_unused]] ResultOrErrorType execute(StatementType const& statement) const; /** - * @brief Execute a batch of (bound or simple) statements asynchronously + * @brief Execute a batch of (bound or simple) statements asynchronously. * + * @param statements The statements to execute * @return A future */ [[nodiscard]] FutureType asyncExecute(std::vector const& statements) const; /** - * @brief Synchonous version of the above + * @brief Synchonous version of the above. * - * See @ref asyncExecute(std::vector const&) const for how - * this works. + * See @ref asyncExecute(std::vector const&) const for how this works. + * + * @param statements The statements to execute + * @return Possibly an error */ [[maybe_unused]] MaybeErrorType execute(std::vector const& statements) const; /** - * @brief Execute a batch of (bound or simple) statements asynchronously - * with a completion callback + * @brief Execute a batch of (bound or simple) statements asynchronously with a completion callback. * + * @param statements The statements to execute + * @param cb The callback to execute when data is ready * @return A future that holds onto the callback provided */ [[nodiscard]] FutureWithCallbackType asyncExecute(std::vector const& statements, std::function&& cb) const; /** - * @brief Prepare a statement + * @brief Prepare a statement. * - * @return A @ref PreparedStatementType + * @param query + * @return A prepared statement * @throws std::runtime_error with underlying error description on failure */ [[nodiscard]] PreparedStatementType @@ -278,12 +314,13 @@ public: }; /** - * @brief Extracts the results into series of std::tuple by creating a - * simple wrapper with an STL input iterator inside. + * @brief Extracts the results into series of std::tuple by creating a simple wrapper with an STL input + * iterator inside. * * You can call .begin() and .end() in order to iterate as usual. - * This also means that you can use it in a range-based for or with some - * algorithms. + * This also means that you can use it in a range-based for or with some algorithms. + * + * @param result The result to iterate */ template [[nodiscard]] detail::ResultExtractor diff --git a/src/data/cassandra/Schema.h b/src/data/cassandra/Schema.h index 69b3b424..9c2f7e06 100644 --- a/src/data/cassandra/Schema.h +++ b/src/data/cassandra/Schema.h @@ -38,16 +38,11 @@ template } /** - * @brief Manages the DB schema and provides access to prepared statements + * @brief Manages the DB schema and provides access to prepared statements. */ template class Schema { - // Current schema version. - // Update this everytime you update the schema. - // Migrations will be ran automatically based on this value. - static constexpr uint16_t version = 1u; - util::Logger log_{"Backend"}; std::reference_wrapper settingsProvider_; @@ -261,7 +256,7 @@ public: }(); /** - * @brief Prepared statements holder + * @brief Prepared statements holder. */ class Statements { @@ -641,7 +636,7 @@ public: }; /** - * @brief Recreates the prepared statements + * @brief Recreates the prepared statements. */ void prepareStatements(Handle const& handle) @@ -652,7 +647,7 @@ public: } /** - * @brief Provides access to statements + * @brief Provides access to statements. */ std::unique_ptr const& operator->() const diff --git a/src/data/cassandra/SettingsProvider.h b/src/data/cassandra/SettingsProvider.h index d279e07f..4dfc276a 100644 --- a/src/data/cassandra/SettingsProvider.h +++ b/src/data/cassandra/SettingsProvider.h @@ -28,7 +28,7 @@ namespace data::cassandra { /** - * @brief Provides settings for @ref CassandraBackend + * @brief Provides settings for @ref BasicCassandraBackend. */ class SettingsProvider { @@ -41,34 +41,50 @@ class SettingsProvider Settings settings_; public: + /** + * @brief Create a settings provider from the specified config. + * + * @param cfg The config of Clio to use + * @param ttl Time to live setting + */ explicit SettingsProvider(util::Config const& cfg, uint16_t ttl = 0); - /*! Get the cluster settings */ + /** + * @return The cluster settings + */ [[nodiscard]] Settings getSettings() const; - /*! Get the specified keyspace */ + /** + * @return The specified keyspace + */ [[nodiscard]] inline std::string getKeyspace() const { return keyspace_; } - /*! Get an optional table prefix to use in all queries */ + /** + * @return The optional table prefix to use in all queries + */ [[nodiscard]] inline std::optional getTablePrefix() const { return tablePrefix_; } - /*! Get the replication factor */ + /** + * @return The replication factor + */ [[nodiscard]] inline uint16_t getReplicationFactor() const { return replicationFactor_; } - /*! Get the default time to live to use in all `create` queries */ + /** + * @return The default time to live to use in all `create` queries + */ [[nodiscard]] inline uint16_t getTtl() const { diff --git a/src/data/cassandra/impl/Batch.cpp b/src/data/cassandra/impl/Batch.cpp index 71a667b9..f11aa67c 100644 --- a/src/data/cassandra/impl/Batch.cpp +++ b/src/data/cassandra/impl/Batch.cpp @@ -31,8 +31,7 @@ static constexpr auto batchDeleter = [](CassBatch* ptr) { cass_batch_free(ptr); namespace data::cassandra::detail { -// todo: use an appropritae value instead of CASS_BATCH_TYPE_LOGGED for -// different use cases +// TODO: Use an appropriate value instead of CASS_BATCH_TYPE_LOGGED for different use cases Batch::Batch(std::vector const& statements) : ManagedObject{cass_batch_new(CASS_BATCH_TYPE_LOGGED), batchDeleter} { diff --git a/src/data/cassandra/impl/Cluster.h b/src/data/cassandra/impl/Cluster.h index dc807ade..17c3826e 100644 --- a/src/data/cassandra/impl/Cluster.h +++ b/src/data/cassandra/impl/Cluster.h @@ -33,42 +33,97 @@ namespace data::cassandra::detail { +// TODO: move Settings to public interface, not detail + +/** + * @brief Bundles all cassandra settings in one place. + */ struct Settings { + /** + * @brief Represents the configuration of contact points for cassandra. + */ struct ContactPoints { std::string contactPoints = "127.0.0.1"; // defaults to localhost std::optional port; }; + /** + * @brief Represents the configuration of a secure connection bundle. + */ struct SecureConnectionBundle { std::string bundle; // no meaningful default }; + /** @brief Enables or disables cassandra driver logger */ bool enableLog = false; + + /** @brief Connect timeout specified in milliseconds */ std::chrono::milliseconds connectionTimeout = std::chrono::milliseconds{10000}; + + /** @brief Request timeout specified in milliseconds */ std::chrono::milliseconds requestTimeout = std::chrono::milliseconds{0}; // no timeout at all + + /** @brief Connection information; either ContactPoints or SecureConnectionBundle */ std::variant connectionInfo = ContactPoints{}; + + /** @brief The number of threads for the driver to pool */ uint32_t threads = std::thread::hardware_concurrency(); + + /** @brief The maximum number of outstanding write requests at any given moment */ uint32_t maxWriteRequestsOutstanding = 10'000; + + /** @brief The maximum number of outstanding read requests at any given moment */ uint32_t maxReadRequestsOutstanding = 100'000; + + /** @brief The maximum number of connections per host */ uint32_t maxConnectionsPerHost = 2u; + + /** @brief The number of connection per host to always have active */ uint32_t coreConnectionsPerHost = 2u; + + /** @brief The maximum concurrent requests per connection; new connections will be created when reached */ uint32_t maxConcurrentRequestsThreshold = (maxWriteRequestsOutstanding + maxReadRequestsOutstanding) / coreConnectionsPerHost; + + /** @brief Size of the event queue */ std::optional queueSizeEvent; + + /** @brief Size of the IO queue */ std::optional queueSizeIO; + + /** @brief High watermark for bytes written */ std::optional writeBytesHighWatermark; + + /** @brief Low watermark for bytes written */ std::optional writeBytesLowWatermark; + + /** @brief High watermark for pending requests */ std::optional pendingRequestsHighWatermark; + + /** @brief Low watermark for pending requests */ std::optional pendingRequestsLowWatermark; + + /** @brief Maximum number of requests per flush */ std::optional maxRequestsPerFlush; + + /** @brief Maximum number of connections that will be created concurrently */ std::optional maxConcurrentCreation; + + /** @brief SSL certificate */ std::optional certificate; // ssl context + + /** @brief Username/login */ std::optional username; + + /** @brief Password to match the `username` */ std::optional password; + /** + * @brief Creates a new Settings object as a copy of the current one with overridden contact points. + */ Settings withContactPoints(std::string_view contactPoints) { @@ -77,6 +132,9 @@ struct Settings return tmp; } + /** + * @brief Returns the default settings. + */ static Settings defaultSettings() { diff --git a/src/data/cassandra/impl/ExecutionStrategy.h b/src/data/cassandra/impl/ExecutionStrategy.h index 29c079a1..8a52efef 100644 --- a/src/data/cassandra/impl/ExecutionStrategy.h +++ b/src/data/cassandra/impl/ExecutionStrategy.h @@ -38,12 +38,13 @@ namespace data::cassandra::detail { +// TODO: this could probably be also moved out of detail and into the main cassandra namespace. + /** - * @brief Implements async and sync querying against the cassandra DB with - * support for throttling. + * @brief Implements async and sync querying against the cassandra DB with support for throttling. * - * Note: A lot of the code that uses yield is repeated below. This is ok for now - * because we are hopefully going to be getting rid of it entirely later on. + * Note: A lot of the code that uses yield is repeated below. + * This is ok for now because we are hopefully going to be getting rid of it entirely later on. */ template class DefaultExecutionStrategy @@ -77,6 +78,10 @@ public: using ResultType = typename HandleType::ResultType; using CompletionTokenType = boost::asio::yield_context; + /** + * @param settings The settings to use + * @param handle A handle to the cassandra database + */ DefaultExecutionStrategy(Settings const& settings, HandleType const& handle) : maxWriteRequestsOutstanding_{settings.maxWriteRequestsOutstanding} , maxReadRequestsOutstanding_{settings.maxReadRequestsOutstanding} @@ -96,7 +101,7 @@ public: } /** - * @brief Wait for all async writes to finish before unblocking + * @brief Wait for all async writes to finish before unblocking. */ void sync() @@ -107,6 +112,9 @@ public: log_.debug() << "Sync done."; } + /** + * @return true if outstanding read requests allowance is exhausted; false otherwise + */ bool isTooBusy() const { @@ -114,7 +122,7 @@ public: } /** - * @brief Blocking query execution used for writing data + * @brief Blocking query execution used for writing data. * * Retries forever sleeping for 5 milliseconds between attempts. */ @@ -136,7 +144,7 @@ public: } /** - * @brief Blocking query execution used for writing data + * @brief Blocking query execution used for writing data. * * Retries forever sleeping for 5 milliseconds between attempts. */ @@ -148,11 +156,11 @@ public: } /** - * @brief Non-blocking query execution used for writing data + * @brief Non-blocking query execution used for writing data. * * Retries forever with retry policy specified by @ref AsyncExecutor * - * @param prepradeStatement Statement to prepare and execute + * @param preparedStatement Statement to prepare and execute * @param args Args to bind to the prepared statement * @throw DatabaseTimeout on timeout */ @@ -169,7 +177,7 @@ public: } /** - * @brief Non-blocking batched query execution used for writing data + * @brief Non-blocking batched query execution used for writing data. * * Retries forever with retry policy specified by @ref AsyncExecutor. * @@ -195,7 +203,7 @@ public: * Retries forever until successful or throws an exception on timeout. * * @param token Completion token (yield_context) - * @param prepradeStatement Statement to prepare and execute + * @param preparedStatement Statement to prepare and execute * @param args Args to bind to the prepared statement * @throw DatabaseTimeout on timeout * @return ResultType or error wrapped in Expected diff --git a/src/data/cassandra/impl/Future.cpp b/src/data/cassandra/impl/Future.cpp index ef01b719..f6cf72ca 100644 --- a/src/data/cassandra/impl/Future.cpp +++ b/src/data/cassandra/impl/Future.cpp @@ -73,7 +73,7 @@ void invokeHelper(CassFuture* ptr, void* cbPtr) { // Note: can't use Future{ptr}.get() because double free will occur :/ - auto* cb = static_cast(cbPtr); + auto* cb = static_cast(cbPtr); if (auto const rc = cass_future_error_code(ptr); rc) { auto const errMsg = [&ptr](std::string const& label) { @@ -90,9 +90,8 @@ invokeHelper(CassFuture* ptr, void* cbPtr) } } -// TODO: cb_ can be deleted before cassandra-driver calls it if the user fails to hold onto the future object -/* implicit */ FutureWithCallback::FutureWithCallback(CassFuture* ptr, fn_t&& cb) - : Future{ptr}, cb_{std::make_unique(std::move(cb))} +/* implicit */ FutureWithCallback::FutureWithCallback(CassFuture* ptr, FnType&& cb) + : Future{ptr}, cb_{std::make_unique(std::move(cb))} { // Instead of passing `this` as the userdata void*, we pass the address of // the callback itself which will survive std::move of the diff --git a/src/data/cassandra/impl/Future.h b/src/data/cassandra/impl/Future.h index b2a1c338..0bb41b2f 100644 --- a/src/data/cassandra/impl/Future.h +++ b/src/data/cassandra/impl/Future.h @@ -43,16 +43,16 @@ invokeHelper(CassFuture* ptr, void* self); class FutureWithCallback : public Future { public: - using fn_t = std::function; - using fn_ptr_t = std::unique_ptr; + using FnType = std::function; + using FnPtrType = std::unique_ptr; - /* implicit */ FutureWithCallback(CassFuture* ptr, fn_t&& cb); + /* implicit */ FutureWithCallback(CassFuture* ptr, FnType&& cb); FutureWithCallback(FutureWithCallback const&) = delete; FutureWithCallback(FutureWithCallback&&) = default; private: - /*! Wrapped in a unique_ptr so it can survive std::move :/ */ - fn_ptr_t cb_; + /** Wrapped in a unique_ptr so it can survive std::move :/ */ + FnPtrType cb_; }; } // namespace data::cassandra::detail diff --git a/src/data/cassandra/impl/Result.h b/src/data/cassandra/impl/Result.h index b9b12ead..7a93a539 100644 --- a/src/data/cassandra/impl/Result.h +++ b/src/data/cassandra/impl/Result.h @@ -51,11 +51,11 @@ extractColumn(CassRow const* row, std::size_t idx) } }; - using decayed_t = std::decay_t; - using uint_tuple_t = std::tuple; - using uchar_vector_t = std::vector; + using DecayedType = std::decay_t; + using UintTupleType = std::tuple; + using UCharVectorType = std::vector; - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { cass_byte_t const* buf; std::size_t bufSize; @@ -63,7 +63,7 @@ extractColumn(CassRow const* row, std::size_t idx) throwErrorIfNeeded(rc, "Extract ripple::uint256"); output = ripple::uint256::fromVoid(buf); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { cass_byte_t const* buf; std::size_t bufSize; @@ -71,20 +71,20 @@ extractColumn(CassRow const* row, std::size_t idx) throwErrorIfNeeded(rc, "Extract ripple::AccountID"); output = ripple::AccountID::fromVoid(buf); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { cass_byte_t const* buf; std::size_t bufSize; auto const rc = cass_value_get_bytes(cass_row_get_column(row, idx), &buf, &bufSize); throwErrorIfNeeded(rc, "Extract vector"); - output = uchar_vector_t{buf, buf + bufSize}; + output = UCharVectorType{buf, buf + bufSize}; } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const* tuple = cass_row_get_column(row, idx); output = TupleIterator::fromTuple(tuple).extract(); } - else if constexpr (std::is_convertible_v) + else if constexpr (std::is_convertible_v) { char const* value; std::size_t len; @@ -92,7 +92,7 @@ extractColumn(CassRow const* row, std::size_t idx) throwErrorIfNeeded(rc, "Extract string"); output = std::string{value, len}; } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { cass_bool_t flag; auto const rc = cass_value_get_bool(cass_row_get_column(row, idx), &flag); @@ -100,17 +100,17 @@ extractColumn(CassRow const* row, std::size_t idx) output = flag ? true : false; } // clio only uses bigint (int64_t) so we convert any incoming type - else if constexpr (std::is_convertible_v) + else if constexpr (std::is_convertible_v) { int64_t out; auto const rc = cass_value_get_int64(cass_row_get_column(row, idx), &out); throwErrorIfNeeded(rc, "Extract int64"); - output = static_cast(out); + output = static_cast(out); } else { // type not supported for extraction - static_assert(unsupported_v); + static_assert(unsupported_v); } return output; diff --git a/src/data/cassandra/impl/Statement.h b/src/data/cassandra/impl/Statement.h index 8f242f8f..ac52ed4d 100644 --- a/src/data/cassandra/impl/Statement.h +++ b/src/data/cassandra/impl/Statement.h @@ -44,7 +44,7 @@ class Statement : public ManagedObject public: /** - * @brief Construct a new statement with optionally provided arguments + * @brief Construct a new statement with optionally provided arguments. * * Note: it's up to the user to make sure the bound parameters match * the format of the query (e.g. amount of '?' matches count of args). @@ -66,6 +66,11 @@ public: Statement(Statement&&) = default; + /** + * @brief Binds the given arguments to the statement. + * + * @param args Arguments to bind + */ template void bind(Args&&... args) const @@ -74,6 +79,12 @@ public: (this->bindAt(idx++, std::forward(args)), ...); } + /** + * @brief Binds an argument to a specific index. + * + * @param idx The index of the argument + * @param value The value to bind it to + */ template void bindAt(std::size_t const idx, Type&& value) const @@ -88,48 +99,48 @@ public: return cass_statement_bind_bytes(*this, idx, static_cast(data), size); }; - using decayed_t = std::decay_t; - using uchar_vec_t = std::vector; - using uint_tuple_t = std::tuple; + using DecayedType = std::decay_t; + using UCharVectorType = std::vector; + using UintTupleType = std::tuple; - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { auto const rc = bindBytes(value.data(), value.size()); throwErrorIfNeeded(rc, "Bind ripple::uint256"); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const rc = bindBytes(value.data(), value.size()); throwErrorIfNeeded(rc, "Bind ripple::AccountID"); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const rc = bindBytes(value.data(), value.size()); throwErrorIfNeeded(rc, "Bind vector"); } - else if constexpr (std::is_convertible_v) + else if constexpr (std::is_convertible_v) { // reinterpret_cast is needed here :'( auto const rc = bindBytes(reinterpret_cast(value.data()), value.size()); throwErrorIfNeeded(rc, "Bind string (as bytes)"); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const rc = cass_statement_bind_tuple(*this, idx, Tuple{std::move(value)}); throwErrorIfNeeded(rc, "Bind tuple"); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const rc = cass_statement_bind_bool(*this, idx, value ? cass_true : cass_false); throwErrorIfNeeded(rc, "Bind bool"); } - else if constexpr (std::is_same_v) + else if constexpr (std::is_same_v) { auto const rc = cass_statement_bind_int32(*this, idx, value.limit); throwErrorIfNeeded(rc, "Bind limit (int32)"); } // clio only uses bigint (int64_t) so we convert any incoming type - else if constexpr (std::is_convertible_v) + else if constexpr (std::is_convertible_v) { auto const rc = cass_statement_bind_int64(*this, idx, value); throwErrorIfNeeded(rc, "Bind int64"); @@ -137,11 +148,16 @@ public: else { // type not supported for binding - static_assert(unsupported_v); + static_assert(unsupported_v); } } }; +/** + * @brief Represents a prepared statement on the DB side. + * + * This is used to produce Statement objects that can be executed. + */ class PreparedStatement : public ManagedObject { static constexpr auto deleter = [](CassPrepared const* ptr) { cass_prepared_free(ptr); }; @@ -151,6 +167,12 @@ public: { } + /** + * @brief Bind the given arguments and produce a ready to execute Statement. + * + * @param args The arguments to bind + * @return A bound and ready to execute Statement object + */ template Statement bind(Args&&... args) const diff --git a/src/data/cassandra/impl/Tuple.h b/src/data/cassandra/impl/Tuple.h index ffe729a3..1cad636e 100644 --- a/src/data/cassandra/impl/Tuple.h +++ b/src/data/cassandra/impl/Tuple.h @@ -68,15 +68,15 @@ public: } }; - using decayed_t = std::decay_t; + using DecayedType = std::decay_t; - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { auto const rc = cass_tuple_set_bool(*this, idx, value ? cass_true : cass_false); throwErrorIfNeeded(rc, "Bind bool"); } // clio only uses bigint (int64_t) so we convert any incoming type - else if constexpr (std::is_convertible_v) + else if constexpr (std::is_convertible_v) { auto const rc = cass_tuple_set_int64(*this, idx, value); throwErrorIfNeeded(rc, "Bind int64"); @@ -84,7 +84,7 @@ public: else { // type not supported for binding - static_assert(unsupported_v); + static_assert(unsupported_v); } } }; @@ -126,20 +126,20 @@ private: } }; - using decayed_t = std::decay_t; + using DecayedType = std::decay_t; // clio only uses bigint (int64_t) so we convert any incoming type - if constexpr (std::is_convertible_v) + if constexpr (std::is_convertible_v) { int64_t out; auto const rc = cass_value_get_int64(cass_iterator_get_value(*this), &out); throwErrorIfNeeded(rc, "Extract int64 from tuple"); - output = static_cast(out); + output = static_cast(out); } else { // type not supported for extraction - static_assert(unsupported_v); + static_assert(unsupported_v); } return output; diff --git a/src/etl/ETLHelpers.h b/src/etl/ETLHelpers.h index 3cb4ccbe..567febaa 100644 --- a/src/etl/ETLHelpers.h +++ b/src/etl/ETLHelpers.h @@ -17,6 +17,7 @@ */ //============================================================================== +/** @file */ #pragma once #include @@ -44,6 +45,9 @@ class NetworkValidatedLedgers std::condition_variable cv_; public: + /** + * @brief A factory function for NetworkValidatedLedgers. + */ static std::shared_ptr make_ValidatedLedgers() { @@ -51,9 +55,9 @@ public: } /** - * @brief Notify the datastructure that idx has been validated by the network + * @brief Notify the datastructure that idx has been validated by the network. * - * @param idx sequence validated by network + * @param idx Sequence validated by network */ void push(uint32_t idx) @@ -69,7 +73,7 @@ public: * * If no ledgers are known to have been validated, this function waits until the next ledger is validated * - * @return sequence of most recently validated ledger. empty optional if the datastructure has been stopped + * @return Sequence of most recently validated ledger. empty optional if the datastructure has been stopped */ std::optional getMostRecent() @@ -80,9 +84,9 @@ public: } /** - * @brief Waits for the sequence to be validated by the network + * @brief Waits for the sequence to be validated by the network. * - * @param sequence to wait for + * @param sequence The sequence to wait for * @return true if sequence was validated, false otherwise a return value of false means the datastructure has been * stopped */ @@ -101,7 +105,7 @@ public: // TODO: does the note make sense? lockfree queues provide the same blocking behaviour just without mutex, don't they? /** - * @brief Generic thread-safe queue with a max capacity + * @brief Generic thread-safe queue with a max capacity. * * @note (original note) We can't use a lockfree queue here, since we need the ability to wait for an element to be * added or removed from the queue. These waits are blocking calls. @@ -117,21 +121,21 @@ class ThreadSafeQueue public: /** - * @brief Create an instance of the queue + * @brief Create an instance of the queue. * * @param maxSize maximum size of the queue. Calls that would cause the queue to exceed this size will block until - * free space is available + * free space is available. */ ThreadSafeQueue(uint32_t maxSize) : maxSize_(maxSize) { } /** - * @brief Push element onto the queue + * @brief Push element onto the queue. * - * Note: This method will block until free space is available + * Note: This method will block until free space is available. * - * @param elt element to push onto queue + * @param elt Element to push onto queue */ void push(T const& elt) @@ -143,11 +147,11 @@ public: } /** - * @brief Push element onto the queue + * @brief Push element onto the queue. * * Note: This method will block until free space is available * - * @param elt element to push onto queue. elt is moved from + * @param elt Element to push onto queue. Ownership is transferred */ void push(T&& elt) @@ -159,11 +163,11 @@ public: } /** - * @brief Pop element from the queue + * @brief Pop element from the queue. * - * Note: Will block until queue is non-empty + * Note: Will block until queue is non-empty. * - * @return element popped from queue + * @return Element popped from queue */ T pop() @@ -179,9 +183,9 @@ public: } /** - * @brief Attempt to pop an element + * @brief Attempt to pop an element. * - * @return element popped from queue or empty optional if queue was empty + * @return Element popped from queue or empty optional if queue was empty */ std::optional tryPop() @@ -201,7 +205,7 @@ public: /** * @brief Parititions the uint256 keyspace into numMarkers partitions, each of equal size. * - * @param numMarkers total markers to partition for + * @param numMarkers Total markers to partition for */ inline std::vector getMarkers(size_t numMarkers) diff --git a/src/etl/ETLService.h b/src/etl/ETLService.h index ebae62c5..3cf73291 100644 --- a/src/etl/ETLService.h +++ b/src/etl/ETLService.h @@ -46,6 +46,9 @@ namespace feed { class SubscriptionManager; } +/** + * @brief This namespace contains everything to do with the ETL and ETL sources. + */ namespace etl { /** @@ -98,7 +101,7 @@ class ETLService public: /** - * @brief Create an instance of ETLService + * @brief Create an instance of ETLService. * * @param config The configuration to use * @param ioc io context to run on @@ -115,6 +118,18 @@ public: std::shared_ptr balancer, std::shared_ptr ledgers); + /** + * @brief A factory function to spawn new ETLService instances. + * + * Creates and runs the ETL service. + * + * @param config The configuration to use + * @param ioc io context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param balancer Load balancer to use + * @param ledgers The network validated ledgers datastructure + */ static std::shared_ptr make_ETLService( util::Config const& config, @@ -131,7 +146,7 @@ public: } /** - * @brief Stops components and joins worker thread + * @brief Stops components and joins worker thread. */ ~ETLService() { @@ -148,7 +163,7 @@ public: } /** - * @brief Get time passed since last ledger close, in seconds + * @brief Get time passed since last ledger close, in seconds. */ std::uint32_t lastCloseAgeSeconds() const @@ -229,7 +244,7 @@ private: } /** - * @brief Get the number of markers to use during the initial ledger download + * @brief Get the number of markers to use during the initial ledger download. * * This is equivelent to the degree of parallelism during the initial ledger download. * @@ -242,19 +257,19 @@ private: } /** - * @brief Start all components to run ETL service + * @brief Start all components to run ETL service. */ void run(); /** - * @brief Spawn the worker thread and start monitoring + * @brief Spawn the worker thread and start monitoring. */ void doWork(); /** - * @brief Sets amendment blocked flag + * @brief Sets amendment blocked flag. */ void setAmendmentBlocked() @@ -262,4 +277,4 @@ private: state_.isAmendmentBlocked = true; } }; -} // namespace etl \ No newline at end of file +} // namespace etl diff --git a/src/etl/LoadBalancer.cpp b/src/etl/LoadBalancer.cpp index 14b3be34..70c1d26f 100644 --- a/src/etl/LoadBalancer.cpp +++ b/src/etl/LoadBalancer.cpp @@ -43,15 +43,13 @@ namespace etl { std::unique_ptr LoadBalancer::make_Source( Config const& config, - boost::asio::io_context& ioContext, + boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr networkValidatedLedgers, + std::shared_ptr validatedLedgers, LoadBalancer& balancer) { - auto src = - std::make_unique(config, ioContext, backend, subscriptions, networkValidatedLedgers, balancer); - + auto src = std::make_unique(config, ioc, backend, subscriptions, validatedLedgers, balancer); src->run(); return src; @@ -70,10 +68,10 @@ LoadBalancer::make_LoadBalancer( LoadBalancer::LoadBalancer( Config const& config, - boost::asio::io_context& ioContext, + boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr nwvl) + std::shared_ptr validatedLedgers) { if (auto value = config.maybeValue("num_markers"); value) downloadRanges_ = std::clamp(*value, 1u, 256u); @@ -82,13 +80,18 @@ LoadBalancer::LoadBalancer( for (auto const& entry : config.array("etl_sources")) { - std::unique_ptr source = make_Source(entry, ioContext, backend, subscriptions, nwvl, *this); + std::unique_ptr source = make_Source(entry, ioc, backend, subscriptions, validatedLedgers, *this); sources_.push_back(std::move(source)); log_.info() << "Added etl source - " << sources_.back()->toString(); } } +LoadBalancer::~LoadBalancer() +{ + sources_.clear(); +} + std::pair, bool> LoadBalancer::loadInitialLedger(uint32_t sequence, bool cacheOnly) { @@ -235,4 +238,4 @@ LoadBalancer::execute(Func f, uint32_t ledgerSequence) } return true; } -} // namespace etl \ No newline at end of file +} // namespace etl diff --git a/src/etl/LoadBalancer.h b/src/etl/LoadBalancer.h index 0014d167..cb159009 100644 --- a/src/etl/LoadBalancer.h +++ b/src/etl/LoadBalancer.h @@ -33,14 +33,15 @@ namespace etl { class Source; class ProbingSource; } // namespace etl + namespace feed { class SubscriptionManager; -} +} // namespace feed namespace etl { /** - * @brief This class is used to manage connections to transaction processing processes + * @brief This class is used to manage connections to transaction processing processes. * * This class spawns a listener for each etl source, which listens to messages on the ledgers stream (to keep track of * which ledgers have been validated by the network, and the range of ledgers each etl source has). This class also @@ -56,25 +57,34 @@ public: private: util::Logger log_{"ETL"}; std::vector> sources_; - std::uint32_t downloadRanges_ = 16; + std::uint32_t downloadRanges_ = 16; /*< The number of markers to use when downloading intial ledger */ public: /** - * @brief Create an instance of the load balancer + * @brief Create an instance of the load balancer. * * @param config The configuration to use - * @param ioContext io context to run on + * @param ioc The io_context to run on * @param backend BackendInterface implementation * @param subscriptions Subscription manager - * @param nwvl The network validated ledgers datastructure + * @param validatedLedgers The network validated ledgers datastructure */ LoadBalancer( util::Config const& config, - boost::asio::io_context& ioContext, + boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr nwvl); + std::shared_ptr validatedLedgers); + /** + * @brief A factory function for the load balancer. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + */ static std::shared_ptr make_LoadBalancer( util::Config const& config, @@ -83,37 +93,46 @@ public: std::shared_ptr subscriptions, std::shared_ptr validatedLedgers); + /** + * @brief A factory function for the ETL source. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param balancer The load balancer + */ static std::unique_ptr make_Source( util::Config const& config, - boost::asio::io_context& ioContext, + boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr networkValidatedLedgers, + std::shared_ptr validatedLedgers, LoadBalancer& balancer); - ~LoadBalancer() - { - sources_.clear(); - } + ~LoadBalancer(); /** - * @brief Load the initial ledger, writing data to the queue + * @brief Load the initial ledger, writing data to the queue. * - * @param sequence sequence of ledger to download + * @param sequence Sequence of ledger to download + * @param cacheOnly Whether to only write to cache and not to the DB; defaults to false */ std::pair, bool> loadInitialLedger(uint32_t sequence, bool cacheOnly = false); /** - * @brief Fetch data for a specific ledger + * @brief Fetch data for a specific ledger. * * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger * is found in the database, or the server is shutting down. * - * @param ledgerSequence sequence of ledger to fetch data for - * @param getObjects if true, fetch diff between specified ledger and previous - * @return the extracted data, if extraction was successful. If the ledger was found in the database or the server + * @param ledgerSequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one + * @param getObjectNeighbors Whether to request object neighbors + * @return The extracted data, if extraction was successful. If the ledger was found in the database or the server * is shutting down, the optional will be empty */ OptionalGetLedgerResponseType @@ -133,16 +152,18 @@ public: shouldPropagateTxnStream(Source* in) const; /** - * @return JSON representation of the state of this load balancer + * @return JSON representation of the state of this load balancer. */ boost::json::value toJson() const; /** - * @brief Forward a JSON RPC request to a randomly selected rippled node + * @brief Forward a JSON RPC request to a randomly selected rippled node. * - * @param request JSON-RPC request - * @return response received from rippled node + * @param request JSON-RPC request to forward + * @param clientIp The IP address of the peer + * @param yield The coroutine context + * @return Response received from rippled node as JSON object on success; nullopt on failure */ std::optional forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield) @@ -150,13 +171,13 @@ public: private: /** - * @brief Execute a function on a randomly selected source + * @brief Execute a function on a randomly selected source. * * @note f is a function that takes an Source as an argument and returns a bool. * Attempt to execute f for one randomly chosen Source that has the specified ledger. If f returns false, another * randomly chosen Source is used. The process repeats until f returns true. * - * @param f function to execute. This function takes the ETL source as an argument, and returns a bool. + * @param f Function to execute. This function takes the ETL source as an argument, and returns a bool * @param ledgerSequence f is executed for each Source that has this ledger * @return true if f was eventually executed successfully. false if the ledger was found in the database or the * server is shutting down @@ -165,4 +186,4 @@ private: bool execute(Func f, uint32_t ledgerSequence); }; -} // namespace etl \ No newline at end of file +} // namespace etl diff --git a/src/etl/NFTHelpers.h b/src/etl/NFTHelpers.h index ed56ba19..776aaa0d 100644 --- a/src/etl/NFTHelpers.h +++ b/src/etl/NFTHelpers.h @@ -17,6 +17,7 @@ */ //============================================================================== +/** @file */ #pragma once #include @@ -27,13 +28,22 @@ namespace etl { /** - * @brief Pull NFT data from TX via ETLService + * @brief Pull NFT data from TX via ETLService. + * + * @param txMeta Transaction metadata + * @param sttx The transaction + * @return NFT transactions data as a pair of transactions and optional NFTsData */ std::pair, std::optional> getNFTDataFromTx(ripple::TxMeta const& txMeta, ripple::STTx const& sttx); /** - * @brief Pull NFT data from ledger object via loadInitialLedger + * @brief Pull NFT data from ledger object via loadInitialLedger. + * + * @param seq The ledger sequence to pull for + * @param key The owner key + * @param blob Object data as blob + * @return The NFT data as a vector */ std::vector getNFTDataFromObj(std::uint32_t const seq, std::string const& key, std::string const& blob); diff --git a/src/etl/ProbingSource.cpp b/src/etl/ProbingSource.cpp index 47a562d7..ccb878c2 100644 --- a/src/etl/ProbingSource.cpp +++ b/src/etl/ProbingSource.cpp @@ -105,19 +105,19 @@ ProbingSource::token() const } std::pair, bool> -ProbingSource::loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly) +ProbingSource::loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly) { if (!currentSrc_) return {{}, false}; - return currentSrc_->loadInitialLedger(ledgerSequence, numMarkers, cacheOnly); + return currentSrc_->loadInitialLedger(sequence, numMarkers, cacheOnly); } std::pair -ProbingSource::fetchLedger(uint32_t ledgerSequence, bool getObjects, bool getObjectNeighbors) +ProbingSource::fetchLedger(uint32_t sequence, bool getObjects, bool getObjectNeighbors) { if (!currentSrc_) return {}; - return currentSrc_->fetchLedger(ledgerSequence, getObjects, getObjectNeighbors); + return currentSrc_->fetchLedger(sequence, getObjects, getObjectNeighbors); } std::optional diff --git a/src/etl/ProbingSource.h b/src/etl/ProbingSource.h index 9bb7631f..ca135cec 100644 --- a/src/etl/ProbingSource.h +++ b/src/etl/ProbingSource.h @@ -56,7 +56,7 @@ private: public: /** - * @brief Create an instance of the probing source + * @brief Create an instance of the probing source. * * @param config The configuration to use * @param ioc io context to run on @@ -99,10 +99,10 @@ public: toString() const override; std::pair, bool> - loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override; + loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) override; std::pair - fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override; + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) override; std::optional forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield) diff --git a/src/etl/README.md b/src/etl/README.md index b52c28d5..2280cd55 100644 --- a/src/etl/README.md +++ b/src/etl/README.md @@ -1,3 +1,5 @@ +# ETL subsystem + A single clio node has one or more ETL sources, specified in the config file. clio will subscribe to the `ledgers` stream of each of the ETL sources. This stream sends a message whenever a new ledger is validated. Upon diff --git a/src/etl/Source.h b/src/etl/Source.h index 7050878e..70d607ac 100644 --- a/src/etl/Source.h +++ b/src/etl/Source.h @@ -53,47 +53,98 @@ class SubscriptionManager; namespace etl { /** - * @brief Base class for all ETL sources + * @brief Base class for all ETL sources. + * + * Note: Since sources below are implemented via CRTP, it sort of makes no sense to have a virtual base class. + * We should consider using a vector of ProbingSources instead of vector of unique ptrs to this virtual base. */ class Source { public: + /** @return true if source is connected; false otherwise */ virtual bool isConnected() const = 0; + /** @return JSON representation of the source */ virtual boost::json::object toJson() const = 0; + /** @brief Runs the source */ virtual void run() = 0; + /** @brief Request to pause the source (i.e. disconnect and do nothing) */ virtual void pause() = 0; + /** @brief Reconnect and resume this source */ virtual void resume() = 0; + /** @return String representation of the source (for debug) */ virtual std::string toString() const = 0; + /** + * @brief Check if ledger is known by this source. + * + * @param sequence The ledger sequence to check + * @return true if ledger is in the range of this source; false otherwise + */ virtual bool hasLedger(uint32_t sequence) const = 0; + /** + * @brief Fetch data for a specific ledger. + * + * This function will continuously try to fetch data for the specified ledger until the fetch succeeds, the ledger + * is found in the database, or the server is shutting down. + * + * @param sequence Sequence of the ledger to fetch + * @param getObjects Whether to get the account state diff between this ledger and the prior one; defaults to true + * @param getObjectNeighbors Whether to request object neighbors; defaults to false + * @return A std::pair of the response status and the response itself + */ virtual std::pair - fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) = 0; + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) = 0; + /** + * @brief Download a ledger in full. + * + * @param sequence Sequence of the ledger to download + * @param numMarkers Number of markers to generate for async calls + * @param cacheOnly Only insert into cache, not the DB; defaults to false + * @return A std::pair of the data and a bool indicating whether the download was successfull + */ virtual std::pair, bool> loadInitialLedger(uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) = 0; + /** + * @brief Forward a request to rippled. + * + * @param request The request to forward + * @param clientIp IP of the client forwarding this request + * @param yield The coroutine context + * @return Response wrapped in an optional on success; nullopt otherwise + */ virtual std::optional forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield) const = 0; + /** + * @return A token that uniquely identifies this source instance. + */ virtual boost::uuids::uuid token() const = 0; virtual ~Source() = default; + /** + * @brief Comparison is done via comparing tokens provided by the token() function. + * + * @param other The other source to compare to + * @return true if sources are equal; false otherwise + */ bool operator==(Source const& other) const { @@ -115,7 +166,7 @@ private: }; /** - * @brief Hooks for source events such as connects and disconnects + * @brief Hooks for source events such as connects and disconnects. */ struct SourceHooks { @@ -126,7 +177,9 @@ struct SourceHooks }; /** - * @brief Base implementation of shared source logic (using CRTP) + * @brief Base implementation of shared source logic. + * + * @tparam Derived The derived class for CRTP */ template class SourceImpl : public Source @@ -174,25 +227,30 @@ protected: public: /** - * @brief Create ETL source without gRPC endpoint + * @brief Create the base portion of ETL source. * - * Fetch ledger and load initial ledger will fail for this source. - * Primarly used in read-only mode, to monitor when ledgers are validated. + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param balancer Load balancer to use + * @param hooks Hooks to use for connect/disconnect events */ SourceImpl( util::Config const& config, - boost::asio::io_context& ioContext, + boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr networkValidatedLedgers, + std::shared_ptr validatedLedgers, LoadBalancer& balancer, SourceHooks hooks) - : networkValidatedLedgers_(networkValidatedLedgers) + : networkValidatedLedgers_(validatedLedgers) , backend_(backend) , subscriptions_(subscriptions) , balancer_(balancer) - , forwardCache_(config, ioContext, *this) - , strand_(boost::asio::make_strand(ioContext)) + , forwardCache_(config, ioc, *this) + , strand_(boost::asio::make_strand(ioc)) , timer_(strand_) , resolver_(strand_) , hooks_(hooks) @@ -241,20 +299,6 @@ public: return uuid_; } - std::chrono::system_clock::time_point - getLastMsgTime() const - { - std::lock_guard lck(lastMsgTimeMtx_); - return lastMsgTime_; - } - - void - setLastMsgTime() - { - std::lock_guard lck(lastMsgTimeMtx_); - lastMsgTime_ = std::chrono::system_clock::now(); - } - std::optional requestFromRippled( boost::json::object const& request, @@ -337,10 +381,6 @@ public: } } - /** - * @param sequence ledger sequence to check for - * @return true if this source has the desired ledger - */ bool hasLedger(uint32_t sequence) const override { @@ -362,64 +402,8 @@ public: return false; } - /** - * @brief Process the validated range received on the ledgers stream. set the appropriate member variable - * - * @param range validated range received on ledgers stream - */ - void - setValidatedRange(std::string const& range) - { - std::vector> pairs; - std::vector ranges; - boost::split(ranges, range, boost::is_any_of(",")); - for (auto& pair : ranges) - { - std::vector minAndMax; - - boost::split(minAndMax, pair, boost::is_any_of("-")); - - if (minAndMax.size() == 1) - { - uint32_t sequence = std::stoll(minAndMax[0]); - pairs.push_back(std::make_pair(sequence, sequence)); - } - else - { - assert(minAndMax.size() == 2); - uint32_t min = std::stoll(minAndMax[0]); - uint32_t max = std::stoll(minAndMax[1]); - pairs.push_back(std::make_pair(min, max)); - } - } - std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; }); - - // we only hold the lock here, to avoid blocking while string processing - std::lock_guard lck(mtx_); - validatedLedgers_ = std::move(pairs); - validatedLedgersRaw_ = range; - } - - /** - * @return the validated range of this source - * @note this is only used by server_info - */ - std::string - getValidatedRange() const - { - std::lock_guard lck(mtx_); - return validatedLedgersRaw_; - } - - /** - * @brief Fetch the specified ledger - * - * @param ledgerSequence sequence of the ledger to fetch @getObjects whether to get the account state diff between - * this ledger and the prior one - * @return the extracted data and the result status - */ std::pair - fetchLedger(uint32_t ledgerSequence, bool getObjects = true, bool getObjectNeighbors = false) override + fetchLedger(uint32_t sequence, bool getObjects = true, bool getObjectNeighbors = false) override { org::xrpl::rpc::v1::GetLedgerResponse response; if (!stub_) @@ -429,7 +413,7 @@ public: org::xrpl::rpc::v1::GetLedgerRequest request; grpc::ClientContext context; - request.mutable_ledger()->set_sequence(ledgerSequence); + request.mutable_ledger()->set_sequence(sequence); request.set_transactions(true); request.set_expand(true); request.set_get_objects(getObjects); @@ -448,9 +432,6 @@ public: return {status, std::move(response)}; } - /** - * @brief Produces a human-readable string with info about the source - */ std::string toString() const override { @@ -458,10 +439,6 @@ public: ", grpc port: " + grpcPort_ + "}"; } - /** - * @brief Produces stats for this source in a json object - * @return json object with stats - */ boost::json::object toJson() const override { @@ -482,15 +459,8 @@ public: return res; } - /** - * @brief Download a ledger in full - * - * @param ledgerSequence sequence of the ledger to download - * @param writeQueue queue to push downloaded ledger objects - * @return true if the download was successful - */ std::pair, bool> - loadInitialLedger(std::uint32_t ledgerSequence, std::uint32_t numMarkers, bool cacheOnly = false) override + loadInitialLedger(std::uint32_t sequence, std::uint32_t numMarkers, bool cacheOnly = false) override { if (!stub_) return {{}, false}; @@ -508,10 +478,10 @@ public: if (i + 1 < markers.size()) nextMarker = markers[i + 1]; - calls.emplace_back(ledgerSequence, markers[i], nextMarker); + calls.emplace_back(sequence, markers[i], nextMarker); } - log_.debug() << "Starting data download for ledger " << ledgerSequence << ". Using source = " << toString(); + log_.debug() << "Starting data download for ledger " << sequence << ". Using source = " << toString(); for (auto& c : calls) c.call(stub_, cq); @@ -564,60 +534,19 @@ public: return {std::move(edgeKeys), !abort}; } - /** - * @brief Attempt to reconnect to the ETL source - */ - void - reconnect(boost::beast::error_code ec) + std::optional + forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield) + const override { - if (paused_) - return; - - if (isConnected()) - hooks_.onDisconnected(ec); - - connected_ = false; - readBuffer_ = {}; - - // These are somewhat normal errors. operation_aborted occurs on shutdown, - // when the timer is cancelled. connection_refused will occur repeatedly - std::string err = ec.message(); - // if we cannot connect to the transaction processing process - if (ec.category() == boost::asio::error::get_ssl_category()) + if (auto resp = forwardCache_.get(request); resp) { - err = std::string(" (") + boost::lexical_cast(ERR_GET_LIB(ec.value())) + "," + - boost::lexical_cast(ERR_GET_REASON(ec.value())) + ") "; - - // ERR_PACK /* crypto/err/err.h */ - char buf[128]; - ::ERR_error_string_n(ec.value(), buf, sizeof(buf)); - err += buf; - - log_.error() << err; + log_.debug() << "request hit forwardCache"; + return resp; } - if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused) - { - log_.error() << "error code = " << ec << " - " << toString(); - } - else - { - log_.warn() << "error code = " << ec << " - " << toString(); - } - - // exponentially increasing timeouts, with a max of 30 seconds - size_t waitTime = std::min(pow(2, numFailures_), 30.0); - numFailures_++; - timer_.expires_after(boost::asio::chrono::seconds(waitTime)); - timer_.async_wait([this](auto ec) { - bool startAgain = (ec != boost::asio::error::operation_aborted); - derived().close(startAgain); - }); + return requestFromRippled(request, clientIp, yield); } - /** - * @brief Pause the source effectively stopping it from trying to reconnect - */ void pause() override { @@ -625,9 +554,6 @@ public: derived().close(false); } - /** - * @brief Resume the source allowing it to reconnect again - */ void resume() override { @@ -636,7 +562,10 @@ public: } /** - * @brief Callback for resolving the server host + * @brief Callback for resolving the server host. + * + * @param ec The error code + * @param results Result of the resolve operation */ void onResolve(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type results) @@ -656,7 +585,9 @@ public: } /** - * @brief Callback for handshake with the server + * @brief Callback for handshake with the server. + * + * @param ec The error code */ void onHandshake(boost::beast::error_code ec) @@ -692,10 +623,13 @@ public: } /** - * @brief Callback for writing data + * @brief Callback for writing data. + * + * @param ec The error code + * @param size Amount of bytes written */ void - onWrite(boost::beast::error_code ec, size_t size) + onWrite(boost::beast::error_code ec, [[maybe_unused]] size_t size) { if (ec) reconnect(ec); @@ -704,7 +638,10 @@ public: } /** - * @brief Callback for data available to read + * @brief Callback for data available to read. + * + * @param ec The error code + * @param size Amount of bytes read */ void onRead(boost::beast::error_code ec, size_t size) @@ -721,8 +658,10 @@ public: } /** - * @brief Handle the most recently received message - * @return true if the message was handled successfully. false on error + * @brief Handle the most recently received message. + * + * @param size Amount of bytes available in the read buffer + * @return true if the message was handled successfully; false otherwise */ bool handleMessage(size_t size) @@ -802,23 +741,6 @@ public: } } - /** - * @brief Forward a request to rippled - * @return response wrapped in an optional on success; nullopt otherwise - */ - std::optional - forwardToRippled(boost::json::object const& request, std::string const& clientIp, boost::asio::yield_context yield) - const override - { - if (auto resp = forwardCache_.get(request); resp) - { - log_.debug() << "request hit forwardCache"; - return resp; - } - - return requestFromRippled(request, clientIp, yield); - } - protected: Derived& derived() @@ -831,47 +753,172 @@ protected: { resolver_.async_resolve(ip_, wsPort_, [this](auto ec, auto results) { onResolve(ec, results); }); } + + void + reconnect(boost::beast::error_code ec) + { + if (paused_) + return; + + if (isConnected()) + hooks_.onDisconnected(ec); + + connected_ = false; + readBuffer_ = {}; + + // These are somewhat normal errors. operation_aborted occurs on shutdown, + // when the timer is cancelled. connection_refused will occur repeatedly + std::string err = ec.message(); + // if we cannot connect to the transaction processing process + if (ec.category() == boost::asio::error::get_ssl_category()) + { + err = std::string(" (") + boost::lexical_cast(ERR_GET_LIB(ec.value())) + "," + + boost::lexical_cast(ERR_GET_REASON(ec.value())) + ") "; + + // ERR_PACK /* crypto/err/err.h */ + char buf[128]; + ::ERR_error_string_n(ec.value(), buf, sizeof(buf)); + err += buf; + + log_.error() << err; + } + + if (ec != boost::asio::error::operation_aborted && ec != boost::asio::error::connection_refused) + { + log_.error() << "error code = " << ec << " - " << toString(); + } + else + { + log_.warn() << "error code = " << ec << " - " << toString(); + } + + // exponentially increasing timeouts, with a max of 30 seconds + size_t waitTime = std::min(pow(2, numFailures_), 30.0); + numFailures_++; + timer_.expires_after(boost::asio::chrono::seconds(waitTime)); + timer_.async_wait([this](auto ec) { + bool startAgain = (ec != boost::asio::error::operation_aborted); + derived().close(startAgain); + }); + } + +private: + void + setLastMsgTime() + { + std::lock_guard lck(lastMsgTimeMtx_); + lastMsgTime_ = std::chrono::system_clock::now(); + } + + std::chrono::system_clock::time_point + getLastMsgTime() const + { + std::lock_guard lck(lastMsgTimeMtx_); + return lastMsgTime_; + } + + void + setValidatedRange(std::string const& range) + { + std::vector> pairs; + std::vector ranges; + boost::split(ranges, range, boost::is_any_of(",")); + for (auto& pair : ranges) + { + std::vector minAndMax; + + boost::split(minAndMax, pair, boost::is_any_of("-")); + + if (minAndMax.size() == 1) + { + uint32_t sequence = std::stoll(minAndMax[0]); + pairs.push_back(std::make_pair(sequence, sequence)); + } + else + { + assert(minAndMax.size() == 2); + uint32_t min = std::stoll(minAndMax[0]); + uint32_t max = std::stoll(minAndMax[1]); + pairs.push_back(std::make_pair(min, max)); + } + } + std::sort(pairs.begin(), pairs.end(), [](auto left, auto right) { return left.first < right.first; }); + + // we only hold the lock here, to avoid blocking while string processing + std::lock_guard lck(mtx_); + validatedLedgers_ = std::move(pairs); + validatedLedgersRaw_ = range; + } + + std::string + getValidatedRange() const + { + std::lock_guard lck(mtx_); + return validatedLedgersRaw_; + } }; +/** + * @brief Implementation of a source that uses a regular, non-secure websocket connection. + */ class PlainSource : public SourceImpl { using StreamType = boost::beast::websocket::stream; std::unique_ptr ws_; public: + /** + * @brief Create a non-secure ETL source. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param balancer Load balancer to use + * @param hooks Hooks to use for connect/disconnect events + */ PlainSource( util::Config const& config, boost::asio::io_context& ioc, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr nwvl, + std::shared_ptr validatedLedgers, LoadBalancer& balancer, SourceHooks hooks) - : SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks)) + : SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks)) , ws_(std::make_unique(strand_)) { } /** - * @brief Callback for connection to the server + * @brief Callback for connection to the server. + * + * @param ec The error code + * @param endpoint The resolved endpoint */ void onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint); /** - * @brief Close the websocket - * @param startAgain whether to reconnect + * @brief Close the websocket. + * + * @param startAgain Whether to automatically reconnect */ void close(bool startAgain); - boost::beast::websocket::stream& + /** @return The underlying TCP stream */ + StreamType& ws() { return *ws_; } }; +/** + * @brief Implementation of a source that uses a secure websocket connection. + */ class SslSource : public SourceImpl { using StreamType = boost::beast::websocket::stream>; @@ -879,44 +926,64 @@ class SslSource : public SourceImpl std::unique_ptr ws_; public: + /** + * @brief Create a secure ETL source. + * + * @param config The configuration to use + * @param ioc The io_context to run on + * @param sslCtx The SSL context if any + * @param backend BackendInterface implementation + * @param subscriptions Subscription manager + * @param validatedLedgers The network validated ledgers datastructure + * @param balancer Load balancer to use + * @param hooks Hooks to use for connect/disconnect events + */ SslSource( util::Config const& config, boost::asio::io_context& ioc, std::optional> sslCtx, std::shared_ptr backend, std::shared_ptr subscriptions, - std::shared_ptr nwvl, + std::shared_ptr validatedLedgers, LoadBalancer& balancer, SourceHooks hooks) - : SourceImpl(config, ioc, backend, subscriptions, nwvl, balancer, std::move(hooks)) + : SourceImpl(config, ioc, backend, subscriptions, validatedLedgers, balancer, std::move(hooks)) , sslCtx_(sslCtx) , ws_(std::make_unique(strand_, *sslCtx_)) { } /** - * @brief Callback for connection to the server + * @brief Callback for connection to the server. + * + * @param ec The error code + * @param endpoint The resolved endpoint */ void onConnect(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint); /** - * @brief Callback for SSL handshake completion + * @brief Callback for SSL handshake completion. + * + * @param ec The error code + * @param endpoint The resolved endpoint */ void onSslHandshake(boost::beast::error_code ec, boost::asio::ip::tcp::resolver::results_type::endpoint_type endpoint); /** - * @brief Close the websocket - * @param startAgain whether to reconnect + * @brief Close the websocket. + * + * @param startAgain Whether to automatically reconnect */ void close(bool startAgain); + /** @return The underlying SSL stream */ StreamType& ws() { return *ws_; } }; -} // namespace etl \ No newline at end of file +} // namespace etl diff --git a/src/etl/SystemState.h b/src/etl/SystemState.h index f4c39eb7..eef728a6 100644 --- a/src/etl/SystemState.h +++ b/src/etl/SystemState.h @@ -23,36 +23,23 @@ namespace etl { +/** + * @brief Represents the state of the ETL subsystem. + */ struct SystemState { /** - * @brief Whether the process is in strict read-only mode + * @brief Whether the process is in strict read-only mode. * * In strict read-only mode, the process will never attempt to become the ETL writer, and will only publish ledgers * as they are written to the database. */ bool isReadOnly = false; - /** - * @brief Whether the process is writing to the database. - * - * Used by server_info - */ - std::atomic_bool isWriting = false; - - /** - * @brief Whether the software is stopping - */ - std::atomic_bool isStopping = false; - - /** - * @brief Whether a write conflict was detected - */ - std::atomic_bool writeConflict = false; - - /** - * @brief Whether we detected an amendment block - */ - std::atomic_bool isAmendmentBlocked = false; + std::atomic_bool isWriting = false; /**< @brief Whether the process is writing to the database. */ + std::atomic_bool isStopping = false; /**< @brief Whether the software is stopping. */ + std::atomic_bool writeConflict = false; /**< @brief Whether a write conflict was detected. */ + std::atomic_bool isAmendmentBlocked = false; /**< @brief Whether we detected an amendment block. */ }; + } // namespace etl diff --git a/src/etl/impl/ForwardCache.cpp b/src/etl/impl/ForwardCache.cpp index af5fcf22..cc67fc19 100644 --- a/src/etl/impl/ForwardCache.cpp +++ b/src/etl/impl/ForwardCache.cpp @@ -70,7 +70,7 @@ ForwardCache::get(boost::json::object const& request) const if (!command) return {}; - if (RPC::specifiesCurrentOrClosedLedger(request)) + if (rpc::specifiesCurrentOrClosedLedger(request)) return {}; std::shared_lock lk(mtx_); diff --git a/src/etl/impl/LedgerFetcher.h b/src/etl/impl/LedgerFetcher.h index 500df0a3..0b0524ea 100644 --- a/src/etl/impl/LedgerFetcher.h +++ b/src/etl/impl/LedgerFetcher.h @@ -64,11 +64,11 @@ public: * @return ledger header and transaction+metadata blobs; empty optional if the server is shutting down */ OptionalGetLedgerResponseType - fetchData(uint32_t seq) + fetchData(uint32_t sequence) { - log_.debug() << "Attempting to fetch ledger with sequence = " << seq; + log_.debug() << "Attempting to fetch ledger with sequence = " << sequence; - auto response = loadBalancer_->fetchLedger(seq, false, false); + auto response = loadBalancer_->fetchLedger(sequence, false, false); if (response) log_.trace() << "GetLedger reply = " << response->DebugString(); return response; @@ -85,12 +85,12 @@ public: * this ledger and the parent; Empty optional if the server is shutting down */ OptionalGetLedgerResponseType - fetchDataAndDiff(uint32_t seq) + fetchDataAndDiff(uint32_t sequence) { - log_.debug() << "Attempting to fetch ledger with sequence = " << seq; + log_.debug() << "Attempting to fetch ledger with sequence = " << sequence; auto response = loadBalancer_->fetchLedger( - seq, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= seq); + sequence, true, !backend_->cache().isFull() || backend_->cache().latestLedgerSequence() >= sequence); if (response) log_.trace() << "GetLedger reply = " << response->DebugString(); diff --git a/src/etl/impl/LedgerLoader.h b/src/etl/impl/LedgerLoader.h index 28578d17..ddbaa4b7 100644 --- a/src/etl/impl/LedgerLoader.h +++ b/src/etl/impl/LedgerLoader.h @@ -31,6 +31,9 @@ #include +/** + * @brief Account transactions, NFT transactions and NFT data bundled togeher. + */ struct FormattedTransactionsData { std::vector accountTxData; diff --git a/src/etl/impl/Transformer.h b/src/etl/impl/Transformer.h index 5d321561..f11cde80 100644 --- a/src/etl/impl/Transformer.h +++ b/src/etl/impl/Transformer.h @@ -45,7 +45,7 @@ namespace etl::detail { */ /** - * @brief Transformer thread that prepares new ledger out of raw data from GRPC + * @brief Transformer thread that prepares new ledger out of raw data from GRPC. */ template class Transformer @@ -66,7 +66,7 @@ class Transformer public: /** - * @brief Create an instance of the transformer + * @brief Create an instance of the transformer. * * This spawns a new thread that reads from the data pipe and writes ledgers to the DB using LedgerLoader and * LedgerPublisher. @@ -89,7 +89,7 @@ public: } /** - * @brief Joins the transformer thread + * @brief Joins the transformer thread. */ ~Transformer() { @@ -98,7 +98,7 @@ public: } /** - * @brief Block calling thread until transformer thread exits + * @brief Block calling thread until transformer thread exits. */ void waitTillFinished() @@ -155,13 +155,12 @@ private: } } - // TODO update this documentation /** * @brief Build the next ledger using the previous ledger and the extracted data. * @note rawData should be data that corresponds to the ledger immediately following the previous seq. * - * @param rawData data extracted from an ETL source - * @return the newly built ledger and data to write to the database + * @param rawData Data extracted from an ETL source + * @return The newly built ledger and data to write to the database */ std::pair buildNextLedger(GetLedgerResponseType& rawData) @@ -342,7 +341,7 @@ private: } /** - * @brief Write successors info into DB + * @brief Write successors info into DB. * * @param lgrInfo Ledger info * @param rawData Ledger data from GRPC @@ -399,24 +398,38 @@ private: } } + /** @return true if the transformer is stopping; false otherwise */ bool isStopping() const { return state_.get().isStopping; } + /** @return true if there was a write conflict; false otherwise */ bool hasWriteConflict() const { return state_.get().writeConflict; } + /** + * @brief Sets the write conflict flag. + * + * @param conflict The value to set + */ void setWriteConflict(bool conflict) { state_.get().writeConflict = conflict; } + /** + * @brief Sets the amendment blocked flag. + * + * Being amendment blocked means that Clio was compiled with libxrpl that does not yet support some field that + * arrived from rippled and therefore can't extract the ledger diff. When this happens, Clio can't proceed with ETL + * and should log this error and only handle RPC requests. + */ void setAmendmentBlocked() { diff --git a/src/feed/Message.h b/src/feed/Message.h deleted file mode 100644 index d08d8921..00000000 --- a/src/feed/Message.h +++ /dev/null @@ -1,59 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of clio: https://github.com/XRPLF/clio - Copyright (c) 2022, the clio developers. - - Permission to use, copy, modify, and distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#pragma once - -#include - -namespace feed { -// This class should only be constructed once, then it can -// be read from in parallel by many websocket senders -class Message -{ - std::string message_; - -public: - Message() = delete; - Message(std::string&& message) : message_(std::move(message)) - { - } - - Message(Message const&) = delete; - Message(Message&&) = delete; - Message& - operator=(Message const&) = delete; - Message& - operator=(Message&&) = delete; - - ~Message() = default; - - char* - data() - { - return message_.data(); - } - - std::size_t - size() - { - return message_.size(); - } -}; - -} // namespace feed \ No newline at end of file diff --git a/src/feed/SubscriptionManager.cpp b/src/feed/SubscriptionManager.cpp index b0fab108..859bf25e 100644 --- a/src/feed/SubscriptionManager.cpp +++ b/src/feed/SubscriptionManager.cpp @@ -55,9 +55,9 @@ getLedgerPubMessage( pubMsg["ledger_hash"] = to_string(lgrInfo.hash); pubMsg["ledger_time"] = lgrInfo.closeTime.time_since_epoch().count(); - pubMsg["fee_base"] = RPC::toBoostJson(fees.base.jsonClipped()); - pubMsg["reserve_base"] = RPC::toBoostJson(fees.reserve.jsonClipped()); - pubMsg["reserve_inc"] = RPC::toBoostJson(fees.increment.jsonClipped()); + pubMsg["fee_base"] = rpc::toBoostJson(fees.base.jsonClipped()); + pubMsg["reserve_base"] = rpc::toBoostJson(fees.reserve.jsonClipped()); + pubMsg["reserve_inc"] = rpc::toBoostJson(fees.increment.jsonClipped()); pubMsg["validated_ledgers"] = ledgerRange; pubMsg["txn_count"] = txnCount; @@ -159,11 +159,11 @@ SubscriptionManager::pubLedger( void SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, ripple::LedgerHeader const& lgrInfo) { - auto [tx, meta] = RPC::deserializeTxPlusMeta(blobs, lgrInfo.seq); + auto [tx, meta] = rpc::deserializeTxPlusMeta(blobs, lgrInfo.seq); boost::json::object pubObj; - pubObj["transaction"] = RPC::toJson(*tx); - pubObj["meta"] = RPC::toJson(*meta); - RPC::insertDeliveredAmount(pubObj["meta"].as_object(), tx, meta, blobs.date); + pubObj["transaction"] = rpc::toJson(*tx); + pubObj["meta"] = rpc::toJson(*meta); + rpc::insertDeliveredAmount(pubObj["meta"].as_object(), tx, meta, blobs.date); pubObj["type"] = "transaction"; pubObj["validated"] = true; pubObj["status"] = "closed"; @@ -187,7 +187,7 @@ SubscriptionManager::pubTransaction(data::TransactionAndMetadata const& blobs, r ripple::STAmount ownerFunds; auto fetchFundsSynchronous = [&]() { data::synchronous([&](boost::asio::yield_context yield) { - ownerFunds = RPC::accountFunds(*backend_, lgrInfo.seq, amount, account, yield); + ownerFunds = rpc::accountFunds(*backend_, lgrInfo.seq, amount, account, yield); }); }; @@ -248,7 +248,7 @@ SubscriptionManager::pubBookChanges( ripple::LedgerHeader const& lgrInfo, std::vector const& transactions) { - auto const json = RPC::computeBookChanges(lgrInfo, transactions); + auto const json = rpc::computeBookChanges(lgrInfo, transactions); auto const bookChangesMsg = std::make_shared(boost::json::serialize(json)); bookChangesSubscribers_.publish(bookChangesMsg); } @@ -260,7 +260,7 @@ SubscriptionManager::forwardProposedTransaction(boost::json::object const& respo txProposedSubscribers_.publish(pubMsg); auto transaction = response.at("transaction").as_object(); - auto accounts = RPC::getAccountsFromTransaction(transaction); + auto accounts = rpc::getAccountsFromTransaction(transaction); for (ripple::AccountID const& account : accounts) accountProposedSubscribers_.publish(pubMsg, account); @@ -367,4 +367,4 @@ SubscriptionManager::cleanup(SessionPtrType session) cleanupFuncs_.erase(session); } -} // namespace feed \ No newline at end of file +} // namespace feed diff --git a/src/feed/SubscriptionManager.h b/src/feed/SubscriptionManager.h index 8a5845c1..76e37b0e 100644 --- a/src/feed/SubscriptionManager.h +++ b/src/feed/SubscriptionManager.h @@ -28,85 +28,20 @@ #include +/** + * @brief This namespace deals with subscriptions. + */ namespace feed { using SessionPtrType = std::shared_ptr; -class Subscription -{ - boost::asio::strand strand_; - std::unordered_set subscribers_ = {}; - std::atomic_uint64_t subCount_ = 0; - -public: - Subscription() = delete; - Subscription(Subscription&) = delete; - Subscription(Subscription&&) = delete; - - explicit Subscription(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc)) - { - } - - ~Subscription() = default; - - void - subscribe(SessionPtrType const& session); - - void - unsubscribe(SessionPtrType const& session); - - void - publish(std::shared_ptr const& message); - - std::uint64_t - count() const - { - return subCount_.load(); - } - - bool - empty() const - { - return count() == 0; - } -}; - -template -class SubscriptionMap -{ - using subscribers = std::set; - - boost::asio::strand strand_; - std::unordered_map subscribers_ = {}; - std::atomic_uint64_t subCount_ = 0; - -public: - SubscriptionMap() = delete; - SubscriptionMap(SubscriptionMap&) = delete; - SubscriptionMap(SubscriptionMap&&) = delete; - - explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc)) - { - } - - ~SubscriptionMap() = default; - - void - subscribe(SessionPtrType const& session, Key const& key); - - void - unsubscribe(SessionPtrType const& session, Key const& key); - - void - publish(std::shared_ptr const& message, Key const& key); - - std::uint64_t - count() const - { - return subCount_.load(); - } -}; - +/** + * @brief Sends a message to subscribers. + * + * @param message The message to send + * @param subscribers The subscription stream to send the message to + * @param counter The subscription counter to decrement if session is detected as dead + */ template inline void sendToSubscribers(std::shared_ptr const& message, T& subscribers, std::atomic_uint64_t& counter) @@ -127,6 +62,13 @@ sendToSubscribers(std::shared_ptr const& message, T& subscribers, s } } +/** + * @brief Adds a session to the subscription stream. + * + * @param session The session to add + * @param subscribers The stream to subscribe to + * @param counter The counter representing the current total subscribers + */ template inline void addSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter) @@ -138,6 +80,13 @@ addSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter } } +/** + * @brief Removes a session from the subscription stream. + * + * @param session The session to remove + * @param subscribers The stream to unsubscribe from + * @param counter The counter representing the current total subscribers + */ template inline void removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& counter) @@ -149,47 +98,170 @@ removeSession(SessionPtrType session, T& subscribers, std::atomic_uint64_t& coun } } -template -void -SubscriptionMap::subscribe(SessionPtrType const& session, Key const& account) +/** + * @brief Represents a subscription stream. + */ +class Subscription { - boost::asio::post(strand_, [this, session, account]() { addSession(session, subscribers_[account], subCount_); }); -} + boost::asio::strand strand_; + std::unordered_set subscribers_ = {}; + std::atomic_uint64_t subCount_ = 0; +public: + Subscription() = delete; + Subscription(Subscription&) = delete; + Subscription(Subscription&&) = delete; + + /** + * @brief Create a new subscription stream. + * + * @param ioc The io_context to run on + */ + explicit Subscription(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc)) + { + } + + ~Subscription() = default; + + /** + * @brief Adds the given session to the subscribers set. + * + * @param session The session to add + */ + void + subscribe(SessionPtrType const& session); + + /** + * @brief Removes the given session from the subscribers set. + * + * @param session The session to remove + */ + void + unsubscribe(SessionPtrType const& session); + + /** + * @brief Sends the given message to all subscribers. + * + * @param message The message to send + */ + void + publish(std::shared_ptr const& message); + + /** + * @return Total subscriber count on this stream. + */ + std::uint64_t + count() const + { + return subCount_.load(); + } + + /** + * @return true if the stream currently has no subscribers; false otherwise + */ + bool + empty() const + { + return count() == 0; + } +}; + +/** + * @brief Represents a collection of subscriptions where each stream is mapped to a key. + */ template -void -SubscriptionMap::unsubscribe(SessionPtrType const& session, Key const& account) +class SubscriptionMap { - boost::asio::post(strand_, [this, account, session]() { - if (!subscribers_.contains(account)) - return; + using SubscribersType = std::set; - if (!subscribers_[account].contains(session)) - return; + boost::asio::strand strand_; + std::unordered_map subscribers_ = {}; + std::atomic_uint64_t subCount_ = 0; - --subCount_; +public: + SubscriptionMap() = delete; + SubscriptionMap(SubscriptionMap&) = delete; + SubscriptionMap(SubscriptionMap&&) = delete; - subscribers_[account].erase(session); + /** + * @brief Create a new subscription map. + * + * @param ioc The io_context to run on + */ + explicit SubscriptionMap(boost::asio::io_context& ioc) : strand_(boost::asio::make_strand(ioc)) + { + } - if (subscribers_[account].size() == 0) - { - subscribers_.erase(account); - } - }); -} + ~SubscriptionMap() = default; -template -void -SubscriptionMap::publish(std::shared_ptr const& message, Key const& account) -{ - boost::asio::post(strand_, [this, account, message]() { - if (!subscribers_.contains(account)) - return; + /** + * @brief Subscribe to a specific stream by its key. + * + * @param session The session to add + * @param key The key for the subscription to subscribe to + */ + void + subscribe(SessionPtrType const& session, Key const& key) + { + boost::asio::post(strand_, [this, session, key]() { addSession(session, subscribers_[key], subCount_); }); + } - sendToSubscribers(message, subscribers_[account], subCount_); - }); -} + /** + * @brief Unsubscribe from a specific stream by its key. + * + * @param session The session to remove + * @param key The key for the subscription to unsubscribe from + */ + void + unsubscribe(SessionPtrType const& session, Key const& key) + { + boost::asio::post(strand_, [this, key, session]() { + if (!subscribers_.contains(key)) + return; + if (!subscribers_[key].contains(session)) + return; + + --subCount_; + subscribers_[key].erase(session); + + if (subscribers_[key].size() == 0) + { + subscribers_.erase(key); + } + }); + } + + /** + * @brief Sends the given message to all subscribers. + * + * @param message The message to send + * @param key The key for the subscription to send the message to + */ + void + publish(std::shared_ptr const& message, Key const& key) + { + boost::asio::post(strand_, [this, key, message]() { + if (!subscribers_.contains(key)) + return; + + sendToSubscribers(message, subscribers_[key], subCount_); + }); + } + + /** + * @return Total subscriber count on all streams in the collection. + */ + std::uint64_t + count() const + { + return subCount_.load(); + } +}; + +/** + * @brief Manages subscriptions. + */ class SubscriptionManager { util::Logger log_{"Subscriptions"}; @@ -212,14 +284,26 @@ class SubscriptionManager std::shared_ptr backend_; public: + /** + * @brief A factory function that creates a new subscription manager configured from the config provided. + * + * @param config The configuration to use + * @param backend The backend to use + */ static std::shared_ptr - make_SubscriptionManager(util::Config const& config, std::shared_ptr const& b) + make_SubscriptionManager(util::Config const& config, std::shared_ptr const& backend) { auto numThreads = config.valueOr("subscription_workers", 1); - return std::make_shared(numThreads, b); + return std::make_shared(numThreads, backend); } - SubscriptionManager(std::uint64_t numThreads, std::shared_ptr const& b) + /** + * @brief Creates a new instance of the subscription manager. + * + * @param numThreads The number of worker threads to manage subscriptions + * @param backend The backend to use + */ + SubscriptionManager(std::uint64_t numThreads, std::shared_ptr const& backend) : ledgerSubscribers_(ioc_) , txSubscribers_(ioc_) , txProposedSubscribers_(ioc_) @@ -229,7 +313,7 @@ public: , accountSubscribers_(ioc_) , accountProposedSubscribers_(ioc_) , bookSubscribers_(ioc_) - , backend_(b) + , backend_(backend) { work_.emplace(ioc_); @@ -243,6 +327,7 @@ public: workers_.emplace_back([this] { ioc_.run(); }); } + /** @brief Stops the worker threads of the subscription manager. */ ~SubscriptionManager() { work_.reset(); @@ -252,9 +337,24 @@ public: worker.join(); } + /** + * @brief Subscribe to the ledger stream. + * + * @param yield The coroutine context + * @param session The session to subscribe to the stream + * @return JSON object representing the first message to be sent to the new subscriber + */ boost::json::object subLedger(boost::asio::yield_context yield, SessionPtrType session); + /** + * @brief Publish to the ledger stream. + * + * @param lgrInfo The ledger header to serialize + * @param fees The fees to serialize + * @param ledgerRange The ledger range this message applies to + * @param txnCount The total number of transactions to serialize + */ void pubLedger( ripple::LedgerHeader const& lgrInfo, @@ -262,97 +362,216 @@ public: std::string const& ledgerRange, std::uint32_t txnCount); + /** + * @brief Publish to the book changes stream. + * + * @param lgrInfo The ledger header to serialize + * @param transactions The transactions to serialize + */ void pubBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector const& transactions); + /** + * @brief Unsubscribe from the ledger stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubLedger(SessionPtrType session); + /** + * @brief Subscribe to the transactions stream. + * + * @param session The session to subscribe to the stream + */ void subTransactions(SessionPtrType session); + /** + * @brief Unsubscribe from the transactions stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubTransactions(SessionPtrType session); + /** + * @brief Publish to the book changes stream. + * + * @param blobs The transactions to serialize + * @param lgrInfo The ledger header to serialize + */ void pubTransaction(data::TransactionAndMetadata const& blobs, ripple::LedgerHeader const& lgrInfo); + /** + * @brief Subscribe to the account changes stream. + * + * @param account The account to monitor changes for + * @param session The session to subscribe to the stream + */ void subAccount(ripple::AccountID const& account, SessionPtrType const& session); + /** + * @brief Unsubscribe from the account changes stream. + * + * @param account The account the stream is for + * @param session The session to unsubscribe from the stream + */ void unsubAccount(ripple::AccountID const& account, SessionPtrType const& session); + /** + * @brief Subscribe to a specific book changes stream. + * + * @param book The book to monitor changes for + * @param session The session to subscribe to the stream + */ void subBook(ripple::Book const& book, SessionPtrType session); + /** + * @brief Unsubscribe from the specific book changes stream. + * + * @param book The book to stop monitoring changes for + * @param session The session to unsubscribe from the stream + */ void unsubBook(ripple::Book const& book, SessionPtrType session); + /** + * @brief Subscribe to the book changes stream. + * + * @param session The session to subscribe to the stream + */ void subBookChanges(SessionPtrType session); + /** + * @brief Unsubscribe from the book changes stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubBookChanges(SessionPtrType session); + /** + * @brief Subscribe to the manifest stream. + * + * @param session The session to subscribe to the stream + */ void subManifest(SessionPtrType session); + /** + * @brief Unsubscribe from the manifest stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubManifest(SessionPtrType session); + /** + * @brief Subscribe to the validation stream. + * + * @param session The session to subscribe to the stream + */ void subValidation(SessionPtrType session); + /** + * @brief Unsubscribe from the validation stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubValidation(SessionPtrType session); + /** + * @brief Publish proposed transactions and proposed accounts from a JSON response. + * + * @param response The JSON response to use + */ void forwardProposedTransaction(boost::json::object const& response); + /** + * @brief Publish manifest updates from a JSON response. + * + * @param response The JSON response to use + */ void forwardManifest(boost::json::object const& response); + /** + * @brief Publish validation updates from a JSON response. + * + * @param response The JSON response to use + */ void forwardValidation(boost::json::object const& response); + /** + * @brief Subscribe to the proposed account stream. + * + * @param account The account to monitor + * @param session The session to subscribe to the stream + */ void subProposedAccount(ripple::AccountID const& account, SessionPtrType session); + /** + * @brief Unsubscribe from the proposed account stream. + * + * @param account The account the stream is for + * @param session The session to unsubscribe from the stream + */ void unsubProposedAccount(ripple::AccountID const& account, SessionPtrType session); + /** + * @brief Subscribe to the processed transactions stream. + * + * @param session The session to subscribe to the stream + */ void subProposedTransactions(SessionPtrType session); + /** + * @brief Unsubscribe from the proposed transactions stream. + * + * @param session The session to unsubscribe from the stream + */ void unsubProposedTransactions(SessionPtrType session); + /** @brief Clenup the session on removal. */ void cleanup(SessionPtrType session); + /** + * @brief Generate a JSON report on the current state of the subscriptions. + * + * @return The report as a JSON object + */ boost::json::object report() const { - boost::json::object counts = {}; - - counts["ledger"] = ledgerSubscribers_.count(); - counts["transactions"] = txSubscribers_.count(); - counts["transactions_proposed"] = txProposedSubscribers_.count(); - counts["manifests"] = manifestSubscribers_.count(); - counts["validations"] = validationsSubscribers_.count(); - counts["account"] = accountSubscribers_.count(); - counts["accounts_proposed"] = accountProposedSubscribers_.count(); - counts["books"] = bookSubscribers_.count(); - counts["book_changes"] = bookChangesSubscribers_.count(); - - return counts; + return { + {"ledger", ledgerSubscribers_.count()}, + {"transactions", txSubscribers_.count()}, + {"transactions_proposed", txProposedSubscribers_.count()}, + {"manifests", manifestSubscribers_.count()}, + {"validations", validationsSubscribers_.count()}, + {"account", accountSubscribers_.count()}, + {"accounts_proposed", accountProposedSubscribers_.count()}, + {"books", bookSubscribers_.count()}, + {"book_changes", bookChangesSubscribers_.count()}, + }; } private: - void - sendAll(std::string const& pubMsg, std::unordered_set& subs); - using CleanupFunction = std::function; void @@ -362,14 +581,11 @@ private: void subscribeHelper(SessionPtrType const& session, Key const& k, SubscriptionMap& subs, CleanupFunction&& func); - /** - * This is how we chose to cleanup subscriptions that have been closed. - * Each time we add a subscriber, we add the opposite lambda that - * unsubscribes that subscriber when cleanup is called with the session that - * closed. - */ + // This is how we chose to cleanup subscriptions that have been closed. + // Each time we add a subscriber, we add the opposite lambda that unsubscribes that subscriber when cleanup is + // called with the session that closed. std::mutex cleanupMtx_; std::unordered_map> cleanupFuncs_ = {}; }; -} // namespace feed \ No newline at end of file +} // namespace feed diff --git a/src/main/Main.cpp b/src/main/Main.cpp index 06e5d418..dd55b4f1 100644 --- a/src/main/Main.cpp +++ b/src/main/Main.cpp @@ -201,15 +201,15 @@ try // ETL is responsible for writing and publishing to streams. In read-only mode, ETL only publishes auto etl = etl::ETLService::make_ETLService(config, ioc, backend, subscriptions, balancer, ledgers); - auto workQueue = WorkQueue::make_WorkQueue(config); - auto counters = RPC::Counters::make_Counters(workQueue); - auto const handlerProvider = std::make_shared( + auto workQueue = rpc::WorkQueue::make_WorkQueue(config); + auto counters = rpc::Counters::make_Counters(workQueue); + auto const handlerProvider = std::make_shared( config, backend, subscriptions, balancer, etl, counters); - auto const rpcEngine = RPC::RPCEngine::make_RPCEngine( + auto const rpcEngine = rpc::RPCEngine::make_RPCEngine( config, backend, subscriptions, balancer, etl, dosGuard, workQueue, counters, handlerProvider); // init the web server - auto handler = std::make_shared>( + auto handler = std::make_shared>( config, backend, rpcEngine, etl, subscriptions); auto ctx = parseCerts(config); auto const ctxRef = ctx ? std::optional>{ctx.value()} : std::nullopt; diff --git a/src/main/Mainpage.h b/src/main/Mainpage.h new file mode 100644 index 00000000..ee84f51d --- /dev/null +++ b/src/main/Mainpage.h @@ -0,0 +1,43 @@ +//------------------------------------------------------------------------------ +/* + This file is part of clio: https://github.com/XRPLF/clio + Copyright (c) 2023, the clio developers. + + Permission to use, copy, modify, and distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +/** + * @mainpage Clio API server + * + * @section intro Introduction + * + * Clio is an XRP Ledger API server. Clio is optimized for RPC calls, over WebSocket or JSON-RPC. + * + * Validated historical ledger and transaction data are stored in a more space-efficient format, using up to 4 times + * less space than rippled. + * + * Clio can be configured to store data in Apache Cassandra or ScyllaDB, allowing for scalable read throughput. + * Multiple Clio nodes can share access to the same dataset, allowing for a highly available cluster of Clio nodes, + * without the need for redundant data storage or computation. + * + * You can read more general information about Clio and its subsystems from the `Related Pages` section. + * + * @section Develop + * + * As you prepare to develop code for Clio, please be sure you are aware of our current + * Contribution guidelines. + * + * Read `rpc/README.md` carefully to know more about writing your own handlers for + * Clio. + */ diff --git a/src/rpc/Amendments.h b/src/rpc/Amendments.h index c643a7e0..2296bffd 100644 --- a/src/rpc/Amendments.h +++ b/src/rpc/Amendments.h @@ -23,12 +23,19 @@ #include -namespace RPC { +namespace rpc { #define REGISTER_AMENDMENT(name) inline static const ripple::uint256 name = GetAmendmentId(#name); +/** + * @brief Represents a list of amendments in the XRPL. + */ struct Amendments { + /** + * @param name The name of the amendment + * @return The corresponding amendment Id + */ static ripple::uint256 const GetAmendmentId(std::string_view const name) { @@ -38,4 +45,4 @@ struct Amendments REGISTER_AMENDMENT(DisallowIncoming) REGISTER_AMENDMENT(Clawback) }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/BookChangesHelper.h b/src/rpc/BookChangesHelper.h index 18d4c13b..49cc4337 100644 --- a/src/rpc/BookChangesHelper.h +++ b/src/rpc/BookChangesHelper.h @@ -17,13 +17,14 @@ */ //============================================================================== +/** @file */ #pragma once #include #include -namespace RPC { +namespace rpc { /** * @brief Represents an entry in the book_changes' changes array. @@ -178,7 +179,7 @@ private: void handleBookChange(data::TransactionAndMetadata const& blob) { - auto const [tx, meta] = RPC::deserializeTxPlusMeta(blob); + auto const [tx, meta] = rpc::deserializeTxPlusMeta(blob); if (!tx || !meta || !tx->isFieldPresent(ripple::sfTransactionType)) return; @@ -205,6 +206,12 @@ private: }; }; +/** + * @brief Implementation of value_from for BookChange type. + * + * @param jv The JSON value to populate + * @param change The BookChange to serialize + */ inline void tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookChange const& change) { @@ -228,7 +235,13 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, BookChange const }; } +/** + * @brief Computes all book changes for the given ledger header and transactions. + * + * @param lgrInfo The ledger header + * @param transactions The vector of transactions with heir metadata + */ [[nodiscard]] boost::json::object const computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector const& transactions); -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Counters.cpp b/src/rpc/Counters.cpp index 03889bb9..dcd07e63 100644 --- a/src/rpc/Counters.cpp +++ b/src/rpc/Counters.cpp @@ -21,7 +21,7 @@ #include #include -namespace RPC { +namespace rpc { void Counters::rpcFailed(std::string const& method) @@ -137,4 +137,4 @@ Counters::report() const return obj; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Counters.h b/src/rpc/Counters.h index c9cbf4cd..db4dc8f4 100644 --- a/src/rpc/Counters.h +++ b/src/rpc/Counters.h @@ -28,10 +28,16 @@ #include #include -namespace RPC { +namespace rpc { +/** + * @brief Holds information about successful, failed, forwarded, etc. RPC handler calls. + */ class Counters { + /** + * @brief All counters the system keeps track of for each RPC method. + */ struct MethodInfo { std::uint64_t started = 0u; @@ -57,49 +63,92 @@ class Counters std::chrono::time_point startupTime_; public: + /** + * @brief Creates a new counters instance that operates on the given WorkQueue. + * + * @param wq The work queue to operate on + */ Counters(WorkQueue const& wq) : workQueue_(std::cref(wq)), startupTime_{std::chrono::system_clock::now()} {}; + /** + * @brief A factory function that creates a new counters instance. + * + * @param wq The work queue to operate on + * @return The new instance + */ static Counters make_Counters(WorkQueue const& wq) { return Counters{wq}; } + /** + * @brief Increments the failed count for a particular RPC method. + * + * @param method The method to increment the count for + */ void rpcFailed(std::string const& method); + /** + * @brief Increments the errored count for a particular RPC method. + * + * @param method The method to increment the count for + */ void rpcErrored(std::string const& method); + /** + * @brief Increments the completed count for a particular RPC method. + * + * @param method The method to increment the count for + */ void rpcComplete(std::string const& method, std::chrono::microseconds const& rpcDuration); + /** + * @brief Increments the forwarded count for a particular RPC method. + * + * @param method The method to increment the count for + */ void rpcForwarded(std::string const& method); + /** + * @brief Increments the failed to forward count for a particular RPC method. + * + * @param method The method to increment the count for + */ void rpcFailedToForward(std::string const& method); + /** @brief Increments the global too busy counter. */ void onTooBusy(); + /** @brief Increments the global not ready counter. */ void onNotReady(); + /** @brief Increments the global bad syntax counter. */ void onBadSyntax(); + /** @brief Increments the global unknown command/method counter. */ void onUnknownCommand(); + /** @brief Increments the global internal error counter. */ void onInternalError(); + /** @return Uptime of this instance in seconds. */ std::chrono::seconds uptime() const; + /** @return A JSON report with current state of all counters for every method. */ boost::json::object report() const; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Errors.cpp b/src/rpc/Errors.cpp index bed0dacb..f1cfec18 100644 --- a/src/rpc/Errors.cpp +++ b/src/rpc/Errors.cpp @@ -36,7 +36,7 @@ template overloadSet(Ts...) -> overloadSet; } // namespace -namespace RPC { +namespace rpc { WarningInfo const& getWarningInfo(WarningCode code) @@ -152,4 +152,4 @@ makeError(Status const& status) return res; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Errors.h b/src/rpc/Errors.h index d15c3274..e0864f57 100644 --- a/src/rpc/Errors.h +++ b/src/rpc/Errors.h @@ -17,6 +17,7 @@ */ //============================================================================== +/** @file */ #pragma once #include @@ -28,11 +29,9 @@ #include #include -namespace RPC { +namespace rpc { -/** - * @brief Custom clio RPC Errors. - */ +/** @brief Custom clio RPC Errors. */ enum class ClioError { // normal clio errors start with 5000 rpcMALFORMED_CURRENCY = 5000, @@ -51,9 +50,7 @@ enum class ClioError { rpcPARAMS_UNPARSEABLE = 6004, }; -/** - * @brief Holds info about a particular @ref ClioError. - */ +/** @brief Holds info about a particular @ref ClioError. */ struct ClioErrorInfo { ClioError const code; @@ -61,9 +58,7 @@ struct ClioErrorInfo std::string_view const message; }; -/** - * @brief Clio uses compatible Rippled error codes for most RPC errors. - */ +/** @brief Clio uses compatible Rippled error codes for most RPC errors. */ using RippledError = ripple::error_code_i; /** @@ -74,9 +69,7 @@ using RippledError = ripple::error_code_i; */ using CombinedError = std::variant; -/** - * @brief A status returned from any RPC handler. - */ +/** @brief A status returned from any RPC handler. */ struct Status { CombinedError code = RippledError::rpcSUCCESS; @@ -102,9 +95,7 @@ struct Status { } - /** - * @brief Returns true if the Status is *not* OK. - */ + /** @brief Returns true if the Status is *not* OK. */ operator bool() const { if (auto err = std::get_if(&code)) @@ -114,10 +105,10 @@ struct Status } /** - * @brief Returns true if the Status contains the desired @ref RippledError + * @brief Returns true if the @ref rpc::Status contains the desired @ref rpc::RippledError * - * @param other The RippledError to match - * @return bool true if status matches given error; false otherwise + * @param other The @ref rpc::RippledError to match + * @return true if status matches given error; false otherwise */ bool operator==(RippledError other) const @@ -132,7 +123,7 @@ struct Status * @brief Returns true if the Status contains the desired @ref ClioError * * @param other The RippledError to match - * @return bool true if status matches given error; false otherwise + * @return true if status matches given error; false otherwise */ bool operator==(ClioError other) const @@ -144,14 +135,10 @@ struct Status } }; -/** - * @brief Warning codes that can be returned by clio. - */ +/** @brief Warning codes that can be returned by clio. */ enum WarningCode { warnUNKNOWN = -1, warnRPC_CLIO = 2001, warnRPC_OUTDATED = 2002, warnRPC_RATE_LIMIT = 2003 }; -/** - * @brief Holds information about a clio warning. - */ +/** @brief Holds information about a clio warning. */ struct WarningInfo { constexpr WarningInfo() = default; @@ -163,9 +150,7 @@ struct WarningInfo std::string_view const message = "unknown warning"; }; -/** - * @brief Invalid parameters error. - */ +/** @brief Invalid parameters error. */ class InvalidParamsError : public std::exception { std::string msg; @@ -182,9 +167,7 @@ public: } }; -/** - * @brief Account not found error. - */ +/** @brief Account not found error. */ class AccountNotFoundError : public std::exception { std::string account; @@ -201,16 +184,14 @@ public: } }; -/** - * @brief A globally available @ref Status that represents a successful state - */ +/** @brief A globally available @ref rpc::Status that represents a successful state. */ static Status OK; /** * @brief Get the warning info object from a warning code. * * @param code The warning code - * @return WarningInfo const& A reference to the static warning info + * @return A reference to the static warning info */ WarningInfo const& getWarningInfo(WarningCode code); @@ -219,34 +200,34 @@ getWarningInfo(WarningCode code); * @brief Get the error info object from an clio-specific error code. * * @param code The error code - * @return ClioErrorInfo const& A reference to the static error info + * @return A reference to the static error info */ ClioErrorInfo const& getErrorInfo(ClioError code); /** - * @brief Generate JSON from a warning code. + * @brief Generate JSON from a @ref rpc::WarningCode. * - * @param code The @ref WarningCode - * @return boost::json::object The JSON output + * @param code The warning code + * @return The JSON output */ boost::json::object makeWarning(WarningCode code); /** - * @brief Generate JSON from a @ref Status. + * @brief Generate JSON from a @ref rpc::Status. * - * @param status The @ref Status - * @return boost::json::object The JSON output + * @param status The status object + * @return The JSON output */ boost::json::object makeError(Status const& status); /** - * @brief Generate JSON from a @ref RippledError. + * @brief Generate JSON from a @ref rpc::RippledError. * - * @param status The rippled @ref RippledError - * @return boost::json::object The JSON output + * @param err The rippled error + * @return The JSON output */ boost::json::object makeError( @@ -255,10 +236,10 @@ makeError( std::optional customMessage = std::nullopt); /** - * @brief Generate JSON from a @ref ClioError. + * @brief Generate JSON from a @ref rpc::ClioError. * - * @param status The clio's custom @ref ClioError - * @return boost::json::object The JSON output + * @param err The clio's custom error + * @return The JSON output */ boost::json::object makeError( @@ -266,4 +247,4 @@ makeError( std::optional customError = std::nullopt, std::optional customMessage = std::nullopt); -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Factories.cpp b/src/rpc/Factories.cpp index cf562ff9..8c58b553 100644 --- a/src/rpc/Factories.cpp +++ b/src/rpc/Factories.cpp @@ -23,7 +23,7 @@ using namespace std; using namespace util; -namespace RPC { +namespace rpc { util::Expected make_WsContext( @@ -94,4 +94,4 @@ make_HttpContext( return web::Context(yc, command, *apiVersion, array.at(0).as_object(), nullptr, tagFactory, range, clientIp); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/Factories.h b/src/rpc/Factories.h index 2f4817d4..a7591233 100644 --- a/src/rpc/Factories.h +++ b/src/rpc/Factories.h @@ -36,11 +36,22 @@ * This file contains various classes necessary for executing RPC handlers. * Context gives the handlers access to various other parts of the application Status is used to report errors. * And lastly, there are various functions for making Contexts, Statuses and serializing Status to JSON. - * This file is meant to contain any class or function that code outside of the rpc folder needs to use. For helper - * functions or classes used within the rpc folder, use RPCHelpers.h. + * This file is meant to contain any class or function that code outside of the rpc folder needs to use. + * For helper functions or classes used within the rpc folder, use RPCHelpers.h. */ -namespace RPC { +namespace rpc { +/** + * @brief A factory function that creates a Websocket context. + * + * @param yc The coroutine context + * @param request The request as JSON object + * @param session The connection + * @param tagFactory A factory that provides tags to track requests + * @param range The ledger range that is available at request time + * @param clientIp The IP address of the connected client + * @param apiVersionParser A parser that is used to parse out the "api_version" field + */ util::Expected make_WsContext( boost::asio::yield_context yc, @@ -51,6 +62,16 @@ make_WsContext( std::string const& clientIp, std::reference_wrapper apiVersionParser); +/** + * @brief A factory function that creates a HTTP context. + * + * @param yc The coroutine context + * @param request The request as JSON object + * @param tagFactory A factory that provides tags to track requests + * @param range The ledger range that is available at request time + * @param clientIp The IP address of the connected client + * @param apiVersionParser A parser that is used to parse out the "api_version" field + */ util::Expected make_HttpContext( boost::asio::yield_context yc, @@ -60,4 +81,4 @@ make_HttpContext( std::string const& clientIp, std::reference_wrapper apiVersionParser); -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/JS.h b/src/rpc/JS.h index b9df8486..e627745d 100644 --- a/src/rpc/JS.h +++ b/src/rpc/JS.h @@ -21,9 +21,8 @@ #include -// Useful macro for borrowing from ripple::jss -// static strings. (J)son (S)trings +/** @brief Helper macro for borrowing from ripple::jss static (J)son (S)trings. */ #define JS(x) ripple::jss::x.c_str() -// Access (SF)ield name (S)trings +/** @brief Provides access to (SF)ield name (S)trings. */ #define SFS(x) ripple::x.jsonName.c_str() diff --git a/src/rpc/README.md b/src/rpc/README.md index f959e5fd..4e9777f5 100644 --- a/src/rpc/README.md +++ b/src/rpc/README.md @@ -1,29 +1,22 @@ -# Clio RPC subsystem +# RPC subsystem + ## Background The RPC subsystem is where the common framework for handling incoming JSON requests is implemented. -Currently the NextGen RPC framework is a work in progress and the handlers are not yet implemented using the new common framework classes. - -## Integration plan -- Implement base framework - **done** -- Migrate handlers one by one, making them injectable, adding unit-tests - **in progress** -- Integrate all new handlers into clio in one go -- Cover the rest with unit-tests -- Release first time with new subsystem active ## Components See `common` subfolder. - **AnyHandler**: The type-erased wrapper that allows for storing different handlers in one map/vector. - **RpcSpec/FieldSpec**: The RPC specification classes, used to specify how incoming JSON is to be validated before it's parsed and passed on to individual handler implementations. -- **Validators**: A bunch of supported validators that can be specified as requirements for each **`FieldSpec`** to make up the final **`RpcSpec`** of any given RPC handler. +- **Validators/Modifiers**: A bunch of supported validators and modifiers that can be specified as requirements for each **`FieldSpec`** to make up the final **`RpcSpec`** of any given RPC handler. -## Implementing a (NextGen) handler +## Implementing a handler See `unittests/rpc` for exmaples. -Handlers need to fulfil the requirements specified by the **`Handler`** concept (see `rpc/common/Concepts.h`): +Handlers need to fulfil the requirements specified by the **`SomeHandler`** concept (see `rpc/common/Concepts.h`): - Expose types: * `Input` - The POD struct which acts as input for the handler * `Output` - The POD struct which acts as output of a valid handler invocation -- Have a `spec(uint32_t)` member function returning a const reference to an **`RpcSpec`** describing the JSON input. +- Have a `spec(uint32_t)` member function returning a const reference to an **`RpcSpec`** describing the JSON input for the specified API version. - Have a `process(Input)` member function that operates on `Input` POD and returns `HandlerReturnType` - Implement `value_from` and `value_to` support using `tag_invoke` as per `boost::json` documentation for these functions. diff --git a/src/rpc/RPCEngine.h b/src/rpc/RPCEngine.h index f39ff2ae..1c8a7f3e 100644 --- a/src/rpc/RPCEngine.h +++ b/src/rpc/RPCEngine.h @@ -52,10 +52,13 @@ class LoadBalancer; class ETLService; } // namespace etl -namespace RPC { +/** + * @brief This namespace contains all the RPC logic and handlers. + */ +namespace rpc { /** - * @brief The RPC engine that ties all RPC-related functionality together + * @brief The RPC engine that ties all RPC-related functionality together. */ template class RPCEngineBase @@ -113,8 +116,10 @@ public: } /** - * @brief Main request processor routine + * @brief Main request processor routine. + * * @param ctx The @ref Context of the request + * @return A result which can be an error status or a valid JSON response */ Result buildResponse(web::Context const& ctx) @@ -171,22 +176,24 @@ public: } /** - * @brief Used to schedule request processing onto the work queue + * @brief Used to schedule request processing onto the work queue. + * + * @tparam FnType The type of function * @param func The lambda to execute when this request is handled * @param ip The ip address for which this request is being executed */ - template + template bool - post(Fn&& func, std::string const& ip) + post(FnType&& func, std::string const& ip) { - return workQueue_.get().postCoro(std::forward(func), dosGuard_.get().isWhiteListed(ip)); + return workQueue_.get().postCoro(std::forward(func), dosGuard_.get().isWhiteListed(ip)); } /** - * @brief Notify the system that specified method was executed + * @brief Notify the system that specified method was executed. + * * @param method - * @param duration The time it took to execute the method specified in - * microseconds + * @param duration The time it took to execute the method specified in microseconds */ void notifyComplete(std::string const& method, std::chrono::microseconds const& duration) @@ -196,7 +203,7 @@ public: } /** - * @brief Notify the system that specified method failed to execute due to a recoverable user error + * @brief Notify the system that specified method failed to execute due to a recoverable user error. * * Used for errors based on user input, not actual failures of the db or clio itself. * @@ -211,7 +218,7 @@ public: } /** - * @brief Notify the system that specified method failed due to some unrecoverable error + * @brief Notify the system that specified method failed due to some unrecoverable error. * * Used for erors such as database timeout, internal errors, etc. * @@ -225,7 +232,7 @@ public: } /** - * @brief Notify the system that the RPC system is too busy to handle an incoming request + * @brief Notify the system that the RPC system is too busy to handle an incoming request. */ void notifyTooBusy() @@ -234,7 +241,7 @@ public: } /** - * @brief Notify the system that the RPC system was not ready to handle an incoming request + * @brief Notify the system that the RPC system was not ready to handle an incoming request. * * This happens when the backend is not yet have a ledger range */ @@ -245,7 +252,7 @@ public: } /** - * @brief Notify the system that the incoming request did not specify the RPC method/command + * @brief Notify the system that the incoming request did not specify the RPC method/command. */ void notifyBadSyntax() @@ -254,7 +261,7 @@ public: } /** - * @brief Notify the system that the incoming request specified an unknown/unsupported method/command + * @brief Notify the system that the incoming request specified an unknown/unsupported method/command. */ void notifyUnknownCommand() @@ -263,7 +270,7 @@ public: } /** - * @brief Notify the system that the incoming request lead to an internal error (unrecoverable) + * @brief Notify the system that the incoming request lead to an internal error (unrecoverable). */ void notifyInternalError() @@ -281,4 +288,4 @@ private: using RPCEngine = RPCEngineBase; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/RPCHelpers.cpp b/src/rpc/RPCHelpers.cpp index 66e26f61..d03530ba 100644 --- a/src/rpc/RPCHelpers.cpp +++ b/src/rpc/RPCHelpers.cpp @@ -35,7 +35,7 @@ namespace { util::Logger gLog{"RPC"}; } // namespace -namespace RPC { +namespace rpc { std::optional parseAccountCursor(std::optional jsonCursor) @@ -1340,4 +1340,4 @@ isAmendmentEnabled( return std::find(listAmendments.begin(), listAmendments.end(), amendmentId) != listAmendments.end(); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/RPCHelpers.h b/src/rpc/RPCHelpers.h index ee217fd5..1b94b7f6 100644 --- a/src/rpc/RPCHelpers.h +++ b/src/rpc/RPCHelpers.h @@ -17,6 +17,7 @@ */ //============================================================================== +/** @file */ #pragma once /* @@ -37,7 +38,7 @@ #include -namespace RPC { +namespace rpc { enum class NFTokenjson { ENABLE, DISABLE }; @@ -246,4 +247,4 @@ logDuration(web::Context const& ctx, T const& dur) log.info() << ctx.tag() << msg; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/WorkQueue.cpp b/src/rpc/WorkQueue.cpp index d43f371b..909d75ab 100644 --- a/src/rpc/WorkQueue.cpp +++ b/src/rpc/WorkQueue.cpp @@ -19,6 +19,8 @@ #include +namespace rpc { + WorkQueue::WorkQueue(std::uint32_t numWorkers, uint32_t maxSize) : ioc_{numWorkers} { if (maxSize != 0) @@ -29,3 +31,5 @@ WorkQueue::~WorkQueue() { ioc_.join(); } + +} // namespace rpc diff --git a/src/rpc/WorkQueue.h b/src/rpc/WorkQueue.h index 08ec654d..7cfe4e7b 100644 --- a/src/rpc/WorkQueue.h +++ b/src/rpc/WorkQueue.h @@ -32,6 +32,11 @@ #include #include +namespace rpc { + +/** + * @brief An asynchronous, thread-safe queue for RPC requests. + */ class WorkQueue { // these are cumulative for the lifetime of the process @@ -45,9 +50,20 @@ class WorkQueue boost::asio::thread_pool ioc_; public: + /** + * @brief Create an we instance of the work queue. + * + * @param numWorkers The amount of threads to spawn in the pool + * @param maxSize The maximum capacity of the queue; 0 means unlimited + */ WorkQueue(std::uint32_t numWorkers, uint32_t maxSize = 0); ~WorkQueue(); + /** + * @brief A factory function that creates the work queue based on a config. + * + * @param config The Clio config to use + */ static WorkQueue make_WorkQueue(util::Config const& config) { @@ -60,9 +76,19 @@ public: return WorkQueue{numThreads, maxQueueSize}; } - template + /** + * @brief Submit a job to the work queue. + * + * The job will be rejected if isWhiteListed is set to false and the current size of the queue reached capacity. + * + * @tparam FnType The function object type + * @param func The function object to queue as a job + * @param isWhiteListed Whether the queue capacity applies to this job + * @return true if the job was successfully queued; false otherwise + */ + template bool - postCoro(F&& f, bool isWhiteListed) + postCoro(FnType&& func, bool isWhiteListed) { if (curSize_ >= maxSize_ && !isWhiteListed) { @@ -75,7 +101,8 @@ public: // Each time we enqueue a job, we want to post a symmetrical job that will dequeue and run the job at the front // of the job queue. boost::asio::spawn( - ioc_, [this, f = std::forward(f), start = std::chrono::system_clock::now()](auto yield) mutable { + ioc_, + [this, func = std::forward(func), start = std::chrono::system_clock::now()](auto yield) mutable { auto const run = std::chrono::system_clock::now(); auto const wait = std::chrono::duration_cast(run - start).count(); @@ -83,13 +110,18 @@ public: durationUs_ += wait; log_.info() << "WorkQueue wait time = " << wait << " queue size = " << curSize_; - f(yield); + func(yield); --curSize_; }); return true; } + /** + * @brief Generate a report of the work queue state. + * + * @return The report as a JSON object. + */ boost::json::object report() const { @@ -103,3 +135,5 @@ public: return obj; } }; + +} // namespace rpc diff --git a/src/rpc/common/APIVersion.h b/src/rpc/common/APIVersion.h index c4a668da..146d45d5 100644 --- a/src/rpc/common/APIVersion.h +++ b/src/rpc/common/APIVersion.h @@ -26,7 +26,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief Default API version to use if no version is specified by clients @@ -54,7 +54,13 @@ class APIVersionParser public: virtual ~APIVersionParser() = default; + /** + * @brief Extracts API version information from a JSON object. + * + * @param request A JSON object representing the request + * @return The specified API version if contained in the JSON object; error string otherwise + */ util::Expected virtual parse(boost::json::object const& request) const = 0; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/AnyHandler.h b/src/rpc/common/AnyHandler.h index 8ea39e16..d1bfe375 100644 --- a/src/rpc/common/AnyHandler.h +++ b/src/rpc/common/AnyHandler.h @@ -23,7 +23,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief A type-erased Handler that can contain any (NextGen) RPC handler class @@ -40,9 +40,9 @@ public: * * @tparam HandlerType The real type of wrapped handler class * @tparam ProcessingStrategy A strategy that implements how processing of JSON is to be done - * @param handler The handler to wrap. Required to fulfil the @ref Handler concept. + * @param handler The handler to wrap. Required to fulfil the @ref rpc::SomeHandler concept. */ - template > + template > /* implicit */ AnyHandler(HandlerType&& handler) : pimpl_{std::make_unique>(std::forward(handler))} { @@ -117,4 +117,4 @@ private: std::unique_ptr pimpl_; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/Concepts.h b/src/rpc/common/Concepts.h index 754f7f42..04ce17ed 100644 --- a/src/rpc/common/Concepts.h +++ b/src/rpc/common/Concepts.h @@ -26,65 +26,87 @@ #include -namespace RPC { +namespace rpc { struct RpcSpec; /** - * @brief A concept that specifies what a requirement used with @ref FieldSpec - * must provide + * @brief Specifies what a requirement used with @ref rpc::FieldSpec must provide. */ // clang-format off template -concept Requirement = requires(T a, boost::json::value lval) { +concept SomeRequirement = requires(T a, boost::json::value lval) { { a.verify(lval, std::string{}) } -> std::same_as; }; // clang-format on +/** + * @brief Specifies what a modifier used with @ref rpc::FieldSpec must provide. + */ // clang-format off template -concept Modifier = requires(T a, boost::json::value lval) { +concept SomeModifier = requires(T a, boost::json::value lval) { { a.modify(lval, std::string{}) } -> std::same_as; }; // clang-format on +/** + * @brief The requirements of a processor to be used with @ref rpc::FieldSpec. + */ template -concept Processor = (Requirement or Modifier); +concept SomeProcessor = (SomeRequirement or SomeModifier); /** - * @brief A concept that specifies what a Handler type must provide - * - * Note that value_from and value_to should be implemented using tag_invoke - * as per boost::json documentation for these functions. + * @brief A process function that expects both some Input and a Context. */ // clang-format off template -concept ContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) { +concept SomeContextProcessWithInput = requires(T a, typename T::Input in, typename T::Output out, Context const& ctx) { { a.process(in, ctx) } -> std::same_as>; }; +// clang-format on +/** + * @brief A process function that expects no Input but does take a Context. + */ +// clang-format off template -concept ContextProcessWithoutInput = requires(T a, typename T::Output out, Context const& ctx) { +concept SomeContextProcessWithoutInput = requires(T a, typename T::Output out, Context const& ctx) { { a.process(ctx) } -> std::same_as>; }; +// clang-format on +/** + * @brief Specifies what a Handler with Input must provide. + */ +// clang-format off template -concept HandlerWithInput = requires(T a, uint32_t version) { +concept SomeHandlerWithInput = requires(T a, uint32_t version) { { a.spec(version) } -> std::same_as; } -and ContextProcessWithInput +and SomeContextProcessWithInput and boost::json::has_value_to::value; +// clang-format on +/** + * @brief Specifies what a Handler without Input must provide. + */ +// clang-format off template -concept HandlerWithoutInput = ContextProcessWithoutInput; +concept SomeHandlerWithoutInput = SomeContextProcessWithoutInput; +// clang-format on +/** + * @brief Specifies what a Handler type must provide. + */ +// clang-format off template -concept Handler = +concept SomeHandler = ( - HandlerWithInput or - HandlerWithoutInput + SomeHandlerWithInput or + SomeHandlerWithoutInput ) and boost::json::has_value_from::value; // clang-format on -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/MetaProcessors.cpp b/src/rpc/common/MetaProcessors.cpp index 66813d71..460e8166 100644 --- a/src/rpc/common/MetaProcessors.cpp +++ b/src/rpc/common/MetaProcessors.cpp @@ -24,7 +24,7 @@ #include -namespace RPC::meta { +namespace rpc::meta { [[nodiscard]] MaybeError Section::verify(boost::json::value& value, std::string_view key) const @@ -68,4 +68,4 @@ ValidateArrayAt::verify(boost::json::value& value, std::string_view key) const return {}; } -} // namespace RPC::meta +} // namespace rpc::meta diff --git a/src/rpc/common/MetaProcessors.h b/src/rpc/common/MetaProcessors.h index 76749fd0..11b15196 100644 --- a/src/rpc/common/MetaProcessors.h +++ b/src/rpc/common/MetaProcessors.h @@ -26,10 +26,10 @@ #include -namespace RPC::meta { +namespace rpc::meta { /** - * @brief A meta-processor that acts as a spec for a sub-object/section + * @brief A meta-processor that acts as a spec for a sub-object/section. */ class Section final { @@ -37,7 +37,7 @@ class Section final public: /** - * @brief Construct new section validator from a list of specs + * @brief Construct new section validator from a list of specs. * * @param specs List of specs @ref FieldSpec */ @@ -46,17 +46,18 @@ public: } /** - * @brief Verify that the JSON value representing the section is valid according to the given specs + * @brief Verify that the JSON value representing the section is valid according to the given specs. * * @param value The JSON value representing the outer object * @param key The key used to retrieve the section from the outer object + * @return Possibly an error */ [[nodiscard]] MaybeError verify(boost::json::value& value, std::string_view key) const; }; /** - * @brief A meta-processor that specifies a list of specs to run against the object at the given index in the array + * @brief A meta-processor that specifies a list of specs to run against the object at the given index in the array. */ class ValidateArrayAt final { @@ -65,7 +66,7 @@ class ValidateArrayAt final public: /** - * @brief Constructs a processor that validates the specified element of a JSON array + * @brief Constructs a processor that validates the specified element of a JSON array. * * @param idx The index inside the array to validate * @param specs The specifications to validate against @@ -75,10 +76,11 @@ public: } /** - * @brief Verify that the JSON array element at given index is valid according the stored specs + * @brief Verify that the JSON array element at given index is valid according the stored specs. * * @param value The JSON value representing the outer object * @param key The key used to retrieve the array from the outer object + * @return Possibly an error */ [[nodiscard]] MaybeError verify(boost::json::value& value, std::string_view key) const; @@ -86,17 +88,17 @@ public: /** * @brief A meta-processor that specifies a list of requirements to run against when the type matches the template - * parameter + * parameter. */ template class IfType final { public: /** - * @brief Constructs a validator that validates the specs if the type matches + * @brief Constructs a validator that validates the specs if the type matches. * @param requirements The requirements to validate against */ - template + template IfType(Requirements&&... requirements) { processor_ = [... r = std::forward(requirements)]( @@ -122,11 +124,11 @@ public: } /** - * @brief Verify that the element is valid - * according the stored requirements when type matches + * @brief Verify that the element is valid according to the stored requirements when type matches. * * @param value The JSON value representing the outer object * @param key The key used to retrieve the element from the outer object + * @return Possibly an error */ [[nodiscard]] MaybeError verify(boost::json::value& value, std::string_view key) const @@ -134,7 +136,7 @@ public: if (not value.is_object() or not value.as_object().contains(key.data())) return {}; // ignore. field does not exist, let 'required' fail instead - if (not RPC::validation::checkType(value.as_object().at(key.data()))) + if (not rpc::validation::checkType(value.as_object().at(key.data()))) return {}; // ignore if type does not match return processor_(value, key); @@ -145,23 +147,30 @@ private: }; /** - * @brief A meta-processor that wraps another validator and produces a custom error in case the wrapped validator fails + * @brief A meta-processor that wraps a validator and produces a custom error in case the wrapped validator fails. */ -template +template class WithCustomError final { - Requirement requirement; + SomeRequirement requirement; Status error; public: /** * @brief Constructs a validator that calls the given validator `req` and returns a custom error `err` in case `req` - * fails + * fails. */ - WithCustomError(Requirement req, Status err) : requirement{std::move(req)}, error{err} + WithCustomError(SomeRequirement req, Status err) : requirement{std::move(req)}, error{err} { } + /** + * @brief Runs the stored validator and produces a custom error if the wrapped validator fails. + * + * @param value The JSON value representing the outer object + * @param key The key used to retrieve the element from the outer object + * @return Possibly an error + */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const { @@ -172,4 +181,4 @@ public: } }; -} // namespace RPC::meta +} // namespace rpc::meta diff --git a/src/rpc/common/Modifiers.h b/src/rpc/common/Modifiers.h index 3971d411..70abd78a 100644 --- a/src/rpc/common/Modifiers.h +++ b/src/rpc/common/Modifiers.h @@ -23,7 +23,7 @@ #include #include -namespace RPC::modifiers { +namespace rpc::modifiers { /** * @brief Clamp value between min and max. @@ -36,7 +36,7 @@ class Clamp final public: /** - * @brief Construct the modifier storing min and max values + * @brief Construct the modifier storing min and max values. * * @param min * @param max @@ -50,6 +50,7 @@ public: * * @param value The JSON value representing the outer object * @param key The key used to retrieve the modified value from the outer object + * @return Possibly an error */ [[nodiscard]] MaybeError modify(boost::json::value& value, std::string_view key) const @@ -67,4 +68,4 @@ public: } }; -} // namespace RPC::modifiers +} // namespace rpc::modifiers diff --git a/src/rpc/common/Specs.cpp b/src/rpc/common/Specs.cpp index 2ef0c5cd..00984efb 100644 --- a/src/rpc/common/Specs.cpp +++ b/src/rpc/common/Specs.cpp @@ -21,7 +21,7 @@ #include -namespace RPC { +namespace rpc { [[nodiscard]] MaybeError FieldSpec::process(boost::json::value& value) const @@ -39,4 +39,4 @@ RpcSpec::process(boost::json::value& value) const return {}; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/Specs.h b/src/rpc/common/Specs.h index e49f46c0..1adaa68e 100644 --- a/src/rpc/common/Specs.h +++ b/src/rpc/common/Specs.h @@ -26,28 +26,28 @@ #include #include -namespace RPC { +namespace rpc { /** - * @brief Represents a Specification for one field of an RPC command + * @brief Represents a Specification for one field of an RPC command. */ struct FieldSpec final { /** - * @brief Construct a field specification out of a set of processors + * @brief Construct a field specification out of a set of processors. * - * @tparam Processors The types of processors @ref Processor + * @tparam Processors The types of processors * @param key The key in a JSON object that the field validates - * @param processors The processors, each of them have to fulfil the @ref Processor concept + * @param processors The processors, each of them have to fulfil the @ref rpc::SomeProcessor concept */ - template + template FieldSpec(std::string const& key, Processors&&... processors) : processor_{detail::makeFieldProcessor(key, std::forward(processors)...)} { } /** - * @brief Processos the passed JSON value using the stored processors + * @brief Processos the passed JSON value using the stored processors. * * @param value The JSON value to validate and/or modify * @return Nothing on success; @ref Status on error @@ -60,7 +60,7 @@ private: }; /** - * @brief Represents a Specification of an entire RPC command + * @brief Represents a Specification of an entire RPC command. * * Note: this should really be all constexpr and handlers would expose * static constexpr RpcSpec spec instead. Maybe some day in the future. @@ -68,7 +68,7 @@ private: struct RpcSpec final { /** - * @brief Construct a full RPC request specification + * @brief Construct a full RPC request specification. * * @param fields The fields of the RPC specification @ref FieldSpec */ @@ -77,7 +77,7 @@ struct RpcSpec final } /** - * @brief Processos the passed JSON value using the stored field specs + * @brief Processos the passed JSON value using the stored field specs. * * @param value The JSON value to validate and/or modify * @return Nothing on success; @ref Status on error @@ -89,4 +89,4 @@ private: std::vector fields_; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/Types.h b/src/rpc/common/Types.h index 7953114e..e2519eac 100644 --- a/src/rpc/common/Types.h +++ b/src/rpc/common/Types.h @@ -36,9 +36,12 @@ namespace feed { class SubscriptionManager; } -namespace RPC { +namespace rpc { class Counters; +struct RpcSpec; +struct FieldSpec; +class AnyHandler; /** * @brief Return type used for Validators that can return error but don't have @@ -62,15 +65,21 @@ using HandlerReturnType = util::Expected; */ using ReturnType = util::Expected; -struct RpcSpec; -struct FieldSpec; - +/** + * @brief An alias for a const reference to @ref RpcSpec. + */ using RpcSpecConstRef = RpcSpec const&; +/** + * @brief An empty type used as Output for handlers than don't actually produce output. + */ struct VoidOutput { }; +/** + * @brief Context of an RPC call. + */ struct Context { boost::asio::yield_context yield; @@ -80,8 +89,14 @@ struct Context uint32_t apiVersion = 0u; // invalid by default }; +/** + * @brief Result type used to return responses or error statuses to the Webserver subsystem. + */ using Result = std::variant; +/** + * @brief A cursor object used to traverse nodes owned by an account. + */ struct AccountCursor { ripple::uint256 index; @@ -100,8 +115,9 @@ struct AccountCursor } }; -class AnyHandler; - +/** + * @brief Interface for the provider of RPC handlers. + */ class HandlerProvider { public: @@ -123,4 +139,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, VoidOutput const jv = boost::json::object{}; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/common/Validators.cpp b/src/rpc/common/Validators.cpp index 58ed1cf9..35fca1c0 100644 --- a/src/rpc/common/Validators.cpp +++ b/src/rpc/common/Validators.cpp @@ -26,7 +26,7 @@ #include #include -namespace RPC::validation { +namespace rpc::validation { [[nodiscard]] MaybeError Required::verify(boost::json::value const& value, std::string_view key) const @@ -202,4 +202,4 @@ CustomValidator SubscribeAccountsValidator = return MaybeError{}; }}; -} // namespace RPC::validation +} // namespace rpc::validation diff --git a/src/rpc/common/Validators.h b/src/rpc/common/Validators.h index c058fe40..1a3db81b 100644 --- a/src/rpc/common/Validators.h +++ b/src/rpc/common/Validators.h @@ -25,10 +25,10 @@ #include -namespace RPC::validation { +namespace rpc::validation { /** - * @brief Check that the type is the same as what was expected + * @brief Check that the type is the same as what was expected. * * @tparam Expected The expected type that value should be convertible to * @param value The json value to check the type of @@ -79,7 +79,7 @@ template } /** - * @brief A validator that simply requires a field to be present + * @brief A validator that simply requires a field to be present. */ struct Required final { @@ -88,15 +88,16 @@ struct Required final }; /** - * @brief A validator that forbids a field to be present - * If there is a value provided, it will forbid the field only when the value equals - * If there is no value provided, it will forbid the field when the field shows up + * @brief A validator that forbids a field to be present. + * + * If there is a value provided, it will forbid the field only when the value equals. + * If there is no value provided, it will forbid the field when the field shows up. */ template class NotSupported; /** - * @brief A specialized NotSupported validator that forbids a field to be present when the value equals the given value + * @brief A specialized NotSupported validator that forbids a field to be present when the value equals the given value. */ template class NotSupported final @@ -104,6 +105,22 @@ class NotSupported final T value_; public: + /** + * @brief Constructs a new NotSupported validator. + * + * @param val The value to store and verify against + */ + NotSupported(T val) : value_(val) + { + } + + /** + * @brief Verify whether the field is supported or not. + * + * @param value The JSON value representing the outer object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcNOT_SUPPORTED` if the value matched; otherwise no error is returned + */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const { @@ -118,19 +135,22 @@ public: } return {}; } - - NotSupported(T val) : value_(val) - { - } }; /** - * @brief A specialized NotSupported validator that forbids a field to be present + * @brief A specialized NotSupported validator that forbids a field to be present. */ template <> class NotSupported<> final { public: + /** + * @brief Verify whether the field is supported or not. + * + * @param value The JSON value representing the outer object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcNOT_SUPPORTED` if the field is found; otherwise no error is returned + */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const { @@ -141,22 +161,24 @@ public: } }; -// deduction guide to avoid having to specify the template arguments +/** + * @brief Deduction guide to avoid having to specify the template arguments. + */ template NotSupported(T&&... t) -> NotSupported; /** - * @brief Validates that the type of the value is one of the given types + * @brief Validates that the type of the value is one of the given types. */ template struct Type final { /** - * @brief Verify that the JSON value is (one) of specified type(s) + * @brief Verify that the JSON value is (one) of specified type(s). * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -175,7 +197,7 @@ struct Type final }; /** - * @brief Validate that value is between specified min and max + * @brief Validate that value is between specified min and max. */ template class Between final @@ -185,7 +207,7 @@ class Between final public: /** - * @brief Construct the validator storing min and max values + * @brief Construct the validator storing min and max values. * * @param min * @param max @@ -195,11 +217,11 @@ public: } /** - * @brief Verify that the JSON value is within a certain range + * @brief Verify that the JSON value is within a certain range. * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -221,7 +243,7 @@ public: }; /** - * @brief Validate that value is equal or greater than the specified min + * @brief Validate that value is equal or greater than the specified min. */ template class Min final @@ -230,7 +252,7 @@ class Min final public: /** - * @brief Construct the validator storing min value + * @brief Construct the validator storing min value. * * @param min */ @@ -242,8 +264,8 @@ public: * @brief Verify that the JSON value is not smaller than min * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -263,7 +285,7 @@ public: }; /** - * @brief Validate that value is not greater than max + * @brief Validate that value is not greater than max. */ template class Max final @@ -272,7 +294,7 @@ class Max final public: /** - * @brief Construct the validator storing max value + * @brief Construct the validator storing max value. * * @param max */ @@ -281,10 +303,11 @@ public: } /** - * @brief Verify that the JSON value is not greater than max + * @brief Verify that the JSON value is not greater than max. * * @param value The JSON value representing the outer object * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -304,7 +327,7 @@ public: }; /** - * @brief Validates that the value is equal to the one passed in + * @brief Validates that the value is equal to the one passed in. */ template class EqualTo final @@ -313,7 +336,7 @@ class EqualTo final public: /** - * @brief Construct the validator with stored original value + * @brief Construct the validator with stored original value. * * @param original The original value to store */ @@ -322,11 +345,11 @@ public: } /** - * @brief Verify that the JSON value is equal to the stored original + * @brief Verify that the JSON value is equal to the stored original. * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -345,13 +368,12 @@ public: }; /** - * @brief Deduction guide to help disambiguate what it means to EqualTo a - * "string" without specifying the type. + * @brief Deduction guide to help disambiguate what it means to EqualTo a "string" without specifying the type. */ EqualTo(char const*)->EqualTo; /** - * @brief Validates that the value is one of the values passed in + * @brief Validates that the value is one of the values passed in. */ template class OneOf final @@ -360,7 +382,7 @@ class OneOf final public: /** - * @brief Construct the validator with stored options of initializer list + * @brief Construct the validator with stored options of initializer list. * * @param options The list of allowed options */ @@ -369,7 +391,7 @@ public: } /** - * @brief Construct the validator with stored options of other container + * @brief Construct the validator with stored options of other container. * * @param begin,end the range to copy the elements from */ @@ -379,11 +401,11 @@ public: } /** - * @brief Verify that the JSON value is one of the stored options + * @brief Verify that the JSON value is one of the stored options. * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return `RippledError::rpcINVALID_PARAMS` if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const @@ -402,13 +424,12 @@ public: }; /** - * @brief Deduction guide to help disambiguate what it means to OneOf a - * few "strings" without specifying the type. + * @brief Deduction guide to help disambiguate what it means to OneOf a few "strings" without specifying the type. */ OneOf(std::initializer_list)->OneOf; /** - * @brief A meta-validator that allows to specify a custom validation function + * @brief A meta-validator that allows to specify a custom validation function. */ class CustomValidator final { @@ -416,7 +437,7 @@ class CustomValidator final public: /** - * @brief Constructs a custom validator from any supported callable + * @brief Constructs a custom validator from any supported callable. * * @tparam Fn The type of callable * @param fn The callable/function object @@ -427,79 +448,83 @@ public: } /** - * @brief Verify that the JSON value is valid according to the custom - * validation function stored + * @brief Verify that the JSON value is valid according to the custom validation function stored. * * @param value The JSON value representing the outer object - * @param key The key used to retrieve the tested value from the outer - * object + * @param key The key used to retrieve the tested value from the outer object + * @return Any compatible user-provided error if validation failed; otherwise no error is returned */ [[nodiscard]] MaybeError verify(boost::json::value const& value, std::string_view key) const; }; /** - * @brief Helper function to check if sv is an uint32 number or not + * @brief Helper function to check if input value is an uint32 number or not. + * + * @param sv The input value as a string_view + * @return true if the string can be converted to a uint32; false otherwise */ [[nodiscard]] bool checkIsU32Numeric(std::string_view sv); /** - * @brief Provide a common used validator for ledger index - * LedgerIndex must be a string or int - * If the specified LedgerIndex is a string, it's value must be either + * @brief Provides a commonly used validator for ledger index. + * + * LedgerIndex must be a string or an int. If the specified LedgerIndex is a string, its value must be either * "validated" or a valid integer value represented as a string. */ extern CustomValidator LedgerIndexValidator; /** - * @brief Provide a common used validator for account - * Account must be a string and the converted public key is valid + * @brief Provides a commonly used validator for accounts. + * + * Account must be a string and the converted public key is valid. */ extern CustomValidator AccountValidator; /** - * @brief Provide a common used validator for account - * Account must be a string and can convert to base58 + * @brief Provides a commonly used validator for accounts. + * + * Account must be a string and can convert to base58. */ extern CustomValidator AccountBase58Validator; /** - * @brief Provide a common used validator for marker - * Marker is composed of a comma separated index and start hint. The - * former will be read as hex, and the latter can cast to uint64. + * @brief Provides a commonly used validator for markers. + * + * A marker is composed of a comma-separated index and a start hint. + * The former will be read as hex, and the latter can be cast to uint64. */ extern CustomValidator AccountMarkerValidator; /** - * @brief Provide a common used validator for uint256 hex string - * It must be a string and hex - * Transaction index, ledger hash all use this validator + * @brief Provides a commonly used validator for uint256 hex string. + * + * It must be a string and also a decodable hex. + * Transaction index, ledger hash all use this validator. */ extern CustomValidator Uint256HexStringValidator; /** - * @brief Provide a common used validator for currency - * including standard currency code and token code + * @brief Provides a commonly used validator for currency, including standard currency code and token code. */ extern CustomValidator CurrencyValidator; /** - * @brief Provide a common used validator for issuer type - * It must be a hex string or base58 string + * @brief Provides a commonly used validator for issuer type. + * + * It must be a hex string or base58 string. */ extern CustomValidator IssuerValidator; /** - * @brief Provide a validator for validating valid streams used in - * subscribe/unsubscribe + * @brief Provides a validator for validating streams used in subscribe/unsubscribe. */ extern CustomValidator SubscribeStreamValidator; /** - * @brief Provide a validator for validating valid accounts used in - * subscribe/unsubscribe + * @brief Provides a validator for validating accounts used in subscribe/unsubscribe. */ extern CustomValidator SubscribeAccountsValidator; -} // namespace RPC::validation +} // namespace rpc::validation diff --git a/src/rpc/common/impl/APIVersionParser.cpp b/src/rpc/common/impl/APIVersionParser.cpp index 8bd66c7e..f6bfaf5f 100644 --- a/src/rpc/common/impl/APIVersionParser.cpp +++ b/src/rpc/common/impl/APIVersionParser.cpp @@ -24,7 +24,7 @@ using namespace std; -namespace RPC::detail { +namespace rpc::detail { ProductionAPIVersionParser::ProductionAPIVersionParser(util::Config const& config) : ProductionAPIVersionParser( @@ -58,4 +58,4 @@ ProductionAPIVersionParser::parse(boost::json::object const& request) const return defaultVersion_; } -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/APIVersionParser.h b/src/rpc/common/impl/APIVersionParser.h index 13749e5d..e788a9e4 100644 --- a/src/rpc/common/impl/APIVersionParser.h +++ b/src/rpc/common/impl/APIVersionParser.h @@ -27,7 +27,7 @@ #include #include -namespace RPC::detail { +namespace rpc::detail { class ProductionAPIVersionParser : public APIVersionParser { @@ -94,4 +94,4 @@ public: } }; -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/AdminVerificationStrategy.h b/src/rpc/common/impl/AdminVerificationStrategy.h index e1aeec87..daa477d5 100644 --- a/src/rpc/common/impl/AdminVerificationStrategy.h +++ b/src/rpc/common/impl/AdminVerificationStrategy.h @@ -21,7 +21,7 @@ #include -namespace RPC::detail { +namespace rpc::detail { class IPAdminVerificationStrategy final { @@ -39,4 +39,4 @@ public: } }; -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/Factories.h b/src/rpc/common/impl/Factories.h index e34a0e8f..378cf159 100644 --- a/src/rpc/common/impl/Factories.h +++ b/src/rpc/common/impl/Factories.h @@ -26,12 +26,12 @@ #include -namespace RPC::detail { +namespace rpc::detail { template static constexpr bool unsupported_v = false; -template +template [[nodiscard]] auto makeFieldProcessor(std::string const& key, Processors&&... procs) { @@ -45,10 +45,10 @@ makeFieldProcessor(std::string const& key, Processors&&... procs) if (firstFailure) return; // already failed earlier - skip - if constexpr (Requirement) { + if constexpr (SomeRequirement) { if (auto const res = req->verify(j, key); not res) firstFailure = res.error(); - } else if constexpr (Modifier) { + } else if constexpr (SomeModifier) { if (auto const res = req->modify(j, key); not res) firstFailure = res.error(); } else { @@ -65,4 +65,4 @@ makeFieldProcessor(std::string const& key, Processors&&... procs) }; } -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/ForwardingProxy.h b/src/rpc/common/impl/ForwardingProxy.h index aafdeb0b..59a68264 100644 --- a/src/rpc/common/impl/ForwardingProxy.h +++ b/src/rpc/common/impl/ForwardingProxy.h @@ -30,7 +30,7 @@ #include #include -namespace RPC::detail { +namespace rpc::detail { template class ForwardingProxy @@ -148,4 +148,4 @@ private: } }; -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/HandlerProvider.cpp b/src/rpc/common/impl/HandlerProvider.cpp index ab8fb23e..09178923 100644 --- a/src/rpc/common/impl/HandlerProvider.cpp +++ b/src/rpc/common/impl/HandlerProvider.cpp @@ -53,7 +53,7 @@ #include #include -namespace RPC::detail { +namespace rpc::detail { ProductionHandlerProvider::ProductionHandlerProvider( util::Config const& config, @@ -117,4 +117,4 @@ ProductionHandlerProvider::isClioOnly(std::string const& command) const return handlerMap_.contains(command) && handlerMap_.at(command).isClioOnly; } -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/HandlerProvider.h b/src/rpc/common/impl/HandlerProvider.h index c1101eb3..b3ac6094 100644 --- a/src/rpc/common/impl/HandlerProvider.h +++ b/src/rpc/common/impl/HandlerProvider.h @@ -32,14 +32,14 @@ namespace etl { class ETLService; class LoadBalancer; } // namespace etl -namespace RPC { +namespace rpc { class Counters; } namespace feed { class SubscriptionManager; } -namespace RPC::detail { +namespace rpc::detail { class ProductionHandlerProvider final : public HandlerProvider { @@ -70,4 +70,4 @@ public: isClioOnly(std::string const& command) const override; }; -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/common/impl/Processors.h b/src/rpc/common/impl/Processors.h index 48ca5d00..dadd356c 100644 --- a/src/rpc/common/impl/Processors.h +++ b/src/rpc/common/impl/Processors.h @@ -23,12 +23,12 @@ #include #include -namespace RPC::detail { +namespace rpc::detail { template static constexpr bool unsupported_handler_v = false; -template +template struct DefaultProcessor final { [[nodiscard]] ReturnType @@ -36,7 +36,7 @@ struct DefaultProcessor final { using boost::json::value_from; using boost::json::value_to; - if constexpr (HandlerWithInput) + if constexpr (SomeHandlerWithInput) { // first we run validation against specified API version auto const spec = handler.spec(ctx.apiVersion); @@ -54,7 +54,7 @@ struct DefaultProcessor final else return value_from(ret.value()); } - else if constexpr (HandlerWithoutInput) + else if constexpr (SomeHandlerWithoutInput) { // no input to pass, ignore the value if (auto const ret = handler.process(ctx); not ret) @@ -64,11 +64,10 @@ struct DefaultProcessor final } else { - // when concept HandlerWithInput and HandlerWithoutInput not cover - // all Handler case + // when concept SomeHandlerWithInput and SomeHandlerWithoutInput not cover all Handler case static_assert(unsupported_handler_v); } } }; -} // namespace RPC::detail +} // namespace rpc::detail diff --git a/src/rpc/handlers/AccountChannels.cpp b/src/rpc/handlers/AccountChannels.cpp index e0d22d67..b21e2140 100644 --- a/src/rpc/handlers/AccountChannels.cpp +++ b/src/rpc/handlers/AccountChannels.cpp @@ -20,7 +20,7 @@ #include #include -namespace RPC { +namespace rpc { void AccountChannelsHandler::addChannel(std::vector& jsonChannels, ripple::SLE const& channelSle) const @@ -188,4 +188,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, AccountChannelsH jv = std::move(obj); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountChannels.h b/src/rpc/handlers/AccountChannels.h index 84bd142d..480f821b 100644 --- a/src/rpc/handlers/AccountChannels.h +++ b/src/rpc/handlers/AccountChannels.h @@ -27,7 +27,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The account_channels method returns information about an account's Payment Channels. This includes only @@ -126,4 +126,4 @@ private: friend void tag_invoke(boost::json::value_from_tag, boost::json::value& jv, ChannelResponse const& channel); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountCurrencies.cpp b/src/rpc/handlers/AccountCurrencies.cpp index a4590159..99c5a455 100644 --- a/src/rpc/handlers/AccountCurrencies.cpp +++ b/src/rpc/handlers/AccountCurrencies.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { AccountCurrenciesHandler::Result AccountCurrenciesHandler::process(AccountCurrenciesHandler::Input input, Context const& ctx) const { @@ -114,4 +114,4 @@ tag_invoke(boost::json::value_to_tag, boost::js return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountCurrencies.h b/src/rpc/handlers/AccountCurrencies.h index 3af7586d..572d43f0 100644 --- a/src/rpc/handlers/AccountCurrencies.h +++ b/src/rpc/handlers/AccountCurrencies.h @@ -27,7 +27,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The account_currencies command retrieves a list of currencies that an account can send or receive, @@ -87,4 +87,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountInfo.cpp b/src/rpc/handlers/AccountInfo.cpp index 5344bb68..86326508 100644 --- a/src/rpc/handlers/AccountInfo.cpp +++ b/src/rpc/handlers/AccountInfo.cpp @@ -21,7 +21,7 @@ #include -namespace RPC { +namespace rpc { AccountInfoHandler::Result AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx) const { @@ -51,10 +51,10 @@ AccountInfoHandler::process(AccountInfoHandler::Input input, Context const& ctx) return Error{Status{RippledError::rpcDB_DESERIALIZATION}}; auto const isDisallowIncomingEnabled = - RPC::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, RPC::Amendments::DisallowIncoming); + rpc::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, rpc::Amendments::DisallowIncoming); auto const isClawbackEnabled = - RPC::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, RPC::Amendments::Clawback); + rpc::isAmendmentEnabled(sharedPtrBackend_, ctx.yield, lgrInfo.seq, rpc::Amendments::Clawback); // Return SignerList(s) if that is requested. if (input.signerLists) @@ -173,4 +173,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::va return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountInfo.h b/src/rpc/handlers/AccountInfo.h index fe72ae68..e4b2c3e7 100644 --- a/src/rpc/handlers/AccountInfo.h +++ b/src/rpc/handlers/AccountInfo.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The account_info command retrieves information about an account, its activity, and its XRP balance. @@ -106,4 +106,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountLines.cpp b/src/rpc/handlers/AccountLines.cpp index 87ccc901..a4a40db3 100644 --- a/src/rpc/handlers/AccountLines.cpp +++ b/src/rpc/handlers/AccountLines.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { void AccountLinesHandler::addLine( @@ -236,4 +236,4 @@ tag_invoke( jv = std::move(obj); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountLines.h b/src/rpc/handlers/AccountLines.h index 21e02aaf..9de21559 100644 --- a/src/rpc/handlers/AccountLines.h +++ b/src/rpc/handlers/AccountLines.h @@ -28,7 +28,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The account_lines method returns information about an account's trust lines, which contain balances in all @@ -135,4 +135,4 @@ private: tag_invoke(boost::json::value_from_tag, boost::json::value& jv, LineResponse const& line); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountNFTs.cpp b/src/rpc/handlers/AccountNFTs.cpp index 16872f67..47c93207 100644 --- a/src/rpc/handlers/AccountNFTs.cpp +++ b/src/rpc/handlers/AccountNFTs.cpp @@ -22,7 +22,7 @@ #include #include -namespace RPC { +namespace rpc { AccountNFTsHandler::Result AccountNFTsHandler::process(AccountNFTsHandler::Input input, Context const& ctx) const @@ -146,4 +146,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::va return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountNFTs.h b/src/rpc/handlers/AccountNFTs.h index 5c5681a3..d06001ae 100644 --- a/src/rpc/handlers/AccountNFTs.h +++ b/src/rpc/handlers/AccountNFTs.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The account_nfts method returns a list of NFToken objects for the specified account. @@ -96,4 +96,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountObjects.cpp b/src/rpc/handlers/AccountObjects.cpp index 0fead55a..a30e3b0d 100644 --- a/src/rpc/handlers/AccountObjects.cpp +++ b/src/rpc/handlers/AccountObjects.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { std::unordered_map const AccountObjectsHandler::TYPESMAP{ {"state", ripple::ltRIPPLE_STATE}, @@ -168,4 +168,4 @@ tag_invoke(boost::json::value_to_tag, boost::json: return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountObjects.h b/src/rpc/handlers/AccountObjects.h index 9ca9f8f8..f93c3c4b 100644 --- a/src/rpc/handlers/AccountObjects.h +++ b/src/rpc/handlers/AccountObjects.h @@ -27,7 +27,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The account_objects command returns the raw ledger format for all objects owned by an account. @@ -122,4 +122,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountOffers.cpp b/src/rpc/handlers/AccountOffers.cpp index 1ae3e538..ccc7befa 100644 --- a/src/rpc/handlers/AccountOffers.cpp +++ b/src/rpc/handlers/AccountOffers.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { void AccountOffersHandler::addOffer(std::vector& offers, ripple::SLE const& offerSle) const @@ -157,4 +157,4 @@ tag_invoke(boost::json::value_to_tag, boost::json:: return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountOffers.h b/src/rpc/handlers/AccountOffers.h index 611c4fe4..9327cb00 100644 --- a/src/rpc/handlers/AccountOffers.h +++ b/src/rpc/handlers/AccountOffers.h @@ -26,7 +26,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The account_offers method retrieves a list of offers made by a given account. @@ -111,4 +111,4 @@ private: friend void tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Offer const& offer); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountTx.cpp b/src/rpc/handlers/AccountTx.cpp index 7a2b64d6..4fab8e37 100644 --- a/src/rpc/handlers/AccountTx.cpp +++ b/src/rpc/handlers/AccountTx.cpp @@ -20,7 +20,7 @@ #include #include -namespace RPC { +namespace rpc { // TODO: this is currently very similar to nft_history but its own copy for time // being. we should aim to reuse common logic in some way in the future. @@ -211,4 +211,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::valu return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/AccountTx.h b/src/rpc/handlers/AccountTx.h index d2a15685..bbf31084 100644 --- a/src/rpc/handlers/AccountTx.h +++ b/src/rpc/handlers/AccountTx.h @@ -27,7 +27,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The account_tx method retrieves a list of transactions that involved the specified account. @@ -127,4 +127,4 @@ private: friend void tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Marker const& marker); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/BookChanges.cpp b/src/rpc/handlers/BookChanges.cpp index 2df73297..4813c5c6 100644 --- a/src/rpc/handlers/BookChanges.cpp +++ b/src/rpc/handlers/BookChanges.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { BookChangesHandler::Result BookChangesHandler::process(BookChangesHandler::Input input, Context const& ctx) const @@ -92,4 +92,4 @@ computeBookChanges(ripple::LedgerHeader const& lgrInfo, std::vector #include -namespace RPC { +namespace rpc { /** * @brief BookChangesHandler returns the order book changes for a given ledger. @@ -81,4 +81,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/BookOffers.cpp b/src/rpc/handlers/BookOffers.cpp index f3bef77a..015b1f5a 100644 --- a/src/rpc/handlers/BookOffers.cpp +++ b/src/rpc/handlers/BookOffers.cpp @@ -20,7 +20,7 @@ #include #include -namespace RPC { +namespace rpc { BookOffersHandler::Result BookOffersHandler::process(Input input, Context const& ctx) const @@ -98,4 +98,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::val return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/BookOffers.h b/src/rpc/handlers/BookOffers.h index 80e22238..94ff517c 100644 --- a/src/rpc/handlers/BookOffers.h +++ b/src/rpc/handlers/BookOffers.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The book_offers method retrieves a list of Offers between two currencies, also known as an order book. @@ -117,4 +117,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/DepositAuthorized.cpp b/src/rpc/handlers/DepositAuthorized.cpp index bd164626..aa5570bb 100644 --- a/src/rpc/handlers/DepositAuthorized.cpp +++ b/src/rpc/handlers/DepositAuthorized.cpp @@ -20,7 +20,7 @@ #include #include -namespace RPC { +namespace rpc { DepositAuthorizedHandler::Result DepositAuthorizedHandler::process(DepositAuthorizedHandler::Input input, Context const& ctx) const @@ -112,4 +112,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, DepositAuthorize }; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/DepositAuthorized.h b/src/rpc/handlers/DepositAuthorized.h index c732039e..4f32f75c 100644 --- a/src/rpc/handlers/DepositAuthorized.h +++ b/src/rpc/handlers/DepositAuthorized.h @@ -26,7 +26,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The deposit_authorized command indicates whether one account is authorized to send payments directly to @@ -90,4 +90,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/GatewayBalances.cpp b/src/rpc/handlers/GatewayBalances.cpp index ad15a88f..53da2f99 100644 --- a/src/rpc/handlers/GatewayBalances.cpp +++ b/src/rpc/handlers/GatewayBalances.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { GatewayBalancesHandler::Result GatewayBalancesHandler::process(GatewayBalancesHandler::Input input, Context const& ctx) const @@ -228,4 +228,4 @@ tag_invoke(boost::json::value_to_tag, boost::json return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/GatewayBalances.h b/src/rpc/handlers/GatewayBalances.h index b70f0a65..f5ea0270 100644 --- a/src/rpc/handlers/GatewayBalances.h +++ b/src/rpc/handlers/GatewayBalances.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * The gateway_balances command calculates the total balances issued by a given account, optionally excluding amounts @@ -121,4 +121,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Ledger.cpp b/src/rpc/handlers/Ledger.cpp index abc8d570..fbd1b84b 100644 --- a/src/rpc/handlers/Ledger.cpp +++ b/src/rpc/handlers/Ledger.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { LedgerHandler::Result LedgerHandler::process(LedgerHandler::Input input, Context const& ctx) const { @@ -85,7 +85,7 @@ LedgerHandler::process(LedgerHandler::Input input, Context const& ctx) const if (input.ownerFunds) { // check the type of tx - auto const [tx, meta] = RPC::deserializeTxPlusMeta(obj); + auto const [tx, meta] = rpc::deserializeTxPlusMeta(obj); if (tx and tx->isFieldPresent(ripple::sfTransactionType) and tx->getTxnType() == ripple::ttOFFER_CREATE) { @@ -204,4 +204,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::value c return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Ledger.h b/src/rpc/handlers/Ledger.h index a3933625..53438b69 100644 --- a/src/rpc/handlers/Ledger.h +++ b/src/rpc/handlers/Ledger.h @@ -24,7 +24,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief Retrieve information about the public ledger. @@ -94,4 +94,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerData.cpp b/src/rpc/handlers/LedgerData.cpp index 8ce77156..f4463fb5 100644 --- a/src/rpc/handlers/LedgerData.cpp +++ b/src/rpc/handlers/LedgerData.cpp @@ -23,7 +23,7 @@ #include -namespace RPC { +namespace rpc { std::unordered_map const LedgerDataHandler::TYPES_MAP{ {JS(account), ripple::ltACCOUNT_ROOT}, @@ -254,4 +254,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::val return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerData.h b/src/rpc/handlers/LedgerData.h index 5ad670bf..5ecc53eb 100644 --- a/src/rpc/handlers/LedgerData.h +++ b/src/rpc/handlers/LedgerData.h @@ -28,7 +28,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The ledger_data method retrieves contents of the specified ledger. You can iterate through several calls to @@ -92,11 +92,7 @@ public: {"out_of_order", validation::Type{}}, {JS(ledger_hash), validation::Uint256HexStringValidator}, {JS(ledger_index), validation::LedgerIndexValidator}, - { - JS(limit), - validation::Type{}, - validation::Min(1u), - }, + {JS(limit), validation::Type{}, validation::Min(1u)}, {JS(marker), validation::Type{}, meta::IfType{validation::Uint256HexStringValidator}}, @@ -122,4 +118,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerEntry.cpp b/src/rpc/handlers/LedgerEntry.cpp index 78a651d4..244f7ae2 100644 --- a/src/rpc/handlers/LedgerEntry.cpp +++ b/src/rpc/handlers/LedgerEntry.cpp @@ -21,7 +21,7 @@ #include -namespace RPC { +namespace rpc { LedgerEntryHandler::Result LedgerEntryHandler::process(LedgerEntryHandler::Input input, Context const& ctx) const @@ -237,4 +237,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::va return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerEntry.h b/src/rpc/handlers/LedgerEntry.h index a0d7e242..3c2deea2 100644 --- a/src/rpc/handlers/LedgerEntry.h +++ b/src/rpc/handlers/LedgerEntry.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The ledger_entry method returns a single ledger object from the XRP Ledger in its raw format. @@ -187,4 +187,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerRange.cpp b/src/rpc/handlers/LedgerRange.cpp index 20a6b9df..f8b0654b 100644 --- a/src/rpc/handlers/LedgerRange.cpp +++ b/src/rpc/handlers/LedgerRange.cpp @@ -22,7 +22,7 @@ #include -namespace RPC { +namespace rpc { LedgerRangeHandler::Result LedgerRangeHandler::process([[maybe_unused]] Context const& ctx) const @@ -40,4 +40,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, LedgerRangeHandl }; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/LedgerRange.h b/src/rpc/handlers/LedgerRange.h index 98a6ae92..163fb502 100644 --- a/src/rpc/handlers/LedgerRange.h +++ b/src/rpc/handlers/LedgerRange.h @@ -26,7 +26,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The ledger_range command returns the index number of the earliest and latest ledgers that the server has. @@ -56,4 +56,4 @@ private: friend void tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTBuyOffers.cpp b/src/rpc/handlers/NFTBuyOffers.cpp index 6bd2799e..a8966bff 100644 --- a/src/rpc/handlers/NFTBuyOffers.cpp +++ b/src/rpc/handlers/NFTBuyOffers.cpp @@ -24,7 +24,7 @@ using namespace ripple; -namespace RPC { +namespace rpc { NFTBuyOffersHandler::Result NFTBuyOffersHandler::process(NFTBuyOffersHandler::Input input, Context const& ctx) const @@ -34,4 +34,4 @@ NFTBuyOffersHandler::process(NFTBuyOffersHandler::Input input, Context const& ct return iterateOfferDirectory(input, tokenID, directory, ctx.yield); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTBuyOffers.h b/src/rpc/handlers/NFTBuyOffers.h index 5fda3c41..7c9eea76 100644 --- a/src/rpc/handlers/NFTBuyOffers.h +++ b/src/rpc/handlers/NFTBuyOffers.h @@ -22,7 +22,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The nft_buy_offers method returns a list of buy offers for a given NFToken object. @@ -40,4 +40,4 @@ public: Result process(Input input, Context const& ctx) const; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTHistory.cpp b/src/rpc/handlers/NFTHistory.cpp index 837b837e..6f07c2b8 100644 --- a/src/rpc/handlers/NFTHistory.cpp +++ b/src/rpc/handlers/NFTHistory.cpp @@ -22,7 +22,7 @@ #include -namespace RPC { +namespace rpc { // TODO: this is currently very similar to account_tx but its own copy for time // being. we should aim to reuse common logic in some way in the future. @@ -209,4 +209,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::val return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTHistory.h b/src/rpc/handlers/NFTHistory.h index b59b378a..89851333 100644 --- a/src/rpc/handlers/NFTHistory.h +++ b/src/rpc/handlers/NFTHistory.h @@ -27,7 +27,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The nft_history command asks the Clio server for past transaction metadata for the NFT being queried. @@ -126,4 +126,4 @@ private: tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Marker const& marker); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTInfo.cpp b/src/rpc/handlers/NFTInfo.cpp index 90fa4aed..94414e83 100644 --- a/src/rpc/handlers/NFTInfo.cpp +++ b/src/rpc/handlers/NFTInfo.cpp @@ -24,9 +24,9 @@ #include using namespace ripple; -using namespace ::RPC; +using namespace ::rpc; -namespace RPC { +namespace rpc { NFTInfoHandler::Result NFTInfoHandler::process(NFTInfoHandler::Input input, Context const& ctx) const @@ -103,4 +103,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::value return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTInfo.h b/src/rpc/handlers/NFTInfo.h index 7b03418a..7bbcc904 100644 --- a/src/rpc/handlers/NFTInfo.h +++ b/src/rpc/handlers/NFTInfo.h @@ -24,7 +24,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The nft_info command asks the Clio server for information about the NFT being queried. @@ -90,4 +90,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTOffersCommon.cpp b/src/rpc/handlers/NFTOffersCommon.cpp index 38bafdca..20fac538 100644 --- a/src/rpc/handlers/NFTOffersCommon.cpp +++ b/src/rpc/handlers/NFTOffersCommon.cpp @@ -23,7 +23,7 @@ #include using namespace ripple; -using namespace ::RPC; +using namespace ::rpc; namespace ripple { @@ -51,7 +51,7 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, SLE const& offer } // namespace ripple -namespace RPC { +namespace rpc { NFTOffersHandlerBase::Result NFTOffersHandlerBase::iterateOfferDirectory( @@ -183,4 +183,4 @@ tag_invoke(boost::json::value_to_tag, boost::json:: return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTOffersCommon.h b/src/rpc/handlers/NFTOffersCommon.h index ab81a347..555c5674 100644 --- a/src/rpc/handlers/NFTOffersCommon.h +++ b/src/rpc/handlers/NFTOffersCommon.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { class NFTOffersHandlerBase { @@ -96,4 +96,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTSellOffers.cpp b/src/rpc/handlers/NFTSellOffers.cpp index 3e279cf9..127c42b0 100644 --- a/src/rpc/handlers/NFTSellOffers.cpp +++ b/src/rpc/handlers/NFTSellOffers.cpp @@ -24,7 +24,7 @@ using namespace ripple; -namespace RPC { +namespace rpc { NFTSellOffersHandler::Result NFTSellOffersHandler::process(NFTSellOffersHandler::Input input, Context const& ctx) const @@ -35,4 +35,4 @@ NFTSellOffersHandler::process(NFTSellOffersHandler::Input input, Context const& return iterateOfferDirectory(input, tokenID, directory, ctx.yield); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NFTSellOffers.h b/src/rpc/handlers/NFTSellOffers.h index 877560b7..0d880f53 100644 --- a/src/rpc/handlers/NFTSellOffers.h +++ b/src/rpc/handlers/NFTSellOffers.h @@ -22,7 +22,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The nft_sell_offers method returns a list of sell offers for a given NFToken object. @@ -41,4 +41,4 @@ public: process(Input input, Context const& ctx) const; }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NoRippleCheck.cpp b/src/rpc/handlers/NoRippleCheck.cpp index 88154e8f..8d8146d6 100644 --- a/src/rpc/handlers/NoRippleCheck.cpp +++ b/src/rpc/handlers/NoRippleCheck.cpp @@ -22,7 +22,7 @@ #include -namespace RPC { +namespace rpc { NoRippleCheckHandler::Result NoRippleCheckHandler::process(NoRippleCheckHandler::Input input, Context const& ctx) const @@ -201,4 +201,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, NoRippleCheckHan jv = std::move(obj); } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/NoRippleCheck.h b/src/rpc/handlers/NoRippleCheck.h index 40a8a1e8..16cc6b0c 100644 --- a/src/rpc/handlers/NoRippleCheck.h +++ b/src/rpc/handlers/NoRippleCheck.h @@ -28,7 +28,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The noripple_check command provides a quick way to check the status of the Default Ripple field for an account @@ -104,4 +104,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Ping.h b/src/rpc/handlers/Ping.h index 56898458..bde4b195 100644 --- a/src/rpc/handlers/Ping.h +++ b/src/rpc/handlers/Ping.h @@ -21,7 +21,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The ping command returns an acknowledgement, so that clients can test the connection status and latency. @@ -41,4 +41,4 @@ public: } }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Random.cpp b/src/rpc/handlers/Random.cpp index 48d338c5..b64e1354 100644 --- a/src/rpc/handlers/Random.cpp +++ b/src/rpc/handlers/Random.cpp @@ -23,7 +23,7 @@ #include #include -namespace RPC { +namespace rpc { RandomHandler::Result RandomHandler::process([[maybe_unused]] Context const& ctx) const @@ -42,4 +42,4 @@ tag_invoke(boost::json::value_from_tag, boost::json::value& jv, RandomHandler::O }; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Random.h b/src/rpc/handlers/Random.h index 089954a1..6cf0e197 100644 --- a/src/rpc/handlers/Random.h +++ b/src/rpc/handlers/Random.h @@ -25,7 +25,7 @@ #include -namespace RPC { +namespace rpc { /** * @brief The random command provides a random number to be used as a source of entropy for random number generation by @@ -51,4 +51,4 @@ private: tag_invoke(boost::json::value_from_tag, boost::json::value& jv, Output const& output); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/ServerInfo.h b/src/rpc/handlers/ServerInfo.h index c9570f3f..1fc42f36 100644 --- a/src/rpc/handlers/ServerInfo.h +++ b/src/rpc/handlers/ServerInfo.h @@ -39,11 +39,11 @@ class LoadBalancer; namespace feed { class SubscriptionManager; } -namespace RPC { +namespace rpc { class Counters; } -namespace RPC { +namespace rpc { template class BaseServerInfoHandler @@ -120,7 +120,7 @@ public: Result process(Context const& ctx) const { - using namespace RPC; + using namespace rpc; using namespace std::chrono; auto const range = backend_->fetchLedgerRange(); @@ -258,4 +258,4 @@ private: using ServerInfoHandler = BaseServerInfoHandler; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Subscribe.h b/src/rpc/handlers/Subscribe.h index 28e73bc4..376b2120 100644 --- a/src/rpc/handlers/Subscribe.h +++ b/src/rpc/handlers/Subscribe.h @@ -28,7 +28,7 @@ namespace feed { class SubscriptionManager; } -namespace RPC { +namespace rpc { template class BaseSubscribeHandler { @@ -324,4 +324,4 @@ private: */ using SubscribeHandler = BaseSubscribeHandler; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/TransactionEntry.cpp b/src/rpc/handlers/TransactionEntry.cpp index be7fe3b9..4de83b24 100644 --- a/src/rpc/handlers/TransactionEntry.cpp +++ b/src/rpc/handlers/TransactionEntry.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { TransactionEntryHandler::Result TransactionEntryHandler::process(TransactionEntryHandler::Input input, Context const& ctx) const @@ -90,4 +90,4 @@ tag_invoke(boost::json::value_to_tag, boost::jso return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/TransactionEntry.h b/src/rpc/handlers/TransactionEntry.h index 3e5a88ac..7c607ded 100644 --- a/src/rpc/handlers/TransactionEntry.h +++ b/src/rpc/handlers/TransactionEntry.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The transaction_entry method retrieves information on a single transaction from a specific ledger version. @@ -87,4 +87,4 @@ private: tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Tx.cpp b/src/rpc/handlers/Tx.cpp index c24a9db2..b49300af 100644 --- a/src/rpc/handlers/Tx.cpp +++ b/src/rpc/handlers/Tx.cpp @@ -19,7 +19,7 @@ #include -namespace RPC { +namespace rpc { TxHandler::Result TxHandler::process(Input input, Context const& ctx) const @@ -118,4 +118,4 @@ tag_invoke(boost::json::value_to_tag, boost::json::value const return input; } -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Tx.h b/src/rpc/handlers/Tx.h index 3a2ce9d8..a5478918 100644 --- a/src/rpc/handlers/Tx.h +++ b/src/rpc/handlers/Tx.h @@ -24,7 +24,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The tx method retrieves information on a single transaction, by its identifying hash. @@ -85,4 +85,4 @@ private: friend Input tag_invoke(boost::json::value_to_tag, boost::json::value const& jv); }; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/Unsubscribe.h b/src/rpc/handlers/Unsubscribe.h index 7d2188eb..5e3f1e62 100644 --- a/src/rpc/handlers/Unsubscribe.h +++ b/src/rpc/handlers/Unsubscribe.h @@ -28,7 +28,7 @@ namespace feed { class SubscriptionManager; } -namespace RPC { +namespace rpc { template class BaseUnsubscribeHandler @@ -225,4 +225,4 @@ private: */ using UnsubscribeHandler = BaseUnsubscribeHandler; -} // namespace RPC +} // namespace rpc diff --git a/src/rpc/handlers/VersionHandler.h b/src/rpc/handlers/VersionHandler.h index 12567ff7..53ff86f0 100644 --- a/src/rpc/handlers/VersionHandler.h +++ b/src/rpc/handlers/VersionHandler.h @@ -25,7 +25,7 @@ #include #include -namespace RPC { +namespace rpc { /** * @brief The version command returns the min,max and current api Version we are using @@ -33,7 +33,7 @@ namespace RPC { */ class VersionHandler { - RPC::detail::ProductionAPIVersionParser apiVersionParser_; + rpc::detail::ProductionAPIVersionParser apiVersionParser_; public: struct Output @@ -56,7 +56,7 @@ public: Result process([[maybe_unused]] Context const& ctx) const { - using namespace RPC; + using namespace rpc; auto output = Output{}; output.currVersion = apiVersionParser_.getDefaultVersion(); @@ -80,4 +80,4 @@ private: } }; -} // namespace RPC +} // namespace rpc diff --git a/src/util/JsonUtils.h b/src/util/JsonUtils.h index ee6e20fb..4f0b0833 100644 --- a/src/util/JsonUtils.h +++ b/src/util/JsonUtils.h @@ -23,8 +23,17 @@ #include +/** + * @brief This namespace contains various utilities. + */ namespace util { +/** + * @brief Removes any detected secret information from a response JSON object. + * + * @param object The JSON object to remove secrets from + * @return A secret-free copy of the passed object + */ inline boost::json::object removeSecret(boost::json::object const& object) { @@ -40,6 +49,7 @@ removeSecret(boost::json::object const& object) newObject.at("params").as_array()[0].as_object()[secretField] = "*"; } } + // for websocket requests for (auto const& secretField : secretFields) { diff --git a/src/util/LedgerUtils.h b/src/util/LedgerUtils.h index 0cceffad..6e7f7549 100644 --- a/src/util/LedgerUtils.h +++ b/src/util/LedgerUtils.h @@ -22,26 +22,41 @@ #include #include #include +#include #include #include namespace util { +/** + * @brief Deserializes a ripple::LedgerHeader from ripple::Slice of data. + * + * @param data The slice to deserialize + * @return The deserialized ripple::LedgerHeader + */ inline ripple::LedgerHeader deserializeHeader(ripple::Slice data) { - return ripple::deserializeHeader(data, /*hasHash=*/true); + return ripple::deserializeHeader(data, /* hasHash = */ true); } +/** + * @brief A helper function that converts a ripple::LedgerHeader to a string representation. + * + * @param info The ledger header + * @return The string representation of the supplied ledger header + */ inline std::string toString(ripple::LedgerHeader const& info) { - std::stringstream ss; - ss << "LedgerHeader { Sequence : " << info.seq << " Hash : " << ripple::strHex(info.hash) - << " TxHash : " << strHex(info.txHash) << " AccountHash : " << ripple::strHex(info.accountHash) - << " ParentHash : " << strHex(info.parentHash) << " }"; - return ss.str(); + return fmt::format( + "LedgerHeader {{Sequence: {}, Hash: {}, TxHash: {}, AccountHash: {}, ParentHash: {}}}", + info.seq, + ripple::strHex(info.hash), + strHex(info.txHash), + ripple::strHex(info.accountHash), + strHex(info.parentHash)); } } // namespace util diff --git a/src/util/Profiler.h b/src/util/Profiler.h index 8d8bde13..6b85f7b2 100644 --- a/src/util/Profiler.h +++ b/src/util/Profiler.h @@ -26,16 +26,18 @@ namespace util { /** - * @brief Profiler function to measure the time consuming - * @param func function object, can be a lamdba or function wrapper - * @return return a pair if function wrapper has return value: result of - * function wrapper and the elapsed time(ms) during executing the given - * function only return the elapsed time if function wrapper does not have - * return value + * @brief Profiler function to measure the time a function execution consumes. + * + * @tparam U The duration measurement to use; defaults to milliseconds + * @tparam FnType The type of the function object + * @param func Any function object + * @return If the function object has a return value, the result of the function call and the elapsed time(ms) is + * returned as a pair + * @return Only return the elapsed time if passed function object does not have a return value */ -template +template [[nodiscard]] auto -timed(F&& func) +timed(FnType&& func) { auto start = std::chrono::system_clock::now(); @@ -48,7 +50,7 @@ timed(F&& func) { auto ret = func(); auto elapsed = std::chrono::duration_cast(std::chrono::system_clock::now() - start).count(); - return std::make_pair(ret, elapsed); + return std::make_pair(std::move(ret), std::move(elapsed)); } } diff --git a/src/util/Taggable.cpp b/src/util/Taggable.cpp index 07893c2f..ea45df98 100644 --- a/src/util/Taggable.cpp +++ b/src/util/Taggable.cpp @@ -29,14 +29,14 @@ namespace util::detail { -UIntTagGenerator::tag_t +UIntTagGenerator::TagType UIntTagGenerator::next() { static std::atomic_uint64_t num{0}; return num++; } -UUIDTagGenerator::tag_t +UUIDTagGenerator::TagType UUIDTagGenerator::next() { static boost::uuids::random_generator gen{}; @@ -66,7 +66,7 @@ TagDecoratorFactory::make() const } TagDecoratorFactory -TagDecoratorFactory::with(parent_t parent) const noexcept +TagDecoratorFactory::with(ParentType parent) const noexcept { return TagDecoratorFactory(type_, parent); } diff --git a/src/util/Taggable.h b/src/util/Taggable.h index 2a2aa62c..3e20d2d2 100644 --- a/src/util/Taggable.h +++ b/src/util/Taggable.h @@ -46,28 +46,27 @@ struct NullTagGenerator final */ struct UIntTagGenerator final { - using tag_t = std::atomic_uint64_t; + using TagType = std::atomic_uint64_t; - static tag_t + static TagType next(); }; /** - * @brief This strategy uses `boost::uuids::uuid` with a static random generator - * and a mutex + * @brief This strategy uses `boost::uuids::uuid` with a static random generator and a mutex. */ struct UUIDTagGenerator final { - using tag_t = boost::uuids::uuid; + using TagType = boost::uuids::uuid; - static tag_t + static TagType next(); }; } // namespace detail /** - * @brief Represents any tag decorator + * @brief Represents any tag decorator. */ class BaseTagDecorator { @@ -76,6 +75,7 @@ public: /** * @brief Decorates a std::ostream. + * * @param os The stream to decorate */ virtual void @@ -86,7 +86,7 @@ public: * * @param os The stream * @param decorator The decorator - * @return std::ostream& The same stream that we were given + * @return The same stream that we were given */ friend std::ostream& operator<<(std::ostream& os, BaseTagDecorator const& decorator) @@ -98,36 +98,36 @@ public: /** * @brief A decorator that decorates a string (log line) with a unique tag. + * * @tparam Generator The strategy used to generate the tag. */ template class TagDecorator final : public BaseTagDecorator { - using parent_t = std::optional>; - using tag_t = typename Generator::tag_t; + using ParentType = std::optional>; + using TagType = typename Generator::TagType; - parent_t parent_ = std::nullopt; - tag_t tag_ = Generator::next(); + ParentType parent_ = std::nullopt; + TagType tag_ = Generator::next(); public: /** - * @brief Create a new tag decorator with an optional parent + * @brief Create a new tag decorator with an optional parent. * - * If the `parent` is specified it will be streamed out as a chain when this - * decorator will decorate an ostream. + * If the `parent` is specified it will be streamed out as a chain when this decorator will decorate an ostream. * - * Note that if `parent` is specified it is your responsibility that the - * decorator referred to by `parent` outlives this decorator. + * Note that if `parent` is specified it is your responsibility that the decorator referred to by `parent` outlives + * this decorator. * * @param parent An optional parent tag decorator */ - explicit TagDecorator(parent_t parent = std::nullopt) : parent_{parent} + explicit TagDecorator(ParentType parent = std::nullopt) : parent_{parent} { } /** - * @brief Implementation of the decoration. Chaining tags when parent is - * available. + * @brief Implementation of the decoration. Chaining tags when parent is available. + * * @param os The stream to output into */ void @@ -145,8 +145,7 @@ public: /** * @brief Specialization for a nop/null decorator. * - * This generates a pass-thru decorate member function which can be optimized - * away by the compiler. + * This generates a pass-thru decorate member function which can be optimized away by the compiler. */ template <> class TagDecorator final : public BaseTagDecorator @@ -154,6 +153,7 @@ class TagDecorator final : public BaseTagDecorator public: /** * @brief Nop implementation for the decorator. + * * @param os The stream */ void @@ -168,25 +168,26 @@ public: */ class TagDecoratorFactory final { - using parent_t = std::optional>; + using ParentType = std::optional>; /** - * @brief Represents the type of tag decorator + * @brief Represents the type of tag decorator. */ enum class Type { - NONE, /*! No decoration and no tag */ - UUID, /*! Tag based on `boost::uuids::uuid`, thread-safe via mutex */ - UINT /*! atomic_uint64_t tag, thread-safe, lock-free */ + NONE, /**< No decoration and no tag */ + UUID, /**< Tag based on `boost::uuids::uuid`, thread-safe via mutex */ + UINT /**< atomic_uint64_t tag, thread-safe, lock-free */ }; - Type type_; /*! The type of TagDecorator this factory produces */ - parent_t parent_ = std::nullopt; /*! The parent tag decorator to bind */ + Type type_; /*< The type of TagDecorator this factory produces */ + ParentType parent_ = std::nullopt; /*< The parent tag decorator to bind */ public: ~TagDecoratorFactory() = default; /** * @brief Instantiates a tag decorator factory from `clio` configuration. + * * @param config The configuration as a json object */ explicit TagDecoratorFactory(util::Config const& config) : type_{config.valueOr("log_tag_style", Type::NONE)} @@ -194,30 +195,27 @@ public: } private: - TagDecoratorFactory(Type type, parent_t parent) noexcept : type_{type}, parent_{parent} + TagDecoratorFactory(Type type, ParentType parent) noexcept : type_{type}, parent_{parent} { } public: /** - * @brief Instantiates the TagDecorator specified by `type_` with parent - * bound from `parent_`. + * @brief Instantiates the TagDecorator specified by `type_` with parent bound from `parent_`. * - * @return std::unique_ptr An instance of the requested - * decorator + * @return std::unique_ptr An instance of the requested decorator */ std::unique_ptr make() const; /** - * @brief Creates a new tag decorator factory with a bound parent tag - * decorator. + * @brief Creates a new tag decorator factory with a bound parent tag decorator. * * @param parent The parent tag decorator to use - * @return TagDecoratorFactory A new instance of the tag decorator factory + * @return A new instance of the tag decorator factory */ TagDecoratorFactory - with(parent_t parent) const noexcept; + with(ParentType parent) const noexcept; private: friend Type @@ -245,12 +243,13 @@ private: */ class Taggable { - using decorator_t = std::unique_ptr; - decorator_t tagDecorator_; + using DecoratorType = std::unique_ptr; + DecoratorType tagDecorator_; protected: /** - * @brief New Taggable from a specified factory + * @brief New Taggable from a specified factory. + * * @param tagFactory The factory to use */ explicit Taggable(util::TagDecoratorFactory const& tagFactory) : tagDecorator_{tagFactory.make()} @@ -265,7 +264,8 @@ public: /** * @brief Getter for tag decorator. - * @return BaseTagDecorator const& Reference to the tag decorator + * + * @return Reference to the tag decorator */ BaseTagDecorator const& tag() const diff --git a/src/util/config/Config.cpp b/src/util/config/Config.cpp index dd3a9991..80285913 100644 --- a/src/util/config/Config.cpp +++ b/src/util/config/Config.cpp @@ -37,20 +37,20 @@ Config::operator bool() const noexcept } bool -Config::contains(key_type key) const +Config::contains(KeyType key) const { return lookup(key).has_value(); } std::optional -Config::lookup(key_type key) const +Config::lookup(KeyType key) const { if (store_.is_null()) return std::nullopt; std::reference_wrapper cur = std::cref(store_); auto hasBrokenPath = false; - auto tokenized = detail::Tokenizer{key}; + auto tokenized = detail::Tokenizer{key}; std::string subkey{}; auto maybeSection = tokenized.next(); @@ -78,8 +78,8 @@ Config::lookup(key_type key) const return std::make_optional(cur); } -std::optional -Config::maybeArray(key_type key) const +std::optional +Config::maybeArray(KeyType key) const { try { @@ -87,13 +87,13 @@ Config::maybeArray(key_type key) const if (maybe_arr && maybe_arr->is_array()) { auto& arr = maybe_arr->as_array(); - array_type out; + ArrayType out; out.reserve(arr.size()); std::transform(std::begin(arr), std::end(arr), std::back_inserter(out), [](auto&& element) { return Config{std::move(element)}; }); - return std::make_optional(std::move(out)); + return std::make_optional(std::move(out)); } } catch (detail::StoreException const&) @@ -104,24 +104,24 @@ Config::maybeArray(key_type key) const return std::nullopt; } -Config::array_type -Config::array(key_type key) const +Config::ArrayType +Config::array(KeyType key) const { if (auto maybe_arr = maybeArray(key); maybe_arr) return maybe_arr.value(); throw std::logic_error("No array found at '" + key + "'"); } -Config::array_type -Config::arrayOr(key_type key, array_type fallback) const +Config::ArrayType +Config::arrayOr(KeyType key, ArrayType fallback) const { if (auto maybe_arr = maybeArray(key); maybe_arr) return maybe_arr.value(); return fallback; } -Config::array_type -Config::arrayOrThrow(key_type key, std::string_view err) const +Config::ArrayType +Config::arrayOrThrow(KeyType key, std::string_view err) const { try { @@ -134,7 +134,7 @@ Config::arrayOrThrow(key_type key, std::string_view err) const } Config -Config::section(key_type key) const +Config::section(KeyType key) const { auto maybe_element = lookup(key); if (maybe_element && maybe_element->is_object()) @@ -143,7 +143,7 @@ Config::section(key_type key) const } Config -Config::sectionOr(key_type key, boost::json::object fallback) const +Config::sectionOr(KeyType key, boost::json::object fallback) const { auto maybe_element = lookup(key); if (maybe_element && maybe_element->is_object()) @@ -151,13 +151,13 @@ Config::sectionOr(key_type key, boost::json::object fallback) const return Config{std::move(fallback)}; } -Config::array_type +Config::ArrayType Config::array() const { if (not store_.is_array()) throw std::logic_error("_self_ is not an array"); - array_type out; + ArrayType out; auto const& arr = store_.as_array(); out.reserve(arr.size()); diff --git a/src/util/config/Config.h b/src/util/config/Config.h index 79a71ffb..3d09f2cd 100644 --- a/src/util/config/Config.h +++ b/src/util/config/Config.h @@ -41,9 +41,9 @@ class Config final static constexpr char Separator = '.'; public: - using key_type = std::string; /*! The type of key used */ - using array_type = std::vector; /*! The type of array used */ - using write_cursor_type = std::pair>, key_type>; + using KeyType = std::string; /*< The type of key used */ + using ArrayType = std::vector; /*< The type of array used */ + using WriteCursorType = std::pair>, KeyType>; /** * @brief Construct a new Config object. @@ -72,7 +72,7 @@ public: * @throws std::logic_error If the key is of invalid format */ [[nodiscard]] bool - contains(key_type key) const; + contains(KeyType key) const; // // Key value access @@ -95,7 +95,7 @@ public: */ template [[nodiscard]] std::optional - maybeValue(key_type key) const + maybeValue(KeyType key) const { auto maybe_element = lookup(key); if (maybe_element) @@ -121,7 +121,7 @@ public: */ template [[nodiscard]] Result - value(key_type key) const + value(KeyType key) const { return maybeValue(key).value(); } @@ -145,7 +145,7 @@ public: */ template [[nodiscard]] Result - valueOr(key_type key, Result fallback) const + valueOr(KeyType key, Result fallback) const { try { @@ -175,7 +175,7 @@ public: */ template [[nodiscard]] Result - valueOrThrow(key_type key, std::string_view err) const + valueOrThrow(KeyType key, std::string_view err) const { try { @@ -196,11 +196,11 @@ public: * specified key - std::nullopt is returned. * * @param key The key to check - * @return std::optional Optional array + * @return std::optional Optional array * @throws std::logic_error Thrown if the key is of invalid format */ - [[nodiscard]] std::optional - maybeArray(key_type key) const; + [[nodiscard]] std::optional + maybeArray(KeyType key) const; /** * @brief Interface for fetching an array by key. @@ -211,12 +211,12 @@ public: * specified key an std::logic_error is thrown. * * @param key The key to check - * @return array_type The array + * @return ArrayType The array * @throws std::logic_error Thrown if there is no array under the desired * key or the key is of invalid format */ - [[nodiscard]] array_type - array(key_type key) const; + [[nodiscard]] ArrayType + array(KeyType key) const; /** * @brief Interface for fetching an array by key with fallback. @@ -228,11 +228,11 @@ public: * * @param key The key to check * @param fallback The fallback array - * @return array_type The array + * @return ArrayType The array * @throws std::logic_error Thrown if the key is of invalid format */ - [[nodiscard]] array_type - arrayOr(key_type key, array_type fallback) const; + [[nodiscard]] ArrayType + arrayOr(KeyType key, ArrayType fallback) const; /** * @brief Interface for fetching an array by key with custom error handling. @@ -244,12 +244,12 @@ public: * * @param key The key to check * @param err The custom error message - * @return array_type The array + * @return ArrayType The array * @throws std::runtime_error Thrown if there is no array under the desired * key */ - [[nodiscard]] array_type - arrayOrThrow(key_type key, std::string_view err) const; + [[nodiscard]] ArrayType + arrayOrThrow(KeyType key, std::string_view err) const; /** * @brief Interface for fetching a sub section by key. @@ -264,7 +264,7 @@ public: * desired key or the key is of invalid format */ [[nodiscard]] Config - section(key_type key) const; + section(KeyType key) const; /** * @brief Interface for fetching a sub section by key with a fallback object. @@ -274,11 +274,11 @@ public: * stored under the desired key - fallback object is used instead. * * @param key The key to check - * @param fallabkc The fallback object + * @param fallback The fallback object * @return Config Section represented as a separate instance of Config */ [[nodiscard]] Config - sectionOr(key_type key, boost::json::object fallback) const; + sectionOr(KeyType key, boost::json::object fallback) const; // // Direct self-value access @@ -288,7 +288,7 @@ public: * @brief Interface for reading the value directly referred to by the * instance. Wraps as std::optional. * - * See @ref maybeValue(key_type) const for how this works. + * See @ref maybeValue(KeyType) const for how this works. */ template [[nodiscard]] std::optional @@ -303,7 +303,7 @@ public: * @brief Interface for reading the value directly referred to by the * instance. * - * See @ref value(key_type) const for how this works. + * See @ref value(KeyType) const for how this works. */ template [[nodiscard]] Result @@ -316,7 +316,7 @@ public: * @brief Interface for reading the value directly referred to by the * instance with user-specified fallback. * - * See @ref valueOr(key_type, Result) const for how this works. + * See @ref valueOr(KeyType, Result) const for how this works. */ template [[nodiscard]] Result @@ -329,7 +329,7 @@ public: * @brief Interface for reading the value directly referred to by the * instance with user-specified error message. * - * See @ref valueOrThrow(key_type, std::string_view) const for how this + * See @ref valueOrThrow(KeyType, std::string_view) const for how this * works. */ template @@ -350,15 +350,15 @@ public: * @brief Interface for reading the array directly referred to by the * instance. * - * See @ref array(key_type) const for how this works. + * See @ref array(KeyType) const for how this works. */ - [[nodiscard]] array_type + [[nodiscard]] ArrayType array() const; private: template [[nodiscard]] Return - checkedAs(key_type key, boost::json::value const& value) const + checkedAs(KeyType key, boost::json::value const& value) const { using boost::json::value_to; @@ -393,10 +393,10 @@ private: } std::optional - lookup(key_type key) const; + lookup(KeyType key) const; - write_cursor_type - lookupForWrite(key_type key); + WriteCursorType + lookupForWrite(KeyType key); }; /** diff --git a/src/util/log/Logger.cpp b/src/util/log/Logger.cpp index 849f9e26..80bf2b98 100644 --- a/src/util/log/Logger.cpp +++ b/src/util/log/Logger.cpp @@ -143,38 +143,38 @@ LogService::init(util::Config const& config) } Logger::Pump -Logger::trace(source_location_t const& loc) const +Logger::trace(SourceLocationType const& loc) const { return {logger_, Severity::TRC, loc}; }; Logger::Pump -Logger::debug(source_location_t const& loc) const +Logger::debug(SourceLocationType const& loc) const { return {logger_, Severity::DBG, loc}; }; Logger::Pump -Logger::info(source_location_t const& loc) const +Logger::info(SourceLocationType const& loc) const { return {logger_, Severity::NFO, loc}; }; Logger::Pump -Logger::warn(source_location_t const& loc) const +Logger::warn(SourceLocationType const& loc) const { return {logger_, Severity::WRN, loc}; }; Logger::Pump -Logger::error(source_location_t const& loc) const +Logger::error(SourceLocationType const& loc) const { return {logger_, Severity::ERR, loc}; }; Logger::Pump -Logger::fatal(source_location_t const& loc) const +Logger::fatal(SourceLocationType const& loc) const { return {logger_, Severity::FTL, loc}; }; std::string -Logger::Pump::pretty_path(source_location_t const& loc, size_t max_depth) const +Logger::Pump::pretty_path(SourceLocationType const& loc, size_t max_depth) const { auto const file_path = std::string{loc.file_name()}; auto idx = file_path.size(); @@ -187,4 +187,4 @@ Logger::Pump::pretty_path(source_location_t const& loc, size_t max_depth) const return file_path.substr(idx == std::string::npos ? 0 : idx + 1) + ':' + std::to_string(loc.line()); } -} // namespace util \ No newline at end of file +} // namespace util diff --git a/src/util/log/Logger.h b/src/util/log/Logger.h index 13e1b0f0..805ef7d3 100644 --- a/src/util/log/Logger.h +++ b/src/util/log/Logger.h @@ -52,12 +52,12 @@ namespace util { class Config; #if defined(HAS_SOURCE_LOCATION) && __has_builtin(__builtin_source_location) -using source_location_t = std::source_location; -#define CURRENT_SRC_LOCATION source_location_t::current() +using SourceLocationType = std::source_location; +#define CURRENT_SRC_LOCATION SourceLocationType::current() #elif defined(HAS_EXPERIMENTAL_SOURCE_LOCATION) -using source_location_t = std::experimental::source_location; -#define CURRENT_SRC_LOCATION source_location_t::current() +using SourceLocationType = std::experimental::source_location; +#define CURRENT_SRC_LOCATION SourceLocationType::current() #else // A workaround for AppleClang that is lacking source_location atm. @@ -71,23 +71,25 @@ public: SourceLocation(std::string_view file, std::size_t line) : file_{file}, line_{line} { } + std::string_view file_name() const { return file_; } + std::size_t line() const { return line_; } }; -using source_location_t = SourceLocation; -#define CURRENT_SRC_LOCATION source_location_t(__builtin_FILE(), __builtin_LINE()) +using SourceLocationType = SourceLocation; +#define CURRENT_SRC_LOCATION SourceLocationType(__builtin_FILE(), __builtin_LINE()) #endif /** - * @brief Custom severity levels for @ref Logger. + * @brief Custom severity levels for @ref util::Logger. */ enum class Severity { TRC, @@ -121,8 +123,8 @@ operator<<(std::ostream& stream, Severity sev); */ class Logger final { - using logger_t = boost::log::sources::severity_channel_logger_mt; - mutable logger_t logger_; + using LoggerType = boost::log::sources::severity_channel_logger_mt; + mutable LoggerType logger_; friend class LogService; // to expose the Pump interface @@ -131,14 +133,14 @@ class Logger final */ class Pump final { - using pump_opt_t = std::optional>; + using PumpOptType = std::optional>; boost::log::record rec_; - pump_opt_t pump_ = std::nullopt; + PumpOptType pump_ = std::nullopt; public: ~Pump() = default; - Pump(logger_t& logger, Severity sev, source_location_t const& loc) + Pump(LoggerType& logger, Severity sev, SourceLocationType const& loc) : rec_{logger.open_record(boost::log::keywords::severity = sev)} { if (rec_) @@ -174,7 +176,7 @@ class Logger final private: [[nodiscard]] std::string - pretty_path(source_location_t const& loc, size_t max_depth = 3) const; + pretty_path(SourceLocationType const& loc, size_t max_depth = 3) const; /** * @brief Custom JSON parser for @ref Severity. @@ -208,29 +210,29 @@ public: Logger& operator=(Logger&&) = default; - /*! Interface for logging at @ref Severity::TRC severity */ + /** Interface for logging at Severity::TRC severity */ [[nodiscard]] Pump - trace(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + trace(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; - /*! Interface for logging at @ref Severity::DBG severity */ + /** Interface for logging at Severity::DBG severity */ [[nodiscard]] Pump - debug(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + debug(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; - /*! Interface for logging at @ref Severity::INFO severity */ + /** Interface for logging at Severity::INFO severity */ [[nodiscard]] Pump - info(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + info(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; - /*! Interface for logging at @ref Severity::WRN severity */ + /** Interface for logging at Severity::WRN severity */ [[nodiscard]] Pump - warn(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + warn(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; - /*! Interface for logging at @ref Severity::ERR severity */ + /** Interface for logging at Severity::ERR severity */ [[nodiscard]] Pump - error(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + error(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; - /*! Interface for logging at @ref Severity::FTL severity */ + /** Interface for logging at Severity::FTL severity */ [[nodiscard]] Pump - fatal(source_location_t const& loc = CURRENT_SRC_LOCATION) const; + fatal(SourceLocationType const& loc = CURRENT_SRC_LOCATION) const; }; /** @@ -241,8 +243,8 @@ public: */ class LogService { - static Logger general_log_; /*! Global logger for General channel */ - static Logger alert_log_; /*! Global logger for Alerts channel */ + static Logger general_log_; /*< Global logger for General channel */ + static Logger alert_log_; /*< Global logger for Alerts channel */ public: LogService() = delete; @@ -253,51 +255,51 @@ public: static void init(Config const& config); - /*! Globally accesible General logger at @ref Severity::TRC severity */ + /** Globally accesible General logger at Severity::TRC severity */ [[nodiscard]] static Logger::Pump - trace(source_location_t const& loc = CURRENT_SRC_LOCATION) + trace(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.trace(loc); } - /*! Globally accesible General logger at @ref Severity::DBG severity */ + /** Globally accesible General logger at Severity::DBG severity */ [[nodiscard]] static Logger::Pump - debug(source_location_t const& loc = CURRENT_SRC_LOCATION) + debug(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.debug(loc); } - /*! Globally accesible General logger at @ref Severity::NFO severity */ + /** Globally accesible General logger at Severity::NFO severity */ [[nodiscard]] static Logger::Pump - info(source_location_t const& loc = CURRENT_SRC_LOCATION) + info(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.info(loc); } - /*! Globally accesible General logger at @ref Severity::WRN severity */ + /** Globally accesible General logger at Severity::WRN severity */ [[nodiscard]] static Logger::Pump - warn(source_location_t const& loc = CURRENT_SRC_LOCATION) + warn(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.warn(loc); } - /*! Globally accesible General logger at @ref Severity::ERR severity */ + /** Globally accesible General logger at Severity::ERR severity */ [[nodiscard]] static Logger::Pump - error(source_location_t const& loc = CURRENT_SRC_LOCATION) + error(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.error(loc); } - /*! Globally accesible General logger at @ref Severity::FTL severity */ + /** Globally accesible General logger at Severity::FTL severity */ [[nodiscard]] static Logger::Pump - fatal(source_location_t const& loc = CURRENT_SRC_LOCATION) + fatal(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return general_log_.fatal(loc); } - /*! Globally accesible Alert logger */ + /** Globally accesible Alert logger */ [[nodiscard]] static Logger::Pump - alert(source_location_t const& loc = CURRENT_SRC_LOCATION) + alert(SourceLocationType const& loc = CURRENT_SRC_LOCATION) { return alert_log_.warn(loc); } diff --git a/src/web/Context.h b/src/web/Context.h index d8d8f7c4..9506c1d5 100644 --- a/src/web/Context.h +++ b/src/web/Context.h @@ -32,6 +32,9 @@ namespace web { +/** + * @brief Context that is used by the Webserver to pass around information about an incoming request. + */ struct Context : util::Taggable { boost::asio::yield_context yield; @@ -42,6 +45,18 @@ struct Context : util::Taggable data::LedgerRange range; std::string clientIp; + /** + * @brief Create a new Context instance. + * + * @param yield The coroutine context + * @param command The method/command requested + * @param apiVersion The api_version parsed from the request + * @param params Request's parameters/data as a JSON object + * @param session The connection to the peer + * @param tagFactory A factory that is used to generate tags to track requests and connections + * @param range The ledger range that is available at the time of the request + * @param clientIp IP of the peer + */ Context( boost::asio::yield_context yield, std::string const& command, diff --git a/src/web/DOSGuard.h b/src/web/DOSGuard.h index f01b368a..0f7811c3 100644 --- a/src/web/DOSGuard.h +++ b/src/web/DOSGuard.h @@ -32,11 +32,17 @@ namespace web { +/** + * @brief The interface of a denial of service guard. + */ class BaseDOSGuard { public: virtual ~BaseDOSGuard() = default; + /** + * @brief Clears implementation-defined counters. + */ virtual void clear() noexcept = 0; }; @@ -44,8 +50,8 @@ public: /** * @brief A simple denial of service guard used for rate limiting. * - * @tparam Type of the Whitelist Handler - * @tparam Type of the Sweep Handler + * @tparam WhitelistHandlerType The type of the whitelist handler + * @tparam SweepHandlerType The type of the sweep handler */ template class BasicDOSGuard : public BaseDOSGuard @@ -75,8 +81,8 @@ public: * @brief Constructs a new DOS guard. * * @param config Clio config - * @param WhitelistHandlerType Whitelist handler that checks whitelist for ip addresses - * @param SweepHandlerType Sweep handler that implements the sweeping behaviour + * @param whitelistHandler Whitelist handler that checks whitelist for IP addresses + * @param sweepHandler Sweep handler that implements the sweeping behaviour */ BasicDOSGuard( util::Config const& config, @@ -91,7 +97,7 @@ public: } /** - * @brief Check whether an ip address is in the whitelist or not + * @brief Check whether an ip address is in the whitelist or not. * * @param ip The ip address to check * @return true @@ -104,7 +110,7 @@ public: } /** - * @brief Check whether an ip address is currently rate limited or not + * @brief Check whether an ip address is currently rate limited or not. * * @param ip The ip address to check * @return true If not rate limited @@ -143,7 +149,7 @@ public: } /** - * @brief Increment connection count for the given ip address + * @brief Increment connection count for the given ip address. * * @param ip */ @@ -157,7 +163,7 @@ public: } /** - * @brief Decrement connection count for the given ip address + * @brief Decrement connection count for the given ip address. * * @param ip */ @@ -225,8 +231,7 @@ public: } /** - * @brief Instantly clears all fetch counters added by @see add(std::string - * const&, uint32_t) + * @brief Instantly clears all fetch counters added by @see add(std::string const&, uint32_t). */ void clear() noexcept override @@ -261,7 +266,7 @@ class IntervalSweepHandler public: /** - * @brief Construct a new interval-based sweep handler + * @brief Construct a new interval-based sweep handler. * * @param config Clio config * @param ctx The boost::asio::io_context @@ -279,8 +284,7 @@ public: } /** - * @brief This setup member function is called by @ref BasicDOSGuard during - * its initialization. + * @brief This setup member function is called by @ref BasicDOSGuard during its initialization. * * @param guard Pointer to the dos guard */ diff --git a/src/web/HttpSession.h b/src/web/HttpSession.h index bc0eabc0..ae7ec32e 100644 --- a/src/web/HttpSession.h +++ b/src/web/HttpSession.h @@ -27,26 +27,39 @@ namespace web { using tcp = boost::asio::ip::tcp; /** - * @brief The HTTP session class - * It will handle the upgrade to WebSocket, pass the ownership of the socket to the upgrade session. + * @brief Represents a HTTP connection established by a client. + * + * It will handle the upgrade to websocket, pass the ownership of the socket to the upgrade session. * Otherwise, it will pass control to the base class. + * + * @tparam HandlerType The type of the server handler to use */ -template -class HttpSession : public detail::HttpBase, - public std::enable_shared_from_this> +template +class HttpSession : public detail::HttpBase, + public std::enable_shared_from_this> { boost::beast::tcp_stream stream_; std::reference_wrapper tagFactory_; public: + /** + * @brief Create a new session. + * + * @param socket The socket. Ownership is transferred to HttpSession + * @param ip Client's IP address + * @param tagFactory A factory that is used to generate tags to track requests and sessions + * @param dosGuard The denial of service guard to use + * @param handler The server handler to use + * @param buffer Buffer with initial data received from the peer + */ explicit HttpSession( tcp::socket&& socket, std::string const& ip, std::reference_wrapper tagFactory, std::reference_wrapper dosGuard, - std::shared_ptr const& handler, + std::shared_ptr const& handler, boost::beast::flat_buffer buffer) - : detail::HttpBase(ip, tagFactory, dosGuard, handler, std::move(buffer)) + : detail::HttpBase(ip, tagFactory, dosGuard, handler, std::move(buffer)) , stream_(std::move(socket)) , tagFactory_(tagFactory) { @@ -54,21 +67,24 @@ public: ~HttpSession() = default; + /** @return The TCP stream */ boost::beast::tcp_stream& stream() { return stream_; } + /** @brief Starts reading from the stream. */ void run() { boost::asio::dispatch( stream_.get_executor(), boost::beast::bind_front_handler( - &detail::HttpBase::doRead, this->shared_from_this())); + &detail::HttpBase::doRead, this->shared_from_this())); } + /** @brief Closes the underlying socket. */ void doClose() { @@ -76,10 +92,11 @@ public: stream_.socket().shutdown(tcp::socket::shutdown_send, ec); } + /** @brief Upgrade to WebSocket connection. */ void upgrade() { - std::make_shared>( + std::make_shared>( std::move(stream_), this->clientIp, tagFactory_, diff --git a/src/web/PlainWsSession.h b/src/web/PlainWsSession.h index 419dff08..e50b39e7 100644 --- a/src/web/PlainWsSession.h +++ b/src/web/PlainWsSession.h @@ -24,43 +24,59 @@ namespace web { /** - * @brief The plain WebSocket session class, just to hold the plain stream. Other operations will be handled by the base - * class + * @brief Represents a non-secure websocket session. + * + * Majority of the operations are handled by the base class. */ -template -class PlainWsSession : public detail::WsBase +template +class PlainWsSession : public detail::WsBase { - boost::beast::websocket::stream ws_; + using StreamType = boost::beast::websocket::stream; + StreamType ws_; public: + /** + * @brief Create a new non-secure websocket session. + * + * @param socket The socket. Ownership is transferred + * @param ip Client's IP address + * @param tagFactory A factory that is used to generate tags to track requests and sessions + * @param dosGuard The denial of service guard to use + * @param handler The server handler to use + * @param buffer Buffer with initial data received from the peer + */ explicit PlainWsSession( boost::asio::ip::tcp::socket&& socket, std::string ip, std::reference_wrapper tagFactory, std::reference_wrapper dosGuard, - std::shared_ptr const& callback, + std::shared_ptr const& handler, boost::beast::flat_buffer&& buffer) - : detail::WsBase(ip, tagFactory, dosGuard, callback, std::move(buffer)) + : detail::WsBase(ip, tagFactory, dosGuard, handler, std::move(buffer)) , ws_(std::move(socket)) { } - boost::beast::websocket::stream& + ~PlainWsSession() = default; + + /** @return The websocket stream. */ + StreamType& ws() { return ws_; } - - ~PlainWsSession() = default; }; /** - * @brief The plain WebSocket upgrader class, upgrade from http session to websocket session. + * @brief The websocket upgrader class, upgrade from an HTTP session to a non-secure websocket session. + * * Pass the socket to the session class after upgrade. */ -template -class WsUpgrader : public std::enable_shared_from_this> +template +class WsUpgrader : public std::enable_shared_from_this> { + using std::enable_shared_from_this>::shared_from_this; + boost::beast::tcp_stream http_; boost::optional> parser_; boost::beast::flat_buffer buffer_; @@ -68,33 +84,47 @@ class WsUpgrader : public std::enable_shared_from_this> std::reference_wrapper dosGuard_; http::request req_; std::string ip_; - std::shared_ptr const handler_; + std::shared_ptr const handler_; public: + /** + * @brief Create a new upgrader to non-secure websocket. + * + * @param stream The TCP stream. Ownership is transferred + * @param ip Client's IP address + * @param tagFactory A factory that is used to generate tags to track requests and sessions + * @param dosGuard The denial of service guard to use + * @param handler The server handler to use + * @param buffer Buffer with initial data received from the peer. Ownership is transferred + * @param request The request. Ownership is transferred + */ WsUpgrader( boost::beast::tcp_stream&& stream, std::string ip, std::reference_wrapper tagFactory, std::reference_wrapper dosGuard, - std::shared_ptr const& handler, - boost::beast::flat_buffer&& b, - http::request req) + std::shared_ptr const& handler, + boost::beast::flat_buffer&& buffer, + http::request request) : http_(std::move(stream)) - , buffer_(std::move(b)) + , buffer_(std::move(buffer)) , tagFactory_(tagFactory) , dosGuard_(dosGuard) - , req_(std::move(req)) + , req_(std::move(request)) , ip_(ip) , handler_(handler) { } + WsUpgrader() = default; + + /** @brief Initiate the upgrade. */ void run() { boost::asio::dispatch( http_.get_executor(), - boost::beast::bind_front_handler(&WsUpgrader::doUpgrade, this->shared_from_this())); + boost::beast::bind_front_handler(&WsUpgrader::doUpgrade, shared_from_this())); } private: @@ -103,26 +133,23 @@ private: { parser_.emplace(); - constexpr static auto MaxBobySize = 10000; - parser_->body_limit(MaxBobySize); + constexpr static auto maxBodySize = 10000; + parser_->body_limit(maxBodySize); boost::beast::get_lowest_layer(http_).expires_after(std::chrono::seconds(30)); - onUpgrade(); } void onUpgrade() { - // See if it is a WebSocket Upgrade if (!boost::beast::websocket::is_upgrade(req_)) return; - // Disable the timeout. - // The websocket::stream uses its own timeout settings. + // Disable the timeout. The websocket::stream uses its own timeout settings. boost::beast::get_lowest_layer(http_).expires_never(); - std::make_shared>( + std::make_shared>( http_.release_socket(), ip_, tagFactory_, dosGuard_, handler_, std::move(buffer_)) ->run(std::move(req_)); } diff --git a/src/web/README.md b/src/web/README.md index 6d3d9d6d..012cf47f 100644 --- a/src/web/README.md +++ b/src/web/README.md @@ -1,8 +1,11 @@ +# Webserver subsystem + This folder contains all of the classes for running the webserver. -The webserver handles JSON-RPC and websocket requests. -The webserver supports SSL if a cert and key file are specified in the config. -The webserver handles all types of requests on a single port. +The webserver subsystem +- Handles JSON-RPC and websocket requests. +- Supports SSL if a cert and key file are specified in the config. +- Handles all types of requests on a single port. Each request is handled asynchronously using boost asio. diff --git a/src/web/RPCServerHandler.h b/src/web/RPCServerHandler.h index d9c28c8d..6d734a5c 100644 --- a/src/web/RPCServerHandler.h +++ b/src/web/RPCServerHandler.h @@ -29,31 +29,42 @@ #include +namespace web { + /** - * @brief The server handler for RPC requests called by web server + * @brief The server handler for RPC requests called by web server. * - * Note: see ServerHandler concept + * Note: see @ref web::SomeServerHandler concept */ -template +template class RPCServerHandler { std::shared_ptr const backend_; - std::shared_ptr const rpcEngine_; - std::shared_ptr const etl_; + std::shared_ptr const rpcEngine_; + std::shared_ptr const etl_; // subscription manager holds the shared_ptr of this class std::weak_ptr const subscriptions_; util::TagDecoratorFactory const tagFactory_; - RPC::detail::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed + rpc::detail::ProductionAPIVersionParser apiVersionParser_; // can be injected if needed util::Logger log_{"RPC"}; util::Logger perfLog_{"Performance"}; public: + /** + * @brief Create a new server handler. + * + * @param config Clio config to use + * @param backend The backend to use + * @param rpcEngine The RPC engine to use + * @param etl The ETL to use + * @param subscriptions The subscription manager to use + */ RPCServerHandler( util::Config const& config, std::shared_ptr const& backend, - std::shared_ptr const& rpcEngine, - std::shared_ptr const& etl, + std::shared_ptr const& rpcEngine, + std::shared_ptr const& etl, std::shared_ptr const& subscriptions) : backend_(backend) , rpcEngine_(rpcEngine) @@ -65,16 +76,17 @@ public: } /** - * @brief The callback when server receives a request - * @param req The request + * @brief The callback when server receives a request. + * + * @param request The request * @param connection The connection */ void - operator()(std::string const& reqStr, std::shared_ptr const& connection) + operator()(std::string const& request, std::shared_ptr const& connection) { try { - auto req = boost::json::parse(reqStr).as_object(); + auto req = boost::json::parse(request).as_object(); perfLog_.debug() << connection->tag() << "Adding to work queue"; if (not connection->upgraded and not req.contains("params")) @@ -112,12 +124,14 @@ public: /** * @brief The callback when there is an error. - * Remove the session shared ptr from subscription manager - * @param _ The error code + * + * Remove the session shared ptr from subscription manager. + * + * @param ec The error code * @param connection The connection */ void - operator()(boost::beast::error_code _, std::shared_ptr const& connection) + operator()([[maybe_unused]] boost::beast::error_code ec, std::shared_ptr const& connection) { if (auto manager = subscriptions_.lock(); manager) manager->cleanup(connection); @@ -146,7 +160,7 @@ private: auto const context = [&] { if (connection->upgraded) - return RPC::make_WsContext( + return rpc::make_WsContext( yield, request, connection, @@ -155,7 +169,7 @@ private: connection->clientIp, std::cref(apiVersionParser_)); else - return RPC::make_HttpContext( + return rpc::make_HttpContext( yield, request, tagFactory_.with(connection->tag()), @@ -179,10 +193,10 @@ private: auto [v, timeDiff] = util::timed([&]() { return rpcEngine_->buildResponse(*context); }); auto us = std::chrono::duration(timeDiff); - RPC::logDuration(*context, us); + rpc::logDuration(*context, us); boost::json::object response; - if (auto const status = std::get_if(&v)) + if (auto const status = std::get_if(&v)) { // note: error statuses are counted/notified in buildResponse itself response = web::detail::ErrorHelper(connection, request).composeError(*status); @@ -234,10 +248,10 @@ private: } boost::json::array warnings; - warnings.emplace_back(RPC::makeWarning(RPC::warnRPC_CLIO)); + warnings.emplace_back(rpc::makeWarning(rpc::warnRPC_CLIO)); if (etl_->lastCloseAgeSeconds() >= 60) - warnings.emplace_back(RPC::makeWarning(RPC::warnRPC_OUTDATED)); + warnings.emplace_back(rpc::makeWarning(rpc::warnRPC_OUTDATED)); response["warnings"] = warnings; connection->send(boost::json::serialize(response)); @@ -254,3 +268,5 @@ private: } } }; + +} // namespace web diff --git a/src/web/Server.h b/src/web/Server.h index 6937e870..e18b37cf 100644 --- a/src/web/Server.h +++ b/src/web/Server.h @@ -26,35 +26,55 @@ #include +/** + * @brief This namespace implements the web server and related components. + * + * The web server is leveraging the power of `boost::asio` with it's coroutine support thru `boost::asio::yield_context` + * and `boost::asio::spawn`. + * + * Majority of the code is based on examples that came with boost. + */ namespace web { /** * @brief The Detector class to detect if the connection is a ssl or not. - * If it is a ssl connection, it will pass the ownership of the socket to SslSession, otherwise to PlainSession. - * @tparam PlainSession The plain session type - * @tparam SslSession The ssl session type - * @tparam Handler The executor to handle the requests + * + * If it is an SSL connection, the Detector will pass the ownership of the socket to SslSessionType, otherwise to + * PlainSessionType. + * + * @tparam PlainSessionType The plain session type + * @tparam SslSessionType The SSL session type + * @tparam HandlerType The executor to handle the requests */ -template