From 056e170a56768c0ad51de79fe5faa36944b6a2b9 Mon Sep 17 00:00:00 2001 From: CJ Cobb Date: Wed, 23 Jun 2021 14:43:29 +0000 Subject: [PATCH] Cleanup, documentation, rename some things, cmake changes --- CMakeLists.txt | 83 +++--- deps/Postgres.cmake | 10 +- deps/cassandra.cmake | 7 +- {backend => src/backend}/BackendFactory.h | 8 +- {backend => src/backend}/BackendIndexer.cpp | 1 + src/backend/BackendIndexer.h | 104 +++++++ {backend => src/backend}/BackendInterface.cpp | 0 {backend => src/backend}/BackendInterface.h | 281 +++++++----------- {backend => src/backend}/CassandraBackend.cpp | 0 {backend => src/backend}/CassandraBackend.h | 0 {backend => src/backend}/DBHelpers.cpp | 0 {backend => src/backend}/DBHelpers.h | 0 {backend => src/backend}/Pg.cpp | 0 {backend => src/backend}/Pg.h | 0 {backend => src/backend}/PostgresBackend.cpp | 0 {backend => src/backend}/PostgresBackend.h | 0 {backend => src/backend}/README.md | 0 {etl => src/etl}/ETLHelpers.h | 0 {etl => src/etl}/ETLSource.cpp | 0 {etl => src/etl}/ETLSource.h | 0 {etl => src/etl}/ReportingETL.cpp | 0 {etl => src/etl}/ReportingETL.h | 0 .../handlers}/AccountChannels.cpp | 0 .../handlers}/AccountCurrencies.cpp | 0 {handlers => src/handlers}/AccountInfo.cpp | 0 {handlers => src/handlers}/AccountLines.cpp | 0 {handlers => src/handlers}/AccountObjects.cpp | 0 {handlers => src/handlers}/AccountOffers.cpp | 0 {handlers => src/handlers}/AccountTx.cpp | 0 {handlers => src/handlers}/BookOffers.cpp | 0 .../handlers}/ChannelAuthorize.cpp | 0 {handlers => src/handlers}/ChannelVerify.cpp | 0 {handlers => src/handlers}/Ledger.cpp | 0 {handlers => src/handlers}/LedgerData.cpp | 0 {handlers => src/handlers}/LedgerEntry.cpp | 0 {handlers => src/handlers}/LedgerRange.cpp | 0 {handlers => src/handlers}/RPCHelpers.cpp | 0 {handlers => src/handlers}/RPCHelpers.h | 0 {handlers => src/handlers}/ServerInfo.cpp | 0 {handlers => src/handlers}/Subscribe.cpp | 0 {handlers => src/handlers}/Tx.cpp | 0 {server => src/server}/DOSGuard.h | 0 {server => src/server}/Handlers.cpp | 0 {server => src/server}/Handlers.h | 0 {server => src/server}/HttpBase.h | 0 {server => src/server}/HttpSession.h | 0 {server => src/server}/PlainWsSession.h | 0 {server => src/server}/SslHttpSession.h | 0 {server => src/server}/SslWsSession.h | 0 .../server}/SubscriptionManager.cpp | 0 {server => src/server}/SubscriptionManager.h | 0 {server => src/server}/WsBase.h | 0 {server => src/server}/listener.h | 0 .../server/main.cpp | 17 +- test.py | 4 +- 55 files changed, 288 insertions(+), 227 deletions(-) rename {backend => src/backend}/BackendFactory.h (85%) rename {backend => src/backend}/BackendIndexer.cpp (99%) create mode 100644 src/backend/BackendIndexer.h rename {backend => src/backend}/BackendInterface.cpp (100%) rename {backend => src/backend}/BackendInterface.h (69%) rename {backend => src/backend}/CassandraBackend.cpp (100%) rename {backend => src/backend}/CassandraBackend.h (100%) rename {backend => src/backend}/DBHelpers.cpp (100%) rename {backend => src/backend}/DBHelpers.h (100%) rename {backend => src/backend}/Pg.cpp (100%) rename {backend => src/backend}/Pg.h (100%) rename {backend => src/backend}/PostgresBackend.cpp (100%) rename {backend => src/backend}/PostgresBackend.h (100%) rename {backend => src/backend}/README.md (100%) rename {etl => src/etl}/ETLHelpers.h (100%) rename {etl => src/etl}/ETLSource.cpp (100%) rename {etl => src/etl}/ETLSource.h (100%) rename {etl => src/etl}/ReportingETL.cpp (100%) rename {etl => src/etl}/ReportingETL.h (100%) rename {handlers => src/handlers}/AccountChannels.cpp (100%) rename {handlers => src/handlers}/AccountCurrencies.cpp (100%) rename {handlers => src/handlers}/AccountInfo.cpp (100%) rename {handlers => src/handlers}/AccountLines.cpp (100%) rename {handlers => src/handlers}/AccountObjects.cpp (100%) rename {handlers => src/handlers}/AccountOffers.cpp (100%) rename {handlers => src/handlers}/AccountTx.cpp (100%) rename {handlers => src/handlers}/BookOffers.cpp (100%) rename {handlers => src/handlers}/ChannelAuthorize.cpp (100%) rename {handlers => src/handlers}/ChannelVerify.cpp (100%) rename {handlers => src/handlers}/Ledger.cpp (100%) rename {handlers => src/handlers}/LedgerData.cpp (100%) rename {handlers => src/handlers}/LedgerEntry.cpp (100%) rename {handlers => src/handlers}/LedgerRange.cpp (100%) rename {handlers => src/handlers}/RPCHelpers.cpp (100%) rename {handlers => src/handlers}/RPCHelpers.h (100%) rename {handlers => src/handlers}/ServerInfo.cpp (100%) rename {handlers => src/handlers}/Subscribe.cpp (100%) rename {handlers => src/handlers}/Tx.cpp (100%) rename {server => src/server}/DOSGuard.h (100%) rename {server => src/server}/Handlers.cpp (100%) rename {server => src/server}/Handlers.h (100%) rename {server => src/server}/HttpBase.h (100%) rename {server => src/server}/HttpSession.h (100%) rename {server => src/server}/PlainWsSession.h (100%) rename {server => src/server}/SslHttpSession.h (100%) rename {server => src/server}/SslWsSession.h (100%) rename {server => src/server}/SubscriptionManager.cpp (100%) rename {server => src/server}/SubscriptionManager.h (100%) rename {server => src/server}/WsBase.h (100%) rename {server => src/server}/listener.h (100%) rename server/websocket_server_async.cpp => src/server/main.cpp (86%) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb33ca58..55a9f5b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,7 +8,7 @@ # set(CMAKE_VERBOSE_MAKEFILE TRUE) -project(reporting) +project(clio) cmake_minimum_required(VERSION 3.16) set (CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-narrowing") @@ -30,22 +30,23 @@ FetchContent_MakeAvailable(googletest) enable_testing() include(GoogleTest) -add_executable (reporting_main - server/websocket_server_async.cpp +add_executable (clio_server + src/server/main.cpp ) -add_executable (reporting_tests +add_executable (clio_tests unittests/main.cpp ) -add_library(reporting backend/BackendInterface.h) +add_library(clio src/backend/BackendInterface.h) +include_directories(src) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps") include(ExternalProject) message(${CMAKE_CURRENT_BINARY_DIR}) message(${CMAKE_MODULE_PATH}) #include(rippled/Builds/CMake/RippledCore.cmake) add_subdirectory(rippled) -target_link_libraries(reporting PUBLIC xrpl_core grpc_pbufs) -add_dependencies(reporting xrpl_core) -add_dependencies(reporting grpc_pbufs) +target_link_libraries(clio PUBLIC xrpl_core grpc_pbufs) +add_dependencies(clio xrpl_core) +add_dependencies(clio grpc_pbufs) get_target_property(grpc_includes grpc_pbufs INCLUDE_DIRECTORIES) #get_target_property(xrpl_core_includes xrpl_core INCLUDE_DIRECTORIES) # get_target_property(proto_includes protobuf_src INCLUDE_DIRECTORIES) @@ -66,43 +67,43 @@ include(cassandra) include(Postgres) -target_sources(reporting PRIVATE - backend/CassandraBackend.cpp - backend/PostgresBackend.cpp - backend/BackendIndexer.cpp - backend/BackendInterface.cpp - backend/Pg.cpp - backend/DBHelpers.cpp - etl/ETLSource.cpp - etl/ReportingETL.cpp - server/Handlers.cpp - server/SubscriptionManager.cpp - handlers/AccountInfo.cpp - handlers/Tx.cpp - handlers/RPCHelpers.cpp - handlers/AccountTx.cpp - handlers/LedgerData.cpp - handlers/BookOffers.cpp - handlers/LedgerRange.cpp - handlers/Ledger.cpp - handlers/LedgerEntry.cpp - handlers/AccountChannels.cpp - handlers/AccountLines.cpp - handlers/AccountCurrencies.cpp - handlers/AccountOffers.cpp - handlers/AccountObjects.cpp - handlers/ChannelAuthorize.cpp - handlers/ChannelVerify.cpp - handlers/Subscribe.cpp - handlers/ServerInfo.cpp) +target_sources(clio PRIVATE + src/backend/CassandraBackend.cpp + src/backend/PostgresBackend.cpp + src/backend/BackendIndexer.cpp + src/backend/BackendInterface.cpp + src/backend/Pg.cpp + src/backend/DBHelpers.cpp + src/etl/ETLSource.cpp + src/etl/ReportingETL.cpp + src/server/Handlers.cpp + src/server/SubscriptionManager.cpp + src/handlers/AccountInfo.cpp + src/handlers/Tx.cpp + src/handlers/RPCHelpers.cpp + src/handlers/AccountTx.cpp + src/handlers/LedgerData.cpp + src/handlers/BookOffers.cpp + src/handlers/LedgerRange.cpp + src/handlers/Ledger.cpp + src/handlers/LedgerEntry.cpp + src/handlers/AccountChannels.cpp + src/handlers/AccountLines.cpp + src/handlers/AccountCurrencies.cpp + src/handlers/AccountOffers.cpp + src/handlers/AccountObjects.cpp + src/handlers/ChannelAuthorize.cpp + src/handlers/ChannelVerify.cpp + src/handlers/Subscribe.cpp + src/handlers/ServerInfo.cpp) message(${Boost_LIBRARIES}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR} ${Boost_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -TARGET_LINK_LIBRARIES(reporting PUBLIC ${Boost_LIBRARIES}) -TARGET_LINK_LIBRARIES(reporting_main PUBLIC reporting) -TARGET_LINK_LIBRARIES(reporting_tests PUBLIC reporting gtest_main) +TARGET_LINK_LIBRARIES(clio PUBLIC ${Boost_LIBRARIES}) +TARGET_LINK_LIBRARIES(clio_server PUBLIC clio) +TARGET_LINK_LIBRARIES(clio_tests PUBLIC clio gtest_main) -gtest_discover_tests(reporting_tests) +gtest_discover_tests(clio_tests) diff --git a/deps/Postgres.cmake b/deps/Postgres.cmake index 2572599d..b24a7798 100644 --- a/deps/Postgres.cmake +++ b/deps/Postgres.cmake @@ -45,23 +45,23 @@ if(NOT PostgreSQL_FOUND) file(TO_CMAKE_PATH "${postgres_src_SOURCE_DIR}" postgres_src_SOURCE_DIR) - target_link_libraries(reporting PUBLIC postgres) + target_link_libraries(clio PUBLIC postgres) else() message("Found system installed Postgres via find_libary") - target_include_directories(reporting INTERFACE ${libpq-fe}) + target_include_directories(clio INTERFACE ${libpq-fe}) - target_link_libraries(reporting INTERFACE ${postgres}) + target_link_libraries(clio INTERFACE ${postgres}) endif() else() message("Found system installed Postgres via find_package") message("${PostgreSQL_INCLUDE_DIRS}") - include_directories(reporting INTERFACE "${PostgreSQL_INCLUDE_DIRS}") + include_directories(clio INTERFACE "${PostgreSQL_INCLUDE_DIRS}") - target_link_libraries(reporting PUBLIC ${PostgreSQL_LIBRARIES}) + target_link_libraries(clio PUBLIC ${PostgreSQL_LIBRARIES}) endif() diff --git a/deps/cassandra.cmake b/deps/cassandra.cmake index b4b459ac..80f2ade5 100644 --- a/deps/cassandra.cmake +++ b/deps/cassandra.cmake @@ -152,13 +152,12 @@ if(NOT cassandra) target_link_libraries(cassandra INTERFACE OpenSSL::SSL) file(TO_CMAKE_PATH "${cassandra_src_SOURCE_DIR}" cassandra_src_SOURCE_DIR) - target_link_libraries(reporting PUBLIC cassandra) + target_link_libraries(clio PUBLIC cassandra) else() message("Found system installed cassandra cpp driver") message(${cassandra}) find_path(cassandra_includes NAMES cassandra.h REQUIRED) - target_link_libraries (reporting PUBLIC ${cassandra}) - target_include_directories(reporting INTERFACE ${cassandra_includes}) + target_link_libraries (clio PUBLIC ${cassandra}) + target_include_directories(clio INTERFACE ${cassandra_includes}) endif() - diff --git a/backend/BackendFactory.h b/src/backend/BackendFactory.h similarity index 85% rename from backend/BackendFactory.h rename to src/backend/BackendFactory.h index c039d1cb..0ba526ef 100644 --- a/backend/BackendFactory.h +++ b/src/backend/BackendFactory.h @@ -7,7 +7,7 @@ #include namespace Backend { -std::unique_ptr +std::shared_ptr make_Backend(boost::json::object const& config) { BOOST_LOG_TRIVIAL(info) << __func__ << ": Constructing BackendInterface"; @@ -20,7 +20,7 @@ make_Backend(boost::json::object const& config) auto type = dbConfig.at("type").as_string(); - std::unique_ptr backend = nullptr; + std::shared_ptr backend = nullptr; if (boost::iequals(type, "cassandra")) { @@ -28,12 +28,12 @@ make_Backend(boost::json::object const& config) dbConfig.at(type).as_object()["ttl"] = config.at("online_delete").as_int64() * 4; backend = - std::make_unique(dbConfig.at(type).as_object()); + std::make_shared(dbConfig.at(type).as_object()); } else if (boost::iequals(type, "postgres")) { backend = - std::make_unique(dbConfig.at(type).as_object()); + std::make_shared(dbConfig.at(type).as_object()); } if (!backend) diff --git a/backend/BackendIndexer.cpp b/src/backend/BackendIndexer.cpp similarity index 99% rename from backend/BackendIndexer.cpp rename to src/backend/BackendIndexer.cpp index c8859326..4aa17fa9 100644 --- a/backend/BackendIndexer.cpp +++ b/src/backend/BackendIndexer.cpp @@ -1,3 +1,4 @@ +#include #include namespace Backend { diff --git a/src/backend/BackendIndexer.h b/src/backend/BackendIndexer.h new file mode 100644 index 00000000..b2c9a51e --- /dev/null +++ b/src/backend/BackendIndexer.h @@ -0,0 +1,104 @@ +#ifndef CLIO_BACKEND_INDEXER_H_INCLUDED +#define CLIO_BACKEND_INDEXER_H_INCLUDED +#include +#include +#include +#include +#include +#include +namespace std { +template <> +struct hash +{ + std::size_t + operator()(const ripple::uint256& k) const noexcept + { + return boost::hash_range(k.begin(), k.end()); + } +}; +} // namespace std +namespace Backend { +// The below two structs exist to prevent developers from accidentally mixing up +// the two indexes. +struct BookIndex +{ + uint32_t bookIndex; + explicit BookIndex(uint32_t v) : bookIndex(v){}; +}; +struct KeyIndex +{ + uint32_t keyIndex; + explicit KeyIndex(uint32_t v) : keyIndex(v){}; +}; +class BackendInterface; +class BackendIndexer +{ + boost::asio::io_context ioc_; + boost::asio::io_context::strand strand_; + std::mutex mutex_; + std::optional work_; + std::thread ioThread_; + + std::atomic_uint32_t indexing_ = 0; + + uint32_t keyShift_ = 20; + std::unordered_set keys; + + mutable bool isFirst_ = true; + void + doKeysRepair( + BackendInterface const& backend, + std::optional sequence); + void + writeKeyFlagLedger( + uint32_t ledgerSequence, + BackendInterface const& backend); + +public: + BackendIndexer(boost::json::object const& config); + ~BackendIndexer(); + + void + addKey(ripple::uint256&& key); + + void + finish(uint32_t ledgerSequence, BackendInterface const& backend); + void + writeKeyFlagLedgerAsync( + uint32_t ledgerSequence, + BackendInterface const& backend); + void + doKeysRepairAsync( + BackendInterface const& backend, + std::optional sequence); + uint32_t + getKeyShift() + { + return keyShift_; + } + std::optional + getCurrentlyIndexing() + { + uint32_t cur = indexing_.load(); + if (cur != 0) + return cur; + return {}; + } + KeyIndex + getKeyIndexOfSeq(uint32_t seq) const + { + if (isKeyFlagLedger(seq)) + return KeyIndex{seq}; + auto incr = (1 << keyShift_); + KeyIndex index{(seq >> keyShift_ << keyShift_) + incr}; + assert(isKeyFlagLedger(index.keyIndex)); + return index; + } + bool + isKeyFlagLedger(uint32_t ledgerSequence) const + { + return (ledgerSequence % (1 << keyShift_)) == 0; + } +}; +} // namespace Backend +#endif diff --git a/backend/BackendInterface.cpp b/src/backend/BackendInterface.cpp similarity index 100% rename from backend/BackendInterface.cpp rename to src/backend/BackendInterface.cpp diff --git a/backend/BackendInterface.h b/src/backend/BackendInterface.h similarity index 69% rename from backend/BackendInterface.h rename to src/backend/BackendInterface.h index 39a0ef23..f528f38d 100644 --- a/backend/BackendInterface.h +++ b/src/backend/BackendInterface.h @@ -2,20 +2,17 @@ #define RIPPLE_APP_REPORTING_BACKENDINTERFACE_H_INCLUDED #include #include +#include #include -namespace std { -template <> -struct hash -{ - std::size_t - operator()(const ripple::uint256& k) const noexcept - { - return boost::hash_range(k.begin(), k.end()); - } -}; -} // namespace std +class ReportingETL; +class AsyncCallData; +class BackendTest_Basic_Test; namespace Backend { + +// *** return types + using Blob = std::vector; + struct LedgerObject { ripple::uint256 key; @@ -55,19 +52,6 @@ struct LedgerRange uint32_t maxSequence; }; -// The below two structs exist to prevent developers from accidentally mixing up -// the two indexes. -struct BookIndex -{ - uint32_t bookIndex; - explicit BookIndex(uint32_t v) : bookIndex(v){}; -}; -struct KeyIndex -{ - uint32_t keyIndex; - explicit KeyIndex(uint32_t v) : keyIndex(v){}; -}; - class DatabaseTimeout : public std::exception { const char* @@ -76,76 +60,6 @@ class DatabaseTimeout : public std::exception return "Database read timed out. Please retry the request"; } }; -class BackendInterface; -class BackendIndexer -{ - boost::asio::io_context ioc_; - boost::asio::io_context::strand strand_; - std::mutex mutex_; - std::optional work_; - std::thread ioThread_; - - std::atomic_uint32_t indexing_ = 0; - - uint32_t keyShift_ = 20; - std::unordered_set keys; - - mutable bool isFirst_ = true; - void - doKeysRepair( - BackendInterface const& backend, - std::optional sequence); - void - writeKeyFlagLedger( - uint32_t ledgerSequence, - BackendInterface const& backend); - -public: - BackendIndexer(boost::json::object const& config); - ~BackendIndexer(); - - void - addKey(ripple::uint256&& key); - - void - finish(uint32_t ledgerSequence, BackendInterface const& backend); - void - writeKeyFlagLedgerAsync( - uint32_t ledgerSequence, - BackendInterface const& backend); - void - doKeysRepairAsync( - BackendInterface const& backend, - std::optional sequence); - uint32_t - getKeyShift() - { - return keyShift_; - } - std::optional - getCurrentlyIndexing() - { - uint32_t cur = indexing_.load(); - if (cur != 0) - return cur; - return {}; - } - KeyIndex - getKeyIndexOfSeq(uint32_t seq) const - { - if (isKeyFlagLedger(seq)) - return KeyIndex{seq}; - auto incr = (1 << keyShift_); - KeyIndex index{(seq >> keyShift_ << keyShift_) + incr}; - assert(isKeyFlagLedger(index.keyIndex)); - return index; - } - bool - isKeyFlagLedger(uint32_t ledgerSequence) const - { - return (ledgerSequence % (1 << keyShift_)) == 0; - } -}; class BackendInterface { @@ -154,10 +68,12 @@ protected: mutable bool isFirst_ = true; public: - // read methods BackendInterface(boost::json::object const& config) : indexer_(config) { } + virtual ~BackendInterface() + { + } BackendIndexer& getIndexer() const @@ -165,75 +81,34 @@ public: return indexer_; } - void - checkFlagLedgers() const; - - std::optional - getKeyIndexOfSeq(uint32_t seq) const; - - bool - finishWrites(uint32_t ledgerSequence) const; - - virtual std::optional - fetchLatestLedgerSequence() const = 0; + // *** public read methods *** + // All of these reads methods can throw DatabaseTimeout. When writing code + // in an RPC handler, this exception does not need to be caught: when an RPC + // results in a timeout, an error is returned to the client +public: + // *** ledger methods virtual std::optional fetchLedgerBySequence(uint32_t sequence) const = 0; + virtual std::optional + fetchLatestLedgerSequence() const = 0; + virtual std::optional fetchLedgerRange() const = 0; + // Doesn't throw DatabaseTimeout. Should be used with care. std::optional fetchLedgerRangeNoThrow() const; - virtual std::optional - fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence) const = 0; + // *** transaction methods - // returns a transaction, metadata pair virtual std::optional fetchTransaction(ripple::uint256 const& hash) const = 0; - virtual std::vector - fetchAllTransactionsInLedger(uint32_t ledgerSequence) const = 0; - - virtual std::vector - fetchAllTransactionHashesInLedger(uint32_t ledgerSequence) const = 0; - - LedgerPage - fetchLedgerPage( - std::optional const& cursor, - std::uint32_t ledgerSequence, - std::uint32_t limit, - std::uint32_t limitHint = 0) const; - - bool - isLedgerIndexed(std::uint32_t ledgerSequence) const; - - std::optional - fetchSuccessor(ripple::uint256 key, uint32_t ledgerSequence) const; - - virtual LedgerPage - doFetchLedgerPage( - std::optional const& cursor, - std::uint32_t ledgerSequence, - std::uint32_t limit) const = 0; - - // TODO add warning for incomplete data - BookOffersPage - fetchBookOffers( - ripple::uint256 const& book, - uint32_t ledgerSequence, - std::uint32_t limit, - std::optional const& cursor = {}) const; - virtual std::vector fetchTransactions(std::vector const& hashes) const = 0; - virtual std::vector - fetchLedgerObjects( - std::vector const& keys, - uint32_t sequence) const = 0; - virtual std::pair< std::vector, std::optional> @@ -242,7 +117,58 @@ public: std::uint32_t limit, std::optional const& cursor = {}) const = 0; - // write methods + virtual std::vector + fetchAllTransactionsInLedger(uint32_t ledgerSequence) const = 0; + + virtual std::vector + fetchAllTransactionHashesInLedger(uint32_t ledgerSequence) const = 0; + + // *** state data methods + + virtual std::optional + fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence) const = 0; + + virtual std::vector + fetchLedgerObjects( + std::vector const& keys, + uint32_t sequence) const = 0; + + // Fetches a page of ledger objects, ordered by key/index. + // Used by ledger_data + LedgerPage + fetchLedgerPage( + std::optional const& cursor, + std::uint32_t ledgerSequence, + std::uint32_t limit, + std::uint32_t limitHint = 0) const; + + // Fetches the successor to key/index. key need not actually be a valid + // key/index. + std::optional + fetchSuccessor(ripple::uint256 key, uint32_t ledgerSequence) const; + + BookOffersPage + fetchBookOffers( + ripple::uint256 const& book, + uint32_t ledgerSequence, + std::uint32_t limit, + std::optional const& cursor = {}) const; + + // Methods related to the indexer + bool + isLedgerIndexed(std::uint32_t ledgerSequence) const; + + std::optional + getKeyIndexOfSeq(uint32_t seq) const; + + // *** protected write methods +protected: + friend class ::ReportingETL; + friend class BackendIndexer; + friend class ::AsyncCallData; + friend std::shared_ptr + make_Backend(boost::json::object const& config); + friend class ::BackendTest_Basic_Test; virtual void writeLedger( @@ -259,15 +185,6 @@ public: bool isDeleted, std::optional&& book) const; - virtual void - doWriteLedgerObject( - std::string&& key, - uint32_t seq, - std::string&& blob, - bool isCreated, - bool isDeleted, - std::optional&& book) const = 0; - virtual void writeTransaction( std::string&& hash, @@ -279,7 +196,26 @@ public: writeAccountTransactions( std::vector&& data) const = 0; - // other database methods + // TODO: this function, or something similar, could be called internally by + // writeLedgerObject + virtual bool + writeKeys( + std::unordered_set const& keys, + KeyIndex const& index, + bool isAsync = false) const = 0; + + // Tell the database we are about to begin writing data for a particular + // ledger. + virtual void + startWrites() const = 0; + + // Tell the database we have finished writing all data for a particular + // ledger + bool + finishWrites(uint32_t ledgerSequence) const; + + virtual bool + doOnlineDelete(uint32_t numLedgersToKeep) const = 0; // Open the database. Set up all of the necessary objects and // datastructures. After this call completes, the database is ready for @@ -291,23 +227,28 @@ public: virtual void close() = 0; + // *** private helper methods +private: + virtual LedgerPage + doFetchLedgerPage( + std::optional const& cursor, + std::uint32_t ledgerSequence, + std::uint32_t limit) const = 0; + virtual void - startWrites() const = 0; + doWriteLedgerObject( + std::string&& key, + uint32_t seq, + std::string&& blob, + bool isCreated, + bool isDeleted, + std::optional&& book) const = 0; virtual bool doFinishWrites() const = 0; - virtual bool - doOnlineDelete(uint32_t numLedgersToKeep) const = 0; - virtual bool - writeKeys( - std::unordered_set const& keys, - KeyIndex const& index, - bool isAsync = false) const = 0; - - virtual ~BackendInterface() - { - } + void + checkFlagLedgers() const; }; } // namespace Backend diff --git a/backend/CassandraBackend.cpp b/src/backend/CassandraBackend.cpp similarity index 100% rename from backend/CassandraBackend.cpp rename to src/backend/CassandraBackend.cpp diff --git a/backend/CassandraBackend.h b/src/backend/CassandraBackend.h similarity index 100% rename from backend/CassandraBackend.h rename to src/backend/CassandraBackend.h diff --git a/backend/DBHelpers.cpp b/src/backend/DBHelpers.cpp similarity index 100% rename from backend/DBHelpers.cpp rename to src/backend/DBHelpers.cpp diff --git a/backend/DBHelpers.h b/src/backend/DBHelpers.h similarity index 100% rename from backend/DBHelpers.h rename to src/backend/DBHelpers.h diff --git a/backend/Pg.cpp b/src/backend/Pg.cpp similarity index 100% rename from backend/Pg.cpp rename to src/backend/Pg.cpp diff --git a/backend/Pg.h b/src/backend/Pg.h similarity index 100% rename from backend/Pg.h rename to src/backend/Pg.h diff --git a/backend/PostgresBackend.cpp b/src/backend/PostgresBackend.cpp similarity index 100% rename from backend/PostgresBackend.cpp rename to src/backend/PostgresBackend.cpp diff --git a/backend/PostgresBackend.h b/src/backend/PostgresBackend.h similarity index 100% rename from backend/PostgresBackend.h rename to src/backend/PostgresBackend.h diff --git a/backend/README.md b/src/backend/README.md similarity index 100% rename from backend/README.md rename to src/backend/README.md diff --git a/etl/ETLHelpers.h b/src/etl/ETLHelpers.h similarity index 100% rename from etl/ETLHelpers.h rename to src/etl/ETLHelpers.h diff --git a/etl/ETLSource.cpp b/src/etl/ETLSource.cpp similarity index 100% rename from etl/ETLSource.cpp rename to src/etl/ETLSource.cpp diff --git a/etl/ETLSource.h b/src/etl/ETLSource.h similarity index 100% rename from etl/ETLSource.h rename to src/etl/ETLSource.h diff --git a/etl/ReportingETL.cpp b/src/etl/ReportingETL.cpp similarity index 100% rename from etl/ReportingETL.cpp rename to src/etl/ReportingETL.cpp diff --git a/etl/ReportingETL.h b/src/etl/ReportingETL.h similarity index 100% rename from etl/ReportingETL.h rename to src/etl/ReportingETL.h diff --git a/handlers/AccountChannels.cpp b/src/handlers/AccountChannels.cpp similarity index 100% rename from handlers/AccountChannels.cpp rename to src/handlers/AccountChannels.cpp diff --git a/handlers/AccountCurrencies.cpp b/src/handlers/AccountCurrencies.cpp similarity index 100% rename from handlers/AccountCurrencies.cpp rename to src/handlers/AccountCurrencies.cpp diff --git a/handlers/AccountInfo.cpp b/src/handlers/AccountInfo.cpp similarity index 100% rename from handlers/AccountInfo.cpp rename to src/handlers/AccountInfo.cpp diff --git a/handlers/AccountLines.cpp b/src/handlers/AccountLines.cpp similarity index 100% rename from handlers/AccountLines.cpp rename to src/handlers/AccountLines.cpp diff --git a/handlers/AccountObjects.cpp b/src/handlers/AccountObjects.cpp similarity index 100% rename from handlers/AccountObjects.cpp rename to src/handlers/AccountObjects.cpp diff --git a/handlers/AccountOffers.cpp b/src/handlers/AccountOffers.cpp similarity index 100% rename from handlers/AccountOffers.cpp rename to src/handlers/AccountOffers.cpp diff --git a/handlers/AccountTx.cpp b/src/handlers/AccountTx.cpp similarity index 100% rename from handlers/AccountTx.cpp rename to src/handlers/AccountTx.cpp diff --git a/handlers/BookOffers.cpp b/src/handlers/BookOffers.cpp similarity index 100% rename from handlers/BookOffers.cpp rename to src/handlers/BookOffers.cpp diff --git a/handlers/ChannelAuthorize.cpp b/src/handlers/ChannelAuthorize.cpp similarity index 100% rename from handlers/ChannelAuthorize.cpp rename to src/handlers/ChannelAuthorize.cpp diff --git a/handlers/ChannelVerify.cpp b/src/handlers/ChannelVerify.cpp similarity index 100% rename from handlers/ChannelVerify.cpp rename to src/handlers/ChannelVerify.cpp diff --git a/handlers/Ledger.cpp b/src/handlers/Ledger.cpp similarity index 100% rename from handlers/Ledger.cpp rename to src/handlers/Ledger.cpp diff --git a/handlers/LedgerData.cpp b/src/handlers/LedgerData.cpp similarity index 100% rename from handlers/LedgerData.cpp rename to src/handlers/LedgerData.cpp diff --git a/handlers/LedgerEntry.cpp b/src/handlers/LedgerEntry.cpp similarity index 100% rename from handlers/LedgerEntry.cpp rename to src/handlers/LedgerEntry.cpp diff --git a/handlers/LedgerRange.cpp b/src/handlers/LedgerRange.cpp similarity index 100% rename from handlers/LedgerRange.cpp rename to src/handlers/LedgerRange.cpp diff --git a/handlers/RPCHelpers.cpp b/src/handlers/RPCHelpers.cpp similarity index 100% rename from handlers/RPCHelpers.cpp rename to src/handlers/RPCHelpers.cpp diff --git a/handlers/RPCHelpers.h b/src/handlers/RPCHelpers.h similarity index 100% rename from handlers/RPCHelpers.h rename to src/handlers/RPCHelpers.h diff --git a/handlers/ServerInfo.cpp b/src/handlers/ServerInfo.cpp similarity index 100% rename from handlers/ServerInfo.cpp rename to src/handlers/ServerInfo.cpp diff --git a/handlers/Subscribe.cpp b/src/handlers/Subscribe.cpp similarity index 100% rename from handlers/Subscribe.cpp rename to src/handlers/Subscribe.cpp diff --git a/handlers/Tx.cpp b/src/handlers/Tx.cpp similarity index 100% rename from handlers/Tx.cpp rename to src/handlers/Tx.cpp diff --git a/server/DOSGuard.h b/src/server/DOSGuard.h similarity index 100% rename from server/DOSGuard.h rename to src/server/DOSGuard.h diff --git a/server/Handlers.cpp b/src/server/Handlers.cpp similarity index 100% rename from server/Handlers.cpp rename to src/server/Handlers.cpp diff --git a/server/Handlers.h b/src/server/Handlers.h similarity index 100% rename from server/Handlers.h rename to src/server/Handlers.h diff --git a/server/HttpBase.h b/src/server/HttpBase.h similarity index 100% rename from server/HttpBase.h rename to src/server/HttpBase.h diff --git a/server/HttpSession.h b/src/server/HttpSession.h similarity index 100% rename from server/HttpSession.h rename to src/server/HttpSession.h diff --git a/server/PlainWsSession.h b/src/server/PlainWsSession.h similarity index 100% rename from server/PlainWsSession.h rename to src/server/PlainWsSession.h diff --git a/server/SslHttpSession.h b/src/server/SslHttpSession.h similarity index 100% rename from server/SslHttpSession.h rename to src/server/SslHttpSession.h diff --git a/server/SslWsSession.h b/src/server/SslWsSession.h similarity index 100% rename from server/SslWsSession.h rename to src/server/SslWsSession.h diff --git a/server/SubscriptionManager.cpp b/src/server/SubscriptionManager.cpp similarity index 100% rename from server/SubscriptionManager.cpp rename to src/server/SubscriptionManager.cpp diff --git a/server/SubscriptionManager.h b/src/server/SubscriptionManager.h similarity index 100% rename from server/SubscriptionManager.h rename to src/server/SubscriptionManager.h diff --git a/server/WsBase.h b/src/server/WsBase.h similarity index 100% rename from server/WsBase.h rename to src/server/WsBase.h diff --git a/server/listener.h b/src/server/listener.h similarity index 100% rename from server/listener.h rename to src/server/listener.h diff --git a/server/websocket_server_async.cpp b/src/server/main.cpp similarity index 86% rename from server/websocket_server_async.cpp rename to src/server/main.cpp index d4312b5c..f2bf90d0 100644 --- a/server/websocket_server_async.cpp +++ b/src/server/main.cpp @@ -119,7 +119,7 @@ main(int argc, char* argv[]) auto const config = parse_config(argv[1]); if (!config) { - std::cerr << "Ccouldnt parse config. Exiting..." << std::endl; + std::cerr << "Couldnt parse config. Exiting..." << std::endl; return EXIT_FAILURE; } initLogLevel(*config); @@ -133,24 +133,39 @@ main(int argc, char* argv[]) } BOOST_LOG_TRIVIAL(info) << "Number of workers = " << threads; + // io context to handle all incoming requests, as well as other things + // This is not the only io context in the application boost::asio::io_context ioc{threads}; + // Rate limiter, to prevent abuse DOSGuard dosGuard{config.value(), ioc}; + // Interface to the database std::shared_ptr backend{Backend::make_Backend(*config)}; + // Manages clients subscribed to streams std::shared_ptr subscriptions{ SubscriptionManager::make_SubscriptionManager()}; + // Tracks which ledgers have been validated by the + // network std::shared_ptr ledgers{ NetworkValidatedLedgers::make_ValidatedLedgers()}; + // Handles the connection to one or more rippled nodes. + // ETL uses the balancer to extract data. + // The server uses the balancer to forward RPCs to a rippled node. + // The balancer itself publishes to streams (transactions_proposed and + // accounts_proposed) auto balancer = ETLLoadBalancer::make_ETLLoadBalancer( *config, ioc, backend, subscriptions, ledgers); + // ETL is responsible for writing and publishing to streams. In read-only + // mode, ETL only publishes auto etl = ReportingETL::make_ReportingETL( *config, ioc, backend, subscriptions, balancer, ledgers); + // The server handles incoming RPCs auto httpServer = Server::make_HttpServer( *config, ioc, backend, subscriptions, balancer, dosGuard); diff --git a/test.py b/test.py index 9b821a46..a6dab041 100755 --- a/test.py +++ b/test.py @@ -681,7 +681,7 @@ import pathlib numCalls = 0 async def ledgers(ip, port, minLedger, maxLedger, transactions, expand, maxCalls): global numCalls - address = 'ws://' + str(ip) + ':' + str(port) + address = 'wss://' + str(ip) + ':' + str(port) random.seed() ledger = 0 ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) @@ -690,7 +690,7 @@ async def ledgers(ip, port, minLedger, maxLedger, transactions, expand, maxCalls ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE try: - async with websockets.connect(address,max_size=1000000000) as ws: + async with websockets.connect(address,max_size=1000000000,ssl=ssl_context) as ws: global numCalls for i in range(0, maxCalls):