Add Reporting Mode

* Add a new operating mode to rippled called reporting mode
* Add ETL mechanism for a reporting node to extract data from a p2p node
* Add new gRPC methods to faciliate ETL
* Use Postgres in place of SQLite in reporting mode
* Add Cassandra as a nodestore option
* Update logic of RPC handlers when running in reporting mode
* Add ability to forward RPCs to a p2p node

Builds/CMake/RippledCore.cmake

@@ -3,7 +3,6 @@
    core functionality, useable by some client software perhaps
 #]===================================================================]
 
-
 file (GLOB_RECURSE rb_headers
   src/ripple/beast/*.h
   src/ripple/beast/*.hpp)
@@ -14,6 +13,7 @@ if (unity)
   set_target_properties(xrpl_core PROPERTIES UNITY_BUILD ON)
 endif ()
 
+
 #[===============================[
     beast/legacy FILES:
     TODO: review these sources for removal or replacement
@@ -126,6 +126,7 @@ target_include_directories (xrpl_core
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/beast/extras>
     $<INSTALL_INTERFACE:include>)
 
+
 target_compile_definitions(xrpl_core
   PUBLIC
     BOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT
@@ -366,6 +367,10 @@ target_sources (rippled PRIVATE
   src/ripple/app/main/Main.cpp
   src/ripple/app/main/NodeIdentity.cpp
   src/ripple/app/main/NodeStoreScheduler.cpp
+  src/ripple/app/reporting/DBHelpers.cpp
+  src/ripple/app/reporting/ReportingETL.cpp
+  src/ripple/app/reporting/ETLSource.cpp
+  src/ripple/app/reporting/P2pProxy.cpp
   src/ripple/app/misc/CanonicalTXSet.cpp
   src/ripple/app/misc/FeeVoteImpl.cpp
   src/ripple/app/misc/HashRouter.cpp
@@ -453,6 +458,7 @@ target_sources (rippled PRIVATE
   src/ripple/core/impl/Stoppable.cpp
   src/ripple/core/impl/TimeKeeper.cpp
   src/ripple/core/impl/Workers.cpp
+  src/ripple/core/Pg.cpp
   #[===============================[
      main sources:
        subdir: consensus
@@ -494,6 +500,7 @@ target_sources (rippled PRIVATE
      main sources:
        subdir: nodestore
   #]===============================]
+  src/ripple/nodestore/backend/CassandraFactory.cpp
   src/ripple/nodestore/backend/MemoryFactory.cpp
   src/ripple/nodestore/backend/NuDBFactory.cpp
   src/ripple/nodestore/backend/NullFactory.cpp
@@ -554,7 +561,6 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/AccountOffers.cpp
   src/ripple/rpc/handlers/AccountTx.cpp
   src/ripple/rpc/handlers/AccountTxOld.cpp
-  src/ripple/rpc/handlers/AccountTxSwitch.cpp
   src/ripple/rpc/handlers/BlackList.cpp
   src/ripple/rpc/handlers/BookOffers.cpp
   src/ripple/rpc/handlers/CanDelete.cpp
@@ -573,6 +579,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/LedgerClosed.cpp
   src/ripple/rpc/handlers/LedgerCurrent.cpp
   src/ripple/rpc/handlers/LedgerData.cpp
+  src/ripple/rpc/handlers/LedgerDiff.cpp
   src/ripple/rpc/handlers/LedgerEntry.cpp
   src/ripple/rpc/handlers/LedgerHandler.cpp
   src/ripple/rpc/handlers/LedgerHeader.cpp
@@ -920,6 +927,7 @@ target_sources (rippled PRIVATE
   src/test/rpc/NoRipple_test.cpp
   src/test/rpc/OwnerInfo_test.cpp
   src/test/rpc/Peers_test.cpp
+  src/test/rpc/ReportingETL_test.cpp
   src/test/rpc/Roles_test.cpp
   src/test/rpc/RPCCall_test.cpp
   src/test/rpc/RPCOverload_test.cpp
@@ -967,6 +975,10 @@ if (is_ci)
   target_compile_definitions(rippled PRIVATE RIPPLED_RUNNING_IN_CI)
 endif ()
 
+if (reporting)
+    target_compile_definitions(rippled PRIVATE RIPPLED_REPORTING)
+endif ()
+
 if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.16)
   # any files that don't play well with unity should be added here
   set_source_files_properties(

Builds/CMake/RippledSettings.cmake

@@ -4,6 +4,8 @@
 
 option (assert "Enables asserts, even in release builds" OFF)
 
+option (reporting "Build rippled with reporting mode enabled" OFF)
+
 option (unity "Creates a build using UNITY support in cmake. This is the default" ON)
 if (unity)
   if (CMAKE_VERSION VERSION_LESS 3.16)

Builds/CMake/deps/Postgres.cmake

@@ -0,0 +1,68 @@
+
+if(reporting)
+
+    find_package(PostgreSQL)
+
+    if(NOT PostgreSQL_FOUND)
+            message("find_package did not find postgres")
+
+
+            find_library(postgres NAMES pq libpq libpq-dev pq-dev postgresql-devel)
+            find_path(libpq-fe NAMES libpq-fe.h PATH_SUFFIXES postgresql pgsql include)
+        if(NOT libpq-fe_FOUND OR NOT postgres_FOUND)
+            message("No system installed Postgres found. Will build")
+
+            add_library(postgres SHARED IMPORTED GLOBAL)
+            ExternalProject_Add(postgres_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/postgres/postgres.git
+                GIT_TAG master
+                CONFIGURE_COMMAND ./configure --without-readline > /dev/null
+                BUILD_COMMAND ${CMAKE_COMMAND} -E env --unset=MAKELEVEL make
+                UPDATE_COMMAND ""
+                BUILD_IN_SOURCE 1
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <BINARY_DIR>/src/interfaces/libpq/${ep_lib_prefix}pq.so
+                LOG_BUILD TRUE
+                )
+
+
+
+            ExternalProject_Get_Property (postgres_src SOURCE_DIR)
+            ExternalProject_Get_Property (postgres_src BINARY_DIR)
+            set (postgres_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${postgres_src_SOURCE_DIR})
+
+            list(APPEND INCLUDE_DIRS ${SOURCE_DIR}/src/include)
+            list(APPEND INCLUDE_DIRS ${SOURCE_DIR}/src/interfaces/libpq)
+
+
+            set_target_properties (postgres PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/src/interfaces/libpq/${ep_lib_prefix}pq.so
+                INTERFACE_INCLUDE_DIRECTORIES
+                "${INCLUDE_DIRS}")
+            add_dependencies(postgres postgres_src)
+
+            file(TO_CMAKE_PATH "${postgres_src_SOURCE_DIR}" postgres_src_SOURCE_DIR)
+
+            target_link_libraries(ripple_libs INTERFACE postgres)
+        else()
+
+            message("Found system installed Postgres via find_libary")
+
+            target_include_directories(ripple_libs INTERFACE ${libpq-fe})
+
+            target_link_libraries(ripple_libs INTERFACE ${postgres})
+        endif()
+
+    else()
+        message("Found system installed Postgres via find_package")
+
+        target_include_directories(ripple_libs INTERFACE ${PostgreSQL_INCLUDE_DIRS})
+
+        target_link_libraries(ripple_libs INTERFACE ${PostgreSQL_LIBRARIES})
+    endif()
+endif()
+
+

Builds/CMake/deps/cassandra.cmake

@@ -0,0 +1,165 @@
+if(reporting)
+    find_library(cassandra NAMES cassandra)
+    if(NOT cassandra)
+
+        message("System installed Cassandra cpp driver not found. Will build")
+
+        find_library(zlib NAMES zlib1g-dev zlib-devel zlib z)
+        if(NOT zlib)
+            message("zlib not found. will build")
+            add_library(zlib STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(zlib_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/madler/zlib.git
+                GIT_TAG master
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}z.a
+                LOG_BUILD TRUE
+                LOG_CONFIGURE TRUE
+                )
+
+
+            ExternalProject_Get_Property (zlib_src SOURCE_DIR)
+            ExternalProject_Get_Property (zlib_src BINARY_DIR)
+            set (zlib_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${zlib_src_SOURCE_DIR}/include)
+
+            set_target_properties (zlib PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/${ep_lib_prefix}z.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(zlib zlib_src)
+
+            file(TO_CMAKE_PATH "${zlib_src_SOURCE_DIR}" zlib_src_SOURCE_DIR)
+        endif()
+
+
+
+
+        find_library(krb5 NAMES krb5-dev libkrb5-dev)
+
+        if(NOT krb5)
+            message("krb5 not found. will build")
+            add_library(krb5 STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(krb5_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/krb5/krb5.git
+                GIT_TAG master
+                UPDATE_COMMAND ""
+                CONFIGURE_COMMAND autoreconf src && ./src/configure --enable-static --disable-shared > /dev/null
+                BUILD_IN_SOURCE 1
+                BUILD_COMMAND make
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <SOURCE_DIR>/lib/${ep_lib_prefix}krb5.a
+                LOG_BUILD TRUE
+                )
+
+            ExternalProject_Get_Property (krb5_src SOURCE_DIR)
+            ExternalProject_Get_Property (krb5_src BINARY_DIR)
+            set (krb5_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${krb5_src_SOURCE_DIR}/include)
+
+            set_target_properties (krb5 PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/lib/${ep_lib_prefix}krb5.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(krb5 krb5_src)
+
+            file(TO_CMAKE_PATH "${krb5_src_SOURCE_DIR}" krb5_src_SOURCE_DIR)
+        endif()
+
+
+        find_library(libuv1 NAMES uv1 libuv1 liubuv1-dev libuv1:amd64)
+
+
+        if(NOT libuv1)
+            message("libuv1 not found, will build")
+            add_library(libuv1 STATIC IMPORTED GLOBAL)
+            ExternalProject_Add(libuv_src
+                PREFIX ${nih_cache_path}
+                GIT_REPOSITORY https://github.com/libuv/libuv.git
+                GIT_TAG v1.x
+                INSTALL_COMMAND ""
+                BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}uv_a.a
+                LOG_BUILD TRUE
+                LOG_CONFIGURE TRUE
+                )
+
+            ExternalProject_Get_Property (libuv_src SOURCE_DIR)
+            ExternalProject_Get_Property (libuv_src BINARY_DIR)
+            set (libuv_src_SOURCE_DIR "${SOURCE_DIR}")
+            file (MAKE_DIRECTORY ${libuv_src_SOURCE_DIR}/include)
+
+            set_target_properties (libuv1 PROPERTIES
+                IMPORTED_LOCATION
+                ${BINARY_DIR}/${ep_lib_prefix}uv_a.a
+                INTERFACE_INCLUDE_DIRECTORIES
+                ${SOURCE_DIR}/include)
+            add_dependencies(libuv1 libuv_src)
+
+            file(TO_CMAKE_PATH "${libuv_src_SOURCE_DIR}" libuv_src_SOURCE_DIR)
+        endif()
+
+        add_library (cassandra STATIC IMPORTED GLOBAL)
+        ExternalProject_Add(cassandra_src
+            PREFIX ${nih_cache_path}
+            GIT_REPOSITORY https://github.com/datastax/cpp-driver.git
+            GIT_TAG master
+            CMAKE_ARGS
+            -DLIBUV_ROOT_DIR=${BINARY_DIR}
+            -DLIBUV_LIBARY=${BINARY_DIR}/libuv_a.a
+            -DLIBUV_INCLUDE_DIR=${SOURCE_DIR}/include
+            -DCASS_BUILD_STATIC=ON
+            INSTALL_COMMAND ""
+            BUILD_BYPRODUCTS <BINARY_DIR>/${ep_lib_prefix}cassandra_static.a
+            LOG_BUILD TRUE
+            LOG_CONFIGURE TRUE
+            )
+
+        ExternalProject_Get_Property (cassandra_src SOURCE_DIR)
+        ExternalProject_Get_Property (cassandra_src BINARY_DIR)
+        set (cassandra_src_SOURCE_DIR "${SOURCE_DIR}")
+        file (MAKE_DIRECTORY ${cassandra_src_SOURCE_DIR}/include)
+
+        set_target_properties (cassandra PROPERTIES
+            IMPORTED_LOCATION
+            ${BINARY_DIR}/${ep_lib_prefix}cassandra_static.a
+            INTERFACE_INCLUDE_DIRECTORIES
+            ${SOURCE_DIR}/include)
+        add_dependencies(cassandra cassandra_src)
+
+        if(NOT libuv1)
+            ExternalProject_Add_StepDependencies(cassandra_src build libuv1)
+            target_link_libraries(cassandra INTERFACE libuv1)
+        else()
+            target_link_libraries(cassandra INTERFACE ${libuv1})
+        endif()
+        if(NOT krb5)
+
+            ExternalProject_Add_StepDependencies(cassandra_src build krb5)
+            target_link_libraries(cassandra INTERFACE krb5)
+        else()
+            target_link_libraries(cassandra INTERFACE ${krb5})
+        endif()
+
+        if(NOT zlib)
+            ExternalProject_Add_StepDependencies(cassandra_src build zlib)
+            target_link_libraries(cassandra INTERFACE zlib)
+        else()
+            target_link_libraries(cassandra INTERFACE ${zlib})
+        endif()
+
+        file(TO_CMAKE_PATH "${cassandra_src_SOURCE_DIR}" cassandra_src_SOURCE_DIR)
+        target_link_libraries(ripple_libs INTERFACE cassandra)
+    else()
+        message("Found system installed cassandra cpp driver")
+
+        find_path(cassandra_includes NAMES cassandra.h REQUIRED)
+        target_link_libraries (ripple_libs INTERFACE ${cassandra})
+        target_include_directories(ripple_libs INTERFACE ${cassandra_includes})
+    endif()
+
+    exclude_if_included (cassandra)
+endif()